locked
Comparison 2 xml file got out of memory if file size exist more than 100MB RRS feed

  • Question

  • User-893002196 posted

    Hi All,

    If my both XML File Less than 100 MB it's work fine. But when my size(S) are 300MB. I got out of Memory message for below code:

    public string DoCompare(string file1, string file2)
            {
                string finalPath = "";
                if (MessageBox.Show("Do you want to continue compare, please wait until loading bar 100% and output will display in IE browser automatically?", "", MessageBoxButtons.YesNo, MessageBoxIcon.Asterisk) == DialogResult.Yes)
                {
                    //Wait 100 milliseconds.
                    //Thread.Sleep(100);
    
                    int i = 20;
    
                    
                    // Report progress.
                    backgroundWorker1.ReportProgress(i);
                    
                    
                    var newXML01 = Path.GetFileName(file1).Replace(".xml", "_1.xml");
                    var newXML02 = Path.GetFileName(file2).Replace(".xml", "_1.xml");
    
                    XmlDocument xml = new XmlDocument();
                    XmlElement root = xml.CreateElement("files");
                    xml.AppendChild(root);
    
                    var CompareViewerLocation = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\TestingTool\\CompareViewer";
                    if (Properties.Settings.Default.CompareViewerLocation == "")
                    {
                        CompareViewerLocation = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\TestingTool\\CompareViewer";
                    }
                    else
                    {
                        CompareViewerLocation = Properties.Settings.Default.CompareViewerLocation;
                    }
    
    
                    XmlElement child = xml.CreateElement("file");
                    child.SetAttribute("href", newXML01);
                    root.AppendChild(child);
    
                    XmlElement child2 = xml.CreateElement("file");
                    child2.SetAttribute("href", newXML02);                
                   
    
                    root.AppendChild(child);
                    root.AppendChild(child2);
                  
    
                   
                    string mon = DateTime.Now.Month.ToString("00").PadRight(2);
                    string day = DateTime.Now.Day.ToString("00").PadRight(2);
                    string yr = DateTime.Now.Year.ToString(); 
    
                    var versions=mon + "" + day + "" + yr;
                    var fileXMLCompare=System.IO.Path.Combine(CompareViewerLocation, "compares.xml");
                    xml.Save(@""+fileXMLCompare);
    
                    i = 45;
                    // Report progress.
                    backgroundWorker1.ReportProgress(i);
    
                    
                    //StreamWriter sw = File.AppendText(@"" + file1);
                    StreamReader streamReader = new StreamReader(@"" + file1);
                    StreamWriter streamWriter = new StreamWriter(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML01));
    
                    //var data=stripDocumentNamespace();
    
                    while (!streamReader.EndOfStream)
                    {
                        string data = streamReader.ReadLine();
                        //data = data.Replace("DataSet1", "DataSet1 xmlns:msdata=\"http://diffgr\" xmlns:diffgr=\"http://diffgr\"");
                        data = data.Replace(" xmlns=\"http://tempuri.org/DataSet1.xsd\"", "").Replace("<diffgr:diffgram xmlns:msdata=\"urn:schemas-microsoft-com:xml-msdata\" xmlns:diffgr=\"urn:schemas-microsoft-com:xml-diffgram-v1\">", "").Replace("</diffgr:diffgram>", "").Replace("diffgr:", "").Replace(" diffgr:", "").Replace("msdata:", "");
                        streamWriter.WriteLine(data);
    
                    }
    
                    streamReader.Close();
                    streamWriter.Close();
                    streamReader.Dispose();
                    streamWriter.Dispose();
    
                    System.Xml.XmlDocument xdoc = new System.Xml.XmlDocument();
                    System.Xml.XmlDocument xdocResult = new System.Xml.XmlDocument();
                    xdoc.Load(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML01));
                    System.Xml.XmlNode ParentNew = xdocResult.CreateNode(System.Xml.XmlNodeType.Element, "DataSet1", string.Empty);
    
                    foreach (System.Xml.XmlNode xnode in xdoc.GetElementsByTagName("DataSet1"))
                    {
                        foreach (System.Xml.XmlNode xchild in xnode.ChildNodes)
                        {
                            xchild.Attributes.RemoveAll();
                            if (xchild.LocalName == "Revenue" || xchild.LocalName == "Currency" || xchild.LocalName == "TargetBudget" || xchild.LocalName == "OperatingExpense" || xchild.LocalName == "RevenueByYear") 
                            {
                                xchild.ParentNode.RemoveChild(xchild);
    
                            }
                            System.Xml.XmlNode importNode = ParentNew.OwnerDocument.ImportNode(xchild, true);
                            ParentNew.AppendChild(importNode);
                        }
                    }
                    xdocResult.AppendChild(ParentNew);
                    xdocResult.Save(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML01));
    
                    i = 65;
                    // Report progress.
                    backgroundWorker1.ReportProgress(i);
    
                    StreamReader streamReader2 = new StreamReader(@"" + file2);
                    StreamWriter streamWriter2 = new StreamWriter(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML02));
    
    
    
                    while (!streamReader2.EndOfStream)
                    {    
                        string data2 = streamReader2.ReadLine();
                        //data2=data2.Replace("DataSet1","DataSet1 xmlns:msdata=\"http://diffgr\" xmlns:diffgr=\"http://diffgr\"");
                        data2 = data2.Replace(" xmlns=\"http://tempuri.org/DataSet1.xsd\"", "").Replace("<diffgr:diffgram xmlns:msdata=\"urn:schemas-microsoft-com:xml-msdata\" xmlns:diffgr=\"urn:schemas-microsoft-com:xml-diffgram-v1\">", "").Replace("</diffgr:diffgram>", "").Replace("diffgr:", "").Replace(" diffgr:", "").Replace("msdata:", "");
                        streamWriter2.WriteLine(data2);
                    }
    
    
                    streamReader2.Close();
                    streamWriter2.Close();
                    streamReader2.Dispose();
                    streamWriter2.Dispose();
    
                                     
                    int newID = 1;
    
    
                    System.Xml.XmlDocument xdoc2 = new System.Xml.XmlDocument();
                    System.Xml.XmlDocument xdocResult2 = new System.Xml.XmlDocument();
                    xdoc2.Load(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML02));
                    System.Xml.XmlNode ParentNew2 = xdocResult2.CreateNode(System.Xml.XmlNodeType.Element, "DataSet1", string.Empty);
    
                    foreach (System.Xml.XmlNode xnode in xdoc2.GetElementsByTagName("DataSet1"))
                    {
                        foreach (System.Xml.XmlNode xchild in xnode.ChildNodes)
                        {
                            xchild.Attributes.RemoveAll();
                            if (xchild.LocalName == "Revenue" || xchild.LocalName == "Currency" || xchild.LocalName == "TargetBudget" || xchild.LocalName == "OperatingExpense" || xchild.LocalName == "RevenueByYear")
                            {
                                xchild.ParentNode.RemoveChild(xchild);
                            }
                            System.Xml.XmlNode importNode = ParentNew2.OwnerDocument.ImportNode(xchild, true);
                            ParentNew2.AppendChild(importNode);
                        }
                    }
                    xdocResult2.AppendChild(ParentNew2);
                    xdocResult2.Save(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML02));
    
                    i = 80;
                    // Report progress.
                    backgroundWorker1.ReportProgress(i);
                   
                    //Linq compare
                    XDocument doc1 = XDocument.Load(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML01));
                    XDocument doc2 = XDocument.Load(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML02));
                    var leftOuterJoin = (from opportunityOld in doc1.Descendants("OpportunityElement")
                                         join opportunityNew in doc2.Descendants("OpportunityElement") on opportunityOld.Element("referencenumber").Value equals opportunityNew.Element("referencenumber").Value
                                         into temp
                                         from opportunityNew in temp.DefaultIfEmpty()
                                         select new
                                         {
                                             Ref = (string)opportunityOld.Element("referencenumber"),
                                             Ostartdate = (string)opportunityOld.Element("startdate"),
                                             Oaccountmanager = (string)opportunityOld.Element("accountmanager"),                                         
                                             Ncos = " ",
                                         });
                   
                    var rightOuterJoin = (from opportunityNew in doc2.Descendants("OpportunityElement")
                                          join opportunityOld in doc1.Descendants("OpportunityElement") on opportunityNew.Element("referencenumber").Value equals opportunityOld.Element("referencenumber").Value
                                          into temp
                                          from opportunityOld in temp.DefaultIfEmpty()
                                          select new
                                          {
    
                                              Ref = (string)opportunityNew.Element("referencenumber"),
                                              Ostartdate = " ",
                                              Oaccountmanager = " ",                                         
                                              Ncos = (string)opportunityNew.Element("cos"),
                                          });
    
                   
                    var joins = leftOuterJoin.Union(rightOuterJoin);
                    var output = (from e in joins
                                  group e by e.Ref into r
                                  select new
                                  {
                                      NStartDate = r.Max(d => d.Nstartdate),
                                      Oaccountmanager = r.Max(d => d.Oaccountmanager),                                  
                                      NCos = r.Max(d => d.Ncos),
                                      Remark = (r.Max(d => d.Nstartdate) == " " && r.Max(d => d.Ncurrencycode) == " " && r.Max(d => d.Nname) == " " && r.Max(d => d.Naccountmanager) == " " && r.Max(d => d.Nfeetypename) == " " && r.Max(d => d.Nfees) == " " && r.Max(d => d.Nduration) == " " && r.Max(d => d.Ncos) == " ") ? (string)"Deleted" : (r.Max(d => d.Ostartdate) == " " && r.Max(d => d.Ocurrencycode) == " " && r.Max(d => d.Oname) == " " && r.Max(d => d.Oaccountmanager) == " " && r.Max(d => d.Ofeetypename) == " " && r.Max(d => d.Ofees) == " " && r.Max(d => d.Oduration) == " " && r.Max(d => d.Ocos) == " ") ? (string)"Created" : (r.Max(d => d.Ostartdate) == r.Max(d => d.Nstartdate) && r.Max(d => d.Ocurrencycode) == r.Max(d => d.Ncurrencycode) && r.Max(d => d.Oname) == r.Max(d => d.Nname) && r.Max(d => d.Oaccountmanager) == r.Max(d => d.Naccountmanager) && r.Max(d => d.Ofeetypename) == r.Max(d => d.Nfeetypename) && r.Max(d => d.Ofees) == r.Max(d => d.Nfees) && r.Max(d => d.Oduration) == r.Max(d => d.Nduration) && r.Max(d => d.Ocos) == r.Max(d => d.Ncos)) ? (string)"No Changes" : "Modified",
                                  }).Where(s => !s.Remark.Equals("No Changes")).ToList();
    
                   
    
                    //..............................Opportunity
                    var leftOuterJoin2 = (from opportunityOld in doc1.Descendants("Opportunity")
                                          join opportunityNew in doc2.Descendants("Opportunity") on opportunityOld.Element("referencenumber").Value equals opportunityNew.Element("referencenumber").Value
                                          into temp
                                          from opportunityNew in temp.DefaultIfEmpty()
                                          select new
                                          {
                                              Ref = (string)opportunityOld.Element("referencenumber"),
                                              Oopportunityid = (string)opportunityOld.Element("opportunityid"),
                                              Ntotalfees = " ",
                                              NTrafficStatus = " "
                                          });
                
                    var rightOuterJoin2 = (from opportunityNew in doc2.Descendants("Opportunity")
                                           join opportunityOld in doc1.Descendants("Opportunity") on opportunityNew.Element("referencenumber").Value equals opportunityOld.Element("referencenumber").Value
                                           into temp
                                           from opportunityOld in temp.DefaultIfEmpty()
                                           select new
                                           {
    
                                               Ref = (string)opportunityNew.Element("referencenumber"),
                                               Oopportunityid = " ",                                          
                                               Ntotalfees = (string)opportunityNew.Element("totalfees"),
                                               NTrafficStatus = (string)opportunityNew.Element("TrafficStatus")
                                           });
    
                    var joins2 = leftOuterJoin2.Union(rightOuterJoin2);
                    var output2 = (from e in joins2
                                   group e by e.Ref into r
                                   select new
                                   {
                                       Nopportunityid = r.Max(d => d.Nopportunityid),
                                       Oopportunityid = r.Max(d => d.Oopportunityid),
                                       Nquotecurrencycode = r.Max(d => d.Nquotecurrencycode),                                   
                                       Naccountmanager = r.Max(d => d.Naccountmanager),
                                       Oaccountmanager = r.Max(d => d.Oaccountmanager),                                  
                                       Ntotalfees = r.Max(d => d.Ntotalfees),
                                       OTrafficStatus = r.Max(d => d.OTrafficStatus),
                                       NTrafficStatus = r.Max(d => d.NTrafficStatus),
                                       Ref = r.Key,
                                       Remark = (r.Max(d => d.Nname) == " " && r.Max(d => d.Nquotecurrencycode) == " ") ? (string)"Deleted" : (r.Max(d => d.Oquotecurrencycode) == " " && r.Max(d => d.Oname) == " ") ? (string)"Created" : (r.Max(d => d.Oquotecurrencycode) == r.Max(d => d.Nquotecurrencycode) && r.Max(d => d.Oname) == r.Max(d => d.Nname) && r.Max(d => d.Oclosedate) == r.Max(d => d.Nclosedate)) ? (string)"No Changes" : "Modified",
                                   }).Where(s => !s.Remark.Equals("No Changes")).ToList();
    
    
                    var Diff1 = new XElement("Root", new XElement("Opportunities",
                                from c1 in output
                                 select ((c1.OCos != c1.NCos) || (c1.OName != c1.NName) || (c1.Oaccountmanager != c1.Naccountmanager)  || (c1.OFees != c1.NFees)) ? new XElement("DealElement",
                                        new XElement("Remark", c1.Remark),
                                       ((c1.OCos != c1.NCos) || (c1.Oaccountmanager != c1.Naccountmanager) || (c1.OName != c1.NName)  || (c1.OFees != c1.NFees)) ? new XElement("Ref", c1.Ref) : null,
                                       (c1.OCos != c1.NCos && c1.Remark == "Created") ? new XElement("Name", c1.NName) : new XElement("Name", c1.OName),
                                       (c1.OCos != c1.NCos && c1.Remark == "Created") ? new XElement("AccountManager", c1.Naccountmanager) : new XElement("AccountManager", c1.Oaccountmanager),
                                       (c1.OCos != c1.NCos && c1.Remark == "Created") ? new XElement("Customer", c1.Ncustomer) : new XElement("Customer", c1.Ocustomer),
                                       (c1.OCos != c1.NCos && c1.Remark == "Modified") ? new XElement("Change", new XAttribute("Id", newID + 0),
                                       (c1.OCos != c1.NCos && c1.Remark == "Modified") ? new XElement("Att", "cos") : null,
                                       (c1.OCos != c1.NCos && c1.Remark == "Modified") ? new XElement("Before", c1.OCos) : null,
                                       (c1.OCos != c1.NCos && c1.Remark == "Modified") ? new XElement("After", c1.NCos) : null) : null,
    
                                      :
    				  :
                                from c in output2
                                select ((c.Oquotecurrencycode != c.Nquotecurrencycode)  || (c.Oclosedate != c.Nclosedate)) ? new XElement("Opportunity",
                                        new XElement("Remark", c.Remark),
                                       ((c.Oquotecurrencycode != c.Nquotecurrencycode) || (c.Oaccountmanager != c.Naccountmanager)  || (c.Oclosedate != c.Nclosedate)) ? new XElement("Ref", c.Ref) : null,
                                       (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Created") ? new XElement("Name", c.NName) : new XElement("Name", c.OName),
                                       (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Created") ? new XElement("AccountManager", c.Naccountmanager) : new XElement("AccountManager", c.Oaccountmanager),
                                       (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Created") ? new XElement("Status", c.Nstatuscodename) : new XElement("Status", c.Ostatuscodename),
                                       (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Created") ? new XElement("Customer", c.Ncustomer) : new XElement("Customer", c.Ocustomer),
                                       (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Modified") ? new XElement("Change", new XAttribute("Id", newID + 0),
                                       (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Modified") ? new XElement("Att", "quotecurrencycode") : null,
                                       (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Modified") ? new XElement("Before", c.Oquotecurrencycode) : null,
                                       (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Modified") ? new XElement("After", c.Nquotecurrencycode) : null) : nul,
    					:
    					:
    				   ));
                   
                                     
                    var comparisonXMLXSLTheader = new XDocument(new XProcessingInstruction(
                     "xml-stylesheet",
                     "type='text/xsl' href='ResultViewer2.xsl'"),
                     Diff1);
    
                    i = 90;
                    // Report progress.
                    backgroundWorker1.ReportProgress(i);
    
                    var mypath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\TestingTool\\CompareViewer";
                    string filename = mypath + "\\ComparisonSnapshot_" + DateTime.Now.ToString("yyyyMMddhhmmssfff") + ".xml";
                    finalPath = mypath + "\\ComparisonSnapshot_" + DateTime.Now.ToString("yyyyMMddhhmmssfff") + ".xml";
                    comparisonXMLXSLTheader.Save(@"" + finalPath);
    
                    i = 95;
                    // Report progress.
                    backgroundWorker1.ReportProgress(i);
                   
                    
                    File.Delete(@"" + file1.Replace(".xml", "_1.xml"));
                    File.Delete(@"" + file2.Replace(".xml", "_2.xml"));
                                    
                }
                return finalPath.ToString();
            }

    Can anyone advise me please how to avoid this message & stop working? I read from post, someone suggest used XMLReader. But how can I convert my code to be XMLReader method?

    Thank you.

    Regards,

    Micheale

    Thursday, July 10, 2014 10:01 PM

Answers

All replies

  • User-1716253493 posted

    Try increase MaxRequestLength value in httpruntime section in web.config

    <httpRuntime maxRequestLength="12345678910" />

    Also try increase MaxAllowedContent lenght under System.WebServer

      <system.webServer>
        <security>
            <requestLimits maxAllowedContentLength="12345678910"/>
          </requestFiltering>
        </security>
      </system.webServer>

    Thursday, July 10, 2014 10:12 PM
  • User-893002196 posted

    Hi,

    This is .exe program writing in C#. Not Web. So I do not have we.config.

    Please advise.

    Thank you.

    Regards,

    Micheale

    Thursday, July 10, 2014 10:52 PM
  • User-1716253493 posted

    I guess this is wrong place to ask

    Ask it in http://social.msdn.microsoft.com/Forums/windows/en-US/home?category=windowsforms

     

    • Marked as answer by Anonymous Thursday, October 7, 2021 12:00 AM
    Thursday, July 10, 2014 11:04 PM