Comparison 2 xml file got out of memory if file size exist more than 100MB

Question

User-893002196 posted

Hi All,

If my both XML File Less than 100 MB it's work fine. But when my size(S) are 300MB. I got out of Memory message for below code:

public string DoCompare(string file1, string file2)
        {
            string finalPath = "";
            if (MessageBox.Show("Do you want to continue compare, please wait until loading bar 100% and output will display in IE browser automatically?", "", MessageBoxButtons.YesNo, MessageBoxIcon.Asterisk) == DialogResult.Yes)
            {
                //Wait 100 milliseconds.
                //Thread.Sleep(100);

                int i = 20;

                
                // Report progress.
                backgroundWorker1.ReportProgress(i);
                
                
                var newXML01 = Path.GetFileName(file1).Replace(".xml", "_1.xml");
                var newXML02 = Path.GetFileName(file2).Replace(".xml", "_1.xml");

                XmlDocument xml = new XmlDocument();
                XmlElement root = xml.CreateElement("files");
                xml.AppendChild(root);

                var CompareViewerLocation = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\TestingTool\\CompareViewer";
                if (Properties.Settings.Default.CompareViewerLocation == "")
                {
                    CompareViewerLocation = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\TestingTool\\CompareViewer";
                }
                else
                {
                    CompareViewerLocation = Properties.Settings.Default.CompareViewerLocation;
                }


                XmlElement child = xml.CreateElement("file");
                child.SetAttribute("href", newXML01);
                root.AppendChild(child);

                XmlElement child2 = xml.CreateElement("file");
                child2.SetAttribute("href", newXML02);                
               

                root.AppendChild(child);
                root.AppendChild(child2);
              

               
                string mon = DateTime.Now.Month.ToString("00").PadRight(2);
                string day = DateTime.Now.Day.ToString("00").PadRight(2);
                string yr = DateTime.Now.Year.ToString(); 

                var versions=mon + "" + day + "" + yr;
                var fileXMLCompare=System.IO.Path.Combine(CompareViewerLocation, "compares.xml");
                xml.Save(@""+fileXMLCompare);

                i = 45;
                // Report progress.
                backgroundWorker1.ReportProgress(i);

                
                //StreamWriter sw = File.AppendText(@"" + file1);
                StreamReader streamReader = new StreamReader(@"" + file1);
                StreamWriter streamWriter = new StreamWriter(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML01));

                //var data=stripDocumentNamespace();

                while (!streamReader.EndOfStream)
                {
                    string data = streamReader.ReadLine();
                    //data = data.Replace("DataSet1", "DataSet1 xmlns:msdata=\"http://diffgr\" xmlns:diffgr=\"http://diffgr\"");
                    data = data.Replace(" xmlns=\"http://tempuri.org/DataSet1.xsd\"", "").Replace("<diffgr:diffgram xmlns:msdata=\"urn:schemas-microsoft-com:xml-msdata\" xmlns:diffgr=\"urn:schemas-microsoft-com:xml-diffgram-v1\">", "").Replace("</diffgr:diffgram>", "").Replace("diffgr:", "").Replace(" diffgr:", "").Replace("msdata:", "");
                    streamWriter.WriteLine(data);

                }

                streamReader.Close();
                streamWriter.Close();
                streamReader.Dispose();
                streamWriter.Dispose();

                System.Xml.XmlDocument xdoc = new System.Xml.XmlDocument();
                System.Xml.XmlDocument xdocResult = new System.Xml.XmlDocument();
                xdoc.Load(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML01));
                System.Xml.XmlNode ParentNew = xdocResult.CreateNode(System.Xml.XmlNodeType.Element, "DataSet1", string.Empty);

                foreach (System.Xml.XmlNode xnode in xdoc.GetElementsByTagName("DataSet1"))
                {
                    foreach (System.Xml.XmlNode xchild in xnode.ChildNodes)
                    {
                        xchild.Attributes.RemoveAll();
                        if (xchild.LocalName == "Revenue" || xchild.LocalName == "Currency" || xchild.LocalName == "TargetBudget" || xchild.LocalName == "OperatingExpense" || xchild.LocalName == "RevenueByYear") 
                        {
                            xchild.ParentNode.RemoveChild(xchild);

                        }
                        System.Xml.XmlNode importNode = ParentNew.OwnerDocument.ImportNode(xchild, true);
                        ParentNew.AppendChild(importNode);
                    }
                }
                xdocResult.AppendChild(ParentNew);
                xdocResult.Save(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML01));

                i = 65;
                // Report progress.
                backgroundWorker1.ReportProgress(i);

                StreamReader streamReader2 = new StreamReader(@"" + file2);
                StreamWriter streamWriter2 = new StreamWriter(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML02));



                while (!streamReader2.EndOfStream)
                {    
                    string data2 = streamReader2.ReadLine();
                    //data2=data2.Replace("DataSet1","DataSet1 xmlns:msdata=\"http://diffgr\" xmlns:diffgr=\"http://diffgr\"");
                    data2 = data2.Replace(" xmlns=\"http://tempuri.org/DataSet1.xsd\"", "").Replace("<diffgr:diffgram xmlns:msdata=\"urn:schemas-microsoft-com:xml-msdata\" xmlns:diffgr=\"urn:schemas-microsoft-com:xml-diffgram-v1\">", "").Replace("</diffgr:diffgram>", "").Replace("diffgr:", "").Replace(" diffgr:", "").Replace("msdata:", "");
                    streamWriter2.WriteLine(data2);
                }


                streamReader2.Close();
                streamWriter2.Close();
                streamReader2.Dispose();
                streamWriter2.Dispose();

                                 
                int newID = 1;


                System.Xml.XmlDocument xdoc2 = new System.Xml.XmlDocument();
                System.Xml.XmlDocument xdocResult2 = new System.Xml.XmlDocument();
                xdoc2.Load(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML02));
                System.Xml.XmlNode ParentNew2 = xdocResult2.CreateNode(System.Xml.XmlNodeType.Element, "DataSet1", string.Empty);

                foreach (System.Xml.XmlNode xnode in xdoc2.GetElementsByTagName("DataSet1"))
                {
                    foreach (System.Xml.XmlNode xchild in xnode.ChildNodes)
                    {
                        xchild.Attributes.RemoveAll();
                        if (xchild.LocalName == "Revenue" || xchild.LocalName == "Currency" || xchild.LocalName == "TargetBudget" || xchild.LocalName == "OperatingExpense" || xchild.LocalName == "RevenueByYear")
                        {
                            xchild.ParentNode.RemoveChild(xchild);
                        }
                        System.Xml.XmlNode importNode = ParentNew2.OwnerDocument.ImportNode(xchild, true);
                        ParentNew2.AppendChild(importNode);
                    }
                }
                xdocResult2.AppendChild(ParentNew2);
                xdocResult2.Save(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML02));

                i = 80;
                // Report progress.
                backgroundWorker1.ReportProgress(i);
               
                //Linq compare
                XDocument doc1 = XDocument.Load(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML01));
                XDocument doc2 = XDocument.Load(@"" + System.IO.Path.Combine(CompareViewerLocation, newXML02));
                var leftOuterJoin = (from opportunityOld in doc1.Descendants("OpportunityElement")
                                     join opportunityNew in doc2.Descendants("OpportunityElement") on opportunityOld.Element("referencenumber").Value equals opportunityNew.Element("referencenumber").Value
                                     into temp
                                     from opportunityNew in temp.DefaultIfEmpty()
                                     select new
                                     {
                                         Ref = (string)opportunityOld.Element("referencenumber"),
                                         Ostartdate = (string)opportunityOld.Element("startdate"),
                                         Oaccountmanager = (string)opportunityOld.Element("accountmanager"),                                         
                                         Ncos = " ",
                                     });
               
                var rightOuterJoin = (from opportunityNew in doc2.Descendants("OpportunityElement")
                                      join opportunityOld in doc1.Descendants("OpportunityElement") on opportunityNew.Element("referencenumber").Value equals opportunityOld.Element("referencenumber").Value
                                      into temp
                                      from opportunityOld in temp.DefaultIfEmpty()
                                      select new
                                      {

                                          Ref = (string)opportunityNew.Element("referencenumber"),
                                          Ostartdate = " ",
                                          Oaccountmanager = " ",                                         
                                          Ncos = (string)opportunityNew.Element("cos"),
                                      });

               
                var joins = leftOuterJoin.Union(rightOuterJoin);
                var output = (from e in joins
                              group e by e.Ref into r
                              select new
                              {
                                  NStartDate = r.Max(d => d.Nstartdate),
                                  Oaccountmanager = r.Max(d => d.Oaccountmanager),                                  
                                  NCos = r.Max(d => d.Ncos),
                                  Remark = (r.Max(d => d.Nstartdate) == " " && r.Max(d => d.Ncurrencycode) == " " && r.Max(d => d.Nname) == " " && r.Max(d => d.Naccountmanager) == " " && r.Max(d => d.Nfeetypename) == " " && r.Max(d => d.Nfees) == " " && r.Max(d => d.Nduration) == " " && r.Max(d => d.Ncos) == " ") ? (string)"Deleted" : (r.Max(d => d.Ostartdate) == " " && r.Max(d => d.Ocurrencycode) == " " && r.Max(d => d.Oname) == " " && r.Max(d => d.Oaccountmanager) == " " && r.Max(d => d.Ofeetypename) == " " && r.Max(d => d.Ofees) == " " && r.Max(d => d.Oduration) == " " && r.Max(d => d.Ocos) == " ") ? (string)"Created" : (r.Max(d => d.Ostartdate) == r.Max(d => d.Nstartdate) && r.Max(d => d.Ocurrencycode) == r.Max(d => d.Ncurrencycode) && r.Max(d => d.Oname) == r.Max(d => d.Nname) && r.Max(d => d.Oaccountmanager) == r.Max(d => d.Naccountmanager) && r.Max(d => d.Ofeetypename) == r.Max(d => d.Nfeetypename) && r.Max(d => d.Ofees) == r.Max(d => d.Nfees) && r.Max(d => d.Oduration) == r.Max(d => d.Nduration) && r.Max(d => d.Ocos) == r.Max(d => d.Ncos)) ? (string)"No Changes" : "Modified",
                              }).Where(s => !s.Remark.Equals("No Changes")).ToList();

               

                //..............................Opportunity
                var leftOuterJoin2 = (from opportunityOld in doc1.Descendants("Opportunity")
                                      join opportunityNew in doc2.Descendants("Opportunity") on opportunityOld.Element("referencenumber").Value equals opportunityNew.Element("referencenumber").Value
                                      into temp
                                      from opportunityNew in temp.DefaultIfEmpty()
                                      select new
                                      {
                                          Ref = (string)opportunityOld.Element("referencenumber"),
                                          Oopportunityid = (string)opportunityOld.Element("opportunityid"),
                                          Ntotalfees = " ",
                                          NTrafficStatus = " "
                                      });
            
                var rightOuterJoin2 = (from opportunityNew in doc2.Descendants("Opportunity")
                                       join opportunityOld in doc1.Descendants("Opportunity") on opportunityNew.Element("referencenumber").Value equals opportunityOld.Element("referencenumber").Value
                                       into temp
                                       from opportunityOld in temp.DefaultIfEmpty()
                                       select new
                                       {

                                           Ref = (string)opportunityNew.Element("referencenumber"),
                                           Oopportunityid = " ",                                          
                                           Ntotalfees = (string)opportunityNew.Element("totalfees"),
                                           NTrafficStatus = (string)opportunityNew.Element("TrafficStatus")
                                       });

                var joins2 = leftOuterJoin2.Union(rightOuterJoin2);
                var output2 = (from e in joins2
                               group e by e.Ref into r
                               select new
                               {
                                   Nopportunityid = r.Max(d => d.Nopportunityid),
                                   Oopportunityid = r.Max(d => d.Oopportunityid),
                                   Nquotecurrencycode = r.Max(d => d.Nquotecurrencycode),                                   
                                   Naccountmanager = r.Max(d => d.Naccountmanager),
                                   Oaccountmanager = r.Max(d => d.Oaccountmanager),                                  
                                   Ntotalfees = r.Max(d => d.Ntotalfees),
                                   OTrafficStatus = r.Max(d => d.OTrafficStatus),
                                   NTrafficStatus = r.Max(d => d.NTrafficStatus),
                                   Ref = r.Key,
                                   Remark = (r.Max(d => d.Nname) == " " && r.Max(d => d.Nquotecurrencycode) == " ") ? (string)"Deleted" : (r.Max(d => d.Oquotecurrencycode) == " " && r.Max(d => d.Oname) == " ") ? (string)"Created" : (r.Max(d => d.Oquotecurrencycode) == r.Max(d => d.Nquotecurrencycode) && r.Max(d => d.Oname) == r.Max(d => d.Nname) && r.Max(d => d.Oclosedate) == r.Max(d => d.Nclosedate)) ? (string)"No Changes" : "Modified",
                               }).Where(s => !s.Remark.Equals("No Changes")).ToList();


                var Diff1 = new XElement("Root", new XElement("Opportunities",
                            from c1 in output
                             select ((c1.OCos != c1.NCos) || (c1.OName != c1.NName) || (c1.Oaccountmanager != c1.Naccountmanager)  || (c1.OFees != c1.NFees)) ? new XElement("DealElement",
                                    new XElement("Remark", c1.Remark),
                                   ((c1.OCos != c1.NCos) || (c1.Oaccountmanager != c1.Naccountmanager) || (c1.OName != c1.NName)  || (c1.OFees != c1.NFees)) ? new XElement("Ref", c1.Ref) : null,
                                   (c1.OCos != c1.NCos && c1.Remark == "Created") ? new XElement("Name", c1.NName) : new XElement("Name", c1.OName),
                                   (c1.OCos != c1.NCos && c1.Remark == "Created") ? new XElement("AccountManager", c1.Naccountmanager) : new XElement("AccountManager", c1.Oaccountmanager),
                                   (c1.OCos != c1.NCos && c1.Remark == "Created") ? new XElement("Customer", c1.Ncustomer) : new XElement("Customer", c1.Ocustomer),
                                   (c1.OCos != c1.NCos && c1.Remark == "Modified") ? new XElement("Change", new XAttribute("Id", newID + 0),
                                   (c1.OCos != c1.NCos && c1.Remark == "Modified") ? new XElement("Att", "cos") : null,
                                   (c1.OCos != c1.NCos && c1.Remark == "Modified") ? new XElement("Before", c1.OCos) : null,
                                   (c1.OCos != c1.NCos && c1.Remark == "Modified") ? new XElement("After", c1.NCos) : null) : null,

                                  :
				  :
                            from c in output2
                            select ((c.Oquotecurrencycode != c.Nquotecurrencycode)  || (c.Oclosedate != c.Nclosedate)) ? new XElement("Opportunity",
                                    new XElement("Remark", c.Remark),
                                   ((c.Oquotecurrencycode != c.Nquotecurrencycode) || (c.Oaccountmanager != c.Naccountmanager)  || (c.Oclosedate != c.Nclosedate)) ? new XElement("Ref", c.Ref) : null,
                                   (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Created") ? new XElement("Name", c.NName) : new XElement("Name", c.OName),
                                   (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Created") ? new XElement("AccountManager", c.Naccountmanager) : new XElement("AccountManager", c.Oaccountmanager),
                                   (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Created") ? new XElement("Status", c.Nstatuscodename) : new XElement("Status", c.Ostatuscodename),
                                   (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Created") ? new XElement("Customer", c.Ncustomer) : new XElement("Customer", c.Ocustomer),
                                   (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Modified") ? new XElement("Change", new XAttribute("Id", newID + 0),
                                   (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Modified") ? new XElement("Att", "quotecurrencycode") : null,
                                   (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Modified") ? new XElement("Before", c.Oquotecurrencycode) : null,
                                   (c.Oquotecurrencycode != c.Nquotecurrencycode && c.Remark == "Modified") ? new XElement("After", c.Nquotecurrencycode) : null) : nul,
					:
					:
				   ));
               
                                 
                var comparisonXMLXSLTheader = new XDocument(new XProcessingInstruction(
                 "xml-stylesheet",
                 "type='text/xsl' href='ResultViewer2.xsl'"),
                 Diff1);

                i = 90;
                // Report progress.
                backgroundWorker1.ReportProgress(i);

                var mypath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\TestingTool\\CompareViewer";
                string filename = mypath + "\\ComparisonSnapshot_" + DateTime.Now.ToString("yyyyMMddhhmmssfff") + ".xml";
                finalPath = mypath + "\\ComparisonSnapshot_" + DateTime.Now.ToString("yyyyMMddhhmmssfff") + ".xml";
                comparisonXMLXSLTheader.Save(@"" + finalPath);

                i = 95;
                // Report progress.
                backgroundWorker1.ReportProgress(i);
               
                
                File.Delete(@"" + file1.Replace(".xml", "_1.xml"));
                File.Delete(@"" + file2.Replace(".xml", "_2.xml"));
                                
            }
            return finalPath.ToString();
        }

Can anyone advise me please how to avoid this message & stop working? I read from post, someone suggest used XMLReader. But how can I convert my code to be XMLReader method?

Thank you.

Regards,

Micheale

Answer 1

User-1716253493 posted

Try increase MaxRequestLength value in httpruntime section in web.config

<httpRuntime maxRequestLength="12345678910" />

Also try increase MaxAllowedContent lenght under System.WebServer

  <system.webServer>
    <security>
        <requestLimits maxAllowedContentLength="12345678910"/>
      </requestFiltering>
    </security>
  </system.webServer>

Answer 2

User-893002196 posted

Hi,

This is .exe program writing in C#. Not Web. So I do not have we.config.

Please advise.

Thank you.

Regards,

Micheale

Answer 3

User-1716253493 posted

I guess this is wrong place to ask

Ask it in http://social.msdn.microsoft.com/Forums/windows/en-US/home?category=windowsforms