none
web.processbatchdata() maximum limit.. optimised way to insert 15000 records RRS feed

  • Question

  • HI,

    what is the maximum records can be added to a list through batch process?

    i have a list of 100 columns uploading from an excel sheet. i m iterating each row of excel sheet and with some logic i contruct xml, with 2000 items and run below code, its creating item no problem, but i want to iterate remaining 13000 records and give to insert, how can i do it in optimized way??

    while debugging i observed that the control waits untill below execute for couple of minute.

    what is maxmimum size of xml can be send to server through StringBuilder? 

    how to execute in thread so that i can loop remaining records?

    web.processbatchdata(sbbulider) 

    batchthreshold i set it to 1999


    Guruprasad Marathe

    if (dtFiltered.Rows.Count > PCDSConstants.BatchThreshhold)
                                    {
                                        for (int i = 0; i < dtFiltered.Rows.Count; i++)
                                        {
                                            presentitration++;
    
                                            if (batchCounter >= PCDSConstants.BatchThreshhold)
                                            {
                                                record++;
                                                if (string.IsNullOrEmpty(Convert.ToString(dtFiltered.Rows[i][PCDSConstants.CorporateID])))
                                                    continue;
                                                int iCount = 1;
                                                DataTable drFilteredRoos = null;// = dataTable.DefaultView.ToTable().Select(string.Format("[" + PCDSConstants.CorporateID + "] = '{0}'", dr[PCDSConstants.CorporateID].ToString()));
                                                var results = from myRow in dataTable.AsEnumerable()
                                                              where myRow[PCDSConstants.CorporateID] == dtFiltered.Rows[i][PCDSConstants.CorporateID]
                                                              select myRow;
                                                drFilteredRoos = results.CopyToDataTable<DataRow>();
                                                if (drFilteredRoos != null)
                                                {
                                                    
                                                    SPUser corpUser;
                                                    string corpUserID = string.Empty;
                                                    SPUser corpL1Manager = null;
                                                    SPUser corpL2Manager = null;
                                                    int ComputerCount = 0;
                                                    if (iCount <= 15)
                                                    {
                                                        corpUser = BuildBatchXmlForCorpID(sbReturnBatch, dtFiltered.Rows[i], ref iCount, drFilteredRoos.Rows, ref corpL1Manager, ref corpL2Manager, ref ComputerCount, SurveyNumber, eweb, userSearch);
                                                    }
                                                }
                                                //batch end tag
                                                sbReturnBatch.Append("</Batch>");
                                                //processbatch
                                                //ParameterizedThreadStart 
                                                //System.Threading.Thread _thread = new Thread(new ThreadStart(InsertAllItemsBatch(sbReturnBatch, eweb)));
                                                //_thred.Start();
                                                erroCode = InsertAllItemsBatch(sbReturnBatch, eweb);
                                                //intiate batch header
                                                sbReturnBatch.Remove(0, sbReturnBatch.Length);
                                                sbReturnBatch.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Batch>");
                                                batchCounter = 0;
                                            }
                                            else
                                            {
                                                int iCount = 1;
                                                DataTable drFilteredRoos = null;// = dataTable.DefaultView.ToTable().Select(string.Format("[" + PCDSConstants.CorporateID + "] = '{0}'", dr[PCDSConstants.CorporateID].ToString()));
                                                var results = from myRow in dataTable.AsEnumerable()
                                                              where myRow[PCDSConstants.CorporateID] == dtFiltered.Rows[i][PCDSConstants.CorporateID]
                                                              select myRow;
                                                drFilteredRoos = results.CopyToDataTable<DataRow>();
                                                if (drFilteredRoos != null)
                                                {
                                                    SPUser corpUser;
                                                    string corpUserID = string.Empty;
                                                    SPUser corpL1Manager = null;
                                                    SPUser corpL2Manager = null;
                                                    int ComputerCount = 0;
                                                    if (iCount <= 15)
                                                    {
                                                        corpUser = BuildBatchXmlForCorpID(sbReturnBatch, dtFiltered.Rows[i], ref iCount, drFilteredRoos.Rows, ref corpL1Manager, ref corpL2Manager, ref ComputerCount, SurveyNumber, eweb, userSearch);
                                                    }
                                                    batchCounter++;
                                                }
                                            }
                                        }
                                        //batch end tag
                                        sbReturnBatch.Append("</Batch>");
                                        //processbatch
                                        erroCode = InsertAllItemsBatch(sbReturnBatch, eweb);
                                        //intiate batch header
                                        sbReturnBatch.Remove(0, sbReturnBatch.Length);
                                        batchCounter = 0;
                                    }



    Thursday, February 14, 2013 12:27 PM

Answers

  • Using ParameterizedThread in long operations, if you are inserting bulk records, web.ProcessbatchData takes longer time to complete and there code will stop execution until all items got created, so you can assign this to thread and continue your work

    private string InsertAllItemsBatch(StringBuilder insertbuilder, SPWeb spWeb)

           {

               ParameterizedThreadStart starter = new ParameterizedThreadStart(Start);

               new Thread(starter).Start(insertbuilder.ToString());

               return null;

           }

           static void Start(object insertbuilder)

           {

            //   StringBuilder insert = (StringBuilder) insertbuilder;

               string methodName = "InsertAllItemsBatch"; string errorCode = string.Empty;

               try

               {

                   SPSite site = new SPSite(PCDSConstants.PCDSSiteUrl);

                   SPWeb spWeb = site.OpenWeb();

                   if (spWeb.AllowUnsafeUpdates == false)

                       spWeb.AllowUnsafeUpdates = true;

                   errorCode = spWeb.ProcessBatchData(insertbuilder.ToString());

                   spWeb.AllowUnsafeUpdates = false;

                  // return errorCode;

               }

               catch (Exception ex)

               {

                   ExceptionHelper.HandleException("Error in " + methodName + ex.Message, ex.StackTrace);

                 //  return null;

               }

           }



    Guruprasad Marathe

    Tuesday, February 19, 2013 5:34 AM