locked
Multithreading issue? Maybe? RRS feed

  • Question

  • User643096697 posted
    I've got a fairly busy image host that gets about 30 image request on avg per second. I'm putting all of these hits into a DataTable in memory with info about the image, user, etc. My problem... well... let me past some code. I have a class called MemoryTables that has the following function that I fire off when the app starts up:
     

     
    private static void BuildHitListTable()
        {
            _HitList = new DataTable();
            lock (_HitList)
            {
                DataColumn dc;
    
                dc = new DataColumn("HitID", typeof(Int32));
                dc.Unique = true;
                dc.AutoIncrement = true;
                dc.AutoIncrementSeed = -1;
                dc.AutoIncrementStep = -1;
                _HitList.Columns.Add(dc);
    
                _HitList.PrimaryKey = new DataColumn[] { dc };
    
                dc = new DataColumn("FileID", typeof(Int32));
                _HitList.Columns.Add(dc);
    
                dc = new DataColumn("FileName", typeof(String));
                _HitList.Columns.Add(dc);
    
                dc = new DataColumn("Username", typeof(String));
                _HitList.Columns.Add(dc);
    
                dc = new DataColumn("MemberID", typeof(Int32));
                _HitList.Columns.Add(dc);
    
                dc = new DataColumn("Success", typeof(Boolean));
                dc.DefaultValue = false;
                _HitList.Columns.Add(dc);
    
                dc = new DataColumn("Bytes", typeof(long));
                _HitList.Columns.Add(dc);
    
                _HitList.TableNewRow += new DataTableNewRowEventHandler(_HitList_TableNewRow);
            }
        }
     

     
    This function fires once the table reaches 10,000 rows:
     

     
    using (SqlBulkCopy copy = new SqlBulkCopy(sourceConnection))
                    {
                        EventLog MyLog = new EventLog();
    
                        #region Dump and clear hit list
                        copy.DestinationTableName = "FileHits";
                        lock (_HitList)
                        {
                            copy.WriteToServer(_HitList);
    
                            try
                            {
                                if (!(EventLog.SourceExists(SourceName)))
                                {
                                    EventLog.CreateEventSource(SourceName, LogName);
                                }
    
                                // Insert into Event Log;   
                                MyLog.Source = SourceName;
                                MyLog.WriteEntry("Hit data has been dumped. HitList Records: " + _HitList.Rows.Count + "; Anonymous HitList Recards: " + _AnonHitList.Rows.Count, EventLogEntryType.Information);
                            }
                            catch (Exception ex)
                            {
                                MyLog.Source = SourceName;
                                MyLog.WriteEntry(ex.ToString(), EventLogEntryType.Error);
                            }
                            finally
                            {
                                _HitList.Clear();
                            }
                        }
                        #endregion
                    }
                }
     

     And this code actually adds the hit to the table in memory (this is in the HttpHandler):
     

     
    DataRow row = MemoryTables.HitList.NewRow();
                                            lock (row)
                                            {
                                                row["FileID"] = Convert.ToInt32(context.Request.QueryString["fi"]);
                                                row["FileName"] = fi.Name;
                                                row["Username"] = _Username;
                                                row["MemberID"] = _MemberID;
                                                row["Success"] = true;
                                                row["Bytes"] = fi.Length;
    
                                                try
                                                {
                                                    lock (MemoryTables.HitList)
                                                    {
    
                                                        //MemoryTables.HitList.BeginLoadData();
                                                        MemoryTables.HitList.Rows.Add(row);
                                                        //MemoryTables.HitList.EndLoadData();
                                                        //MemoryTables.HitList.BeginLoadData();
                                                        //MemoryTables.HitList.LoadDataRow(row, true);
                                                        //MemoryTables.HitList.EndLoadData();
    
    
                                                    }
                                                }
                                                catch (ConstraintException exception)
                                                {
                                                    logMyException(exception);
                                                }
                                            }
     

     
    I've tried everything I can think of. This code seemed to work fine for a few weeks.. now it's bombing left and right. Here are the two main exceptions being thrown:
    Lots of these - Column 'HitID' does not allow nulls.
    And a ton of these kind as well:
    Column 'HitID' is constrained to be unique.  Value '-4862' is already present.
    Column 'HitID' is constrained to be unique.  Value '-4847' is already present.
    Column 'HitID' is constrained to be unique.  Value '-4768' is already present.
     

    As you can see in the table definition above, I have a PrimaryKey, AutoIncrement on, and seed set properly (I think). It's very hard to troublshoot since this is an error that only appears under a production load. I can not reproduce this problem testing on my local machine with just me browsing.

    Does anybody have any ideas on what might be going on, or have a way to troubleshoot this issue better? After a few mins of these errors, the site functionality essentially dies, as the HttpHandler stops working. I can only assume the problem I've just described is what's causing the failure.

     Thanks in advance for any advice or ideas...
     
    Wednesday, November 22, 2006 3:59 PM

Answers

  • User643096697 posted

    I do have my database setup like you suggested. The error isn't thrown when I do my SqlBulkCopy.. it's thrown when I do the Rows.Add to the DataTable in memory. Hoevery... I think I fixed it late last night by switching the order of a few things going on:

    I changed:
     

    DataRow row = MemoryTables.HitList.NewRow();
                                            lock (row)
                                            {
                                                row["FileID"] = Convert.ToInt32(context.Request.QueryString["fi"]);
                                                row["FileName"] = fi.Name;
                                                row["Username"] = _Username;
                                                row["MemberID"] = _MemberID;
                                                row["Success"] = true;
                                                row["Bytes"] = fi.Length;
    
                                                try
                                                {
                                                    lock (MemoryTables.HitList)
                                                    {
    
                                                        //MemoryTables.HitList.BeginLoadData();
                                                        MemoryTables.HitList.Rows.Add(row);
                                                        //MemoryTables.HitList.EndLoadData();
                                                        //MemoryTables.HitList.BeginLoadData();
                                                        //MemoryTables.HitList.LoadDataRow(row, true);
                                                        //MemoryTables.HitList.EndLoadData();
    
    
                                                    }
                                                }
                                                catch (ConstraintException exception)
                                                {
                                                    logMyException(exception);
                                                }
                                            }
     

    to:
     
    lock (MemoryTables.HitList)
                                            {
                                                DataRow row = MemoryTables.HitList.NewRow();
                                                MemoryTables.HitList.Rows.Add(row);
                                                row["FileID"] = Convert.ToInt32(context.Request.QueryString["fi"]);
                                                row["FileName"] = fi.Name;
                                                row["Username"] = _Username;
                                                row["MemberID"] = _MemberID;
                                                row["Success"] = true;
                                                row["Bytes"] = fi.Length;
                                            }
     

     

    And it now seems to be working great! Thnx for the suggestion tho. Hopefully it will keep flying..

    • Marked as answer by Anonymous Thursday, October 7, 2021 12:00 AM
    Thursday, November 23, 2006 8:12 AM

All replies

  • User-158764254 posted

    I dont immediately see whats wrong.
    why not avoid the issue altogether by letting the database generate the unique HitId's?

    I set my Id column in my database transaction log as Identity=Yes, No Nulls
    New transactions to be logged are inserted without an Id.

    Wednesday, November 22, 2006 7:41 PM
  • User643096697 posted

    I do have my database setup like you suggested. The error isn't thrown when I do my SqlBulkCopy.. it's thrown when I do the Rows.Add to the DataTable in memory. Hoevery... I think I fixed it late last night by switching the order of a few things going on:

    I changed:
     

    DataRow row = MemoryTables.HitList.NewRow();
                                            lock (row)
                                            {
                                                row["FileID"] = Convert.ToInt32(context.Request.QueryString["fi"]);
                                                row["FileName"] = fi.Name;
                                                row["Username"] = _Username;
                                                row["MemberID"] = _MemberID;
                                                row["Success"] = true;
                                                row["Bytes"] = fi.Length;
    
                                                try
                                                {
                                                    lock (MemoryTables.HitList)
                                                    {
    
                                                        //MemoryTables.HitList.BeginLoadData();
                                                        MemoryTables.HitList.Rows.Add(row);
                                                        //MemoryTables.HitList.EndLoadData();
                                                        //MemoryTables.HitList.BeginLoadData();
                                                        //MemoryTables.HitList.LoadDataRow(row, true);
                                                        //MemoryTables.HitList.EndLoadData();
    
    
                                                    }
                                                }
                                                catch (ConstraintException exception)
                                                {
                                                    logMyException(exception);
                                                }
                                            }
     

    to:
     
    lock (MemoryTables.HitList)
                                            {
                                                DataRow row = MemoryTables.HitList.NewRow();
                                                MemoryTables.HitList.Rows.Add(row);
                                                row["FileID"] = Convert.ToInt32(context.Request.QueryString["fi"]);
                                                row["FileName"] = fi.Name;
                                                row["Username"] = _Username;
                                                row["MemberID"] = _MemberID;
                                                row["Success"] = true;
                                                row["Bytes"] = fi.Length;
                                            }
     

     

    And it now seems to be working great! Thnx for the suggestion tho. Hopefully it will keep flying..

    • Marked as answer by Anonymous Thursday, October 7, 2021 12:00 AM
    Thursday, November 23, 2006 8:12 AM