locked
trying to speed up some code RRS feed

  • Question

  • User-2012457684 posted

    I have some code that will analyze the IIS log files.  The user selects the date range for the report and then it will use grab all of the entries from those log files and put them in a list.

    Just doing that takes about 10 seconds or more to build a list of 35,000 entries which is liveable.  

    One of the things I want to do is display the browsers used.  to that end I have added some code  using UAParser to parse the UserAgent and get the browser version.  this really slowed things down.  it went from 10 seconds to almost 30 seconds for the same list.

    I realize it will be best to cache the list while the user is building the various reports so that they do not have to rebuild the list each time. so I have set it to cache the list but the initial list build still needs to be faster.

     Here is the class and the method to build the list cache

    public class Logfile
    {
        public string FileName { get; set; }
        public string date { get; set; }
        public string time { get; set; }
        public string ServerIP { get; set; }
        public string Method { get; set; }
        public string URI { get; set; }
        public string Query { get; set; }
        public string Port { get; set; }
        public string Username { get; set; }
        public string IP { get; set; }
        public string UserAgent { get; set; }
        public string Referrer { get; set; }
        public string Status { get; set; }
        public string SubStatus { get; set; }
        public string Win32Status { get; set; }
        public string TimeOnSite { get; set; }
        public string Browser { get; set; }
    
    
    
        public List<String> ListLogs(DateTime StartDate, DateTime EndDate)
    {
    Cache c = new Cache();
    //List<String> logList = new List<String>();
    List<String> logList = c.Get("FileList") as List<String>;
    if (logList == null)
    {
    foreach (DateTime day in EachDay(StartDate, EndDate))
    {
    string d = day.Day.ToString("00");
    string m = day.Month.ToString("00");
    string yr = day.Year.ToString().Substring(2);
    string logfile = "u_ex" + yr + m + d + ".log";
    logList.Add(logfile);
    }
    c.Insert("FileList", logList, null, DateTime.Now.AddMinutes(10), Cache.NoSlidingExpiration);
    }
    return logList;
    } public List<Logfile> GetLogs(List<String> logList, string path) { Cache c = new Cache(); var uaParser = Parser.GetDefault(); //cache the list for use later on List<Logfile> Files = c.Get("logFiles") as List<Logfile>; if (Files == null) { Files = new List<Logfile>(); foreach (string log in logList) { string newFile = path + @"\" + log; if (File.Exists(newFile)) { //Logfile lf = new Logfile(); using (StreamReader sr = new StreamReader(newFile)) { while (sr.Peek() >= 0) { Logfile lf = new Logfile(); string line = sr.ReadLine(); var info = Regex.Split(line, @"\s+"); if (!info[0].StartsWith("#")) { lf.date = info[0]; lf.time = info[1]; lf.ServerIP = info[2]; lf.Method = info[3]; lf.URI = info[4]; lf.Query = info[5]; lf.Port = info[6]; lf.Username = info[7]; lf.IP = info[8]; lf.UserAgent = info[9]; lf.Referrer = info[10]; lf.Status = info[11]; lf.SubStatus = info[12]; lf.Win32Status = info[13]; lf.TimeOnSite = info[14];
    //these next two lines slow everything up!!! ClientInfo c = uaParser.Parse(info[9]); lf.Browser = c.UA.Family; Files.Add(lf); } } } } } c.Insert("logFiles", Files,null, DateTime.Now.AddMinutes(10), Cache.NoSlidingExpiration); } return Files; } private IEnumerable<DateTime> EachDay(DateTime from, DateTime thru) { for (var day = from.Date; day.Date <= thru.Date; day = day.AddDays(1)) { yield return day; } } }

    is there a better way to parse the UserAgent ?

    On caching, the list,  is there a way to override my 10 minute limit if the user changes the dates?

    Thursday, September 3, 2020 10:12 PM

All replies

  • User409696431 posted

    You could try a different UA parser and see if it is any faster.

    https://github.com/totpero/DeviceDetector.NET 

    Friday, September 4, 2020 6:46 AM
  • User1686398519 posted

    Hi mj1223,

    You can modify the synchronous method into an asynchronous method, which can improve efficiency.

    Best Regards,

    YihuiSun

    Friday, September 4, 2020 9:02 AM
  • User-474980206 posted

    while switching to async will improve server efficiency it will not make the query faster. your agent string parser seems to be very slow. depending on your requirements, you might be able to use  simple regex.

    Friday, September 4, 2020 9:55 PM