none
Out Of Memory Exception RRS feed

  • Question

  • Hello all,

    Following problem:

    Multithreading Application, which on request from client get data from database, performs calculations with whole recived data (to save the data in to temp file is not an option, sequential working with data throug reader is also not an option {we need whole data}), send result of calculations to the client, frees the memory (double checked no memory leak at all). We catch the OutOfMemoryException in owr code, but we are not able to handle this exception if it occures on thread syncronisation -> application dies.

    We never know how much memory we need, so using of  MemoryFailPoint Class is not an option. (We get cells with blobs, so we are not able to calculate how much information we will recive). 

    The only possible solution i see, is to define a const value of reserved memory (for example 10 MB) and to check how much memory is free at the point of time, if free memory is lower as reserved const value, abort the execution before reciving the next db row.

    But using Process.WorkingSet64 increase the needed time up to 10 times (from 10 seconds to 100 seconds = 1 Minute 40 seconds).

    Using of

    PerformanceCounter("Process""Working Set", processName);

      is 6 times slower.

    Is there some way to reserve some memory for the CLR or get current ProcessWorkingSet faster?

    Some Code for better understanding:

                 

    private const Int64 X32MaxProcessMemoryLimit = 2147483648; private const Int64 X64MaxProcessMemoryLimit = 8796093022208; public static string JobName = ""; public static int ActivePid = Process.GetCurrentProcess().Id; private static readonly Process CurrentProcess = Process.GetCurrentProcess(); private static bool _askedJob; // private static readonly object Lock = new object();

    private static decimal _jobLimitsBytes; private static decimal _processLimitsBytes; public static bool IsMemoryLimited; private static readonly PerformanceCounter WorkingSetMemoryCounter = new PerformanceCounter("Process""Working Set",

    CurrentProcess.ProcessName);

    		public static Int64 MemoryLimitBytes = 1024*1024;
     
    		public static bool MemoryLimitReached(int maxReserveMem)
    		{
    			QueryInformationJobObject(); // check for external limits
    			bool ret = false;
    			var memList = new List<long> {Environment.Is64BitProcess ? X64MaxProcessMemoryLimit : X32MaxProcessMemoryLimit};
     
    			long memWorkingSet = 0L;
     
    			//lock (Lock)
    			//{
    			//   //CurrentProcess.Refresh();
    			//   //memWorkingSet = CurrentProcess.WorkingSet64; // Slowest variant
     
    			//   memWorkingSet = (long) WorkingSetMemoryCounter.NextValue();
    			//}
     
    			Interlocked.Exchange(ref memWorkingSet, (long) WorkingSetMemoryCounter.NextValue()); // Hot point and very slow
                        
    			if (IsMemoryLimited) // Check for external lmits on process
    			{
    				if (_jobLimitsBytes > 0)
    				{
    					memList.Add(Convert.ToInt64(_jobLimitsBytes));
    				}
    				if (_processLimitsBytes > 0)
    				{
    					memList.Add(Convert.ToInt64(_processLimitsBytes));
    				}
    			}
    			memList.Sort();
    			long limit = memList[0];
    			if (memWorkingSet.CompareTo(limit - (MemoryLimitBytes*Convert.ToInt64(maxReserveMem))) > 0)
    			{
    				ret = true;
    			}
    			return ret;
    		}
               Regards


    • Moved by Leo Liu - MSFT Monday, February 20, 2012 4:50 AM Moved for better support. (From:Visual C# General)
    Friday, February 17, 2012 2:04 PM

Answers

  • Hi All

    Solution is to use native function from psapi.dll GetProcessMemoryInfo

    [StructLayout(LayoutKind.Sequential, Size = 40)] publicstructPROCESS_MEMORY_COUNTERS { publicuint cb; publicuint PageFaultCount; publicIntPtr PeakWorkingSetSize; publicIntPtr WorkingSetSize; publicIntPtr QuotaPeakPagedPoolUsage; publicIntPtr QuotaPagedPoolUsage; publicIntPtr QuotaPeakNonPagedPoolUsage; publicIntPtr QuotaNonPagedPoolUsage; publicIntPtr PagefileUsage; publicIntPtr PeakPagefileUsage; }

                
    		[DllImport("psapi.dll", CharSet = CharSet.Unicode)]
    		public static extern bool GetProcessMemoryInfo(IntPtr hProcess, out Structs.PROCESS_MEMORY_COUNTERS counters, uint size);
    		public static Int64 GetProcessWorkingSet(int pid)
    		{
    			IntPtr handle = IntPtr.Zero;
    			Int64 ret = new Int64();
    			try {
    				handle = GetProcessHandleById(pid, ProcessAccess.QueryInformation);
    				if ( handle != IntPtr.Zero ) {
    					Structs.PROCESS_MEMORY_COUNTERS count = new Structs.PROCESS_MEMORY_COUNTERS();
     
    					bool b = NativeFunctions.GetProcessMemoryInfo(handle, out count, (uint)Marshal.SizeOf(count));
    					if ( !b ) {
    						int err = Marshal.GetLastWin32Error();
    						Trace.WriteLine("Error Code : " + err);
    					}
    					else {
    						ret = count.WorkingSetSize.ToInt64();
    					}
    				}
    			}
    			catch ( Exception ) {
    			}
    			finally {
    				NativeFunctions.CloseHandle(handle);
    			}
     
    			return ret;
    		}
    
    		public static IntPtr GetProcessHandleById(int pid, ProcessAccess access)
    		{
    			return NativeFunctions.OpenProcess(access, false, pid);
    		}
    		[DllImport("kernel32.dll", SetLastError = true)]
    		public static extern IntPtr OpenProcess(ProcessAccess DesiredAccess, bool InheritHandle, int ProcessId);
    		[returnMarshalAs(UnmanagedType.Bool)]
    		[DllImport("kernel32.dll", SetLastError = true)]
    		public static extern bool CloseHandle([InIntPtr Handle);
    		public static Int64 GetProcessWorkingSet(int pid)
    		{
    			IntPtr handle = IntPtr.Zero;
    			Int64 ret = new Int64();
    			try {
    				handle = GetProcessHandleById(pid, ProcessAccess.QueryInformation);
    				if ( handle != IntPtr.Zero ) {
    					Structs.PROCESS_MEMORY_COUNTERS count = new Structs.PROCESS_MEMORY_COUNTERS();
     
    					bool b = NativeFunctions.GetProcessMemoryInfo(handle, out count, (uint)Marshal.SizeOf(count));
    					if ( !b ) {
    						int err = Marshal.GetLastWin32Error();
    						Trace.WriteLine("Error Code : " + err);
    					}
    					else {
    						ret = count.WorkingSetSize.ToInt64();
    					}
    				}
    			}
    			catch ( Exception ) {
    			}
    			finally {
    				NativeFunctions.CloseHandle(handle);
    			}
     
    			return ret;
    		}

    and replace Interlocked.Exchange with

    Interlocked.Exchange(ref memWorkingSet, Native.Objects.Process.GetProcessWorkingSet(ActivePid));
    

    Have Fun

    Iouri Bespalov, alfabet AG, Senior Software Architect

    • Marked as answer by Iouri Bespalov Monday, February 20, 2012 10:28 AM
    Monday, February 20, 2012 10:28 AM

All replies

  • Just a suggestion.  If you've proved Perf. Counters to be too slow, don't use them.  Keep a track of each inbound request using your own counter based on byte count. 

    JP Cowboy Coders Unite!

    Friday, February 17, 2012 2:08 PM
  • Hi All

    Solution is to use native function from psapi.dll GetProcessMemoryInfo

    [StructLayout(LayoutKind.Sequential, Size = 40)] publicstructPROCESS_MEMORY_COUNTERS { publicuint cb; publicuint PageFaultCount; publicIntPtr PeakWorkingSetSize; publicIntPtr WorkingSetSize; publicIntPtr QuotaPeakPagedPoolUsage; publicIntPtr QuotaPagedPoolUsage; publicIntPtr QuotaPeakNonPagedPoolUsage; publicIntPtr QuotaNonPagedPoolUsage; publicIntPtr PagefileUsage; publicIntPtr PeakPagefileUsage; }

                
    		[DllImport("psapi.dll", CharSet = CharSet.Unicode)]
    		public static extern bool GetProcessMemoryInfo(IntPtr hProcess, out Structs.PROCESS_MEMORY_COUNTERS counters, uint size);
    		public static Int64 GetProcessWorkingSet(int pid)
    		{
    			IntPtr handle = IntPtr.Zero;
    			Int64 ret = new Int64();
    			try {
    				handle = GetProcessHandleById(pid, ProcessAccess.QueryInformation);
    				if ( handle != IntPtr.Zero ) {
    					Structs.PROCESS_MEMORY_COUNTERS count = new Structs.PROCESS_MEMORY_COUNTERS();
     
    					bool b = NativeFunctions.GetProcessMemoryInfo(handle, out count, (uint)Marshal.SizeOf(count));
    					if ( !b ) {
    						int err = Marshal.GetLastWin32Error();
    						Trace.WriteLine("Error Code : " + err);
    					}
    					else {
    						ret = count.WorkingSetSize.ToInt64();
    					}
    				}
    			}
    			catch ( Exception ) {
    			}
    			finally {
    				NativeFunctions.CloseHandle(handle);
    			}
     
    			return ret;
    		}
    
    		public static IntPtr GetProcessHandleById(int pid, ProcessAccess access)
    		{
    			return NativeFunctions.OpenProcess(access, false, pid);
    		}
    		[DllImport("kernel32.dll", SetLastError = true)]
    		public static extern IntPtr OpenProcess(ProcessAccess DesiredAccess, bool InheritHandle, int ProcessId);
    		[returnMarshalAs(UnmanagedType.Bool)]
    		[DllImport("kernel32.dll", SetLastError = true)]
    		public static extern bool CloseHandle([InIntPtr Handle);
    		public static Int64 GetProcessWorkingSet(int pid)
    		{
    			IntPtr handle = IntPtr.Zero;
    			Int64 ret = new Int64();
    			try {
    				handle = GetProcessHandleById(pid, ProcessAccess.QueryInformation);
    				if ( handle != IntPtr.Zero ) {
    					Structs.PROCESS_MEMORY_COUNTERS count = new Structs.PROCESS_MEMORY_COUNTERS();
     
    					bool b = NativeFunctions.GetProcessMemoryInfo(handle, out count, (uint)Marshal.SizeOf(count));
    					if ( !b ) {
    						int err = Marshal.GetLastWin32Error();
    						Trace.WriteLine("Error Code : " + err);
    					}
    					else {
    						ret = count.WorkingSetSize.ToInt64();
    					}
    				}
    			}
    			catch ( Exception ) {
    			}
    			finally {
    				NativeFunctions.CloseHandle(handle);
    			}
     
    			return ret;
    		}

    and replace Interlocked.Exchange with

    Interlocked.Exchange(ref memWorkingSet, Native.Objects.Process.GetProcessWorkingSet(ActivePid));
    

    Have Fun

    Iouri Bespalov, alfabet AG, Senior Software Architect

    • Marked as answer by Iouri Bespalov Monday, February 20, 2012 10:28 AM
    Monday, February 20, 2012 10:28 AM
  • Interesting, I wonder why that solution worked?


    JP Cowboy Coders Unite!

    Monday, February 20, 2012 2:25 PM