none
关于disconnectex socket重用的问题 RRS feed

  • 问题

  • 最近写了一个iocp的服务器,使用到了socket池和线程池,并且在socket掉线时对socket进行重用,看似一切都能好好运行,比如设定最大连接数是10,大概连接20来次的时候,就再也连接不上了,使用高大上的procexp_16.05.1446001339.exe查看了进程的属性 tcp/ip 发现不少close_wait状态的连接,主要是这个不是一开始就发生?为什么呢?难道是socket回收的重用的时候有问题?另外还尝试过在重用的时候直接closesocket掉新创建一个socket,最后结果还是一样,这个问题已经找了半个多月没有找到适合的解决方案,哪位朋友有这方面的经验,求助分享!

    付上源码:

    /************************************************************************/
    /* @ iocp project                                                       */
    /************************************************************************/
    
    #include "stdafx.h"
    #include "Winsock2.h"
    #include "Windows.h"
    #include "Winbase.h"
    #include "tlhelp32.h"
    #include "tchar.h"
    #include "Psapi.h"
    #include "Winternl.h"
    #include "Shlwapi.h"
    #include "mstcpip.h"
    #include "mswsock.h"
    #include "ws2tcpip.h"
    #include "time.h"
    
    #pragma comment( lib, "Kernel32.lib" )
    #pragma comment( lib, "Shlwapi.lib" )
    #pragma comment( lib, "Psapi.lib" )
    #pragma comment( lib, "Winmm.lib" )
    #pragma comment( lib, "Ws2_32.lib" )
    
    #define DATA_BUFSIZE   10240
    #define OP_ONCONNECT   1
    #define OP_ONRECV      2
    #define OP_ONSEND      3
    #define OP_DISCONNECT  4
    #define OP_ONACCEPT    5
    
    //iocp struct
    struct iocp_overlapped{
    	OVERLAPPED	m_ol;                          // 
    	int			m_iOpType;                     //do type
    	SOCKET		m_skServer;                    //server socket
    	SOCKET		m_skClient;                    //client
    	DWORD		m_recvBytes;                   //recv msg bytes
    	char		m_pBuf[DATA_BUFSIZE];          //recv buf
    	WSABUF		m_DataBuf;                     //recv data buf
    	int			m_recv_timeout;                //recv timeout
    	int         m_send_timeout;
    	SOCKADDR_IN m_addrClient;                  //client address
    	SOCKADDR_IN m_addrServer;                  //server address
    	int			m_isUsed;                      //client is active 1 yes 0 not
    	time_t      m_active;                      //the last active time
    	int         m_isCrashed;                   //is crashed? 0 not 1 yes
    	int         m_online;                      //is online 1 yes 0 not
    	int         m_usenum;                      //
    
    	//void (*handler)(int,struct tag_socket_data*);   data->handler(res, data);  
    };
    
    
    
    
    static SOCKET  m_sock_listen = INVALID_SOCKET;    //the server listen socket
    
    class WingIOCP{
    private:
    	char* m_listen_ip;        //listen ip
    	int   m_port;             //listen port
    	int   m_max_connect;      //max connection
    	int   m_recv_timeout;     //recv timeout
    	int   m_send_timeout;     //send timeout
    	
    	unsigned long* m_povs;  //clients 
    
    	//iocp worker
    	static VOID CALLBACK worker( 
    		DWORD dwErrorCode,
    		DWORD dwBytesTrans,
    		LPOVERLAPPED lpOverlapped 
    		);
    	//accept ex
    	static BOOL accept(
    			SOCKET sAcceptSocket,
    			PVOID lpOutputBuffer,
    			DWORD dwReceiveDataLength,
    			DWORD dwLocalAddressLength,
    			DWORD dwRemoteAddressLength,
    			LPDWORD lpdwBytesReceived,
    			LPOVERLAPPED lpOverlapped
    		);
    	//disconnect a client socket and reuse it
    	static BOOL disconnect( SOCKET client_socket , LPOVERLAPPED lpOverlapped , DWORD dwFlags = TF_REUSE_SOCKET , DWORD reserved = 0);
    	
    	//event callbacks
    	static void onconnect( iocp_overlapped *&povl );
    	static void ondisconnect( iocp_overlapped *&povl );
    	static void onclose( iocp_overlapped *&povl );
    	static void onrecv( iocp_overlapped *&povl );
    	static void onsend( iocp_overlapped *&povl );
    	static void onrun( iocp_overlapped *&povl, DWORD errorcode, int last_error );
    
    	static void onaccept(iocp_overlapped *&pOL);
    
    public:
    	
    	WingIOCP(
    		const char* listen       = "0.0.0.0",
    		const int   port         = 6998,  
    		const int   max_connect  = 10,
    		const int   recv_timeout = 3000,
    		const int   send_timeout = 3000
    		);
    	~WingIOCP();
    	BOOL start();
    	void wait();
    };
    
    /**
     * @ construct 
     */
    WingIOCP::WingIOCP( 
    	const char* listen,    //listen ip
    	const int port,        //listen port
    	const int max_connect, //max connect
    	const int recv_timeout,//recv timeout in milliseconds 
    	const int send_timeout //send timeout in milliseconds
    )
    { 
    
    	this->m_listen_ip      = _strdup(listen);               //listen ip
    	this->m_port           = port;                          //listen port
    	this->m_max_connect    = max_connect;                   //max connect
    	this->m_recv_timeout   = recv_timeout;                  //recv timeout
    	this->m_send_timeout   = send_timeout;                  //send timeout                               
    	this->m_povs           = new unsigned long[max_connect];//clients 
    
    }
    
    /**
     * @ destruct
     */
    WingIOCP::~WingIOCP(){
    	
    	if( this->m_listen_ip ) 
    	{	
    		free(this->m_listen_ip );
    		this->m_listen_ip = NULL;
    	}
    
    	if( this->m_povs )
    	{
    		delete[] this->m_povs;
    		this->m_povs = NULL;
    	}
    
    	if( m_sock_listen != INVALID_SOCKET )
    	{
    		closesocket( m_sock_listen );
    		m_sock_listen = INVALID_SOCKET;
    	}
    
    	WSACleanup();
    }
    
    /**
     *@wait
     */
    void WingIOCP::wait(){
    	while( true ){
    		Sleep(10);
    	}
    }
    
    //event callbacks
     void WingIOCP::onconnect( iocp_overlapped *&pOL ){
    	 printf("%ld onconnect\r\n",pOL->m_skClient);
    	 pOL->m_online = 1;
    	 pOL->m_active = time(NULL);
    
    	 if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_UPDATE_ACCEPT_CONTEXT,(const char *)&pOL->m_skServer,sizeof(pOL->m_skServer) ) != 0 )
    	 {
    		 //setsockopt fail
    		 //printf("1=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 WSASetLastError(0);
    		 return;
    	 }
    	// BOOL bReuse = 1;
    	// if( 0 != ::setsockopt( pOL->m_skClient, SOL_SOCKET, SO_REUSEADDR,(LPCSTR)&bReuse, sizeof(BOOL) ) )
    	 //{
    		 //some error happened
    		 
    	 //}
    
    	 // set send timeout
    	 if( pOL->m_send_timeout > 0 )
    	 {
    		 if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_SNDTIMEO, (const char*)&pOL->m_send_timeout,sizeof(pOL->m_send_timeout)) !=0 )
    		 {
    			 //setsockopt fail
    			// printf("2=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 }
    	 }
    	 if( pOL->m_recv_timeout > 0 )
    	 {
    		 if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_RCVTIMEO, (const char*)&pOL->m_recv_timeout,sizeof(pOL->m_recv_timeout)) != 0 )
    		 {
    			 //setsockopt fail
    			// printf("3=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 }
    	 }
    
    	 linger so_linger;
    	 so_linger.l_onoff	= TRUE;
    	 so_linger.l_linger	= 5; // without close wait status
    	 if( setsockopt( pOL->m_skClient,SOL_SOCKET,SO_LINGER,(const char*)&so_linger,sizeof(so_linger) ) != 0 ){
    		 printf("31=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    	 } 
    
    	 //get client ip and port
    	 int client_size = sizeof(pOL->m_addrClient);  
    	 ZeroMemory( &pOL->m_addrClient , sizeof(pOL->m_addrClient) );
    
    	 if( getpeername( pOL->m_skClient , (SOCKADDR *)&pOL->m_addrClient , &client_size ) != 0 ) 
    	 {
    		 //getpeername fail
    		// printf("4=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    	 }
    
    	// printf("%s %d connect\r\n",inet_ntoa(pOL->m_addrClient.sin_addr), pOL->m_addrClient.sin_port);
    
    	 //keepalive open
    	 int dt		= 1;
    	 DWORD dw	= 0;
    	 tcp_keepalive live,liveout ;     
    	 live.keepaliveinterval	= 1000;     //连接之后 多长时间发现无活动 开始发送心跳吧 单位为毫秒 
    	 live.keepalivetime		= 200;     //多长时间发送一次心跳包 1分钟是 60000 以此类推     
    	 live.onoff				= TRUE;     //是否开启 keepalive
    
    	 if( setsockopt( pOL->m_skClient, SOL_SOCKET, SO_KEEPALIVE, (char *)&dt, sizeof(dt) ) == 0 )
    	 {
    		 //setsockopt fail
    		 if( WSAIoctl(   pOL->m_skClient, SIO_KEEPALIVE_VALS, &live, sizeof(live), &liveout, sizeof(liveout), &dw, &pOL->m_ol , NULL ) == SOCKET_ERROR )
    		 {
    			 //WSAIoctl error
    			 printf("6=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 }
    	 } else  printf("5=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    
    	 memset(pOL->m_pBuf,0,DATA_BUFSIZE);
    	 //post recv
    	 pOL->m_DataBuf.buf	= pOL->m_pBuf;  
    	 pOL->m_DataBuf.len	= DATA_BUFSIZE;  
    	 pOL->m_iOpType		= OP_ONRECV;
    
    	 DWORD RecvBytes		= 0;
    	 DWORD Flags			= 0;
    
    	 int code		    = WSARecv(pOL->m_skClient,&(pOL->m_DataBuf),1,&RecvBytes,&Flags,&(pOL->m_ol),NULL);
    	 int error_code	    = WSAGetLastError();
    
    	 if( 0 != code )
    	 {
    		 if( WSA_IO_PENDING != error_code ) 
    		 {
    			// printf("7=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    			 return;
    		 }
    	 }
    	 else
    	 {
    		 //recv complete
    		 onrecv( pOL );
    	 }
     }
     void WingIOCP::ondisconnect( iocp_overlapped *&pOL ){
    	// printf("ondisconnect error %d\r\n",WSAGetLastError());
    	 WSASetLastError(0);
    	 pOL->m_online   = 0;                                //set offline
    	 pOL->m_active   = time(NULL);                       //the last active time
    	 pOL->m_iOpType	 = OP_ONACCEPT;                      //reset status
    	 pOL->m_isUsed   = 0;                                //
    	 ZeroMemory(pOL->m_pBuf,sizeof(char)*DATA_BUFSIZE);  //clear buf
    
    	 //i try free the socket and create a new one, but still close_wait 
    	 /*closesocket(pOL->m_skClient);
    	 pOL->m_skClient = WSASocket(AF_INET,SOCK_STREAM,IPPROTO_TCP,0,0,WSA_FLAG_OVERLAPPED);
    	 if( INVALID_SOCKET == pOL->m_skClient ) 
    	 {	
    		 return;
    	 }*/
    	 
    	 //post acceptex
    	 int error_code     = accept( pOL->m_skClient,pOL->m_pBuf,0,sizeof(SOCKADDR_IN)+16,sizeof(SOCKADDR_IN)+16,NULL, (LPOVERLAPPED)pOL );
    	 //printf("accept error %d\r\n",WSAGetLastError());
    	 int last_error     = WSAGetLastError() ;
    
    	 if( !error_code && ERROR_IO_PENDING != last_error ){
    
    	 }
    	 //printf("2=>ondisconnect some error happened , error code %d \r\n================================================\r\n\r\n", WSAGetLastError());
    	  
    	  //printf("21=>ondisconnect some error happened , error code %d \r\n================================================\r\n\r\n", WSAGetLastError());
    	 if( !BindIoCompletionCallback( (HANDLE)pOL->m_skClient ,worker,0) ){
    		  printf("BindIoCompletionCallback error %ld\r\n",WSAGetLastError());
    	 }
    	 WSASetLastError(0);
    
     }
    
     void WingIOCP::onaccept(iocp_overlapped *&pOL){
    	 pOL->m_active   = time(NULL);                       //the last active time
    	 pOL->m_iOpType	 = OP_ONCONNECT;                        //reset status
    	 ZeroMemory( &pOL->m_ol,sizeof(OVERLAPPED));
    	 printf("%ld reuse socket real complete , error code %d \r\n", pOL->m_skClient,WSAGetLastError());
    	// WSASetLastError(0);
    	// if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_UPDATE_ACCEPT_CONTEXT,(const char *)&pOL->m_skServer,sizeof(pOL->m_skServer) ) != 0 )
    	 {
    		 //setsockopt fail
    		 //printf("1=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 WSASetLastError(0);
    		 //return;
    	 }
     }
     void WingIOCP::onclose( iocp_overlapped *&pOL ){
    	// printf("%ld close\r\n", pOL->m_skClient);
    
    	 pOL->m_iOpType      = OP_DISCONNECT;
    	 //socket reuse
    	 if( !disconnect( pOL->m_skClient , &pOL->m_ol ) && WSA_IO_PENDING != WSAGetLastError()) {
    		// printf("1=>onclose some error happened , error code %d \r\n", WSAGetLastError());
    	 }
    	 //printf("onclose complete %d \r\n", WSAGetLastError());
     }
     void WingIOCP::onrecv( iocp_overlapped *&pOL ){
    	 pOL->m_active = time(NULL);
    	// printf("recv:\r\n%s\r\n\r\n",pOL->m_pBuf);
    	 ZeroMemory(pOL->m_pBuf,DATA_BUFSIZE);      
     }
     void WingIOCP::onsend( iocp_overlapped *&povl ){
    
     }
     void WingIOCP::onrun( iocp_overlapped *&povl, DWORD errorcode, int last_error ){}
    /**
     * @ acceptex
     */
    BOOL WingIOCP::accept(
    	SOCKET  sAcceptSocket,
    	PVOID   lpOutputBuffer,
    	DWORD   dwReceiveDataLength,
    	DWORD   dwLocalAddressLength,
    	DWORD   dwRemoteAddressLength,
    	LPDWORD lpdwBytesReceived,
    	LPOVERLAPPED lpOverlapped
    )
    {
    	WSASetLastError(0);
    	if( m_sock_listen == INVALID_SOCKET || !lpOverlapped ) 
    	{	
    		return 0;
    	}
    	GUID guidAcceptEx	= WSAID_ACCEPTEX;
    	DWORD dwBytes		= 0;
    	LPFN_ACCEPTEX lpfnAcceptEx;
    
    	int res= WSAIoctl( m_sock_listen, SIO_GET_EXTENSION_FUNCTION_POINTER, &guidAcceptEx, 
    		sizeof(guidAcceptEx), &lpfnAcceptEx, sizeof(lpfnAcceptEx), &dwBytes, NULL, NULL );
    
    	if( 0 != res )
    	{
    		return 0;
    	}
    
    	return lpfnAcceptEx( m_sock_listen, sAcceptSocket, lpOutputBuffer, dwReceiveDataLength,
    		dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, lpOverlapped );        
    }
    
    /**
     * @ disconnect socket and reuse the socket
     */
    BOOL WingIOCP::disconnect( SOCKET client_socket , LPOVERLAPPED lpOverlapped , DWORD dwFlags  , DWORD reserved  )
    {
    	WSASetLastError(0);
    	if( client_socket == INVALID_SOCKET || !lpOverlapped ) 
    	{	
    		return 0;
    	}
    	GUID GuidDisconnectEx = WSAID_DISCONNECTEX;
    	DWORD dwBytes = 0;
    	LPFN_DISCONNECTEX lpfnDisconnectEx; 
    
    	if( 0 != WSAIoctl( client_socket,SIO_GET_EXTENSION_FUNCTION_POINTER,&GuidDisconnectEx,
    		sizeof(GuidDisconnectEx),&lpfnDisconnectEx,sizeof(lpfnDisconnectEx),&dwBytes,NULL,NULL))
    	{
    		return 0;
    	}
    	return lpfnDisconnectEx(client_socket,lpOverlapped,/*TF_REUSE_SOCKET*/dwFlags,reserved);
    }
    
    
    	
    /**
     * @ iocp worker thread
     */
    VOID CALLBACK WingIOCP::worker( DWORD dwErrorCode,DWORD dwBytesTrans,LPOVERLAPPED lpOverlapped )
    {
    	//why here get the error code 87 ?
    	//printf("worker error %d\r\n",WSAGetLastError());
    	if( NULL == lpOverlapped  )
    	{
    		//not real complete
    		SleepEx(20,TRUE);//set warn status
    		WSASetLastError(0);
    		return;
    	}
    
    	//get overlapped data
    	iocp_overlapped*  pOL = CONTAINING_RECORD(lpOverlapped, iocp_overlapped, m_ol);
    	
    	//just a test
    	//onrun( pOL, dwErrorCode, WSAGetLastError() );
    	
    	switch( pOL->m_iOpType )
    	{
    	case OP_DISCONNECT:
    			ondisconnect( pOL );
    		break;
    	case OP_ONACCEPT:
    			onaccept( pOL );
    		break;
    	case OP_ONCONNECT: 
    		{
    			//new client connect
    			onconnect( pOL );
    		}
    		break;
    	case OP_ONRECV:
    		{
    			pOL->m_recvBytes = dwBytesTrans;
    			int last_error = WSAGetLastError();
    			//check client offline
    			if( 0 == dwBytesTrans || WSAECONNRESET == last_error  || ERROR_NETNAME_DELETED == last_error )
    			{
    				onclose( pOL );
    			} 
    			else
    			{   //recv msg from client
    				onrecv( pOL );
    			}	
    		}
    		break;
    	case OP_ONSEND:
    		{
    
    		}
    		break;
    	
    	}
    
    	WSASetLastError(0);
    
    }
    
    
    BOOL WingIOCP::start(){	
    
    	do{ 
    
    		WSADATA wsaData; 
    		if( WSAStartup(MAKEWORD(2,2), &wsaData) != 0 )
    		{
    			return FALSE;
    		}
     
    		if(LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2)
    		{
    			break;
    		}  
    
    		m_sock_listen = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED); 
    		if( INVALID_SOCKET == m_sock_listen )
    		{
    			break;
    		}
    
    		//bind the worker thread
    		BOOL bReuse      = TRUE;
    		BOOL bind_status = ::BindIoCompletionCallback((HANDLE)( m_sock_listen ), worker, 0 );
    		if( !bind_status )
    		{
    			break;
    		}
    
    		//set option SO_REUSEADDR 
    		if( 0 != ::setsockopt( m_sock_listen, SOL_SOCKET, SO_REUSEADDR,(LPCSTR)&bReuse, sizeof(BOOL) ) )
    		{
    			//some error happened
    			break;
    		}
    
    
    		struct sockaddr_in ServerAddress; 
    		ZeroMemory(&ServerAddress, sizeof(ServerAddress)); 
    
    		ServerAddress.sin_family		= AF_INET;                    
    		ServerAddress.sin_addr.s_addr	= inet_addr( this->m_listen_ip );          
    		ServerAddress.sin_port			= htons( this->m_port );   
    
    		if ( SOCKET_ERROR == bind( m_sock_listen, (struct sockaddr *) &ServerAddress, sizeof( ServerAddress ) ) )
    		{
    			break;
    		}  
    
    		if( 0 != listen( m_sock_listen , SOMAXCONN ) )
    		{
    			break;
    		}
    		//printf("1=>start get error %d\r\n",WSAGetLastError());
    		WSASetLastError(0);
    		//socket pool
    		for( int i = 0 ; i < this->m_max_connect ; i++ ) 
    		{
    	
    			SOCKET client = WSASocket(AF_INET,SOCK_STREAM,IPPROTO_TCP,0,0,WSA_FLAG_OVERLAPPED);
    			if( INVALID_SOCKET == client ) 
    			{	
    				continue;
    			}
    
    			
    
    			iocp_overlapped *povl = new iocp_overlapped();
    			if( NULL == povl )
    			{
    				closesocket(client);
    				continue;
    			}
    
    			DWORD dwBytes = 0;
    			ZeroMemory(povl,sizeof(iocp_overlapped));
    		
    			povl->m_iOpType			= OP_ONCONNECT;
    			povl->m_skServer			= m_sock_listen;
    			povl->m_skClient			= client;
    			povl->m_recv_timeout		= m_recv_timeout;
    			povl->m_isUsed			= 0;
    			povl->m_active			= 0; 
    			povl->m_isCrashed		= 0;
    			povl->m_online			= 0;
    			povl->m_usenum			= 1;
    
    			int server_size = sizeof(povl->m_addrServer);  
    			ZeroMemory(&povl->m_addrServer,server_size);
    			getpeername(povl->m_skServer,(SOCKADDR *)&povl->m_addrServer,&server_size);  
    
    			int error_code = accept( povl->m_skClient, povl->m_pBuf, 0, sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16, NULL, (LPOVERLAPPED)povl );
    			int last_error = WSAGetLastError() ;
    			if( !error_code && ERROR_IO_PENDING != last_error ) 
    			{
    			
    				closesocket( client );
    				client = povl->m_skClient = INVALID_SOCKET;
    				delete povl;
    				povl = NULL; 
    				//printf("client=>crate error %d\r\n",WSAGetLastError());
    			}else{
    				this->m_povs[i] = (unsigned long)povl;
    			}
    
    
    			if( !BindIoCompletionCallback( (HANDLE)client ,worker,0) )
    			{
    				closesocket(client);
    				//continue;
    			}
    			//here all the last error is 997 , means nothing error happened
    			//printf("client=>start get error %d\r\n",WSAGetLastError());
    			WSASetLastError(0);
    		}
    		//printf("last start get error %d\r\n",WSAGetLastError());
    		WSASetLastError(0);
    		return TRUE;
    
    	} while( 0 );
    
    	if( m_sock_listen != INVALID_SOCKET )
    	{
    		closesocket( m_sock_listen );
    		m_sock_listen = INVALID_SOCKET;
    	}
    	WSACleanup();
    
    	return FALSE;
    }
    
    
    
    
    int _tmain(int argc, _TCHAR* argv[])
    {
    	WingIOCP *iocp  = new WingIOCP();
    	iocp->start();
    	iocp->wait();
    	delete iocp;
    	return 0;
    }
    
    
    
    2016年9月5日 0:42

全部回复

  • i write a iocp server,use socket pool and thread pool,and reuse the socket on close,it seems evething is ok,like that set the max connection is 10,about 20 times to connect,i can not connect server again,so i use procexp_16.05.1446001339.exe to check process proertties  tcp/ip , and i fund that lots of close_wait status,this is not the beginning of the occurrence of the main, why? i think thar is the socket reuse problem, and i had try closesocket and new one socket,the final result is the same,this problem has been looking for more than half a month to find the answer and did not find a suitable solution,who can help me ?

    code:

    /************************************************************************/
    /* @ iocp project                                                       */
    /************************************************************************/
    
    #include "stdafx.h"
    #include "Winsock2.h"
    #include "Windows.h"
    #include "Winbase.h"
    #include "tlhelp32.h"
    #include "tchar.h"
    #include "Psapi.h"
    #include "Winternl.h"
    #include "Shlwapi.h"
    #include "mstcpip.h"
    #include "mswsock.h"
    #include "ws2tcpip.h"
    #include "time.h"
    
    #pragma comment( lib, "Kernel32.lib" )
    #pragma comment( lib, "Shlwapi.lib" )
    #pragma comment( lib, "Psapi.lib" )
    #pragma comment( lib, "Winmm.lib" )
    #pragma comment( lib, "Ws2_32.lib" )
    
    #define DATA_BUFSIZE   10240
    #define OP_ONCONNECT   1
    #define OP_ONRECV      2
    #define OP_ONSEND      3
    #define OP_DISCONNECT  4
    #define OP_ONACCEPT    5
    
    //iocp struct
    struct iocp_overlapped{
    	OVERLAPPED	m_ol;                          // 
    	int			m_iOpType;                     //do type
    	SOCKET		m_skServer;                    //server socket
    	SOCKET		m_skClient;                    //client
    	DWORD		m_recvBytes;                   //recv msg bytes
    	char		m_pBuf[DATA_BUFSIZE];          //recv buf
    	WSABUF		m_DataBuf;                     //recv data buf
    	int			m_recv_timeout;                //recv timeout
    	int         m_send_timeout;
    	SOCKADDR_IN m_addrClient;                  //client address
    	SOCKADDR_IN m_addrServer;                  //server address
    	int			m_isUsed;                      //client is active 1 yes 0 not
    	time_t      m_active;                      //the last active time
    	int         m_isCrashed;                   //is crashed? 0 not 1 yes
    	int         m_online;                      //is online 1 yes 0 not
    	int         m_usenum;                      //
    
    	//void (*handler)(int,struct tag_socket_data*);   data->handler(res, data);  
    };
    
    
    
    
    static SOCKET  m_sock_listen = INVALID_SOCKET;    //the server listen socket
    
    class WingIOCP{
    private:
    	char* m_listen_ip;        //listen ip
    	int   m_port;             //listen port
    	int   m_max_connect;      //max connection
    	int   m_recv_timeout;     //recv timeout
    	int   m_send_timeout;     //send timeout
    	
    	unsigned long* m_povs;  //clients 
    
    	//iocp worker
    	static VOID CALLBACK worker( 
    		DWORD dwErrorCode,
    		DWORD dwBytesTrans,
    		LPOVERLAPPED lpOverlapped 
    		);
    	//accept ex
    	static BOOL accept(
    			SOCKET sAcceptSocket,
    			PVOID lpOutputBuffer,
    			DWORD dwReceiveDataLength,
    			DWORD dwLocalAddressLength,
    			DWORD dwRemoteAddressLength,
    			LPDWORD lpdwBytesReceived,
    			LPOVERLAPPED lpOverlapped
    		);
    	//disconnect a client socket and reuse it
    	static BOOL disconnect( SOCKET client_socket , LPOVERLAPPED lpOverlapped , DWORD dwFlags = TF_REUSE_SOCKET , DWORD reserved = 0);
    	
    	//event callbacks
    	static void onconnect( iocp_overlapped *&povl );
    	static void ondisconnect( iocp_overlapped *&povl );
    	static void onclose( iocp_overlapped *&povl );
    	static void onrecv( iocp_overlapped *&povl );
    	static void onsend( iocp_overlapped *&povl );
    	static void onrun( iocp_overlapped *&povl, DWORD errorcode, int last_error );
    
    	static void onaccept(iocp_overlapped *&pOL);
    
    public:
    	
    	WingIOCP(
    		const char* listen       = "0.0.0.0",
    		const int   port         = 6998,  
    		const int   max_connect  = 10,
    		const int   recv_timeout = 3000,
    		const int   send_timeout = 3000
    		);
    	~WingIOCP();
    	BOOL start();
    	void wait();
    };
    
    /**
     * @ construct 
     */
    WingIOCP::WingIOCP( 
    	const char* listen,    //listen ip
    	const int port,        //listen port
    	const int max_connect, //max connect
    	const int recv_timeout,//recv timeout in milliseconds 
    	const int send_timeout //send timeout in milliseconds
    )
    { 
    
    	this->m_listen_ip      = _strdup(listen);               //listen ip
    	this->m_port           = port;                          //listen port
    	this->m_max_connect    = max_connect;                   //max connect
    	this->m_recv_timeout   = recv_timeout;                  //recv timeout
    	this->m_send_timeout   = send_timeout;                  //send timeout                               
    	this->m_povs           = new unsigned long[max_connect];//clients 
    
    }
    
    /**
     * @ destruct
     */
    WingIOCP::~WingIOCP(){
    	
    	if( this->m_listen_ip ) 
    	{	
    		free(this->m_listen_ip );
    		this->m_listen_ip = NULL;
    	}
    
    	if( this->m_povs )
    	{
    		delete[] this->m_povs;
    		this->m_povs = NULL;
    	}
    
    	if( m_sock_listen != INVALID_SOCKET )
    	{
    		closesocket( m_sock_listen );
    		m_sock_listen = INVALID_SOCKET;
    	}
    
    	WSACleanup();
    }
    
    /**
     *@wait
     */
    void WingIOCP::wait(){
    	while( true ){
    		Sleep(10);
    	}
    }
    
    //event callbacks
     void WingIOCP::onconnect( iocp_overlapped *&pOL ){
    	 printf("%ld onconnect\r\n",pOL->m_skClient);
    	 pOL->m_online = 1;
    	 pOL->m_active = time(NULL);
    
    	 if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_UPDATE_ACCEPT_CONTEXT,(const char *)&pOL->m_skServer,sizeof(pOL->m_skServer) ) != 0 )
    	 {
    		 //setsockopt fail
    		 //printf("1=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 WSASetLastError(0);
    		 return;
    	 }
    	// BOOL bReuse = 1;
    	// if( 0 != ::setsockopt( pOL->m_skClient, SOL_SOCKET, SO_REUSEADDR,(LPCSTR)&bReuse, sizeof(BOOL) ) )
    	 //{
    		 //some error happened
    		 
    	 //}
    
    	 // set send timeout
    	 if( pOL->m_send_timeout > 0 )
    	 {
    		 if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_SNDTIMEO, (const char*)&pOL->m_send_timeout,sizeof(pOL->m_send_timeout)) !=0 )
    		 {
    			 //setsockopt fail
    			// printf("2=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 }
    	 }
    	 if( pOL->m_recv_timeout > 0 )
    	 {
    		 if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_RCVTIMEO, (const char*)&pOL->m_recv_timeout,sizeof(pOL->m_recv_timeout)) != 0 )
    		 {
    			 //setsockopt fail
    			// printf("3=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 }
    	 }
    
    	 linger so_linger;
    	 so_linger.l_onoff	= TRUE;
    	 so_linger.l_linger	= 5; // without close wait status
    	 if( setsockopt( pOL->m_skClient,SOL_SOCKET,SO_LINGER,(const char*)&so_linger,sizeof(so_linger) ) != 0 ){
    		 printf("31=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    	 } 
    
    	 //get client ip and port
    	 int client_size = sizeof(pOL->m_addrClient);  
    	 ZeroMemory( &pOL->m_addrClient , sizeof(pOL->m_addrClient) );
    
    	 if( getpeername( pOL->m_skClient , (SOCKADDR *)&pOL->m_addrClient , &client_size ) != 0 ) 
    	 {
    		 //getpeername fail
    		// printf("4=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    	 }
    
    	// printf("%s %d connect\r\n",inet_ntoa(pOL->m_addrClient.sin_addr), pOL->m_addrClient.sin_port);
    
    	 //keepalive open
    	 int dt		= 1;
    	 DWORD dw	= 0;
    	 tcp_keepalive live,liveout ;     
    	 live.keepaliveinterval	= 1000;     //连接之后 多长时间发现无活动 开始发送心跳吧 单位为毫秒 
    	 live.keepalivetime		= 200;     //多长时间发送一次心跳包 1分钟是 60000 以此类推     
    	 live.onoff				= TRUE;     //是否开启 keepalive
    
    	 if( setsockopt( pOL->m_skClient, SOL_SOCKET, SO_KEEPALIVE, (char *)&dt, sizeof(dt) ) == 0 )
    	 {
    		 //setsockopt fail
    		 if( WSAIoctl(   pOL->m_skClient, SIO_KEEPALIVE_VALS, &live, sizeof(live), &liveout, sizeof(liveout), &dw, &pOL->m_ol , NULL ) == SOCKET_ERROR )
    		 {
    			 //WSAIoctl error
    			 printf("6=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 }
    	 } else  printf("5=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    
    	 memset(pOL->m_pBuf,0,DATA_BUFSIZE);
    	 //post recv
    	 pOL->m_DataBuf.buf	= pOL->m_pBuf;  
    	 pOL->m_DataBuf.len	= DATA_BUFSIZE;  
    	 pOL->m_iOpType		= OP_ONRECV;
    
    	 DWORD RecvBytes		= 0;
    	 DWORD Flags			= 0;
    
    	 int code		    = WSARecv(pOL->m_skClient,&(pOL->m_DataBuf),1,&RecvBytes,&Flags,&(pOL->m_ol),NULL);
    	 int error_code	    = WSAGetLastError();
    
    	 if( 0 != code )
    	 {
    		 if( WSA_IO_PENDING != error_code ) 
    		 {
    			// printf("7=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    			 return;
    		 }
    	 }
    	 else
    	 {
    		 //recv complete
    		 onrecv( pOL );
    	 }
     }
     void WingIOCP::ondisconnect( iocp_overlapped *&pOL ){
    	// printf("ondisconnect error %d\r\n",WSAGetLastError());
    	 WSASetLastError(0);
    	 pOL->m_online   = 0;                                //set offline
    	 pOL->m_active   = time(NULL);                       //the last active time
    	 pOL->m_iOpType	 = OP_ONACCEPT;                      //reset status
    	 pOL->m_isUsed   = 0;                                //
    	 ZeroMemory(pOL->m_pBuf,sizeof(char)*DATA_BUFSIZE);  //clear buf
    
    	 //i try free the socket and create a new one, but still close_wait 
    	 /*closesocket(pOL->m_skClient);
    	 pOL->m_skClient = WSASocket(AF_INET,SOCK_STREAM,IPPROTO_TCP,0,0,WSA_FLAG_OVERLAPPED);
    	 if( INVALID_SOCKET == pOL->m_skClient ) 
    	 {	
    		 return;
    	 }*/
    	 
    	 //post acceptex
    	 int error_code     = accept( pOL->m_skClient,pOL->m_pBuf,0,sizeof(SOCKADDR_IN)+16,sizeof(SOCKADDR_IN)+16,NULL, (LPOVERLAPPED)pOL );
    	 //printf("accept error %d\r\n",WSAGetLastError());
    	 int last_error     = WSAGetLastError() ;
    
    	 if( !error_code && ERROR_IO_PENDING != last_error ){
    
    	 }
    	 //printf("2=>ondisconnect some error happened , error code %d \r\n================================================\r\n\r\n", WSAGetLastError());
    	  
    	  //printf("21=>ondisconnect some error happened , error code %d \r\n================================================\r\n\r\n", WSAGetLastError());
    	 if( !BindIoCompletionCallback( (HANDLE)pOL->m_skClient ,worker,0) ){
    		  printf("BindIoCompletionCallback error %ld\r\n",WSAGetLastError());
    	 }
    	 WSASetLastError(0);
    
     }
    
     void WingIOCP::onaccept(iocp_overlapped *&pOL){
    	 pOL->m_active   = time(NULL);                       //the last active time
    	 pOL->m_iOpType	 = OP_ONCONNECT;                        //reset status
    	 ZeroMemory( &pOL->m_ol,sizeof(OVERLAPPED));
    	 printf("%ld reuse socket real complete , error code %d \r\n", pOL->m_skClient,WSAGetLastError());
    	// WSASetLastError(0);
    	// if( setsockopt( pOL->m_skClient, SOL_SOCKET,SO_UPDATE_ACCEPT_CONTEXT,(const char *)&pOL->m_skServer,sizeof(pOL->m_skServer) ) != 0 )
    	 {
    		 //setsockopt fail
    		 //printf("1=>onconnect some error happened , error code %d \r\n", WSAGetLastError());
    		 WSASetLastError(0);
    		 //return;
    	 }
     }
     void WingIOCP::onclose( iocp_overlapped *&pOL ){
    	// printf("%ld close\r\n", pOL->m_skClient);
    
    	 pOL->m_iOpType      = OP_DISCONNECT;
    	 //socket reuse
    	 if( !disconnect( pOL->m_skClient , &pOL->m_ol ) && WSA_IO_PENDING != WSAGetLastError()) {
    		// printf("1=>onclose some error happened , error code %d \r\n", WSAGetLastError());
    	 }
    	 //printf("onclose complete %d \r\n", WSAGetLastError());
     }
     void WingIOCP::onrecv( iocp_overlapped *&pOL ){
    	 pOL->m_active = time(NULL);
    	// printf("recv:\r\n%s\r\n\r\n",pOL->m_pBuf);
    	 ZeroMemory(pOL->m_pBuf,DATA_BUFSIZE);      
     }
     void WingIOCP::onsend( iocp_overlapped *&povl ){
    
     }
     void WingIOCP::onrun( iocp_overlapped *&povl, DWORD errorcode, int last_error ){}
    /**
     * @ acceptex
     */
    BOOL WingIOCP::accept(
    	SOCKET  sAcceptSocket,
    	PVOID   lpOutputBuffer,
    	DWORD   dwReceiveDataLength,
    	DWORD   dwLocalAddressLength,
    	DWORD   dwRemoteAddressLength,
    	LPDWORD lpdwBytesReceived,
    	LPOVERLAPPED lpOverlapped
    )
    {
    	WSASetLastError(0);
    	if( m_sock_listen == INVALID_SOCKET || !lpOverlapped ) 
    	{	
    		return 0;
    	}
    	GUID guidAcceptEx	= WSAID_ACCEPTEX;
    	DWORD dwBytes		= 0;
    	LPFN_ACCEPTEX lpfnAcceptEx;
    
    	int res= WSAIoctl( m_sock_listen, SIO_GET_EXTENSION_FUNCTION_POINTER, &guidAcceptEx, 
    		sizeof(guidAcceptEx), &lpfnAcceptEx, sizeof(lpfnAcceptEx), &dwBytes, NULL, NULL );
    
    	if( 0 != res )
    	{
    		return 0;
    	}
    
    	return lpfnAcceptEx( m_sock_listen, sAcceptSocket, lpOutputBuffer, dwReceiveDataLength,
    		dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, lpOverlapped );        
    }
    
    /**
     * @ disconnect socket and reuse the socket
     */
    BOOL WingIOCP::disconnect( SOCKET client_socket , LPOVERLAPPED lpOverlapped , DWORD dwFlags  , DWORD reserved  )
    {
    	WSASetLastError(0);
    	if( client_socket == INVALID_SOCKET || !lpOverlapped ) 
    	{	
    		return 0;
    	}
    	GUID GuidDisconnectEx = WSAID_DISCONNECTEX;
    	DWORD dwBytes = 0;
    	LPFN_DISCONNECTEX lpfnDisconnectEx; 
    
    	if( 0 != WSAIoctl( client_socket,SIO_GET_EXTENSION_FUNCTION_POINTER,&GuidDisconnectEx,
    		sizeof(GuidDisconnectEx),&lpfnDisconnectEx,sizeof(lpfnDisconnectEx),&dwBytes,NULL,NULL))
    	{
    		return 0;
    	}
    	return lpfnDisconnectEx(client_socket,lpOverlapped,/*TF_REUSE_SOCKET*/dwFlags,reserved);
    }
    
    
    	
    /**
     * @ iocp worker thread
     */
    VOID CALLBACK WingIOCP::worker( DWORD dwErrorCode,DWORD dwBytesTrans,LPOVERLAPPED lpOverlapped )
    {
    	//why here get the error code 87 ?
    	//printf("worker error %d\r\n",WSAGetLastError());
    	if( NULL == lpOverlapped  )
    	{
    		//not real complete
    		SleepEx(20,TRUE);//set warn status
    		WSASetLastError(0);
    		return;
    	}
    
    	//get overlapped data
    	iocp_overlapped*  pOL = CONTAINING_RECORD(lpOverlapped, iocp_overlapped, m_ol);
    	
    	//just a test
    	//onrun( pOL, dwErrorCode, WSAGetLastError() );
    	
    	switch( pOL->m_iOpType )
    	{
    	case OP_DISCONNECT:
    			ondisconnect( pOL );
    		break;
    	case OP_ONACCEPT:
    			onaccept( pOL );
    		break;
    	case OP_ONCONNECT: 
    		{
    			//new client connect
    			onconnect( pOL );
    		}
    		break;
    	case OP_ONRECV:
    		{
    			pOL->m_recvBytes = dwBytesTrans;
    			int last_error = WSAGetLastError();
    			//check client offline
    			if( 0 == dwBytesTrans || WSAECONNRESET == last_error  || ERROR_NETNAME_DELETED == last_error )
    			{
    				onclose( pOL );
    			} 
    			else
    			{   //recv msg from client
    				onrecv( pOL );
    			}	
    		}
    		break;
    	case OP_ONSEND:
    		{
    
    		}
    		break;
    	
    	}
    
    	WSASetLastError(0);
    
    }
    
    
    BOOL WingIOCP::start(){	
    
    	do{ 
    
    		WSADATA wsaData; 
    		if( WSAStartup(MAKEWORD(2,2), &wsaData) != 0 )
    		{
    			return FALSE;
    		}
     
    		if(LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2)
    		{
    			break;
    		}  
    
    		m_sock_listen = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED); 
    		if( INVALID_SOCKET == m_sock_listen )
    		{
    			break;
    		}
    
    		//bind the worker thread
    		BOOL bReuse      = TRUE;
    		BOOL bind_status = ::BindIoCompletionCallback((HANDLE)( m_sock_listen ), worker, 0 );
    		if( !bind_status )
    		{
    			break;
    		}
    
    		//set option SO_REUSEADDR 
    		if( 0 != ::setsockopt( m_sock_listen, SOL_SOCKET, SO_REUSEADDR,(LPCSTR)&bReuse, sizeof(BOOL) ) )
    		{
    			//some error happened
    			break;
    		}
    
    
    		struct sockaddr_in ServerAddress; 
    		ZeroMemory(&ServerAddress, sizeof(ServerAddress)); 
    
    		ServerAddress.sin_family		= AF_INET;                    
    		ServerAddress.sin_addr.s_addr	= inet_addr( this->m_listen_ip );          
    		ServerAddress.sin_port			= htons( this->m_port );   
    
    		if ( SOCKET_ERROR == bind( m_sock_listen, (struct sockaddr *) &ServerAddress, sizeof( ServerAddress ) ) )
    		{
    			break;
    		}  
    
    		if( 0 != listen( m_sock_listen , SOMAXCONN ) )
    		{
    			break;
    		}
    		//printf("1=>start get error %d\r\n",WSAGetLastError());
    		WSASetLastError(0);
    		//socket pool
    		for( int i = 0 ; i < this->m_max_connect ; i++ ) 
    		{
    	
    			SOCKET client = WSASocket(AF_INET,SOCK_STREAM,IPPROTO_TCP,0,0,WSA_FLAG_OVERLAPPED);
    			if( INVALID_SOCKET == client ) 
    			{	
    				continue;
    			}
    
    			
    
    			iocp_overlapped *povl = new iocp_overlapped();
    			if( NULL == povl )
    			{
    				closesocket(client);
    				continue;
    			}
    
    			DWORD dwBytes = 0;
    			ZeroMemory(povl,sizeof(iocp_overlapped));
    		
    			povl->m_iOpType			= OP_ONCONNECT;
    			povl->m_skServer			= m_sock_listen;
    			povl->m_skClient			= client;
    			povl->m_recv_timeout		= m_recv_timeout;
    			povl->m_isUsed			= 0;
    			povl->m_active			= 0; 
    			povl->m_isCrashed		= 0;
    			povl->m_online			= 0;
    			povl->m_usenum			= 1;
    
    			int server_size = sizeof(povl->m_addrServer);  
    			ZeroMemory(&povl->m_addrServer,server_size);
    			getpeername(povl->m_skServer,(SOCKADDR *)&povl->m_addrServer,&server_size);  
    
    			int error_code = accept( povl->m_skClient, povl->m_pBuf, 0, sizeof(SOCKADDR_IN)+16, sizeof(SOCKADDR_IN)+16, NULL, (LPOVERLAPPED)povl );
    			int last_error = WSAGetLastError() ;
    			if( !error_code && ERROR_IO_PENDING != last_error ) 
    			{
    			
    				closesocket( client );
    				client = povl->m_skClient = INVALID_SOCKET;
    				delete povl;
    				povl = NULL; 
    				//printf("client=>crate error %d\r\n",WSAGetLastError());
    			}else{
    				this->m_povs[i] = (unsigned long)povl;
    			}
    
    
    			if( !BindIoCompletionCallback( (HANDLE)client ,worker,0) )
    			{
    				closesocket(client);
    				//continue;
    			}
    			//here all the last error is 997 , means nothing error happened
    			//printf("client=>start get error %d\r\n",WSAGetLastError());
    			WSASetLastError(0);
    		}
    		//printf("last start get error %d\r\n",WSAGetLastError());
    		WSASetLastError(0);
    		return TRUE;
    
    	} while( 0 );
    
    	if( m_sock_listen != INVALID_SOCKET )
    	{
    		closesocket( m_sock_listen );
    		m_sock_listen = INVALID_SOCKET;
    	}
    	WSACleanup();
    
    	return FALSE;
    }
    
    
    
    
    int _tmain(int argc, _TCHAR* argv[])
    {
    	WingIOCP *iocp  = new WingIOCP();
    	iocp->start();
    	iocp->wait();
    	delete iocp;
    	return 0;
    }
    
    
    
    • 已合并 lake Xiao 2016年9月6日 2:15 duplicated
    2016年9月5日 3:46
  • Hi 吉利儿,

    感谢在MSDN论坛发帖。

    对于你的问题,建议你使用以下两种方式。

    1.使用Using SO_REUSEADDR和SO_EXCLUSIVEADDRUSE。使用方法参考以下文档。

    https://msdn.microsoft.com/en-us/library/windows/desktop/ms740621(v=vs.85).aspx

    2. 使用Socket::Disconnect函数关闭socket,并允许重用。例如:

    Disconnect( true );

    希望对你有所帮助。

    Best Regards,

    Sera Yu


    We are trying to better understand customer views on social support experience, so your participation in this interview project would be greatly appreciated if you have time. Thanks for helping make community forums a great place.

    2016年9月5日 6:43
  • 谢谢您的回复,我上面的代码在server有设置SO_REUSEADDR,另外我在client也加入过SO_REUSEADDR并进行测试,好像跟这个关系不大,结果还是一样,运行一段时间之后出现close_wait,然后再也无法连接了~
    2016年9月5日 7:24
  • 无法再连接的错误信息是什么?WSAGetLastError()

    Visual C++ enthusiast, like network programming and driver development. At present is being engaged in the WinCE/Windows Mobile platform embedded development.

    2016年9月5日 8:32
    版主
  • 没有错误~ WSAGetLastError()是0,无法再连接的现象就是,比如你用浏览器访问监听的端口,只会看到procexp_16.05.1446001339.exe多了一个close_status,代码里面对应的事件全部都没有触发!
    2016年9月5日 9:32
  • 左边的数字为对应的socket,测试结果表明socket能重用

    2016年9月6日 1:20
  • 把最大连接数修改为2,明显一些,测试结果

    2016年9月6日 1:48
  • Hi 吉利儿,

    感谢在MSDN论坛发帖。

    >>没有错误~ WSAGetLastError()是0,无法再连接的现象就是,比如你用浏览器访问监听的端口,只会看到procexp_16.05.1446001339.exe多了一个close_status,代码里面对应的事件全部都没有触发!

    对于这种情况,建议你使用Winsock Tracing跟踪Socket Close事件。或者使用debug模式查看Socket close消息是从哪边发出来的。方便查找主要原因。

    关于如何使用Winsock Tracing,建议参考以下文档。

    https://msdn.microsoft.com/en-us/library/windows/desktop/bb892103(v=vs.85).aspx

    希望对你有所帮助。

    Best Regards,

    Sera Yu


    We are trying to better understand customer views on social support experience, so your participation in this interview project would be greatly appreciated if you have time. Thanks for helping make community forums a great place.

    2016年9月27日 8:19
  • 挖坟,看到楼主的WingIOCP::worker里面,所有投递IO操作,都用的同一个指针,还不加锁,我就知道要出问题了。

    首先:DisconnectEx会在所有IO投递操作返回之后,才返回,

    并且在多线程的情况下,他们没有什么顺序,你就当它是并发的,

    理想的情况下,一个应答式的服务器,也就是逻辑上跟同步阻塞的方式差不多是一致的方式,一问一答,

    这种情况时,我们能在线程池中保证一个套接字上永远只有一个投递操作,但注意这是理想的情况。

    一旦你考虑了超时套接字的处理,这就是另外一回事了。

    举个例子:

    你先投递了WSARecv,IOCP队列未返回,发现了超时,DisconnectEx,然后,针对楼主的代码,

    m_iOpType,被改变了,而此时,这个套接字上,有两个投递操作,然而你使用了同一个OVERLAPPED去投递,

    然后再是理想的状态下,客户端closesocket了,

    然后你的两个投递操作并发返回,而两个返回的操作都认为自己是DisconnectEx,如果不加锁,不判断,

    然后先返回的那个线程先一步AcceptEx,后返回的线程,再去AcceptEx,结果发现错误,closesocket掉,

    接着重新创建套接字,AcceptEx,然后因为之前已经投递过AcceptEx了,closesocket导致了这个投递操作返回,

    最终开始多线程递归死循环。

    所以,应答式服务器,要用同一个OVERLAPPED去操作,最起码应该保证m_iOpType的改变,是加锁的情况下处理的,

    而在发现套接字超时时,不应该DisconnectEx,而是应该直接关闭掉。

    存在异步行为的情况下:

    AcceptEx, WSARecv可以使用同一个OVERLAPPED,

    WSASend必须是一个独立的OVERLAPPED

    DisconnectEx也必须是一个独立的OVERLAPPED

    在它们几个的包装类中,应该存在OptsType,

    投递WSARecv时,OptTypes|=OP_ONRECV;

    投递WSASend时,OptTypes|=OP_ONWRITE;

    响应的操作时 OptTypes^=重叠结构对应返回的操作码。

    当所有投递操作返回后,才是DisconnectEx与AcceptEx的时机。

    2019年10月26日 2:56