How to access each microphone array of multiple Kinects RRS feed

  • Question

  • I am trying to run 3 parallel applications in C++ on the same computer, I've successfully access the video streams from 3Kinect, by selecting different Kinect index:

    hr_init_0 = NuiCreateSensorByIndex(1, &m_pNuiSensor);

    But when I try to integrate the MFAudioFilter into my application, I don't know how to select each microphone array for the corresponding Kinect.  Is there a similar function as NuiCreateSensorByIndex for the Kinect Audio SDK? Or, how can I manually define which microphone array I could use (to match the microphone array to the same Kinect video stream). 

    The description of the current bug is:

    When I used NuiCreateSensorByIndex (0, &m_pNuiSensnor), I could get the right audio reading from the same Kinect at Index 0.

    When I used NuiCreateSensorByIndex(1, &m_pNuiSensor), I could get the video stream of Kinect at index 1, but I met mistakes when I try to get audio reading; The code stops at :

      hr = pDMO->SetOutputType(0, &mt, 0);

    under the function of 

    HRESULT DShowRecord(IMediaObject* pDMO, IPropertyStore* pPS,
                        const TCHAR* outFile, int  iDuration)
    and in the terminal, it says "Unhandled Exception: System.AccessViolationException:..."

    Lining YAO

    Sunday, April 8, 2012 11:08 PM

All replies

  • Hi Lizziey, 

    Sorry to bother because I didn't see your error before.
    But can you give the entire code for your init and video captures ? It might help to give a targetted answer to your coding ways.

    1st without microphone codes, and 2nd with microphone currently included code.

    Good Luck.

    Tuesday, April 10, 2012 3:51 PM
  • Hi Mickael,

    Sure. The initializing function is pasted below. If you could help to take a look, that would be great! 

    // Nui_Init
    // Initialize reading data from kinect
    HRESULT KinectGrabber::Kinect_Init() {
    	InitializeCriticalSection (&cs);
    	writeTurn = true;
    	HRESULT hr,hr_init_0,hr0,hr_rgb_0;
    	if ( !m_pNuiSensor )
            hr_init_0 = NuiCreateSensorByIndex(0, &m_pNuiSensor);
            if ( FAILED(hr) )
                return hr;
            m_instanceId = m_pNuiSensor->NuiDeviceConnectionId();
        m_hNextVideoFrameEvent = CreateEvent( NULL, TRUE, FALSE, NULL );    
    	m_hNextDepthFrameEvent = CreateEvent( NULL, TRUE, FALSE, NULL );    
    	//m_hNextSkeletonFrameEvent = CreateEvent( NULL, TRUE, FALSE, NULL );
    	m_hNextSkeletonEvent = CreateEvent( NULL, TRUE, FALSE, NULL );
    	//m_hNextDepthPlayerFrameEvent = CreateEvent( NULL, TRUE, FALSE, NULL );
    	int num_kinects;
    	hr = NuiGetSensorCount(&num_kinects);
    	printf(" %d Kinetics found \n\n", num_kinects);
    	//hr_init_0 = NuiCreateSensorByIndex(1,&m_pNuiSensor);  // change this index to pick a different kinect
    	if (hr_init_0 == S_OK) {
    		printf("Created Kinect instance, pointer addres:  %p \n",m_pNuiSensor);
    	else {
    		printf("Creation of Kinetic instance  FAILED. \n");
    	if ( E_NUI_SKELETAL_ENGINE_BUSY == hr )
            hr = m_pNuiSensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_DEPTH |  NUI_INITIALIZE_FLAG_USES_COLOR) ;
    		printf("could not init tracking/n");
        if ( FAILED( hr ) )
            if ( E_NUI_DEVICE_IN_USE == hr )
                		printf("nui device in use/n");
                		printf("problem initializing nui/n");
            return hr;
    	if ( HasSkeletalEngine( m_pNuiSensor ) )
            hr = m_pNuiSensor->NuiSkeletonTrackingEnable( m_hNextSkeletonEvent, 0 );
            if( FAILED( hr ) )
                return hr;
    	m_pNuiSensor-> NuiImageStreamOpen(
            &m_pVideoStreamHandle );
        if( FAILED( hr ) )
    		printf("failed to open NuiImagesStream");
            return hr;
        hr = m_pNuiSensor->NuiImageStreamOpen(
            HasSkeletalEngine(m_pNuiSensor) ? NUI_IMAGE_TYPE_DEPTH_AND_PLAYER_INDEX : NUI_IMAGE_TYPE_DEPTH,
            &m_pDepthStreamHandle );
        if ( FAILED( hr ) )
            printf("failed to open NuiImagesStream");
            return hr;
    	////////////////////////// audio init///////////////////////////////////////
    	minDiscrepancyIdx=7;  //when skeleton is detected, the number should be between 0 to 6. Set it to a random number beyond this range is OK
    	hr = S_OK;
        int  iMicDevIdx = -1; 
    	int  iSpkDevIdx = 0;  //Asume default speakers
        DWORD mmTaskIndex = 0;
        // Set high priority to avoid getting preempted while capturing sound
        mmHandle = AvSetMmThreadCharacteristics(L"Audio", &mmTaskIndex);
        CHECK_BOOL(mmHandle != NULL, "failed to set thread priority\n");
        // DMO initialization
        //CHECKHR(CoCreateInstance(CLSID_CMSRKi new unsigned char [DEPTH_WIDTH*DEPTH_HEIGHT*4]nectAudio, NULL, CLSCTX_INPROC_SERVER, IID_IMediaObject, (void**)&pDMO));
        //CHECKHR(pDMO->QueryInterface(IID_IPropertyStore, (void**)&pPS));
    	INuiAudioBeam* pAudio = NULL;
        CHECKHR(pAudio->QueryInterface(IID_IMediaObject, (void**)&pDMO));
        CHECKHR(pAudio->QueryInterface(IID_IPropertyStore, (void**)&pPS));
    	// Set AEC-MicArray DMO system mode.
        // This must be set for the DMO to work properly
        PROPVARIANT pvSysMode;
        pvSysMode.vt = VT_I4;
        //   SINGLE_CHANNEL_AEC = 0
        //   OPTIBEAM_ARRAY_ONLY = 2
        //   OPTIBEAM_ARRAY_AND_AEC = 4
        //   SINGLE_CHANNEL_NSAGC = 5
        pvSysMode.lVal = (LONG)(2);
    	    DWORD dwWait = 2;
        while (dwWait > 0)
            _tprintf(_T("Device will be ready for recording in %d second(s).\r"), dwWait--);
    	// Tell DMO which capture device to use (we're using whichever device is a microphone array).
        // Default rendering device (speaker) will be used.
        hr = GetMicArrayDeviceIndex(&iMicDevIdx);
        CHECK_RET(hr, "Failed to find microphone array device. Make sure microphone array is properly installed.");
        PROPVARIANT pvDeviceId;
        pvDeviceId.vt = VT_I4;
    	//Speaker index is the two high order bytes and the mic index the two low order ones
        pvDeviceId.lVal = (unsigned long)(iSpkDevIdx<<16) | (unsigned long)(0x0000ffff & iMicDevIdx);
    	////////////////////////// audio init///////////////////////////////////////
    	m_hThNuiProcess = CreateThread( NULL, 0, Kinect_Update, this, 0, NULL );
    	m_hEvNuiProcessStop = CreateEvent( NULL, FALSE, FALSE, NULL );
        //puts("done with initializing kinect sensor, press any key to continue...");
    	return 0;

    Lining YAO

    Tuesday, April 10, 2012 4:41 PM