How to access each microphone array of multiple Kinects

Question

I am trying to run 3 parallel applications in C++ on the same computer, I've successfully access the video streams from 3Kinect, by selecting different Kinect index:

hr_init_0 = NuiCreateSensorByIndex(1, &m_pNuiSensor);

But when I try to integrate the MFAudioFilter into my application, I don't know how to select each microphone array for the corresponding Kinect.  Is there a similar function as NuiCreateSensorByIndex for the Kinect Audio SDK? Or, how can I manually define which microphone array I could use (to match the microphone array to the same Kinect video stream). 

The description of the current bug is:

When I used NuiCreateSensorByIndex (0, &m_pNuiSensnor), I could get the right audio reading from the same Kinect at Index 0.

When I used NuiCreateSensorByIndex(1, &m_pNuiSensor), I could get the video stream of Kinect at index 1, but I met mistakes when I try to get audio reading; The code stops at :

  hr = pDMO->SetOutputType(0, &mt, 0);

under the function of 

HRESULT DShowRecord(IMediaObject* pDMO, IPropertyStore* pPS,
                    const TCHAR* outFile, int  iDuration)

and in the terminal, it says "Unhandled Exception: System.AccessViolationException:..."

---

Lining YAO

Answer 1

Hi Lizziey, 

Sorry to bother because I didn't see your error before.
But can you give the entire code for your init and video captures ? It might help to give a targetted answer to your coding ways.

1st without microphone codes, and 2nd with microphone currently included code.

Good Luck.

Answer 2

Hi Mickael,

Sure. The initializing function is pasted below. If you could help to take a look, that would be great! 

//-------------------------------------------------------------------
// Nui_Init
//-------------------------------------------------------------------
// Initialize reading data from kinect
HRESULT KinectGrabber::Kinect_Init() {


	InitializeCriticalSection (&cs);
	writeTurn = true;
	HRESULT hr,hr_init_0,hr0,hr_rgb_0;

	if ( !m_pNuiSensor )
    {
        hr_init_0 = NuiCreateSensorByIndex(0, &m_pNuiSensor);

        if ( FAILED(hr) )
        {
            return hr;
        }

        SysFreeString(m_instanceId);

        m_instanceId = m_pNuiSensor->NuiDeviceConnectionId();
    }

    m_hNextVideoFrameEvent = CreateEvent( NULL, TRUE, FALSE, NULL );    
	m_hNextDepthFrameEvent = CreateEvent( NULL, TRUE, FALSE, NULL );    
	//m_hNextSkeletonFrameEvent = CreateEvent( NULL, TRUE, FALSE, NULL );
	m_hNextSkeletonEvent = CreateEvent( NULL, TRUE, FALSE, NULL );
	//m_hNextDepthPlayerFrameEvent = CreateEvent( NULL, TRUE, FALSE, NULL );

	//add
	int num_kinects;
	hr = NuiGetSensorCount(&num_kinects);
	printf(" %d Kinetics found \n\n", num_kinects);

	//m_pNuiSensor=NULL;
	//hr_init_0 = NuiCreateSensorByIndex(1,&m_pNuiSensor);  // change this index to pick a different kinect

	if (hr_init_0 == S_OK) {

		printf("Created Kinect instance, pointer addres:  %p \n",m_pNuiSensor);
	}
	else {

		printf("Creation of Kinetic instance  FAILED. \n");
	}

	hr0 = m_pNuiSensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_DEPTH_AND_PLAYER_INDEX | NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_SKELETON| NUI_INITIALIZE_FLAG_USES_AUDIO);

	if ( E_NUI_SKELETAL_ENGINE_BUSY == hr )
    {
        hr = m_pNuiSensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_DEPTH |  NUI_INITIALIZE_FLAG_USES_COLOR) ;
		printf("could not init tracking/n");
    }
    if ( FAILED( hr ) )
    {
        if ( E_NUI_DEVICE_IN_USE == hr )
        {
            		printf("nui device in use/n");
        }
        else
        {
            		printf("problem initializing nui/n");
        }
        return hr;
    }


	if ( HasSkeletalEngine( m_pNuiSensor ) )
    {
        hr = m_pNuiSensor->NuiSkeletonTrackingEnable( m_hNextSkeletonEvent, 0 );
        if( FAILED( hr ) )
        {
            return hr;
        }
    }


	m_pNuiSensor-> NuiImageStreamOpen(
        NUI_IMAGE_TYPE_COLOR,
        NUI_IMAGE_RESOLUTION_640x480,
        0,
        2,
        m_hNextVideoFrameEvent,
        &m_pVideoStreamHandle );
    if( FAILED( hr ) )
    {
		printf("failed to open NuiImagesStream");
        return hr;
    }



    hr = m_pNuiSensor->NuiImageStreamOpen(
        HasSkeletalEngine(m_pNuiSensor) ? NUI_IMAGE_TYPE_DEPTH_AND_PLAYER_INDEX : NUI_IMAGE_TYPE_DEPTH,
        NUI_IMAGE_RESOLUTION_640x480,
        0,
        2,
        m_hNextDepthFrameEvent,
        &m_pDepthStreamHandle );

    if ( FAILED( hr ) )
    {
        printf("failed to open NuiImagesStream");
        return hr;
    }





	////////////////////////// audio init///////////////////////////////////////

	minDiscrepancyIdx=7;  //when skeleton is detected, the number should be between 0 to 6. Set it to a random number beyond this range is OK
	hr = S_OK;
    CoInitialize(NULL);
    int  iMicDevIdx = -1; 
	int  iSpkDevIdx = 0;  //Asume default speakers
    DWORD mmTaskIndex = 0;

    // Set high priority to avoid getting preempted while capturing sound
    mmHandle = AvSetMmThreadCharacteristics(L"Audio", &mmTaskIndex);
    CHECK_BOOL(mmHandle != NULL, "failed to set thread priority\n");

    // DMO initialization
    //CHECKHR(CoCreateInstance(CLSID_CMSRKi new unsigned char [DEPTH_WIDTH*DEPTH_HEIGHT*4]nectAudio, NULL, CLSCTX_INPROC_SERVER, IID_IMediaObject, (void**)&pDMO));
    //CHECKHR(pDMO->QueryInterface(IID_IPropertyStore, (void**)&pPS));
	printf("..............");
	INuiAudioBeam* pAudio = NULL;
    CHECKHR(NuiGetAudioSource(&pAudio));
    CHECKHR(pAudio->QueryInterface(IID_IMediaObject, (void**)&pDMO));
    CHECKHR(pAudio->QueryInterface(IID_IPropertyStore, (void**)&pPS));
    SAFE_RELEASE(pAudio);	


	// Set AEC-MicArray DMO system mode.
    // This must be set for the DMO to work properly
    PROPVARIANT pvSysMode;
    PropVariantInit(&pvSysMode);
    pvSysMode.vt = VT_I4;
    //   SINGLE_CHANNEL_AEC = 0
    //   OPTIBEAM_ARRAY_ONLY = 2
    //   OPTIBEAM_ARRAY_AND_AEC = 4
    //   SINGLE_CHANNEL_NSAGC = 5
    pvSysMode.lVal = (LONG)(2);
    CHECKHR(pPS->SetValue(MFPKEY_WMAAECMA_SYSTEM_MODE, pvSysMode));
    PropVariantClear(&pvSysMode);

	    DWORD dwWait = 2;
    while (dwWait > 0)
    {
        _tprintf(_T("Device will be ready for recording in %d second(s).\r"), dwWait--);
        Sleep(1000);
    }
	/*
	// Tell DMO which capture device to use (we're using whichever device is a microphone array).
    // Default rendering device (speaker) will be used.
    hr = GetMicArrayDeviceIndex(&iMicDevIdx);
    CHECK_RET(hr, "Failed to find microphone array device. Make sure microphone array is properly installed.");
   
    PROPVARIANT pvDeviceId;
    PropVariantInit(&pvDeviceId);
    pvDeviceId.vt = VT_I4;
	//Speaker index is the two high order bytes and the mic index the two low order ones
    pvDeviceId.lVal = (unsigned long)(iSpkDevIdx<<16) | (unsigned long)(0x0000ffff & iMicDevIdx);
    CHECKHR(pPS->SetValue(MFPKEY_WMAAECMA_DEVICE_INDEXES, pvDeviceId));
    PropVariantClear(&pvDeviceId);
	*/

	////////////////////////// audio init///////////////////////////////////////


	m_hThNuiProcess = CreateThread( NULL, 0, Kinect_Update, this, 0, NULL );
	m_hEvNuiProcessStop = CreateEvent( NULL, FALSE, FALSE, NULL );

    //puts("done with initializing kinect sensor, press any key to continue...");
    //_getch();


	exit:
    puts(""); 
	return 0;
}

---

Lining YAO