locked
Uncompressed video sample (IMFSample/RGB32) to bitmap thumbnail

    Question

  • I am trying to generate thumbnails for a video in Windows Runtime C++ Project. I am passing in the input video file as IRandomAccessStream as a parameter. I am able to get IMFSamples using SourceReader but I am not sure how to pass back the data to the caller as an image thumbnail to be consumed in XAML/C# app.

    Please disregard some of the hard-coded values and for simplicity reasons, I am just getting the first thumbnail at around 0.5 sec mark. I wasn't sure what the best way to convert from IMFSample to WriteBitmap was. Here's what I've tried so far... not even sure if I am in the right direction...

    IAsyncOperation<WriteableBitmap^>^ MediaTransformer::GenerateThumbnailAsync(IRandomAccessStream^ inputStream)

    {

           WriteableBitmap^ bitmap = ref new WriteableBitmap(1280, 720);

           IUnknown* pUnknown = reinterpret_cast<IUnknown*>(bitmap->PixelBuffer);

           IBufferByteAccess* pBufferByteAccess = nullptr;

           CHK(pUnknown->QueryInterface(IID_PPV_ARGS(&pBufferByteAccess)));

           pUnknown->Release();

           return create_async([this, bitmap, pBufferByteAccess, inputStream](cancellation_token token) -> WriteableBitmap^

           {

                  HRESULT hr = S_OK;

                  AutoMF mf;

                  //

                  // Create the ReadWriteClassFactory

                  //

                  CoCreateInstance(CLSID_MFReadWriteClassFactory, NULL, CLSCTX_INPROC_SERVER, IID_IMFReadWriteClassFactory, (void**)(&spClassFactory));

                  //

                  // Create the Source Reader

                  //

                  ComPtr<IMFByteStream> spInputByteStream;

                  CHK(MFCreateMFByteStreamOnStreamEx((IUnknown*)inputStream, &spInputByteStream));

                  ComPtr<IMFAttributes> spReaderAttributes;

                  CHK(MFCreateAttributes(&spReaderAttributes, 10));

                  CHK(spReaderAttributes->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, true));

                  CHK(spReaderAttributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, true));

                  CHK(spClassFactory->CreateInstanceFromObject(CLSID_MFSourceReader, spInputByteStream.Get(), spReaderAttributes.Get(), IID_IMFSourceReader, &spSourceReader));

                  ComPtr<IMFMediaType> spType = NULL;

                  // Configure the source reader to give us progressive RGB32 frames.

                  // The source reader will load the decoder if needed.

                  CHK(MFCreateMediaType(&spType));

                  CHK(spType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video))

                  CHK(spType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32));

                  CHK(spSourceReader->SetCurrentMediaType(

                               (DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,

                               NULL

                               spType.Get()));

                  CHK(spSourceReader->SetStreamSelection(

                               (DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM

                               TRUE));

                  

                  //hr = GetVideoFormat(&m_format);

                  BOOL bCanSeek = FALSE;

                  LONGLONG hnsDuration = 0;

                  LONGLONG hnsRangeStart = 0;

                  LONGLONG hnsRangeEnd = 0;

                  LONGLONG hnsIncrement = 5000000;

                  CHK(CanSeek(&bCanSeek));

                  if (bCanSeek)

                  {

                         CHK(MediaTransformer::GetDuration(spSourceReader.Get(), &hnsDuration));

                         //hnsRangeStart = 0;

                         //hnsRangeEnd = hnsDuration;

                         //// We have the file duration , so we'll take bitmaps from

                         //// several positions in the file. Occasionally, the first frame

                         //// in a video is black, so we don't start at time 0.

                         //hnsIncrement = (hnsRangeEnd - hnsRangeStart) / (5 + 1);

                  }

                  LONGLONG hnsPos = hnsIncrement;

                  DWORD       dwFlags = 0;

                  BYTE        *pBitmapData = NULL;    // Bitmap data

                  DWORD       cbBitmapData = 0;       // Size of data, in bytes

                  LONGLONG    hnsTimeStamp = 0;

                  DWORD       cSkipped = 0;           // Number of skipped frames

                  DWORD         streamIndex = 0;

                  IMFSample *pSample = NULL;

                  IMFMediaBuffer *pBuffer = 0;

                  IMF2DBuffer2 *pImageBuffer = 0;

                  //ID2D1Bitmap *pBitmap = NULL;

                  LONG strideSize = 0;

                  DWORD bufferLength = 0;

                  BYTE *pBufferStart;

                  if (bCanSeek && (hnsPos > 0))

                  {

                         PROPVARIANT var;

                         PropVariantInit(&var);

                         var.vt = VT_I8;

                         var.hVal.QuadPart = hnsPos;

                         CHK(spSourceReader->SetCurrentPosition(GUID_NULL, var));

                  }

                  // Pulls video frames from the source reader.

                  // NOTE: Seeking might be inaccurate, depending on the container

                  //       format and how the file was indexed. Therefore, the first

                  //       frame that we get might be earlier than the desired time.

                  //       If so, we skip up to MAX_FRAMES_TO_SKIP frames.

                  while (1)

                  {

                         IMFSample *pSampleTmp = NULL;

                         hr = spSourceReader->ReadSample(

                               (DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,

                               0,

                               &streamIndex,

                               &dwFlags,

                               NULL,

                               &pSampleTmp

                               );

                         if (FAILED(hr)) { goto done; }

                         if (dwFlags & MF_SOURCE_READERF_ENDOFSTREAM)

                         {

                               break;

                         }

                         if (dwFlags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED)

                         {

                               // Type change. Get the new format.

                               //hr = GetVideoFormat(&m_format);

                               if (FAILED(hr)) { goto done; }

                         }

                         if (pSampleTmp == NULL)

                         {

                               continue;

                         }

                         // We got a sample. Hold onto it.

                         SafeRelease(&pSample);

                         pSample = pSampleTmp;

                         pSample->AddRef();

                         if (SUCCEEDED( pSample->GetSampleTime(&hnsTimeStamp) ))

                         {

                               // Keep going until we get a frame that is within tolerance of the

                               // desired seek position, or until we skip MAX_FRAMES_TO_SKIP frames.

                               // During this process, we might reach the end of the file, so we

                               // always cache the last sample that we got (pSample).

                               if ( (cSkipped < MAX_FRAMES_TO_SKIP) &&

                                      (hnsTimeStamp + SEEK_TOLERANCE < hnsPos) )

                               {

                                      SafeRelease(&pSampleTmp);

                                      ++cSkipped;

                                      continue;

                               }

                         }

                         SafeRelease(&pSampleTmp);

                         hnsPos = hnsTimeStamp;

                         break;

                  }

                  if (pSample)

                  {

                         CHK(pSample->ConvertToContiguousBuffer(&pBuffer));

                         CHK(pBuffer->QueryInterface(IID_IMF2DBuffer2, (void**)&pImageBuffer));

                         hr = pImageBuffer->Lock2DSize(MF2DBuffer_LockFlags_Read, &pBitmapData, &strideSize, &pBufferStart, &bufferLength);

                         hr = pBufferByteAccess->Buffer(&pBitmapData);

                         CHK(MFCopyImage(pBitmapData, 0, pBufferStart, strideSize, 1280, 720));

                  }

                  else

                  {

                         hr = MF_E_END_OF_STREAM;

                  }

    done:

                  if (pBitmapData)

                  {

                         pImageBuffer->Unlock2D();

                  }

                  SafeRelease(&pBuffer);

                  SafeRelease(&pSample);

                  SafeRelease(&pImageBuffer);

                  SafeRelease(&pBufferStart);

                  return bitmap;

           });

    }

    Friday, September 14, 2012 4:10 AM

All replies

  • I didn't run your code, it looks similar to how I would do it.

    One note though...

    As far as I know for gpu acceleration (or at least to have control of your d3d device), the IMFSourceReader needs a IMFDXGIManager initialized (ResetDevice) with a D3D device (with ID3D10Multithread::SetMultithread(TRUE) ).  The IMFDXGIManager will be set to your spReaderAttributes with the MF_SOURCE_READER_D3D_MANAGER key.

    Also you can get a IMFDXGIBuffer from your sample and render it using a SurfaceImageSource (D3D interop with xaml).


    • Proposed as answer by Tyrone Fa Thursday, May 16, 2013 8:04 AM
    Monday, September 17, 2012 11:31 PM