I'm trying to re-write my software-only logic for displaying video feed from a web camera. This time using hardware and DirectX. (I need to preface this by saying that this is the first time that I'm writing something for DirectX.)
I have the following declared on the global scale:
IMFMediaSource* g_pMediaSource = NULL;
ReaderCallback* gpRdrCallback = NULL;
IMFSourceReader* g_pSrcReader = NULL;
//DirectX stuff:
IDXGISwapChain1* g_pHW_SwapChain = NULL;
ID3D11Device1* g_pHW_D3DDevice = NULL;
ID3D11DeviceContext1* g_pHW_ImmContext = NULL;
IMFDXGIDeviceManager* g_pHW_DXGIDevMgr = NULL;
ID3D11RenderTargetView* g_pHW_RenderTargetView = NULL;
ID3D11Texture2D* g_pHW_BackBuffer = NULL;
I get the IMFMediaSource for the webcam, using this code when the app starts:
//Error handling is omitted for readability
CComPtr<IMFAttributes> com_attributes;
hr = MFCreateAttributes(&com_attributes, 2);
hr = com_attributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID);
hr = com_attributes->SetString(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK,
strWebcamSymLink.c_str());
CComPtr<IMFActivate> com_Activ;
hr = MFCreateDeviceSourceActivate(com_attributes, &com_Activ);
hr = com_Activ->ActivateObject(IID_PPV_ARGS(&g_pMediaSource));
Then I do the following (once) during initialization to set up DirectX:
(Error handling is omitted for readability.)
//For this test, the input parameters are:
//(As they are received from a webcam)
//
// szFrameW = 160;
// szFrameH = 90;
// nFrameRateNumer = 5;
// nFrameRateDenom = 1;
// nAspectRatioNumer = 1;
// nAspectRatioDenom = 1;
// dwIdxDev = 0; //Webcam device index
//
DXGI_SWAP_CHAIN_DESC sd = {};
sd.BufferCount = 1;
sd.BufferDesc.Width = szFrameW;
sd.BufferDesc.Height = szFrameH;
sd.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
sd.BufferDesc.ScanlineOrdering = DXGI_MODE_SCANLINE_ORDER_PROGRESSIVE;
sd.BufferDesc.Scaling = DXGI_MODE_SCALING_CENTERED;
sd.BufferDesc.RefreshRate.Numerator = nFrameRateNumer;
sd.BufferDesc.RefreshRate.Denominator = nFrameRateDenom;
sd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
sd.OutputWindow = ghWnd; //Main window handle
sd.Windowed = TRUE;
sd.SampleDesc.Count = 1;
sd.SampleDesc.Quality = 0;
D3D_FEATURE_LEVEL d3dFeatureLvls[] = {
D3D_FEATURE_LEVEL_11_1,
};
UINT dwNumLvlsRequested = _countof(d3dFeatureLvls);
D3D_FEATURE_LEVEL FeatureLevelsSupported;
CComPtr<IDXGISwapChain> com_SwapChain;
CComPtr<ID3D11Device> com_Dev;
CComPtr<ID3D11DeviceContext> com_Ctx;
hr = D3D11CreateDeviceAndSwapChain(NULL,
D3D_DRIVER_TYPE_HARDWARE,
NULL,
D3D11_CREATE_DEVICE_BGRA_SUPPORT |
#ifdef _DEBUG
D3D11_CREATE_DEVICE_DEBUG
#else
0
#endif
,
d3dFeatureLvls,
dwNumLvlsRequested,
D3D11_SDK_VERSION,
&sd,
&com_SwapChain,
&com_Dev,
&FeatureLevelsSupported,
&com_Ctx);
//Get version 1.0 of interfaces - I'm not sure if I need it here?
hr = com_SwapChain.QueryInterface(&g_pHW_SwapChain);
hr = com_Dev.QueryInterface(&g_pHW_D3DDevice);
hr = com_Ctx.QueryInterface(&g_pHW_ImmContext);
hr = g_pHW_SwapChain->GetBuffer(0, IID_PPV_ARGS(&g_pHW_BackBuffer));
hr = g_pHW_D3DDevice->CreateRenderTargetView(g_pHW_BackBuffer, NULL, &g_pHW_RenderTargetView);
g_pHW_ImmContext->OMSetRenderTargets(1, &g_pHW_RenderTargetView, NULL);
D3D11_VIEWPORT vp;
vp.Width = (FLOAT)szFrameW;
vp.Height = (FLOAT)szFrameH;
vp.MinDepth = 0.0f;
vp.MaxDepth = 1.0f;
vp.TopLeftX = 0;
vp.TopLeftY = 0;
g_pHW_ImmContext->RSSetViewports( 1, &vp );
CComPtr<IMFTransform> com_Transform;
hr = g_pMediaSource->QueryInterface(IID_PPV_ARGS(&com_Transform));
UINT uiToken = 0;
hr = MFCreateDXGIDeviceManager(&uiToken, &g_pHW_DXGIDevMgr);
hr = g_pHW_DXGIDevMgr->ResetDevice(g_pHW_D3DDevice, uiToken);
hr = com_Transform->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER,
(ULONG_PTR)g_pHW_DXGIDevMgr);
Then to initiate the Microsoft Media Foundation for asynchronous rendering from a webcam:
//Error handling is omitted for readability
hr = MFCreateAttributes(&com_attributes, 3);
gpRdrCallback = new ReaderCallback();
hr = com_attributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, gpRdrCallback);
hr = com_attributes->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, TRUE);
hr = com_attributes->SetUINT32(MF_SOURCE_READER_DISCONNECT_MEDIASOURCE_ON_SHUTDOWN, TRUE);
hr = com_attributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, g_pHW_DXGIDevMgr);
hr = MFCreateSourceReaderFromMediaSource(g_pMediaSource, com_attributes, &g_pSrcReader);
CComPtr<IMFMediaType> com_vid_output;
hr = MFCreateMediaType(&com_vid_output);
hr = com_vid_output->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
hr = com_vid_output->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32);
hr = com_vid_output->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
hr = com_vid_output->SetUINT64(MF_MT_FRAME_SIZE,
PackSize(szFrameW, szFrameH));
hr = com_vid_output->SetUINT64(MF_MT_FRAME_RATE,
PackSize(nFrameRateNumer, nFrameRateDenom));
hr = MFSetAttributeRatio(com_vid_output,
MF_MT_PIXEL_ASPECT_RATIO,
nAspectRatioNumer,
nAspectRatioDenom);
hr = g_pSrcReader->SetCurrentMediaType(dwIdxDev, NULL, com_vid_output);
//Initiate the read from a webcam (async)
hr = g_pSrcReader->SetStreamSelection(dwIdxDev, TRUE);
hr = g_pSrcReader->ReadSample(dwIdxDev,
0,
NULL, NULL, NULL, NULL);
After that from my ReaderCallback when a frame is read:
//Error handling is omitted for readability
HRESULT ReaderCallback::OnReadSample(HRESULT hrStatus,
DWORD dwStreamIndex,
DWORD dwStreamFlags,
LONGLONG llTimestamp,
IMFSample* pSample)
{
if(SUCCEEDED(hrStatus))
{
CComPtr<IMFMediaBuffer> com_buffer;
hr = pSample->ConvertToContiguousBuffer(&com_buffer);
com_buffer->AddRef(); /pseudo-call
remember_video_buffer_for_later_processing(com_buffer);
}
//Initiate reading of another frame
hr = g_pSrcReader->ReadSample(dwIdxDev,
0,
NULL, NULL, NULL, NULL);
return S_OK;
}
Finally, when I am ready to render the video frame onto the window:
(The frame is received from my ReaderCallback::OnReadSample above.)
//Error handling is omitted for readability
void renderFromRawVideoPixels(IMFMediaBuffer* pMediaBuffer)
{
CComPtr<IMFDXGIBuffer> com_DxgBuffer;
hr = pMediaBuffer->QueryInterface(IID_PPV_ARGS(&com_DxgBuffer));
CComPtr<ID3D11Texture2D> com_Texture;
hr = com_DxgBuffer->GetResource(IID_PPV_ARGS(&com_Texture));
UINT nSubindex;
hr = com_DxgBuffer->GetSubresourceIndex(&nSubindex);
g_pHW_ImmContext->CopySubresourceRegion(g_pHW_BackBuffer,
0, 0 ,0, 0,
com_Texture,
nSubindex,
NULL);
DXGI_PRESENT_PARAMETERS dpp = {};
hr = g_pHW_SwapChain->Present1(0, 0, &dpp);
}
The call to g_pHW_ImmContext->CopySubresourceRegion above leaves the following error in the debugging output window and I get a black screen:
D3D11 ERROR: ID3D11DeviceContext::CopySubresourceRegion: Cannot invoke CopySubresourceRegion when the Formats of each Resource are not the same or at least castable to each other, unless one format is compressed (DXGI_FORMAT_R9G9B9E5_SHAREDEXP, or DXGI_FORMAT_BC[1,2,3,4,5,6,7]_* ) and the source format is similar to the dest according to: BC[1|4] ~= R16G16B16A16|R32G32, BC[2|3|5|6|7] ~= R32G32B32A32, R9G9B9E5_SHAREDEXP ~= R32. [ RESOURCE_MANIPULATION ERROR #281: COPYSUBRESOURCEREGION_INVALIDSOURCE]
I have no idea what it means.
Any help on how to resolve this?
As the system tells you, the formats between what Media Foundation gives you back and what the DirectX swapchain expects are not compatible.
You can make them compatible with two changes:
A) change the swapchain format, change this:
sd.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
into this:
sd.BufferDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
B) change MF output video format, change this:
hr = com_vid_output->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32);
=> MF will give you back DXGI_FORMAT_B8G8R8X8_UNORM
into this:
hr = com_vid_output->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_ARGB32);
=> MF will give you back DXGI_FORMAT_B8G8R8A8_UNORM