videoh.264ms-media-foundation

Is there min size of IMFSample when ProcessInput?


now my question is when I first receive data and ProcessInput it is all right. but then I will receive some small data (may be only 3kb?) and the image is not right and will lack something(only part of the whole right image).

so I think it does not receive the whole data needed to to IMFTransform so I will keep the data until it is 30kb(or other larger) and then go to MFCreateMemoryBuffer.

yes now the image frame is right until the last frame.

so I guess I do not get the whole right data of one Frame right? how to get the right size of one completely IMFSample(Frame)?

I do as @Simon Mourier 's code but there is error image. enter image description here


Solution

  • Here is a sample code that decodes an H264 stream of bytes using Media Foundation H.264 Video Decoder transform.

    It simulates the stream of various-sized chunks of bytes using a file but the principles are the same.

    Key points:

    #include <windows.h>
    #include <atlbase.h>
    #include <mfapi.h>
    #include <mferror.h>
    #include <mfidl.h>
    #include <mftransform.h>
    #include <cstdlib>
    
    #pragma comment(lib, "mfplat.lib")
    #pragma comment(lib, "mfuuid.lib")
    
    #define HRCHECK(__expr) {auto __hr=(__expr);if(FAILED(__hr)){wprintf(L"FAILURE 0x%08X (%i)\n\tline: %u file: '%s'\n\texpr: '" _CRT_WIDE(#__expr) L"'\n",__hr,__hr,__LINE__,_CRT_WIDE(__FILE__));_CrtDbgBreak();}}
    #define WIN32CHECK(__expr) {if(!(__expr)){auto __hr=HRESULT_FROM_WIN32(GetLastError());{wprintf(L"FAILURE 0x%08X (%i)\n\tline: %u file: '%s'\n\texpr: '" _CRT_WIDE(#__expr) L"'\n",__hr,__hr,__LINE__,_CRT_WIDE(__FILE__));_CrtDbgBreak();}}}
    
    static HRESULT SetOutputType(IMFTransform* transform, GUID format)
    {
        DWORD index = 0;
        do
        {
            CComPtr<IMFMediaType> outputType;
            auto hr = transform->GetOutputAvailableType(0, index++, &outputType);
            if (FAILED(hr))
                return hr;
    
            GUID guid;
            if (SUCCEEDED(outputType->GetGUID(MF_MT_SUBTYPE, &guid)) && guid == format)
            {
                HRCHECK(transform->SetOutputType(0, outputType, 0));
                return S_OK;
            }
        } while (true);
    }
    
    int main()
    {
        HRCHECK(CoInitialize(nullptr));
        {
            HRCHECK(MFStartup(MF_VERSION));
            // open file
            auto file = CreateFile(L"h264.h264", GENERIC_READ, 0, nullptr, OPEN_EXISTING, 0, nullptr);
            WIN32CHECK(file != INVALID_HANDLE_VALUE);
    
            // create H264 transform https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-decoder
            CComPtr<IMFTransform> decoder;
            HRCHECK(decoder.CoCreateInstance(CLSID_MSH264DecoderMFT));
    
            // You can check in decoder attributes that MF_MT_FIXED_SIZE_SAMPLES is set to TRUE.
            // Calling GetOutputStreamInfo this will tell you the MFT cannot provide samples
            // as MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES and MFT_OUTPUT_STREAM_PROVIDES_SAMPLES are not set
    
            // So we don't know enough information yet, we'll feed input samples until we get MF_E_TRANSFORM_STREAM_CHANGE and then we'll provide a sample as per doc:
            //   "If the input type contains only these two attributes, the decoder will offer a default output type, which acts as a placeholder."
            //   "When the decoder receives enough input samples to produce an output frame, it signals a format change by returning MF_E_TRANSFORM_STREAM_CHANGE"
            UINT32 sampleSize = 0;
    
            // input type is H264
            CComPtr<IMFMediaType> inputType;
            HRCHECK(MFCreateMediaType(&inputType));
            HRCHECK(inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video));
            HRCHECK(inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264));
            HRCHECK(decoder->SetInputType(0, inputType, 0)); // video is id 0
    
            // get (and set) NV12 output type (could be I420, IYUV, YUY2, YV12)
            HRCHECK(SetOutputType(decoder, MFVideoFormat_NV12));
    
            do
            {
                // get a random chunk size between 500 and 1500
                DWORD chunkSize = 500 + (1000 * (RAND_MAX - std::rand())) / RAND_MAX;
    
                // create an MF input buffer & read into it
                CComPtr<IMFMediaBuffer> inputBuffer;
                HRCHECK(MFCreateMemoryBuffer(chunkSize, &inputBuffer));
                BYTE* chunk;
                HRCHECK(inputBuffer->Lock(&chunk, nullptr, nullptr));
                DWORD read;
                WIN32CHECK(ReadFile(file, chunk, chunkSize, &read, nullptr));
                HRCHECK(inputBuffer->SetCurrentLength(read));
                HRCHECK(inputBuffer->Unlock());
                if (read)
                {
                    CComPtr<IMFSample> inputSample;
                    HRCHECK(MFCreateSample(&inputSample));
                    HRCHECK(inputSample->AddBuffer(inputBuffer));
    
                    auto hr = decoder->ProcessInput(0, inputSample, 0);
                    if (hr != MF_E_NOTACCEPTING) // just go on
                    {
                        HRCHECK(hr);
                    }
                }
                else
                {
                    // end of file, ask decoder to process all data from previous calls
                    HRCHECK(decoder->ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0));
                }
    
                CComPtr<IMFSample> outputSample;
                HRCHECK(MFCreateSample(&outputSample));
                MFT_OUTPUT_DATA_BUFFER outputBuffer{};
                outputBuffer.pSample = outputSample;
    
                if (sampleSize)
                {
                    // now we know the size so we can (and must) allocate the MF output buffer
                    CComPtr<IMFMediaBuffer> outputBuffer;
                    HRCHECK(MFCreateMemoryBuffer(sampleSize, &outputBuffer));
                    HRCHECK(outputSample->AddBuffer(outputBuffer));
                } // else just continue to process
    
                DWORD status = 0;
                auto hr = decoder->ProcessOutput(0, 1, &outputBuffer, &status);
                if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) // just go on
                {
                    if (!read) // file is all read
                        break;
    
                    continue;
                }
    
                // https://learn.microsoft.com/en-us/windows/win32/medfound/handling-stream-changes
                if (hr == MF_E_TRANSFORM_STREAM_CHANGE)
                {
                    // get (and set) NV12 output type (could be I420, IYUV, YUY2, YV12)
                    HRCHECK(SetOutputType(decoder, MFVideoFormat_NV12));
    
                    // now get the sample size
                    CComPtr<IMFMediaType> type;
                    HRCHECK(decoder->GetOutputCurrentType(0, &type));
                    HRCHECK(type->GetUINT32(MF_MT_SAMPLE_SIZE, &sampleSize));
                    continue;
                }
                HRCHECK(hr);
    
                LONGLONG time, duration;
                HRCHECK(outputSample->GetSampleTime(&time));
                HRCHECK(outputSample->GetSampleDuration(&duration));
                wprintf(L"Sample time: %I64u ms duration: %I64u ms\n", time / 10000, duration / 10000);
            } while (true);
    
            // close file
            CloseHandle(file);
            HRCHECK(MFShutdown());
        }
        CoUninitialize();
        return 0;
    }
    

    Full project is available here https://github.com/smourier/MFDecodeH264