I'm attempting to write an application for recording the Windows desktop to a video file. I'm making use of the desktop duplication API and the Media Foundation API. More specifically I'm using the SinkWriter class from Media Foundation. I'm consuming Texture2D objects from the Desktop Duplication API and passing them to the SinkWriter, which makes for a nicely hardware accelerated pipeline. I'm also using C# with SharpDX.
The entire project works very well for the most part. it runs flawlessly on my Surface Pro, capturing fairly consistent fps video without issue.
The trouble has been on other hardware. Specifically I've tested I've tested this on an AMD Ryzen with a discrete Nvidia GPU. This also works fairly well.. except after recording the desktop for usually between 8 and 20 seconds, the program will get locked up on either outputDuplication.ReleaseFrame() or outputDuplication.AcquireNextFrame(). It doesn't crash, it just locks up so I'm unsure how to figure out what's happening. What I do know for sure is it has something to do with passing the desktop texture to the sinkWriter.WriteSample() function. If I remove this line, and keep all else the same, as in continue to create the MediaBuffers and MediaSamples as usual, associate them with my desktop texture, but simply don't pass the sample to the WriteSample method, then the application can happily "record" seemingly indefinitely.
I've checked for every potential mistake I can think of. I made sure all objects are disposed when they should be. There are no memory leaks that I can see. The code uses the SetAllocator() method on the MediaSample, thus the textures I pass to the SinkWriter are recycled. From what I can tell, this all works correctly.
I suspect there's something basic I'm not understanding here. It feels like there's some mistake I've made with managing the unmanaged resources behind SharpDX correctly, but I really don't know where to start to look. Any pointers would be massively appreciated at this point.
Here's the main capture code. This isn't everything, but most of it. I suspect the issue is somewhere in here.
static void WriteFrame(SinkWriter sinkWriter, int steamIndex, long time, long duration, int width, int height, Texture2D texture, TextureAllocator textureAllocator)
{
MediaBuffer textureBuffer;
MediaFactory.CreateDXGISurfaceBuffer(texture.GetType().GUID, texture, 0, false, out textureBuffer);
using (var buffer2d = textureBuffer.QueryInterface<Buffer2D>())
{
textureBuffer.CurrentLength = buffer2d.ContiguousLength;
}
using (var sample = MediaFactory.CreateVideoSampleFromSurface(null))
{
sample.SampleTime = time;
sample.SampleDuration = duration;
sample.AddBuffer(textureBuffer);
using(var trackedSample = sample.QueryInterface<TrackedSample>())
{
trackedSample.SetAllocator(textureAllocator, null);
}
sinkWriter.WriteSample(steamIndex, sample);
}
textureBuffer.Dispose();
}
void captureAction()
{
MediaManager.Startup(true);
int streamIndex = 0;
var whichOutputDevice = 0;
var whichAdapter = 0;
using (var factory = new Factory1())
{
using (var adapter = factory.GetAdapter1(whichAdapter))
{
using (var mDevice = new SharpDX.Direct3D11.Device(adapter))
{
using (var dXGIDeviceManager = new DXGIDeviceManager())
{
dXGIDeviceManager.ResetDevice(mDevice);
using (var output = adapter.GetOutput(whichOutputDevice))
{
using (var output1 = output.QueryInterface<Output1>())
{
var mOutputDesc = output.Description;
var mTextureDesc = new Texture2DDescription()
{
CpuAccessFlags = CpuAccessFlags.Read,
BindFlags = BindFlags.None,
Format = Format.B8G8R8A8_UNorm,
Width = mOutputDesc.DesktopBounds.Right - mOutputDesc.DesktopBounds.Left,
Height = mOutputDesc.DesktopBounds.Bottom - mOutputDesc.DesktopBounds.Top,
OptionFlags = ResourceOptionFlags.None,
MipLevels = 1,
ArraySize = 1,
SampleDescription = { Count = 1, Quality = 0 },
Usage = ResourceUsage.Staging
};
using (var outputDuplication = output1.DuplicateOutput(mDevice))
{
using (var sinkWriter = InitialiseSinkWriter(path, out streamIndex, mTextureDesc.Width, mTextureDesc.Height, frameRate, mDevice, dXGIDeviceManager, preferHardwareAcceleration))
{
using (var rgbToNv12 = new RGBToNV12ConverterD3D11(mDevice, mDevice.ImmediateContext, mTextureDesc.Width, mTextureDesc.Height))
{
var newDesc = mTextureDesc;
if (forceNV12)
newDesc.Format = Format.NV12;
newDesc.BindFlags = BindFlags.RenderTarget;
newDesc.Usage = ResourceUsage.Default;
newDesc.CpuAccessFlags = CpuAccessFlags.None;
var textureAllocator = new TextureAllocator(mDevice, newDesc);
{
var timeStart = 0L;
var frameCount = 0;
var totalAFMilliseconds = 0.0;
var lastTimeStamp = 0L;
var frameAquired = false;
for (int i = 0; !stopCapture; i++)
{
var sleepAmount = Math.Max(0, (1000 / frameRate) / 2);
System.Threading.Thread.Sleep(sleepAmount);
frameCount++;
SharpDX.DXGI.Resource desktopResource = null;
var frameInfo_ = new OutputDuplicateFrameInformation();
var sw = new Stopwatch();
sw.Start();
var outputLoopCount = 0;
if (outputDuplication != null)
{//desktop duplication
int aquireFrameLoopCount = 0;
while (frameInfo_.LastPresentTime == 0)
{
aquireFrameLoopCount++;
if (aquireFrameLoopCount > 1)
{
System.Threading.Thread.Sleep(1);
}
if (frameAquired)
try
{
outputDuplication.ReleaseFrame();
frameAquired = false;
}
catch (Exception e)
{
}
try
{
outputDuplication.AcquireNextFrame(500, out frameInfo_, out desktopResource);
frameAquired = true;
}
catch (Exception e)
{
}
outputLoopCount++;
}
Debug.Assert(frameInfo_.LastPresentTime != 0);
Debug.Assert(frameInfo_.AccumulatedFrames != 0);
}
sw.Stop();
var frameCaptureTime = frameCount * TimeSpan.FromSeconds(1.0 / frameRate).Ticks;
frameCaptureTime = DateTime.Now.ToFileTimeUtc();
var aqTime = TimeSpan.FromSeconds((double)sw.ElapsedTicks / Stopwatch.Frequency);
totalAFMilliseconds += aqTime.TotalMilliseconds;
if (desktopResource != null)
using (var desktopTexture = desktopResource?.QueryInterface<Texture2D>())
{
Texture2D desktopTextureCopy = null;
try
{
desktopTextureCopy = textureAllocator.AllocateTexture();
}
catch (SharpDX.SharpDXException e)
{
var result = mDevice.DeviceRemovedReason;
result.CheckError();
throw e;
}
if (outputDuplication != null)
rgbToNv12.ConvertRGBToNV12(desktopTexture, desktopTextureCopy);
if (frameCount == 2)
timeStart = lastTimeStamp;
Console.WriteLine("Writing frame {0}", TimeSpan.FromTicks(frameCaptureTime - lastTimeStamp).ToString());
if (frameCount > 0)
{
var frameDuration = TimeSpan.FromSeconds(1.0 / frameRate).Ticks;
var time = (frameCount - 2) * frameDuration;
WriteFrame(sinkWriter, streamIndex, lastTimeStamp - timeStart, frameCaptureTime - lastTimeStamp, desktopTextureCopy.Description.Width, desktopTextureCopy.Description.Height, desktopTextureCopy, textureAllocator);
Console.WriteLine("frame written");
}
desktopTextureCopy.Dispose();
}
lastTimeStamp = frameCaptureTime;
CaptureTime = TimeSpan.FromTicks(lastTimeStamp - timeStart);
if (CaptureFramesChanged != null)
{
CaptureFramesChanged.Invoke(null, new EventArgs());
}
Console.WriteLine("end of loop");
}
stopCapture = false;
Console.WriteLine("Total AquireFrame time: {0}", totalAFMilliseconds / (300 * (1000 / frameRate)));
Console.WriteLine("Begining finalise");
sinkWriter.Finalize();
}
}
}
}
}
}
}
}
}
}
MediaManager.Shutdown();
}
Thanks
I fixed it. Seems it was some multithreading issue and was fixed by adding these lines:
using(var multiThread = mDevice.QueryInterface<Multithread>())
{
multiThread.SetMultithreadProtected(true);
}
Right after the creating the device.