I want to read and show video from file any format, also edit video frame data, for example, drawing rects.
OpenCV is banned
How I image solution.
We need sequence of elements:
filesrc -> decodebin -> videoconvert -> autovideosink
decodebin decodes video data from any format to x-raw and sends to videoconvert.
videoconvert converts video frames from any frame format to specific format (for example, I want to RGB).
We can use Pad Probe mechanism to connect our callback where we can edit video. I want to set RGB format to videoconvert output and work with that format in my callback.
int main(int argc, char **argv)
{
gst_init(&argc, &argv);
GstElement *pipeline = gst_pipeline_new("videoshow");
GstElement *filesrc = gst_element_factory_make("filesrc", "videofile");
g_object_set(filesrc, "location", "video.mp4", NULL);
GstElement *decodebin = gst_element_factory_make("decodebin", "decoder");
GstElement *videoconvert = gst_element_factory_make("videoconvert", "converter");
GstElement *videosink = gst_element_factory_make("xvimagesink", "videosink");
gst_bin_add_many(GST_BIN(pipeline), filesrc, decodebin, videoconvert, videosink, NULL);
if (gst_element_link(filesrc, decodebin) != TRUE ||
gst_element_link(videoconvert, videosink) != TRUE)
{
g_printerr ("Elements could not be linked.\n");
gst_object_unref(pipeline);
return -1;
}
// link decodebin and videoconvert after some time, when decodebin read file metadata
g_signal_connect(decodebin, "pad-added", G_CALLBACK(on_decodebin_ready), videoconvert);
// setting our callback for edit video
GstPad *vc_output = gst_element_get_static_pad(videoconvert, "src");
gst_pad_add_probe(vc_output, GST_PAD_PROBE_TYPE_BUFFER, on_pad_probe, NULL, NULL);
gst_object_unref(vc_output);
GstStateChangeReturn ret = gst_element_set_state(pipeline, GST_STATE_PLAYING);
if (ret == GST_STATE_CHANGE_FAILURE)
{
g_printerr ("Unable to set the pipeline to the playing state.\n");
gst_object_unref(pipeline);
return -1;
}
GstBus *bus = gst_element_get_bus(pipeline);
GstMessage *msg = gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, GstMessageType(GST_MESSAGE_ERROR | GST_MESSAGE_EOS));
if (msg != NULL) {
GError *err;
gchar *debug_info;
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_ERROR:
gst_message_parse_error (msg, &err, &debug_info);
g_printerr ("Error received from element %s: %s\n",
GST_OBJECT_NAME (msg->src), err->message);
g_printerr ("Debugging information: %s\n",
debug_info ? debug_info : "none");
g_clear_error (&err);
g_free (debug_info);
break;
case GST_MESSAGE_EOS:
g_print ("End-Of-Stream reached.\n");
break;
default:
/* We should not reach here because we only asked for ERRORs and EOS */
g_printerr ("Unexpected message received.\n");
break;
}
gst_message_unref (msg);
}
gst_object_unref(bus);
gst_element_set_state(pipeline, GST_STATE_NULL);
gst_object_unref(pipeline);
return 0;
}
static void on_decodebin_ready(GstElement *decodebin, GstPad *new_pad, GstElement *videoconvert)
{
GstPad *vc_input = gst_element_get_static_pad(videoconvert, "sink");
if (gst_pad_is_linked(vc_input))
{
gst_object_unref(vc_input);
g_print ("We are already linked. Ignoring.\n");
return;
}
g_print("Received new pad '%s' from '%s':\n", GST_PAD_NAME(new_pad), GST_ELEMENT_NAME(decodebin));
GstCaps *new_pad_caps = gst_pad_get_current_caps(new_pad);
GstStructure *new_pad_struct = gst_caps_get_structure(new_pad_caps, 0);
const gchar *new_pad_type = gst_structure_get_name(new_pad_struct);
if (g_str_has_prefix(new_pad_type, "video"))
{
gint w, h;
gst_structure_get_int(new_pad_struct, "width", &w);
gst_structure_get_int(new_pad_struct, "height", &h);
GstPad *vc_output = gst_element_get_static_pad(videoconvert, "src");
GstCaps *new_output_caps = gst_caps_new_simple("video/x-raw",
"format", G_TYPE_STRING, "RGB",
"width", G_TYPE_INT, w,
"height", G_TYPE_INT, h, NULL);
/* it's true but <videosink:sink> caps video/x-raw, format=(string)RGB, width=(int)640, height=(int)360 not accepted */
bool ok = gst_pad_set_caps(vc_output, new_output_caps);
gst_caps_unref(new_output_caps);
GstPadLinkReturn ret = gst_pad_link(new_pad, vc_input);
if (GST_PAD_LINK_FAILED(ret))
g_print("Type is '%s' but link failed.\n", new_pad_type);
else
g_print("Link succeeded (type '%s').\n", new_pad_type);
}
gst_caps_unref(new_pad_caps);
gst_object_unref(vc_input);
}
static GstPadProbeReturn on_pad_probe(GstPad *pad, GstPadProbeInfo *info, gpointer user_data)
{
/* just i check video format */
GstCaps *caps = gst_pad_get_current_caps(pad);
gchar *str = gst_caps_to_string(caps);
g_print(str, NULL);
g_free(str);
gst_caps_unref(caps);
return GST_PAD_PROBE_OK;
}
decodebin has I420 format.
And I awaiting src pad of videoconvert will be RGB format, but on_pad_probe prints YV12 (why?).
Also in console I got lines:
<videosink:sink> caps video/x-raw, format=(string)RGB, width=(int)640, height=(int)360 not accepted
and
<converter:src> Sticky event misordering, got 'caps' before 'stream-start'
I just need a place where I can edit video frames. And I just can’t get the image into the format I need.
Your videosink probably doesn't support RGB format as input. You would add a second converter for further converting RGB into a format supported by the videosink such as:
#include <stdio.h>
#include <gst/gst.h>
/* This callback function will be called each time the videoconverter has prepared its output frame */
static GstPadProbeReturn
cb_have_data (GstPad *pad,
GstPadProbeInfo *info,
gpointer user_data)
{
GstMapInfo map;
guint32 *ptr;
GstBuffer *buffer;
buffer = GST_PAD_PROBE_INFO_BUFFER (info);
buffer = gst_buffer_make_writable (buffer);
/* Making a buffer writable can fail (for example if it
* cannot be copied and is used more than once)
*/
if (buffer == NULL) {
g_print("Writable buffer creation failed. Aborted");
return GST_PAD_PROBE_OK;
}
/* Mapping a buffer can fail (non-writable) */
if (gst_buffer_map (buffer, &map, GST_MAP_WRITE)) {
ptr = (guint32 *) map.data;
/* Process your frame here. This example just ouputs a dot. */
g_print(".");
gst_buffer_unmap (buffer, &map);
}
GST_PAD_PROBE_INFO_DATA (info) = buffer;
return GST_PAD_PROBE_OK;
}
gint
main (gint argc,
gchar *argv[])
{
GMainLoop *loop;
GstElement *pipeline;
gst_init (&argc, &argv);
loop = g_main_loop_new (NULL, FALSE);
pipeline = gst_parse_launch("filesrc location=video.mp4 ! decodebin ! videoconvert name=conv ! video/x-raw,format=RGB ! videoconvert ! autovideosink", NULL);
if (pipeline == NULL)
g_error ("Failed to launch pipeline");
GstElement *conv = gst_bin_get_by_name(GST_BIN(pipeline), "conv");
if (conv == NULL)
g_error ("Failed to find conv in pipeline");
GstPad *pad = gst_element_get_static_pad (conv, "src");
gulong probe_id = gst_pad_add_probe (pad, GST_PAD_PROBE_TYPE_BUFFER,(GstPadProbeCallback) cb_have_data, NULL, NULL);
gst_object_unref (pad);
gst_object_unref (conv);
/* run */
gst_element_set_state (pipeline, GST_STATE_PLAYING);
/* wait until it's up and running or failed */
if (gst_element_get_state (pipeline, NULL, NULL, -1) == GST_STATE_CHANGE_FAILURE) {
g_error ("Failed to go into PLAYING state");
}
g_print ("Running ...\n");
g_main_loop_run (loop);
/* exit */
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
return 0;
}