Gstreamer with GTK3 using Wayland

Hi,
I am using gstreamer to play recorded video with audio in colibri imx6dl processor by cross compiling the below code

#include <gst/gst.h>

void on_pad_added(GstElement *src, GstPad *new_pad, GstElement *pipeline);

int main(int argc, char *argv) {
GstElement *pipeline, *filesrc, *decodebin, *video_queue, *v4l2convert, *videosink, *audio_queue, *audioconvert, *audioresample, *audiosink;
GstBus *bus;
GstMessage *msg;

// Initialize GStreamer
gst_init(&argc, &argv);

// Create the elements
filesrc = gst_element_factory_make("filesrc", "file-source");
decodebin = gst_element_factory_make("decodebin", "decoder");
video_queue = gst_element_factory_make("queue", "video-queue");
v4l2convert = gst_element_factory_make("v4l2convert", "v4l2-converter");
videosink = gst_element_factory_make("autovideosink", "video-output");
audio_queue = gst_element_factory_make("queue", "audio-queue");
audioconvert = gst_element_factory_make("audioconvert", "audio-converter");
audioresample = gst_element_factory_make("audioresample", "audio-resampler");
audiosink = gst_element_factory_make("autoaudiosink", "audio-output");

// Check if elements are created properly
if (!filesrc || !decodebin || !video_queue || !v4l2convert || !videosink || !audio_queue || !audioconvert || !audioresample || !audiosink) {
    g_printerr("Not all elements could be created.\n");
    return -1;
}

// Create the empty pipeline
pipeline = gst_pipeline_new("video-audio-player");

if (!pipeline) {
    g_printerr("Pipeline could not be created.\n");
    return -1;
}

// Set the source file location
g_object_set(G_OBJECT(filesrc), "location", "video.mp4", NULL);

// Build the pipeline
gst_bin_add_many(GST_BIN(pipeline), filesrc, decodebin, video_queue, v4l2convert, videosink, audio_queue, audioconvert, audioresample, audiosink, NULL);

// Link the file source to the decoder
if (!gst_element_link(filesrc, decodebin)) {
    g_printerr("File source and decoder could not be linked.\n");
    gst_object_unref(pipeline);
    return -1;
}

// Link the video elements (the actual linking happens in the decodebin's pad-added signal handler)
if (!gst_element_link_many(video_queue, v4l2convert, videosink, NULL)) {
    g_printerr("Video elements could not be linked.\n");
    gst_object_unref(pipeline);
    return -1;
}

// Link the audio elements (the actual linking happens in the decodebin's pad-added signal handler)
if (!gst_element_link_many(audio_queue, audioconvert, audioresample, audiosink, NULL)) {
    g_printerr("Audio elements could not be linked.\n");
    gst_object_unref(pipeline);
    return -1;
}

// Connect to the pad-added signal of decodebin
g_signal_connect(decodebin, "pad-added", G_CALLBACK(on_pad_added), pipeline);

// Start playing
gst_element_set_state(pipeline, GST_STATE_PLAYING);

// Wait until error or EOS (End of Stream)
bus = gst_element_get_bus(pipeline);
msg = gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, GST_MESSAGE_ERROR | GST_MESSAGE_EOS);

// Parse message
if (msg != NULL) {
    GError *err;
    gchar *debug_info;

    switch (GST_MESSAGE_TYPE(msg)) {
        case GST_MESSAGE_ERROR:
            gst_message_parse_error(msg, &err, &debug_info);
            g_printerr("Error received from element %s: %s\n", GST_OBJECT_NAME(msg->src), err->message);
            g_printerr("Debugging information: %s\n", debug_info ? debug_info : "none");
            g_clear_error(&err);
            g_free(debug_info);
            break;
        case GST_MESSAGE_EOS:
            g_print("End-Of-Stream reached.\n");
            break;
        default:
            g_printerr("Unexpected message received.\n");
            break;
    }
    gst_message_unref(msg);
}

// Free resources
gst_object_unref(bus);
gst_element_set_state(pipeline, GST_STATE_NULL);
gst_object_unref(pipeline);

return 0;

}

void on_pad_added(GstElement *src, GstPad *new_pad, GstElement *pipeline) {
GstPad *sink_pad_video = gst_element_get_static_pad(gst_bin_get_by_name(GST_BIN(pipeline), “video-queue”), “sink”);
GstPad *sink_pad_audio = gst_element_get_static_pad(gst_bin_get_by_name(GST_BIN(pipeline), “audio-queue”), “sink”);
GstPadLinkReturn ret;
GstCaps *new_pad_caps = NULL;
GstStructure *new_pad_struct = NULL;
const gchar *new_pad_type = NULL;

g_print("Received new pad '%s' from '%s':\n", GST_PAD_NAME(new_pad), GST_ELEMENT_NAME(src));

// Check the new pad's type
new_pad_caps = gst_pad_get_current_caps(new_pad);
new_pad_struct = gst_caps_get_structure(new_pad_caps, 0);
new_pad_type = gst_structure_get_name(new_pad_struct);

if (g_str_has_prefix(new_pad_type, "video/x-raw")) {
    // Attempt the link for video
    ret = gst_pad_link(new_pad, sink_pad_video);
    if (GST_PAD_LINK_FAILED(ret)) {
        g_printerr("Type is '%s' but link failed for video.\n", new_pad_type);
    } else {
        g_print("Link succeeded (type '%s' for video).\n", new_pad_type);
    }
} else if (g_str_has_prefix(new_pad_type, "audio/x-raw")) {
    // Attempt the link for audio
    ret = gst_pad_link(new_pad, sink_pad_audio);
    if (GST_PAD_LINK_FAILED(ret)) {
        g_printerr("Type is '%s' but link failed for audio.\n", new_pad_type);
    } else {
        g_print("Link succeeded (type '%s' for audio).\n", new_pad_type);
    }
} else {
    g_print("It has type '%s' which is not handled. Ignoring.\n", new_pad_type);
}

if (new_pad_caps != NULL) {
    gst_caps_unref(new_pad_caps);
}
gst_object_unref(sink_pad_video);
gst_object_unref(sink_pad_audio);

}

Now I need to add UI to play and resume the video . For this I am using GTK3 dependency for UI things . By taking the reference of above logic , I created a sample application to embed gstreamer with GTK3 window by using gtkglsink , gtksinkbin and gtksink . Here I am attaching code regarding to the application to embed gstreamer with gtk3

#include <string.h>
#include <gtk/gtk.h>
#include <gst/gst.h>
#include <gdk/gdk.h>

int play_flag = 0;

/ Structure to contain all our information, so we can pass it around /
typedef struct _CustomData {
** GstElement pipeline; / Our pipeline /*
** GtkWidget sink_widget; / Widget where our video will be displayed /*
** GtkWidget slider; / Slider widget to keep track of current position /*
** GtkWidget streams_list; **
** gulong slider_update_signal_id; /
Signal ID for the slider update signal /*
** GstState state; /* Current state of the pipeline /*
** gint64 duration; /* Duration of the clip, in nanoseconds /*
} CustomData;

**GstElement *filesrc, *decodebin, *video_queue, *v4l2convert, *videosink, *audio_queue, *audioconvert, audioresample, audiosink;

/ This function is called when the PLAY button is clicked /
static void play_cb(GtkButton button, CustomData data) {
** if (play_flag == 1) {

** gst_element_set_state(data->pipeline, GST_STATE_READY);**
** play_flag = 0;**
** }**
** gst_element_set_state(data->pipeline, GST_STATE_PLAYING);**
}

/ This function is called when the PAUSE button is clicked /
static void pause_cb(GtkButton button, CustomData data) {
** gst_element_set_state(data->pipeline, GST_STATE_PAUSED);

}

/ This function is called when the STOP button is clicked /
static void stop_cb(GtkButton button, CustomData data) {
** gst_element_set_state(data->pipeline, GST_STATE_READY);

}

/ This function is called when the main window is closed /
static void delete_event_cb(GtkWidget widget, GdkEvent event, CustomData data) {
** stop_cb(NULL, data);

** gtk_main_quit();
*
}

/ This function is called when the slider changes its position. We perform a seek to the new position here. /
static void slider_cb(GtkRange range, CustomData data) {
** gdouble value = gtk_range_get_value(GTK_RANGE(data->slider));

** gst_element_seek_simple(data->pipeline, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT,**
** (gint64)(value * GST_SECOND));**
}

/ Dynamic pad handler to link decodebin pads /
static void pad_added_handler(GstElement src, GstPad new_pad, CustomData data) {
** GstPad sink_pad = NULL;

** GstCaps new_pad_caps = NULL;

** GstStructure new_pad_struct = NULL;

** const gchar new_pad_type = NULL;*

** new_pad_caps = gst_pad_get_current_caps(new_pad);**
** new_pad_struct = gst_caps_get_structure(new_pad_caps, 0);**
** new_pad_type = gst_structure_get_name(new_pad_struct);**

** if (g_str_has_prefix(new_pad_type, “video/x-raw”)) {**
** sink_pad = gst_element_get_static_pad(video_queue, “sink”);**
** } else if (g_str_has_prefix(new_pad_type, “audio/x-raw”)) {**
** sink_pad = gst_element_get_static_pad(audio_queue, “sink”);**
** }**

** if (gst_pad_is_linked(sink_pad)) {**
** g_object_unref(sink_pad);**
** if (new_pad_caps != NULL)**
** gst_caps_unref(new_pad_caps);**
** return;**
** }**

** if (gst_pad_link(new_pad, sink_pad) != GST_PAD_LINK_OK) {**
** g_printerr(“Type is ‘%s’ but link failed.\n”, new_pad_type);**
** } else {**
** g_print(“Link succeeded (type ‘%s’).\n”, new_pad_type);**
** }**

** if (new_pad_caps != NULL)**
** gst_caps_unref(new_pad_caps);**
** gst_object_unref(sink_pad);**
}

/ This creates all the GTK+ widgets that compose our application, and registers the callbacks /
static void create_ui(CustomData data) {
** GtkWidget main_window; / The uppermost window, containing all other windows /

** GtkWidget main_box; / VBox to hold main_hbox and the controls /*
** GtkWidget main_hbox; / HBox to hold the video sink and the stream info text widget /*
** GtkWidget controls; / HBox to hold the buttons and the slider /*
** GtkWidget *play_button, pause_button, stop_button; / Buttons /

** main_window = gtk_window_new(GTK_WINDOW_TOPLEVEL);**
** g_signal_connect(G_OBJECT(main_window), “delete-event”, G_CALLBACK(delete_event_cb), data);**

** play_button = gtk_button_new_from_icon_name(“media-playback-start”, GTK_ICON_SIZE_SMALL_TOOLBAR);**
** g_signal_connect(G_OBJECT(play_button), “clicked”, G_CALLBACK(play_cb), data);**

** pause_button = gtk_button_new_from_icon_name(“media-playback-pause”, GTK_ICON_SIZE_SMALL_TOOLBAR);**
** g_signal_connect(G_OBJECT(pause_button), “clicked”, G_CALLBACK(pause_cb), data);**

** stop_button = gtk_button_new_from_icon_name(“media-playback-stop”, GTK_ICON_SIZE_SMALL_TOOLBAR);**
** g_signal_connect(G_OBJECT(stop_button), “clicked”, G_CALLBACK(stop_cb), data);**

** data->slider = gtk_scale_new_with_range(GTK_ORIENTATION_HORIZONTAL, 0, 100, 1);**
** gtk_scale_set_draw_value(GTK_SCALE(data->slider), 0);**
** data->slider_update_signal_id = g_signal_connect(G_OBJECT(data->slider), “value-changed”, G_CALLBACK(slider_cb), data);**

** controls = gtk_box_new(GTK_ORIENTATION_HORIZONTAL, 0);**
** gtk_box_pack_start(GTK_BOX(controls), play_button, FALSE, FALSE, 2);**
** gtk_box_pack_start(GTK_BOX(controls), pause_button, FALSE, FALSE, 2);**
** gtk_box_pack_start(GTK_BOX(controls), stop_button, FALSE, FALSE, 2);**
** gtk_box_pack_start(GTK_BOX(controls), data->slider, TRUE, TRUE, 2);**

** main_hbox = gtk_box_new(GTK_ORIENTATION_HORIZONTAL, 0);**
** gtk_box_pack_start(GTK_BOX(main_hbox), data->sink_widget, TRUE, TRUE, 0);**

** main_box = gtk_box_new(GTK_ORIENTATION_VERTICAL, 0);**
** gtk_box_pack_start(GTK_BOX(main_box), main_hbox, TRUE, TRUE, 0);**
** gtk_box_pack_start(GTK_BOX(main_box), controls, FALSE, FALSE, 0);**
** gtk_container_add(GTK_CONTAINER(main_window), main_box);**
** gtk_window_set_default_size(GTK_WINDOW(main_window), 480, 800);**

** gtk_widget_show_all(main_window);**
}

/ This function is called periodically to refresh the GUI /
static gboolean refresh_ui(CustomData data) {
** gint64 current = -1;
*

** /* We do not want to update anything unless we are in the PAUSED or PLAYING states /*
** if (data->state < GST_STATE_PAUSED)**
** return TRUE;**

** /* If we didn’t know it yet, query the stream duration /*
** if (!GST_CLOCK_TIME_IS_VALID(data->duration)) {**
** if (!gst_element_query_duration(data->pipeline, GST_FORMAT_TIME, &data->duration)) {**
** g_printerr(“Could not query current duration.\n”);**
** } else {**
** /* Set the range of the slider to the clip duration, in SECONDS /*
** gtk_range_set_range(GTK_RANGE(data->slider), 0, (gdouble)data->duration / GST_SECOND);**
** }**
** }**

** if (gst_element_query_position(data->pipeline, GST_FORMAT_TIME, &current)) {**
** /* Block the “value-changed” signal, so the slider_cb function is not called**
** * (which would trigger a seek the user has not requested) /*
** g_signal_handler_block(data->slider, data->slider_update_signal_id);**
** /* Set the position of the slider to the current pipeline position, in SECONDS /*
** gtk_range_set_value(GTK_RANGE(data->slider), (gdouble)current / GST_SECOND);**
** /* Re-enable the signal /*
** g_signal_handler_unblock(data->slider, data->slider_update_signal_id);**
** }**
** return TRUE;**
}

/ This function is called when new metadata is discovered in the stream /
static void tags_cb (GstElement pipeline, gint stream, CustomData data) {
** /
We are possibly in a GStreamer working thread, so we notify the main
*
** * thread of this event through a message in the bus /*
** gst_element_post_message (pipeline,**
** gst_message_new_application (GST_OBJECT (pipeline),**
** gst_structure_new_empty (“tags-changed”)));**
}

/ This function is called when an error message is posted on the bus /
*static void error_cb (GstBus bus, GstMessage msg, CustomData data) {
** GError err;

** gchar debug_info;

** /* Print error details on the screen /*
** gst_message_parse_error (msg, &err, &debug_info);**
** g_printerr (“Error received from element %s: %s\n”, GST_OBJECT_NAME (msg->src), err->message);**
** g_printerr (“Debugging information: %s\n”, debug_info ? debug_info : “none”);**
** g_clear_error (&err);**
** g_free (debug_info);**

** /* Set the pipeline to READY (which stops playback) /*
** gst_element_set_state (data->pipeline, GST_STATE_READY);**
}

/ This function is called when an End-Of-Stream message is posted on the bus.*
** * We just set the pipeline to READY (which stops playback) /*
static void eos_cb (GstBus bus, GstMessage msg, CustomData data) {
** g_print (“End-Of-Stream reached.\n”);

** play_flag=1;
*
** // gst_element_set_state (data->pipeline, GST_STATE_READY);**
}

/ This function is called when the pipeline changes states. We use it to*
** * keep track of the current state. /*
static void state_changed_cb (GstBus bus, GstMessage msg, CustomData data) {
** GstState old_state, new_state, pending_state;

** gst_message_parse_state_changed (msg, &old_state, &new_state, &pending_state);
*
** if (GST_MESSAGE_SRC (msg) == GST_OBJECT (data->pipeline)) {**
** data->state = new_state;**
** g_print (“State set to %s\n”, gst_element_state_get_name (new_state));**
** if (old_state == GST_STATE_READY && new_state == GST_STATE_PAUSED) {**
** /* For extra responsiveness, we refresh the GUI as soon as we reach the PAUSED state /*
** refresh_ui (data);**
** }**
** }**
}

int main(int argc, char argv[]) {
** CustomData data;
*
** GstBus bus;*
** GstMessage msg;*
** GstElement gtkglsink;*


** /* Initialize GTK /*
** gtk_init(&argc, &argv);**

** /* Initialize GStreamer /*
** gst_init(&argc, &argv);**

** /* Initialize our data structure /*
** memset(&data, 0, sizeof(data));**
** data.duration = GST_CLOCK_TIME_NONE;**

** /* Create the elements /*
** data.pipeline = gst_pipeline_new(“video-audio-player”);**
** filesrc = gst_element_factory_make(“filesrc”, “file-source”);**
** decodebin = gst_element_factory_make(“decodebin”, “decoder”);**

** video_queue = gst_element_factory_make(“queue”, “video-queue”);**
** v4l2convert = gst_element_factory_make(“videoconvert”, “video-convert”);**
** videosink = gst_element_factory_make(“glsinkbin”, “glsinkbin”); //autovideosink video-output**

** audio_queue = gst_element_factory_make(“queue”, “audio-queue”);**
** audioconvert = gst_element_factory_make(“audioconvert”, “audio-convert”);**
** audioresample = gst_element_factory_make(“audioresample”, “audio-resample”);**
** audiosink = gst_element_factory_make(“autoaudiosink”, “audio-output”);**


** gtkglsink = gst_element_factory_make (“gtkglsink”, “gtkglsink”);**


** /* Here we create the GTK Sink element which will provide us with a GTK widget where**
** * GStreamer will render the video at and we can add to our UI.**
** * Try to create the OpenGL version of the video sink, and fallback if that fails /*
** if (gtkglsink != NULL && videosink != NULL) {**
** g_printerr (“Successfully created GTK GL Sink”);**

** g_object_set (videosink, “sink”, gtkglsink, NULL);**

** /* The gtkglsink creates the gtk widget for us. This is accessible through a property.**
** * So we get it and use it later to add it to our gui. /*
** g_object_get (gtkglsink, “widget”, &data.sink_widget, NULL);**
** } else {**
** g_printerr (“Could not create gtkglsink, falling back to gtksink.\n”);**

** videosink = gst_element_factory_make (“gtksink”, “gtksink”);**
** g_object_get (videosink, “widget”, &data.sink_widget, NULL);**
** }**



** if (!data.pipeline || !filesrc || !decodebin || !video_queue || !v4l2convert || !videosink ||**
** !audio_queue || !audioconvert || !audioresample || !audiosink) {**
** g_printerr(“Not all elements could be created.\n”);**
** return -1;**
** }**

** /* Set the file to play /*
** //g_object_set(G_OBJECT(filesrc), “location”, , NULL);**
** g_object_set(G_OBJECT(filesrc), “location”, “video.mp4”, NULL);**


** /* Build the pipeline. We add all elements into the pipeline /*
** gst_bin_add_many(GST_BIN(data.pipeline), filesrc, decodebin, video_queue, v4l2convert, videosink,**
** audio_queue, audioconvert, audioresample, audiosink, NULL);**

** /* Link the file source to the decoder /*
** if (!gst_element_link(filesrc, decodebin)) {**
** g_printerr(“Failed to link filesrc and decodebin.\n”);**
** gst_object_unref(data.pipeline);**
** return -1;**
** }**

** /* Connect to the pad-added signal /*
** g_signal_connect(decodebin, “pad-added”, G_CALLBACK(pad_added_handler), &data);**




** /* Link video elements /*
** if (!gst_element_link_many(video_queue, v4l2convert, videosink, NULL)) {**
** g_printerr(“Video elements could not be linked.\n”);**
** gst_object_unref(data.pipeline);**
** return -1;**
** }**

** /* Link audio elements /*
** if (!gst_element_link_many(audio_queue, audioconvert, audioresample, audiosink, NULL)) {**
** g_printerr(“Audio elements could not be linked.\n”);**
** gst_object_unref(data.pipeline);**
** return -1;**
** }**

** /* Create the GUI /*
** create_ui(&data);**

** /* Instruct the bus to emit signals for each received message, and connect to the interesting signals /*
** bus = gst_element_get_bus(data.pipeline);**
** gst_bus_add_signal_watch(bus);**
** g_signal_connect(bus, “message::error”, G_CALLBACK(error_cb), &data);**
** g_signal_connect(bus, “message::eos”, G_CALLBACK(eos_cb), &data);**
** g_signal_connect (G_OBJECT (bus), “message::state-changed”, (GCallback)state_changed_cb, &data);**

** // Start playing**
** gst_element_set_state(data.pipeline, GST_STATE_PLAYING);**

** /* Register a function that GLib will call every second /*
** g_timeout_add(1000, (GSourceFunc)refresh_ui, &data);**

** /* Start the GTK+ main loop. We will not regain control until gtk_main_quit is called. /*
** gtk_main();**

** /* Free resources /*
** gst_object_unref(bus);**
** gst_element_set_state(data.pipeline, GST_STATE_NULL);**
** gst_object_unref(data.pipeline);**

** return 0;**
}

It worked fine in my host system , but while running the same application in imx6dl processor , I am facing the issue

Could not create gtkglsink, falling back to gtksink.


**(video:816): GLib-GObject-CRITICAL : 05:55:36.716: g_object_get: assertion ‘G_IS_OBJECT (object)’ failed
Not all elements could be created.

So I checked all the plugins for gtkglsink , gtksink . It was not available for gstreamer in imx6dl processor. So I tried to add the plugins manually by using below commands using local.conf in yocto

IMAGE_INSTALL:append = " gstreamer1.0-plugins-bad gstreamer1.0-plugins-good

Even though , these plugins are not installed in the processor. Can you please guide me how to enable these plugins in imx6dl processor using yocto . Or any other alternative methods to merge gstreamer with gtkwindow will highly appreciable.

Thank you in advance.

Thanks & Regards,
Teena .

Hi, I’m the Toradex AI Assistant. I’m in early beta, and I would appreciate your feedback.
The Toradex Support team will follow up with you shortly!

Did you like my answer?

:+1: :-1:


:handshake:

To enable the gtkglsink and gtksink plugins on your Colibri iMX6DL using Yocto, you need to ensure that the necessary GStreamer plugins are included in your build. Here are the steps to achieve this:

  1. Add Required GStreamer Plugins to Your Image:
    Ensure that the gstreamer1.0-plugins-bad and gstreamer1.0-plugins-good packages are included in your Yocto image. You can do this by appending the packages to the IMAGE_INSTALL variable in your local.conf file.

    IMAGE_INSTALL:append = " gstreamer1.0-plugins-bad gstreamer1.0-plugins-good"
    
  2. Include GTK+3 and OpenGL Support:
    Make sure that GTK+3 and OpenGL support are included in your image. You can add the following packages to your local.conf file:

    IMAGE_INSTALL:append = " gtk+3 gstreamer1.0-plugins-base gstreamer1.0-plugins-gl"
    
  3. Rebuild Your Yocto Image:
    After updating the local.conf file, rebuild your Yocto image to include the new packages.

    bitbake <your-image-name>
    
  4. Verify Plugin Availability:
    Once the image is built and flashed to your Colibri iMX6DL, verify that the gtkglsink and gtksink plugins are available by running the following command on the target device:

    gst-inspect-1.0 gtkglsink
    gst-inspect-1.0 gtksink
    

    If the plugins are correctly installed, you should see detailed information about them.

  5. Alternative Method Using Torizon:
    If you prefer not to rebuild the Yocto image, you can use Torizon, which allows you to install the necessary libraries and packages inside Debian containers. You can create a Docker container with the required GStreamer plugins and GTK+3 support.

    Here is an example Dockerfile to create a container with the necessary dependencies:

    FROM torizon/wayland-base:2
    
    RUN apt-get update && apt-get install -y \
        gstreamer1.0-plugins-bad \
        gstreamer1.0

I tried to install the plugins as per your suggestions but I yocto image itself is not compiling after adding the above lines in local.conf . It gives NOTHING PROVIDES error

Hi @Teena ,
Please follow this article for building image using Yocto.
Dont forget to install all the prerequisites.
Let us know if you still face the error