GStreamer基础教程08——pipeline的快捷访问

目标

? ? ? GStreamer建立的pipeline不需要完全关闭。有多种方法可以让数据在任何时候送到pipeline中或者从pipeline中取出。本教程会展示：

? ? ? 如何把外部数据送到pipeline中

? ? ? 如何把数据从pipeline中取出

? ? ? 如何操作这些数据

介绍

? ? ? 有几种方法可以让应用通过pipeline和数据流交互。本教程讲述了最简单的一种，因为使用了专门为这个而创建的element。

? ? ? 专门让应用可以往pipeline里面传入数据的element时appsrc，而appsink就正好相反，让应用可以从pipeline中获得数据。为了避免混淆，我们可以这么来理解，appsrc是一个普通的source element，不过它的数据都是来自外太空，而appsink是一个普通的sink element，数据从这里出去的就消失不见了。

? ? ? appsrc和appsink用得非常多，所以他们都自己提供API，你只要连接了gstreamer-app库，那么就可以访问到。在本教程里，我们会使用一种简单地方法通过信号来实现。

? ? ? appsrc可以有不同的工作模式：在pull模式，在需要时向应用请求数据；在push模式，应用根据自己的节奏把数据推送过来。而且，在push模式，如果已经有了足够的数据，应用可以在push时被阻塞，或者可以经由enough-data和need-data信号来控制。本教程中得例子就采用了这种信号控制的方式，其他没有提及的方法可以在appsrc的文档中查阅。

Buffers

? ? ? 通过pipeline传递的大块数据被称为buffers。因为本例子会制造数据同时也消耗数据，所以我们需要了解GstBuffer。

? ? ? Source Pads负责制造buffer，这些buffer被sink pad消耗掉。GStreamer在一个个element之间传递这些buffer。

? ? ? 一个buffer只能简单地描述一小片数据，不要认为我们所有的buffer都是一样大小的。而且，buffer有一个时间戳和有效期，这个就描述了什么时候buffer里的数据需要渲染出来。时间戳是个非常复杂和精深的话题，但目前这个简单地解释也足够了。

? ? ? 作为一个例子，一个filesrc会提供“ANY”属性的buffers并且没有时间戳信息。在demux（《GStreamer基础教程03——动态pipeline》）之后，buffers会有一些特定的cap了，比如"video/x-h264"，在解码后，每一个buffer都会包含一帧有原始caps的视频帧（比如：video/x-raw-yuv），并且有非常明确地时间戳用来指示这一帧在什么时候显示。

教程

? ? ? 本教程是上一篇教程（《GStreamer基础教程07——多线程和Pad的有效性》）在两个方面的扩展：第一是用appsrc来取代audiotestsrc来生成音频数据；第二是在tee里新加了一个分支，这样流入audio sink和波形显示的数据同样复制了一份传给appsink。这个appsink就把信息回传给应用，应用就可以通知用户收到了数据或者做其他更复杂的工作。

一个粗糙的波形发生器

#include <gst/gst.h>
#include <string.h>
  
#define CHUNK_SIZE 1024   /* Amount of bytes we are sending in each buffer */
#define SAMPLE_RATE 44100 /* Samples per second we are sending */
#define AUDIO_CAPS "audio/x-raw-int,channels=1,rate=%d,signed=(boolean)true,width=16,depth=16,endianness=BYTE_ORDER"
  
/* Structure to contain all our information,so we can pass it to callbacks */
typedef struct _CustomData {
  GstElement *pipeline,*app_source,*tee,*audio_queue,*audio_convert1,*audio_resample,*audio_sink;
  GstElement *video_queue,*audio_convert2,*visual,*video_convert,*video_sink;
  GstElement *app_queue,*app_sink;
  
  guint64 num_samples;   /* Number of samples generated so far (for timestamp generation) */
  gfloat a,b,c,d;     /* For waveform generation */
  
  guint sourceid;        /* To control the GSource */
  
  GMainLoop *main_loop;  /* GLib's Main Loop */
} CustomData;
  
/* This method is called by the idle GSource in the mainloop,to feed CHUNK_SIZE bytes into appsrc.
 * The idle handler is added to the mainloop when appsrc requests us to start sending data (need-data signal)
 * and is removed when appsrc has enough data (enough-data signal).
 */
static gboolean push_data (CustomData *data) {
  GstBuffer *buffer;
  GstFlowReturn ret;
  int i;
  gint16 *raw;
  gint num_samples = CHUNK_SIZE / 2; /* Because each sample is 16 bits */
  gfloat freq;
  
  /* Create a new empty buffer */
  buffer = gst_buffer_new_and_alloc (CHUNK_SIZE);
  
  /* Set its timestamp and duration */
  GST_BUFFER_TIMESTAMP (buffer) = gst_util_uint64_scale (data->num_samples,GST_SECOND,SAMPLE_RATE);
  GST_BUFFER_DURATION (buffer) = gst_util_uint64_scale (CHUNK_SIZE,SAMPLE_RATE);
  
  /* Generate some psychodelic waveforms */
  raw = (gint16 *)GST_BUFFER_DATA (buffer);
  data->c += data->d;
  data->d -= data->c / 1000;
  freq = 1100 + 1000 * data->d;
  for (i = 0; i < num_samples; i++) {
    data->a += data->b;
    data->b -= data->a / freq;
    raw[i] = (gint16)(500 * data->a);
  }
  data->num_samples += num_samples;
  
  /* Push the buffer into the appsrc */
  g_signal_emit_by_name (data->app_source,"push-buffer",buffer,&ret);
  
  /* Free the buffer now that we are done with it */
  gst_buffer_unref (buffer);
  
  if (ret != GST_FLOW_OK) {
    /* We got some error,stop sending data */
    return FALSE;
  }
  
  return TRUE;
}
  
/* This signal callback triggers when appsrc needs data. Here,we add an idle handler
 * to the mainloop to start pushing data into the appsrc */
static void start_feed (GstElement *source,guint size,CustomData *data) {
  if (data->sourceid == 0) {
    g_print ("Start feedingn");
    data->sourceid = g_idle_add ((GSourceFunc) push_data,data);
  }
}
  
/* This callback triggers when appsrc has enough data and we can stop sending.
 * We remove the idle handler from the mainloop */
static void stop_feed (GstElement *source,CustomData *data) {
  if (data->sourceid != 0) {
    g_print ("Stop feedingn");
    g_source_remove (data->sourceid);
    data->sourceid = 0;
  }
}
  
/* The appsink has received a buffer */
static void new_buffer (GstElement *sink,CustomData *data) {
  GstBuffer *buffer;
  
  /* Retrieve the buffer */
  g_signal_emit_by_name (sink,"pull-buffer",&buffer);
  if (buffer) {
    /* The only thing we do in this example is print a * to indicate a received buffer */
    g_print ("*");
    gst_buffer_unref (buffer);
  }
}
  
/* This function is called when an error message is posted on the bus */
static void error_cb (GstBus *bus,GstMessage *msg,CustomData *data) {
  GError *err;
  gchar *debug_info;
  
  /* Print error details on the screen */
  gst_message_parse_error (msg,&err,&debug_info);
  g_printerr ("Error received from element %s: %sn",GST_OBJECT_NAME (msg->src),err->message);
  g_printerr ("Debugging information: %sn",debug_info ? debug_info : "none");
  g_clear_error (&err);
  g_free (debug_info);
  
  g_main_loop_quit (data->main_loop);
}
  
int main(int argc,char *argv[]) {
  CustomData data;
  GstPadTemplate *tee_src_pad_template;
  GstPad *tee_audio_pad,*tee_video_pad,*tee_app_pad;
  GstPad *queue_audio_pad,*queue_video_pad,*queue_app_pad;
  gchar *audio_caps_text;
  GstCaps *audio_caps;
  GstBus *bus;
  
  /* Initialize cumstom data structure */
  memset (&data,sizeof (data));
  data.b = 1; /* For waveform generation */
  data.d = 1;
  
  /* Initialize GStreamer */
  gst_init (&argc,&argv);
  
  /* Create the elements */
  data.app_source = gst_element_factory_make ("appsrc","audio_source");
  data.tee = gst_element_factory_make ("tee","tee");
  data.audio_queue = gst_element_factory_make ("queue","audio_queue");
  data.audio_convert1 = gst_element_factory_make ("audioconvert","audio_convert1");
  data.audio_resample = gst_element_factory_make ("audioresample","audio_resample");
  data.audio_sink = gst_element_factory_make ("autoaudiosink","audio_sink");
  data.video_queue = gst_element_factory_make ("queue","video_queue");
  data.audio_convert2 = gst_element_factory_make ("audioconvert","audio_convert2");
  data.visual = gst_element_factory_make ("wavescope","visual");
  data.video_convert = gst_element_factory_make ("ffmpegcolorspace","csp");
  data.video_sink = gst_element_factory_make ("autovideosink","video_sink");
  data.app_queue = gst_element_factory_make ("queue","app_queue");
  data.app_sink = gst_element_factory_make ("appsink","app_sink");
  
  /* Create the empty pipeline */
  data.pipeline = gst_pipeline_new ("test-pipeline");
  
  if (!data.pipeline || !data.app_source || !data.tee || !data.audio_queue || !data.audio_convert1 ||
      !data.audio_resample || !data.audio_sink || !data.video_queue || !data.audio_convert2 || !data.visual ||
      !data.video_convert || !data.video_sink || !data.app_queue || !data.app_sink) {
    g_printerr ("Not all elements could be created.n");
    return -1;
  }
  
  /* Configure wavescope */
  g_object_set (data.visual,"shader","style",NULL);
  
  /* Configure appsrc */
  audio_caps_text = g_strdup_printf (AUDIO_CAPS,SAMPLE_RATE);
  audio_caps = gst_caps_from_string (audio_caps_text);
  g_object_set (data.app_source,"caps",audio_caps,NULL);
  g_signal_connect (data.app_source,"need-data",G_CALLBACK (start_feed),&data);
  g_signal_connect (data.app_source,"enough-data",G_CALLBACK (stop_feed),&data);
  
  /* Configure appsink */
  g_object_set (data.app_sink,"emit-signals",TRUE,NULL);
  g_signal_connect (data.app_sink,"new-buffer",G_CALLBACK (new_buffer),&data);
  gst_caps_unref (audio_caps);
  g_free (audio_caps_text);
  
  /* Link all elements that can be automatically linked because they have "Always" pads */
  gst_bin_add_many (GST_BIN (data.pipeline),data.app_source,data.tee,data.audio_queue,data.audio_convert1,data.audio_resample,data.audio_sink,data.video_queue,data.audio_convert2,data.visual,data.video_convert,data.video_sink,data.app_queue,data.app_sink,NULL);
  if (gst_element_link_many (data.app_source,NULL) != TRUE ||
      gst_element_link_many (data.audio_queue,NULL) != TRUE ||
      gst_element_link_many (data.video_queue,NULL) != TRUE ||
      gst_element_link_many (data.app_queue,NULL) != TRUE) {
    g_printerr ("Elements could not be linked.n");
    gst_object_unref (data.pipeline);
    return -1;
  }
  
  /* Manually link the Tee,which has "Request" pads */
  tee_src_pad_template = gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (data.tee),"src%d");
  tee_audio_pad = gst_element_request_pad (data.tee,tee_src_pad_template,NULL,NULL);
  g_print ("Obtained request pad %s for audio branch.n",gst_pad_get_name (tee_audio_pad));
  queue_audio_pad = gst_element_get_static_pad (data.audio_queue,"sink");
  tee_video_pad = gst_element_request_pad (data.tee,NULL);
  g_print ("Obtained request pad %s for video branch.n",gst_pad_get_name (tee_video_pad));
  queue_video_pad = gst_element_get_static_pad (data.video_queue,"sink");
  tee_app_pad = gst_element_request_pad (data.tee,NULL);
  g_print ("Obtained request pad %s for app branch.n",gst_pad_get_name (tee_app_pad));
  queue_app_pad = gst_element_get_static_pad (data.app_queue,"sink");
  if (gst_pad_link (tee_audio_pad,queue_audio_pad) != GST_PAD_LINK_OK ||
      gst_pad_link (tee_video_pad,queue_video_pad) != GST_PAD_LINK_OK ||
      gst_pad_link (tee_app_pad,queue_app_pad) != GST_PAD_LINK_OK) {
    g_printerr ("Tee could not be linkedn");
    gst_object_unref (data.pipeline);
    return -1;
  }
  gst_object_unref (queue_audio_pad);
  gst_object_unref (queue_video_pad);
  gst_object_unref (queue_app_pad);
  
  /* Instruct the bus to emit signals for each received message,and connect to the interesting signals */
  bus = gst_element_get_bus (data.pipeline);
  gst_bus_add_signal_watch (bus);
  g_signal_connect (G_OBJECT (bus),"message::error",(GCallback)error_cb,&data);
  gst_object_unref (bus);
  
  /* Start playing the pipeline */
  gst_element_set_state (data.pipeline,GST_STATE_PLAYING);
  
  /* Create a GLib Main Loop and set it to run */
  data.main_loop = g_main_loop_new (NULL,FALSE);
  g_main_loop_run (data.main_loop);
  
  /* Release the request pads from the Tee,and unref them */
  gst_element_release_request_pad (data.tee,tee_audio_pad);
  gst_element_release_request_pad (data.tee,tee_video_pad);
  gst_element_release_request_pad (data.tee,tee_app_pad);
  gst_object_unref (tee_audio_pad);
  gst_object_unref (tee_video_pad);
  gst_object_unref (tee_app_pad);
  
  /* Free resources */
  gst_element_set_state (data.pipeline,GST_STATE_NULL);
  gst_object_unref (data.pipeline);
  return 0;
}

工作流程

? ? ? 创建pipeline段的代码就是上一篇的教程中得例子的扩大版。包括初始或所有的element，连接有Always Pad的element然后手动连接tee element的Request Pad。

? ? ? 下面我们关注一下appsrc和appsink这两个element的配置：

  /* Configure appsrc */
  audio_caps_text = g_strdup_printf (AUDIO_CAPS,&data);

? ? ? 然后我们把need-data和enough-data信号和回调连接起来，这样在appsrc内部的队列里面数据不足或将要满地时候会发送信号，我们就用这些信号来启动/停止我们的信号发生过程。

  /* Configure appsink */
  g_object_set (data.app_sink,&data);
  gst_caps_unref (audio_caps);
  g_free (audio_caps_text);

? ? ? 启动pipeline，等到消息和最后的清理资源都和以前的没什么区别。让我们关注我们刚刚注册的回调吧。

/* This signal callback triggers when appsrc needs data. Here,data);
  }
}

? ? ? 这只是appsrc多种发出数据方法中的一个。特别需要指出的是，buffer不是必须要在主线程中用GLib方法来传递给appsrc的，你也不是一定要用need-data和enough-data信号来同步appsrc的（据说这样最方便）。

? ? ? 我们记录下g_idle_add()的返回的sourceid，这样后面可以关掉它。

/* This callback triggers when appsrc has enough data and we can stop sending.
 * We remove the idle handler from the mainloop */
static void stop_feed (GstElement *source,CustomData *data) {
  if (data->sourceid != 0) {
    g_print ("Stop feedingn");
    g_source_remove (data->sourceid);
    data->sourceid = 0;
  }
}

/* This method is called by the idle GSource in the mainloop,SAMPLE_RATE);
  
  /* Generate some psychodelic waveforms */
  raw = (gint16 *)GST_BUFFER_DATA (buffer);

? ? ? 这里第一步是用gst_buffer_new_and_alloc()方法和给定的大小创建一个新buffer（例子中是1024字节）。

? ? ? 我们计算我们生成的采样数据的数据量，把数据存在CustomData.num_samples里面，这样我们可以用GstBuffer提供的GST_BUFFER_TIMESTAMP宏来生成buffer的时间戳。

? ? ? gst_util_uint64_scale是一个工具函数，用来缩放数据，确保不会溢出。

? ? ? 这些给buffer的数据可以用GstBuffer提供的GST_BUFFER_DATA宏来访问。

? ? ? 我们会跳过波形的生成部分，因为这不是本教程要讲述的内容。

  /* Push the buffer into the appsrc */
  g_signal_emit_by_name (data->app_source,&ret);
  
  /* Free the buffer now that we are done with it */
  gst_buffer_unref (buffer);

/* The appsink has received a buffer */
static void new_buffer (GstElement *sink,&buffer);
  if (buffer) {
    /* The only thing we do in this example is print a * to indicate a received buffer */
    g_print ("*");
    gst_buffer_unref (buffer);
  }
}

? ? 请不要忘记调用gst_buffer_unref()来释放buffer，就讲这么多吧。