vp8enc: finish support for temporally scaled encoding

  - introduce two new properties:

    * temporal-scalability-layer-flags:

      Provide fine-grained control of layer encoding to the
      outside world. The flags sequence should be a multiple of
      the periodicity and is indexed by a running count of encoded
      frames modulo the sequence length.

    * temporal-scalability-layer-sync-flags:

      Specify the pattern of inter-layer synchronisation (i.e.
      which of the frames generated by the layer encoding
      specification represent an inter-layer synchronisation).
      There must be one entry per entry in
      temporal-scalability-layer-flags.

  - apply temporal scalability settings and expose as buffer
    metadata.

    This allows the codec to allocate a given frame to the correct
    internal bitrate allocator. Additionally, all the
    non-bitstream metadata needed to payload a temporally scaled
    stream is now attached to each output buffer as a
    GstVideoVP8Meta.

  - add unit test for temporally scaled encoding.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/728>

1 files changed, 210 insertions, 0 deletions

diff --git a/tests/check/elements/vp8enc.c b/tests/check/elements/vp8enc.c
index 622041335..3221d7885 100644
--- a/tests/check/elements/vp8enc.c
+++ b/tests/check/elements/vp8enc.c
@@ -17,6 +17,9 @@
  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  * Boston, MA 02110-1301, USA.
  */
+
+#define GLIB_DISABLE_DEPRECATION_WARNINGS
+
 #include <gst/check/gstharness.h>
 #include <gst/check/gstcheck.h>
 #include <gst/video/video.h>
@@ -210,6 +213,211 @@ GST_START_TEST (test_autobitrate_changes_with_caps)
 
 GST_END_TEST;
 
+#define verify_meta(buffer, usets, ybit, tid, tl0picindex)               \
+  G_STMT_START {                                                         \
+    gboolean use_temporal_scaling, layer_sync;                           \
+    guint temporal_layer_id, tl0picidx;                                  \
+    GstCustomMeta *meta = gst_buffer_get_custom_meta (buffer,            \
+        "GstVP8Meta");                                                   \
+    GstStructure *s;                                                     \
+    fail_unless (meta != NULL);                                          \
+    s = gst_custom_meta_get_structure (meta);                            \
+    fail_unless (gst_structure_get (s,                                   \
+          "use-temporal-scaling", G_TYPE_BOOLEAN, &use_temporal_scaling, \
+          "layer-sync", G_TYPE_BOOLEAN, &layer_sync,                     \
+          "layer-id", G_TYPE_UINT, &temporal_layer_id,                   \
+          "tl0picidx", G_TYPE_UINT, &tl0picidx, NULL));                  \
+    fail_unless_equals_int (usets, use_temporal_scaling);                \
+    fail_unless_equals_int (ybit, layer_sync);                           \
+    fail_unless_equals_int (tid, temporal_layer_id);                     \
+    fail_unless_equals_int (tl0picindex, tl0picidx);                     \
+  } G_STMT_END
+
+static void
+configure_vp8ts (GstHarness * h)
+{
+  gint i;
+  GValue layer_sync_flags = G_VALUE_INIT;
+  GValueArray *decimators = g_value_array_new (3);
+  GValueArray *layer_ids = g_value_array_new (4);
+  GValueArray *bitrates = g_value_array_new (3);
+  GValue ival = { 0, }, bval = {
+  0,};
+
+  gst_value_array_init (&layer_sync_flags, 8);
+
+  g_value_init (&ival, G_TYPE_INT);
+  for (i = 0; i < 3; i++) {
+    /* 7.5, 15, 30fps */
+    static const gint d[] = { 4, 2, 1 };
+    g_value_set_int (&ival, d[i]);
+    g_value_array_append (decimators, &ival);
+  }
+
+  for (i = 0; i < 4; i++) {
+    static const gint d[] = { 0, 2, 1, 2 };
+    g_value_set_int (&ival, d[i]);
+    g_value_array_append (layer_ids, &ival);
+  }
+
+  for (i = 0; i < 3; i++) {
+    /* Split 512kbps 40%, 20%, 40% */
+    static const gint d[] = { 204800, 307200, 512000 };
+    g_value_set_int (&ival, d[i]);
+    g_value_array_append (bitrates, &ival);
+  }
+
+  gst_util_set_object_arg (G_OBJECT (h->element),
+      "temporal-scalability-layer-flags",
+      /* layer 0 */
+      "<no-ref-golden+no-upd-golden+no-upd-alt,"
+      /* layer 2 (sync) */
+      "no-ref-golden+no-upd-last+no-upd-golden+no-upd-alt+no-upd-entropy,"
+      /* layer 1 (sync) */
+      "no-ref-golden+no-upd-last+no-upd-alt,"
+      /* layer 2 */
+      "no-upd-last+no-upd-golden+no-upd-alt+no-upd-entropy,"
+      /* layer 0 */
+      "no-ref-golden+no-upd-golden+no-upd-alt,"
+      /* layer 2 */
+      "no-upd-last+no-upd-golden+no-upd-alt+no-upd-entropy,"
+      /* layer 1 */
+      "no-upd-last+no-upd-alt,"
+      /* layer 2 */
+      "no-upd-last+no-upd-golden+no-upd-alt+no-upd-entropy>");
+
+  g_value_init (&bval, G_TYPE_BOOLEAN);
+  for (i = 0; i < 8; i++) {
+    /* Reflect pattern above */
+    static const gboolean d[] = {
+      FALSE,
+      TRUE,
+      TRUE,
+      FALSE,
+      FALSE,
+      FALSE,
+      FALSE,
+      FALSE
+    };
+    g_value_set_boolean (&bval, d[i]);
+    gst_value_array_append_value (&layer_sync_flags, &bval);
+  }
+
+  g_object_set_property (G_OBJECT (h->element),
+      "temporal-scalability-layer-sync-flags", &layer_sync_flags);
+
+  g_object_set (h->element,
+      "temporal-scalability-number-layers", decimators->n_values,
+      "temporal-scalability-periodicity", layer_ids->n_values,
+      "temporal-scalability-rate-decimator", decimators,
+      "temporal-scalability-layer-id", layer_ids,
+      "temporal-scalability-target-bitrate", bitrates,
+      "error-resilient", 1, NULL);
+
+  g_value_array_free (decimators);
+  g_value_array_free (layer_ids);
+  g_value_array_free (bitrates);
+  g_value_unset (&layer_sync_flags);
+}
+
+GST_START_TEST (test_encode_temporally_scaled)
+{
+  gint i;
+  struct
+  {
+    gboolean ybit;
+    gint tid;
+    gint tl0picidx;
+    gboolean droppable;
+  } expected[] = {
+    {
+    TRUE, 0, 1, FALSE},         /* This is an intra */
+    {
+    TRUE, 2, 1, TRUE}, {
+    TRUE, 1, 1, FALSE}, {
+    FALSE, 2, 1, TRUE}, {
+    FALSE, 0, 2, FALSE}, {
+    FALSE, 2, 2, TRUE}, {
+    FALSE, 1, 2, FALSE}, {
+    FALSE, 2, 2, TRUE}, {
+    FALSE, 0, 3, FALSE}, {
+    TRUE, 2, 3, TRUE}, {
+    TRUE, 1, 3, FALSE}, {
+    FALSE, 2, 3, TRUE}, {
+    FALSE, 0, 4, FALSE}, {
+    FALSE, 2, 4, TRUE}, {
+    FALSE, 1, 4, FALSE}, {
+  FALSE, 2, 4, TRUE},};
+  GstHarness *h = gst_harness_new ("vp8enc");
+  gst_harness_set_src_caps (h, gst_caps_new_i420 (320, 240));
+  configure_vp8ts (h);
+
+  for (i = 0; i < 16; i++) {
+    GstBuffer *in, *out;
+
+    in = gst_harness_create_video_buffer_full (h, 0x42,
+        320, 240, gst_util_uint64_scale (i, GST_SECOND, 30),
+        gst_util_uint64_scale (1, GST_SECOND, 30));
+    gst_harness_push (h, in);
+
+    out = gst_harness_pull (h);
+    /* Ensure first frame is encoded as an intra */
+    if (i == 0)
+      fail_if (GST_BUFFER_FLAG_IS_SET (out, GST_BUFFER_FLAG_DELTA_UNIT));
+    else
+      fail_unless (GST_BUFFER_FLAG_IS_SET (out, GST_BUFFER_FLAG_DELTA_UNIT));
+    fail_unless_equals_int (expected[i].droppable,
+        GST_BUFFER_FLAG_IS_SET (out, GST_BUFFER_FLAG_DROPPABLE));
+    verify_meta (out, TRUE, expected[i].ybit, expected[i].tid,
+        expected[i].tl0picidx);
+    gst_buffer_unref (out);
+  }
+
+  gst_harness_teardown (h);
+}
+
+GST_END_TEST;
+
+GST_START_TEST (test_encode_fresh_meta)
+{
+  gint i;
+  GstBuffer *buffer;
+  GstHarness *h = gst_harness_new ("vp8enc");
+  GstCustomMeta *meta;
+  GstStructure *s;
+  gst_harness_set_src_caps (h, gst_caps_new_i420_full (320, 240, 25, 1, 1, 1));
+
+  buffer = gst_harness_create_video_buffer_full (h, 0x0,
+      320, 240, gst_util_uint64_scale (0, GST_SECOND, 25),
+      gst_util_uint64_scale (1, GST_SECOND, 25));
+
+  /* Attach bogus meta to input buffer */
+  meta = gst_buffer_add_custom_meta (buffer, "GstVP8Meta");
+  s = gst_custom_meta_get_structure (meta);
+  gst_structure_set (s,
+      "use-temporal-scaling", G_TYPE_BOOLEAN, FALSE,
+      "layer-sync", G_TYPE_BOOLEAN, FALSE,
+      "layer-id", G_TYPE_UINT, 0, "tl0picidx", G_TYPE_UINT, 0, NULL);
+
+  for (i = 0; i < 2; i++) {
+    GstBuffer *out;
+
+    fail_unless_equals_int (GST_FLOW_OK,
+        gst_harness_push (h, gst_buffer_ref (buffer)));
+
+    out = gst_harness_pull (h);
+    /* Ensure that output buffer has fresh meta */
+    verify_meta (out, FALSE, (i == 0), 0, i + 1);
+    gst_buffer_unref (out);
+  }
+
+  gst_buffer_unref (buffer);
+
+  gst_harness_teardown (h);
+}
+
+GST_END_TEST;
+
 static Suite *
 vp8enc_suite (void)
 {
@@ -222,6 +430,8 @@ vp8enc_suite (void)
   tcase_add_test (tc_chain, test_encode_lag_in_frames);
   tcase_add_test (tc_chain, test_encode_simple_when_bitrate_set_to_zero);
   tcase_add_test (tc_chain, test_autobitrate_changes_with_caps);
+  tcase_add_test (tc_chain, test_encode_temporally_scaled);
+  tcase_add_test (tc_chain, test_encode_fresh_meta);
 
   return s;
 }