rtp: Color Space header extension

Implements WebRTC header extension defined in
http://www.webrtc.org/experiments/rtp-hdrext/color-space.

It uses RTP header to communicate color space information and optionally
also metadata that is needed in order to properly render a high dynamic
range (HDR) video stream.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/853>

5 files changed, 509 insertions, 0 deletions

diff --git a/gst/rtp/gstrtp.c b/gst/rtp/gstrtp.c
index 20f3d85f4..9528ffb10 100644
--- a/gst/rtp/gstrtp.c
+++ b/gst/rtp/gstrtp.c
@@ -127,6 +127,7 @@ plugin_init (GstPlugin * plugin)
   ret |= GST_ELEMENT_REGISTER (rtpulpfecdec, plugin);
   ret |= GST_ELEMENT_REGISTER (rtpulpfecenc, plugin);
   ret |= GST_ELEMENT_REGISTER (rtpstorage, plugin);
+  ret |= GST_ELEMENT_REGISTER (rtphdrextcolorspace, plugin);
 
   return ret;
 }
diff --git a/gst/rtp/gstrtpelements.h b/gst/rtp/gstrtpelements.h
index 5f71f1672..9321d3530 100644
--- a/gst/rtp/gstrtpelements.h
+++ b/gst/rtp/gstrtpelements.h
@@ -127,6 +127,7 @@ GST_ELEMENT_REGISTER_DECLARE (rtpreddec);
 GST_ELEMENT_REGISTER_DECLARE (rtpulpfecdec);
 GST_ELEMENT_REGISTER_DECLARE (rtpulpfecenc);
 GST_ELEMENT_REGISTER_DECLARE (rtpstorage);
+GST_ELEMENT_REGISTER_DECLARE (rtphdrextcolorspace);
 
 G_END_DECLS
 
diff --git a/gst/rtp/gstrtphdrext-colorspace.c b/gst/rtp/gstrtphdrext-colorspace.c
new file mode 100644
index 000000000..05fd8ddb0
--- /dev/null
+++ b/gst/rtp/gstrtphdrext-colorspace.c
@@ -0,0 +1,465 @@
+/* GStreamer
+ * Copyright (C) 2020-2021 Collabora Ltd.
+ *   @author: Jakub Adam <jakub.adam@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+/**
+ * SECTION:rtphdrextcolorspace
+ * @title: GstRtphdrext-Colorspace
+ * @short_description: Helper methods for dealing with Color Space RTP header
+ * extension as defined in  http://www.webrtc.org/experiments/rtp-hdrext/color-space
+ * @see_also: #GstRTPHeaderExtension, #GstRTPBasePayload, #GstRTPBaseDepayload
+ *
+ * Since: 1.20
+ */
+
+#include "gstrtphdrext-colorspace.h"
+
+#include "gstrtpelements.h"
+
+#include <gst/base/gstbytereader.h>
+#include <gst/video/video-color.h>
+#include <gst/video/video-hdr.h>
+
+GST_DEBUG_CATEGORY_STATIC (rtphdrext_colorspace_debug);
+#define GST_CAT_DEFAULT (rtphdrext_colorspace_debug)
+
+/**
+ * GstRTPHeaderExtensionColorspace:
+ * @parent: the parent #GstRTPHeaderExtension
+ *
+ * Instance struct for Color Space RTP header extension.
+ *
+ * http://www.webrtc.org/experiments/rtp-hdrext/color-space
+ */
+struct _GstRTPHeaderExtensionColorspace
+{
+  GstRTPHeaderExtension parent;
+
+  GstVideoColorimetry colorimetry;
+  GstVideoChromaSite chroma_site;
+  GstVideoMasteringDisplayInfo mdi;
+  GstVideoContentLightLevel cll;
+  gboolean has_hdr_meta;
+};
+
+G_DEFINE_TYPE_WITH_CODE (GstRTPHeaderExtensionColorspace,
+    gst_rtp_header_extension_colorspace, GST_TYPE_RTP_HEADER_EXTENSION,
+    GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "rtphdrextcolorspace", 0,
+        "RTP Color Space Header Extension");
+    );
+GST_ELEMENT_REGISTER_DEFINE (rtphdrextcolorspace, "rtphdrextcolorspace",
+    GST_RANK_MARGINAL, GST_TYPE_RTP_HEADER_EXTENSION_COLORSPACE);
+
+static void
+gst_rtp_header_extension_colorspace_init (GstRTPHeaderExtensionColorspace *
+    self)
+{
+}
+
+static GstRTPHeaderExtensionFlags
+gst_rtp_header_extension_colorspace_get_supported_flags (GstRTPHeaderExtension *
+    ext)
+{
+  GstRTPHeaderExtensionColorspace *self =
+      GST_RTP_HEADER_EXTENSION_COLORSPACE (ext);
+
+  return self->has_hdr_meta ?
+      GST_RTP_HEADER_EXTENSION_TWO_BYTE : GST_RTP_HEADER_EXTENSION_ONE_BYTE;
+}
+
+static gsize
+gst_rtp_header_extension_colorspace_get_max_size (GstRTPHeaderExtension * ext,
+    const GstBuffer * buffer)
+{
+  GstRTPHeaderExtensionColorspace *self =
+      GST_RTP_HEADER_EXTENSION_COLORSPACE (ext);
+
+  return self->has_hdr_meta ?
+      GST_RTP_HDREXT_COLORSPACE_WITH_HDR_META_SIZE :
+      GST_RTP_HDREXT_COLORSPACE_SIZE;
+}
+
+static gsize
+gst_rtp_header_extension_colorspace_write (GstRTPHeaderExtension * ext,
+    const GstBuffer * input_meta, GstRTPHeaderExtensionFlags write_flags,
+    GstBuffer * output, guint8 * data, gsize size)
+{
+  GstRTPHeaderExtensionColorspace *self =
+      GST_RTP_HEADER_EXTENSION_COLORSPACE (ext);
+  GstRTPBuffer rtp = GST_RTP_BUFFER_INIT;
+  gboolean is_frame_last_buffer;
+  guint8 *ptr = data;
+  guint8 horizontal_site;
+  guint8 vertical_site;
+
+  g_return_val_if_fail (size >=
+      gst_rtp_header_extension_colorspace_get_max_size (ext, NULL), -1);
+  g_return_val_if_fail (write_flags &
+      gst_rtp_header_extension_colorspace_get_supported_flags (ext), -1);
+
+  if (self->colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_UNKNOWN &&
+      self->colorimetry.primaries == GST_VIDEO_COLOR_PRIMARIES_UNKNOWN &&
+      self->colorimetry.range == GST_VIDEO_COLOR_RANGE_UNKNOWN &&
+      self->colorimetry.transfer == GST_VIDEO_TRANSFER_UNKNOWN) {
+    /* Nothing to write. */
+    return 0;
+  }
+
+  gst_rtp_buffer_map (output, GST_MAP_READ, &rtp);
+  is_frame_last_buffer = gst_rtp_buffer_get_marker (&rtp);
+  gst_rtp_buffer_unmap (&rtp);
+
+  if (!is_frame_last_buffer) {
+    /* Only a video frame's final packet should carry color space info. */
+    return 0;
+  }
+
+  *ptr++ = gst_video_color_primaries_to_iso (self->colorimetry.primaries);
+  *ptr++ = gst_video_transfer_function_to_iso (self->colorimetry.transfer);
+  *ptr++ = gst_video_color_matrix_to_iso (self->colorimetry.matrix);
+
+  if (self->chroma_site & GST_VIDEO_CHROMA_SITE_H_COSITED) {
+    horizontal_site = 1;
+  } else if (self->chroma_site & GST_VIDEO_CHROMA_SITE_NONE) {
+    horizontal_site = 2;
+  } else {
+    horizontal_site = 0;
+  }
+
+  if (self->chroma_site & GST_VIDEO_CHROMA_SITE_V_COSITED) {
+    vertical_site = 1;
+  } else if (self->chroma_site & GST_VIDEO_CHROMA_SITE_NONE) {
+    vertical_site = 2;
+  } else {
+    vertical_site = 0;
+  }
+
+  *ptr++ =
+      (self->colorimetry.range << 4) + (horizontal_site << 2) + vertical_site;
+
+  if (self->has_hdr_meta) {
+    guint i;
+
+    GST_WRITE_UINT16_BE (ptr,
+        self->mdi.max_display_mastering_luminance / 10000);
+    ptr += 2;
+    GST_WRITE_UINT16_BE (ptr, self->mdi.min_display_mastering_luminance);
+    ptr += 2;
+
+    for (i = 0; i < 3; ++i) {
+      GST_WRITE_UINT16_BE (ptr, self->mdi.display_primaries[i].x);
+      ptr += 2;
+      GST_WRITE_UINT16_BE (ptr, self->mdi.display_primaries[i].y);
+      ptr += 2;
+    }
+
+    GST_WRITE_UINT16_BE (ptr, self->mdi.white_point.x);
+    ptr += 2;
+    GST_WRITE_UINT16_BE (ptr, self->mdi.white_point.y);
+    ptr += 2;
+
+    GST_WRITE_UINT16_BE (ptr, self->cll.max_content_light_level);
+    ptr += 2;
+    GST_WRITE_UINT16_BE (ptr, self->cll.max_frame_average_light_level);
+    ptr += 2;
+  }
+
+  return ptr - data;
+}
+
+static gboolean
+parse_colorspace (GstByteReader * reader, GstVideoColorimetry * colorimetry,
+    GstVideoChromaSite * chroma_site)
+{
+  guint8 val;
+
+  g_return_val_if_fail (reader != NULL, FALSE);
+  g_return_val_if_fail (colorimetry != NULL, FALSE);
+  g_return_val_if_fail (chroma_site != NULL, FALSE);
+
+  if (gst_byte_reader_get_remaining (reader) < GST_RTP_HDREXT_COLORSPACE_SIZE) {
+    return FALSE;
+  }
+
+  if (!gst_byte_reader_get_uint8 (reader, &val)) {
+    return FALSE;
+  }
+  colorimetry->primaries = gst_video_color_primaries_from_iso (val);
+
+  if (!gst_byte_reader_get_uint8 (reader, &val)) {
+    return FALSE;
+  }
+  colorimetry->transfer = gst_video_transfer_function_from_iso (val);
+
+  if (!gst_byte_reader_get_uint8 (reader, &val)) {
+    return FALSE;
+  }
+  colorimetry->matrix = gst_video_color_matrix_from_iso (val);
+
+  *chroma_site = GST_VIDEO_CHROMA_SITE_UNKNOWN;
+
+  if (!gst_byte_reader_get_uint8 (reader, &val)) {
+    return FALSE;
+  }
+  switch ((val >> 2) & 0x03) {
+    case 1:
+      *chroma_site |= GST_VIDEO_CHROMA_SITE_H_COSITED;
+      break;
+    case 2:
+      *chroma_site |= GST_VIDEO_CHROMA_SITE_NONE;
+      break;
+  }
+
+  switch (val & 0x03) {
+    case 1:
+      *chroma_site |= GST_VIDEO_CHROMA_SITE_V_COSITED;
+      break;
+    case 2:
+      *chroma_site |= GST_VIDEO_CHROMA_SITE_NONE;
+      break;
+  }
+
+  colorimetry->range = val >> 4;
+
+  return TRUE;
+}
+
+static gboolean
+parse_colorspace_with_hdr_meta (GstByteReader * reader,
+    GstVideoColorimetry * colorimetry,
+    GstVideoChromaSite * chroma_site,
+    GstVideoMasteringDisplayInfo * mastering_display_info,
+    GstVideoContentLightLevel * content_light_level)
+{
+  guint i;
+  guint16 val16;
+
+  g_return_val_if_fail (reader != NULL, FALSE);
+  g_return_val_if_fail (mastering_display_info != NULL, FALSE);
+  g_return_val_if_fail (content_light_level != NULL, FALSE);
+
+  if (gst_byte_reader_get_remaining (reader) <
+      GST_RTP_HDREXT_COLORSPACE_WITH_HDR_META_SIZE) {
+    return FALSE;
+  }
+
+  if (!parse_colorspace (reader, colorimetry, chroma_site)) {
+    return FALSE;
+  }
+
+  if (!gst_byte_reader_get_uint16_be (reader, &val16)) {
+    return FALSE;
+  }
+  mastering_display_info->max_display_mastering_luminance = val16 * 10000;
+
+  if (!gst_byte_reader_get_uint16_be (reader, &val16)) {
+    return FALSE;
+  }
+  mastering_display_info->min_display_mastering_luminance = val16;
+
+  for (i = 0; i < 3; ++i) {
+    if (!gst_byte_reader_get_uint16_be (reader,
+            &mastering_display_info->display_primaries[i].x)) {
+      return FALSE;
+    }
+
+    if (!gst_byte_reader_get_uint16_be (reader,
+            &mastering_display_info->display_primaries[i].y)) {
+      return FALSE;
+    }
+  }
+
+  if (!gst_byte_reader_get_uint16_be (reader,
+          &mastering_display_info->white_point.x)) {
+    return FALSE;
+  }
+  if (!gst_byte_reader_get_uint16_be (reader,
+          &mastering_display_info->white_point.y)) {
+    return FALSE;
+  }
+
+  if (!gst_byte_reader_get_uint16_be (reader,
+          &content_light_level->max_content_light_level)) {
+    return FALSE;
+  }
+  if (!gst_byte_reader_get_uint16_be (reader,
+          &content_light_level->max_frame_average_light_level)) {
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_rtp_header_extension_colorspace_read (GstRTPHeaderExtension * ext,
+    GstRTPHeaderExtensionFlags read_flags, const guint8 * data, gsize size,
+    GstBuffer * buffer)
+{
+  GstRTPHeaderExtensionColorspace *self =
+      GST_RTP_HEADER_EXTENSION_COLORSPACE (ext);
+  gboolean has_hdr_meta;
+  GstByteReader *reader;
+  GstVideoColorimetry colorimetry;
+  GstVideoChromaSite chroma_site;
+  GstVideoMasteringDisplayInfo mdi;
+  GstVideoContentLightLevel cll;
+  gboolean caps_update_needed;
+  gboolean result;
+
+  if (size != GST_RTP_HDREXT_COLORSPACE_SIZE &&
+      size != GST_RTP_HDREXT_COLORSPACE_WITH_HDR_META_SIZE) {
+    GST_WARNING_OBJECT (ext, "Invalid Color Space header extension size %"
+        G_GSIZE_FORMAT, size);
+    return FALSE;
+  }
+
+  has_hdr_meta = size == GST_RTP_HDREXT_COLORSPACE_WITH_HDR_META_SIZE;
+
+  reader = gst_byte_reader_new (data, size);
+
+  if (has_hdr_meta) {
+    result = parse_colorspace_with_hdr_meta (reader, &colorimetry, &chroma_site,
+        &mdi, &cll);
+  } else {
+    result = parse_colorspace (reader, &colorimetry, &chroma_site);
+  }
+
+  g_clear_pointer (&reader, gst_byte_reader_free);
+
+  if (!gst_video_colorimetry_is_equal (&self->colorimetry, &colorimetry)) {
+    caps_update_needed = TRUE;
+    self->colorimetry = colorimetry;
+  }
+
+  if (self->chroma_site != chroma_site) {
+    caps_update_needed = TRUE;
+    self->chroma_site = chroma_site;
+  }
+
+  if (self->has_hdr_meta != has_hdr_meta) {
+    caps_update_needed = TRUE;
+    self->has_hdr_meta = has_hdr_meta;
+  }
+
+  if (has_hdr_meta) {
+    if (!gst_video_mastering_display_info_is_equal (&self->mdi, &mdi)) {
+      caps_update_needed = TRUE;
+      self->mdi = mdi;
+    }
+    if (!gst_video_content_light_level_is_equal (&self->cll, &cll)) {
+      caps_update_needed = TRUE;
+      self->cll = cll;
+    }
+  }
+
+  if (caps_update_needed) {
+    gst_rtp_header_extension_set_wants_update_non_rtp_src_caps (ext, TRUE);
+  }
+
+  return result;
+}
+
+static gboolean
+    gst_rtp_header_extension_colorspace_set_non_rtp_sink_caps
+    (GstRTPHeaderExtension * ext, const GstCaps * caps)
+{
+  GstRTPHeaderExtensionColorspace *self =
+      GST_RTP_HEADER_EXTENSION_COLORSPACE (ext);
+  GstStructure *s;
+  const gchar *colorimetry;
+  const gchar *chroma_site;
+
+  s = gst_caps_get_structure (caps, 0);
+
+  colorimetry = gst_structure_get_string (s, "colorimetry");
+  if (colorimetry) {
+    gst_video_colorimetry_from_string (&self->colorimetry, colorimetry);
+
+    self->has_hdr_meta =
+        gst_video_mastering_display_info_from_caps (&self->mdi, caps);
+
+    gst_video_content_light_level_from_caps (&self->cll, caps);
+  }
+
+  chroma_site = gst_structure_get_string (s, "chroma-site");
+  if (chroma_site) {
+    self->chroma_site = gst_video_chroma_from_string (chroma_site);
+  }
+
+  return TRUE;
+}
+
+static gboolean
+    gst_rtp_header_extension_colorspace_update_non_rtp_src_caps
+    (GstRTPHeaderExtension * ext, GstCaps * caps)
+{
+  GstRTPHeaderExtensionColorspace *self =
+      GST_RTP_HEADER_EXTENSION_COLORSPACE (ext);
+
+  gchar *color_str;
+
+  gst_structure_remove_fields (gst_caps_get_structure (caps, 0),
+      "mastering-display-info", "content-light-level", NULL);
+
+  if ((color_str = gst_video_colorimetry_to_string (&self->colorimetry))) {
+    gst_caps_set_simple (caps, "colorimetry", G_TYPE_STRING, color_str, NULL);
+    g_free (color_str);
+  }
+  if (self->chroma_site != GST_VIDEO_CHROMA_SITE_UNKNOWN) {
+    gst_caps_set_simple (caps, "chroma-site", G_TYPE_STRING,
+        gst_video_chroma_to_string (self->chroma_site), NULL);
+  }
+  if (self->has_hdr_meta) {
+    gst_video_mastering_display_info_add_to_caps (&self->mdi, caps);
+    gst_video_content_light_level_add_to_caps (&self->cll, caps);
+  }
+
+  return TRUE;
+}
+
+static void
+    gst_rtp_header_extension_colorspace_class_init
+    (GstRTPHeaderExtensionColorspaceClass * klass)
+{
+  GstRTPHeaderExtensionClass *rtp_hdr_class =
+      GST_RTP_HEADER_EXTENSION_CLASS (klass);
+  GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);
+
+  rtp_hdr_class->get_supported_flags =
+      gst_rtp_header_extension_colorspace_get_supported_flags;
+  rtp_hdr_class->get_max_size =
+      gst_rtp_header_extension_colorspace_get_max_size;
+  rtp_hdr_class->write = gst_rtp_header_extension_colorspace_write;
+  rtp_hdr_class->read = gst_rtp_header_extension_colorspace_read;
+  rtp_hdr_class->set_non_rtp_sink_caps =
+      gst_rtp_header_extension_colorspace_set_non_rtp_sink_caps;
+  rtp_hdr_class->update_non_rtp_src_caps =
+      gst_rtp_header_extension_colorspace_update_non_rtp_src_caps;
+  rtp_hdr_class->set_attributes_from_caps =
+      gst_rtp_header_extension_set_attributes_from_caps_simple_sdp;
+  rtp_hdr_class->set_caps_from_attributes =
+      gst_rtp_header_extension_set_caps_from_attributes_simple_sdp;
+
+  gst_element_class_set_static_metadata (gstelement_class,
+      "Color Space", GST_RTP_HDREXT_ELEMENT_CLASS,
+      "Extends RTP packets with color space and high dynamic range (HDR) information.",
+      "Jakub Adam <jakub.adam@collabora.com>");
+  gst_rtp_header_extension_class_set_uri (rtp_hdr_class,
+      GST_RTP_HDREXT_COLORSPACE_URI);
+}
diff --git a/gst/rtp/gstrtphdrext-colorspace.h b/gst/rtp/gstrtphdrext-colorspace.h
new file mode 100644
index 000000000..c451cc63f
--- /dev/null
+++ b/gst/rtp/gstrtphdrext-colorspace.h
@@ -0,0 +1,41 @@
+/* GStreamer
+ * Copyright (C) 2020-2021 Collabora Ltd.
+ *   @author: Jakub Adam <jakub.adam@collabora.com>
+ *
+ * gstrtphdrext-colorspace.h: Color Space RTP header extension
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __GST_RTPHDREXT_COLORSPACE_H__
+#define __GST_RTPHDREXT_COLORSPACE_H__
+
+#include <gst/rtp/gstrtphdrext.h>
+
+G_BEGIN_DECLS
+
+#define GST_RTP_HDREXT_COLORSPACE_SIZE 4
+#define GST_RTP_HDREXT_COLORSPACE_WITH_HDR_META_SIZE 28
+#define GST_RTP_HDREXT_COLORSPACE_URI "http://www.webrtc.org/experiments/rtp-hdrext/color-space"
+
+#define GST_TYPE_RTP_HEADER_EXTENSION_COLORSPACE (gst_rtp_header_extension_colorspace_get_type())
+
+G_DECLARE_FINAL_TYPE (GstRTPHeaderExtensionColorspace, gst_rtp_header_extension_colorspace,
+  GST, RTP_HEADER_EXTENSION_COLORSPACE, GstRTPHeaderExtension)
+
+G_END_DECLS
+
+#endif /* __GST_RTPHDREXT_COLORSPACE_H__ */
diff --git a/gst/rtp/meson.build b/gst/rtp/meson.build
index 0ad14654e..2710dcc1a 100644
--- a/gst/rtp/meson.build
+++ b/gst/rtp/meson.build
@@ -42,6 +42,7 @@ rtp_sources = [
   'gstrtpgsmpay.c',
   'gstrtpamrdepay.c',
   'gstrtpamrpay.c',
+  'gstrtphdrext-colorspace.c',
   'gstrtph261depay.c',
   'gstrtph261pay.c',
   'gstrtph263pdepay.c',