qtdemux: use unsigned int types to store result of QT_UINT321.20

In a few cases throughout qtdemux, the results of QT_UINT32 were being
stored in a signed integer, which could cause subtle bugs in the case of
an integer overflow, even allowing the the result to equal a negative
number!

This patch prevents this by simply storing the results of this function
call properly in an unsigned integer type. Additionally, we fix up the
length checking with stsd parsing to prevent cases of child atoms
exceeding their parent atom sizes.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3367>

1 files changed, 47 insertions, 29 deletions

diff --git a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c
index 9c631feaab..ae53b329c4 100644
--- a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c
+++ b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c
@@ -10049,8 +10049,8 @@ qtdemux_parse_segments (GstQTDemux * qtdemux, QtDemuxStream * stream,
   stream->segments = NULL;
   if ((edts = qtdemux_tree_get_child_by_type (trak, FOURCC_edts))) {
     GNode *elst;
-    gint n_segments;
-    gint segment_number, entry_size;
+    guint n_segments;
+    guint segment_number, entry_size;
     guint64 time;
     GstClockTime stime;
     const guint8 *buffer;
@@ -10758,7 +10758,7 @@ qtdemux_parse_stereo_svmi_atom (GstQTDemux * qtdemux, QtDemuxStream * stream,
   /*parse svmi header if existing */
   svmi = qtdemux_tree_get_child_by_type (stbl, FOURCC_svmi);
   if (svmi) {
-    guint len = QT_UINT32 ((guint8 *) svmi->data);
+    guint32 len = QT_UINT32 ((guint8 *) svmi->data);
     guint32 version = QT_UINT32 ((guint8 *) svmi->data + 8);
     if (!version) {
       GstVideoMultiviewMode mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
@@ -11165,7 +11165,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
             break;
         }
       } else {
-        gint i, j, start, end;
+        guint i, j, start, end;
 
         if (len < 94)
           goto corrupt_file;
@@ -11281,7 +11281,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
 
       if (pasp) {
         const guint8 *pasp_data = (const guint8 *) pasp->data;
-        gint len = QT_UINT32 (pasp_data);
+        guint len = QT_UINT32 (pasp_data);
 
         if (len == 16) {
           CUR_STREAM (stream)->par_w = QT_UINT32 (pasp_data + 8);
@@ -11297,7 +11297,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
 
       if (fiel) {
         const guint8 *fiel_data = (const guint8 *) fiel->data;
-        gint len = QT_UINT32 (fiel_data);
+        guint len = QT_UINT32 (fiel_data);
 
         if (len == 10) {
           CUR_STREAM (stream)->interlace_mode = GST_READ_UINT8 (fiel_data + 8);
@@ -11307,7 +11307,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
 
       if (colr) {
         const guint8 *colr_data = (const guint8 *) colr->data;
-        gint len = QT_UINT32 (colr_data);
+        guint len = QT_UINT32 (colr_data);
 
         if (len == 19 || len == 18) {
           guint32 color_type = GST_READ_UINT32_LE (colr_data + 8);
@@ -11344,14 +11344,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
           case FOURCC_avc1:
           case FOURCC_avc3:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *avc_data = stsd_entry_data + 0x56;
 
             /* find avcC */
             while (len >= 0x8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (avc_data) <= len)
+              if (QT_UINT32 (avc_data) <= 0x8)
+                size = 0;
+              else if (QT_UINT32 (avc_data) <= len)
                 size = QT_UINT32 (avc_data) - 0x8;
               else
                 size = len - 0x8;
@@ -11458,14 +11461,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
           case FOURCC_dvh1:
           case FOURCC_dvhe:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *hevc_data = stsd_entry_data + 0x56;
 
             /* find hevc */
             while (len >= 0x8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (hevc_data) <= len)
+              if (QT_UINT32 (hevc_data) <= 0x8)
+                size = 0;
+              else if (QT_UINT32 (hevc_data) <= len)
                 size = QT_UINT32 (hevc_data) - 0x8;
               else
                 size = len - 0x8;
@@ -11521,7 +11527,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
             if (glbl) {
               guint8 *data;
               GstBuffer *buf;
-              gint len;
+              guint len;
 
               GST_DEBUG_OBJECT (qtdemux, "found glbl data in stsd");
               data = glbl->data;
@@ -11705,7 +11711,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
             /* add codec_data if provided */
             if (prefix) {
               GstBuffer *buf;
-              gint len;
+              guint len;
 
               GST_DEBUG_OBJECT (qtdemux, "found prefix data in stsd");
               data = prefix->data;
@@ -11727,7 +11733,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
             GstBuffer *buf;
             GstBuffer *seqh = NULL;
             const guint8 *gamma_data = NULL;
-            gint len = QT_UINT32 (stsd_data);   /* FIXME review - why put the whole stsd in codec data? */
+            guint len = QT_UINT32 (stsd_data);  /* FIXME review - why put the whole stsd in codec data? */
 
             qtdemux_parse_svq3_stsd_data (qtdemux, stsd_entry_data, &gamma_data,
                 &seqh);
@@ -11879,14 +11885,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
           }
           case FOURCC_vc_1:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *vc1_data = stsd_entry_data + 0x56;
 
             /* find dvc1 */
             while (len >= 8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (vc1_data) <= len)
+              if (QT_UINT32 (vc1_data) <= 8)
+                size = 0;
+              else if (QT_UINT32 (vc1_data) <= len)
                 size = QT_UINT32 (vc1_data) - 8;
               else
                 size = len - 8;
@@ -11918,14 +11927,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
           }
           case FOURCC_av01:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *av1_data = stsd_entry_data + 0x56;
 
             /* find av1C */
             while (len >= 0x8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (av1_data) <= len)
+              if (QT_UINT32 (av1_data) <= 0x8)
+                size = 0;
+              else if (QT_UINT32 (av1_data) <= len)
                 size = QT_UINT32 (av1_data) - 0x8;
               else
                 size = len - 0x8;
@@ -11997,14 +12009,17 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
              * vp08, vp09, and vp10 fourcc. */
           case FOURCC_vp09:
           {
-            gint len = QT_UINT32 (stsd_entry_data) - 0x56;
+            guint len = QT_UINT32 (stsd_entry_data);
+            len = len <= 0x56 ? 0 : len - 0x56;
             const guint8 *vpcc_data = stsd_entry_data + 0x56;
 
             /* find vpcC */
             while (len >= 0x8) {
-              gint size;
+              guint size;
 
-              if (QT_UINT32 (vpcc_data) <= len)
+              if (QT_UINT32 (vpcc_data) <= 0x8)
+                size = 0;
+              else if (QT_UINT32 (vpcc_data) <= len)
                 size = QT_UINT32 (vpcc_data) - 0x8;
               else
                 size = len - 0x8;
@@ -12152,7 +12167,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
 
     } else if (stream->subtype == FOURCC_soun) {
       GNode *wave;
-      int version, samplesize;
+      guint version, samplesize;
       guint16 compression_id;
       gboolean amrwb = FALSE;
 
@@ -12467,7 +12482,8 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
         }
         case FOURCC_wma_:
         {
-          gint len = QT_UINT32 (stsd_entry_data) - offset;
+          guint len = QT_UINT32 (stsd_entry_data);
+          len = len <= offset ? 0 : len - offset;
           const guint8 *wfex_data = stsd_entry_data + offset;
           const gchar *codec_name = NULL;
           gint version = 1;
@@ -12491,9 +12507,11 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak)
 
           /* find wfex */
           while (len >= 8) {
-            gint size;
+            guint size;
 
-            if (QT_UINT32 (wfex_data) <= len)
+            if (QT_UINT32 (wfex_data) <= 0x8)
+              size = 0;
+            else if (QT_UINT32 (wfex_data) <= len)
               size = QT_UINT32 (wfex_data) - 8;
             else
               size = len - 8;