rtmp: fix multiple broken overflow checks

Sanity checks like `data + size >= data_end || data + size < data' are
broken, because `data + size < data' assumes pointer overflow, which is
undefined behavior in C.  Many compilers such as gcc/clang optimize such
checks away.

Use `size < 0 || size >= data_end - data' instead.

Signed-off-by: Xi Wang <xi.wang@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 902cfe2f74d777a7dc20ac68f2393b9f84b790c1)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 6 insertions, 6 deletions

diff --git a/libavformat/rtmppkt.c b/libavformat/rtmppkt.c
index 7e2ccdc6ac..66bbe5aa52 100644
--- a/libavformat/rtmppkt.c
+++ b/libavformat/rtmppkt.c
@@ -279,11 +279,11 @@ int ff_amf_tag_size(const uint8_t *data, const uint8_t *data_end)
                 data++;
                 break;
             }
-            if (data + size >= data_end || data + size < data)
+            if (size < 0 || size >= data_end - data)
                 return -1;
             data += size;
             t = ff_amf_tag_size(data, data_end);
-            if (t < 0 || data + t >= data_end)
+            if (t < 0 || t >= data_end - data)
                 return -1;
             data += t;
         }
@@ -312,7 +312,7 @@ int ff_amf_get_field_value(const uint8_t *data, const uint8_t *data_end,
         int size = bytestream_get_be16(&data);
         if (!size)
             break;
-        if (data + size >= data_end || data + size < data)
+        if (size < 0 || size >= data_end - data)
             return -1;
         data += size;
         if (size == namelen && !memcmp(data-size, name, namelen)) {
@@ -333,7 +333,7 @@ int ff_amf_get_field_value(const uint8_t *data, const uint8_t *data_end,
             return 0;
         }
         len = ff_amf_tag_size(data, data_end);
-        if (len < 0 || data + len >= data_end || data + len < data)
+        if (len < 0 || len >= data_end - data)
             return -1;
         data += len;
     }
@@ -404,13 +404,13 @@ static void ff_amf_tag_contents(void *ctx, const uint8_t *data, const uint8_t *d
                 data++;
                 break;
             }
-            if (data + size >= data_end || data + size < data)
+            if (size < 0 || size >= data_end - data)
                 return;
             data += size;
             av_log(ctx, AV_LOG_DEBUG, "  %s: ", buf);
             ff_amf_tag_contents(ctx, data, data_end);
             t = ff_amf_tag_size(data, data_end);
-            if (t < 0 || data + t >= data_end)
+            if (t < 0 || t >= data_end - data)
                 return;
             data += t;
         }