decompress: make ZSTD_decompressBound() a true upper bound

lib/decompress/zstd_decompress.c

@@ -751,6 +751,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         const BYTE* const ipstart = ip;
         size_t remainingSize = srcSize;
         size_t nbBlocks = 0;
+        unsigned long long blocksBound = 0;
         ZSTD_FrameHeader zfh;
 
         /* Extract Frame Header */
@@ -777,6 +778,10 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
             ip += ZSTD_blockHeaderSize + cBlockSize;
             remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
             nbBlocks++;
+            /* single-pass decoding accepts Raw and RLE blocks larger than blockSizeMax (#3482) */
+            blocksBound += (blockProperties.blockType == bt_compressed)
+                         ? zfh.blockSizeMax
+                         : MAX(blockProperties.origSize, zfh.blockSizeMax);
 
             if (blockProperties.lastBlock) break;
         }
@@ -792,7 +797,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
         frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
         frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
                                         ? zfh.frameContentSize
-                                        : (unsigned long long)nbBlocks * zfh.blockSizeMax;
+                                        : blocksBound;
         return frameSizeInfo;
     }
 }
@@ -829,6 +834,8 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
         assert(srcSize >= compressedSize);
         src = (const BYTE*)src + compressedSize;
         srcSize -= compressedSize;
+        if (bound + decompressedBound < bound)
+            return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */
         bound += decompressedBound;
     }
     return bound;

lib/zstd.h

@@ -1493,11 +1493,13 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src,
  *  @return : - upper-bound for the decompressed size of all data in all successive frames
  *            - if an error occurred: ZSTD_CONTENTSIZE_ERROR
  *
- *  note 1  : an error can occur if `src` contains an invalid or incorrectly formatted frame.
+ *  note 1  : an error can occur if `src` contains an invalid or incorrectly formatted frame,
+ *            or if the upper-bound does not fit in an `unsigned long long`.
  *  note 2  : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
  *            in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
  *  note 3  : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
  *              upper-bound = # blocks * min(128 KB, Window_Size)
+ *            except that a Raw or RLE block declaring a larger size counts with its declared size.
  */
 ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
 

tests/fuzz/simple_decompress.c

@@ -20,6 +20,7 @@
 #define ZSTD_STATIC_LINKING_ONLY
 
 #include "fuzz_helpers.h"
+#include "zstd_helpers.h"
 #include "zstd.h"
 #include "fuzz_data_producer.h"
 
@@ -46,6 +47,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
             unsigned long long const expectedSize = ZSTD_findDecompressedSize(src, size);
             FUZZ_ASSERT(expectedSize != ZSTD_CONTENTSIZE_ERROR);
             FUZZ_ASSERT(expectedSize == ZSTD_CONTENTSIZE_UNKNOWN || expectedSize == dSize);
+            /* ZSTD_decompressBound() must be an upper bound of what was actually decoded. */
+            if (FUZZ_isCurrentFormat(src, size))
+                FUZZ_ASSERT(dSize <= ZSTD_decompressBound(src, size));
         }
         free(rBuf);
     }

tests/fuzz/zstd_frame_info.c

@@ -33,8 +33,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     ZSTD_getDecompressedSize(src, size);
     ZSTD_findFrameCompressedSize(src, size);
     ZSTD_getDictID_fromFrame(src, size);
-    ZSTD_findDecompressedSize(src, size);
-    ZSTD_decompressBound(src, size);
+    {   unsigned long long const dSize = ZSTD_findDecompressedSize(src, size);
+        unsigned long long const bound = ZSTD_decompressBound(src, size);
+        /* When every frame declares its size, the bound is that exact total,
+         * and an overflowing total is an error for both. */
+        if (dSize != ZSTD_CONTENTSIZE_UNKNOWN && FUZZ_isCurrentFormat(src, size))
+            FUZZ_ASSERT(bound == dSize);
+    }
     ZSTD_frameHeaderSize(src, size);
     ZSTD_isFrame(src, size);
     ZSTD_getFrameHeader(&zfh, src, size);

tests/fuzz/zstd_helpers.c

@@ -206,3 +206,17 @@ FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *pro
 
     return dict;
 }
+
+int FUZZ_isCurrentFormat(void const* src, size_t srcSize)
+{
+    const uint8_t* ip = (const uint8_t*)src;
+    while (srcSize > 0) {
+        ZSTD_FrameHeader zfh;
+        size_t const frameSize = ZSTD_findFrameCompressedSize(ip, srcSize);
+        if (ZSTD_isError(frameSize) || ZSTD_getFrameHeader(&zfh, ip, srcSize) != 0)
+            return 0;
+        ip += frameSize;
+        srcSize -= frameSize;
+    }
+    return 1;
+}

tests/fuzz/zstd_helpers.h

@@ -45,6 +45,12 @@ typedef struct {
  */
 FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer);
 
+/* Returns 1 if src is a series of zstd and skippable frames, 0 if any frame
+ * is invalid or a legacy frame. Legacy decoders are frozen and their size
+ * queries do not bound what they decode.
+ */
+int FUZZ_isCurrentFormat(void const* src, size_t srcSize);
+
 #ifdef FUZZ_THIRD_PARTY_SEQ_PROD
 extern void* FUZZ_seqProdState;
 #endif

tests/fuzzer.c

@@ -572,6 +572,35 @@ static void test_decompressBound(unsigned tnb)
         free(outBuffer);
     }
 
+    /* Two frames declaring 2^63 bytes each : the sum must not wrap around to 0 */
+    {   static const BYTE frames[] = {
+            0x28, 0xB5, 0x2F, 0xFD, 0xE0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x01, 0, 0,
+            0x28, 0xB5, 0x2F, 0xFD, 0xE0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x01, 0, 0 };
+        CHECK_EQ( ZSTD_findDecompressedSize(frames, sizeof(frames)), ZSTD_CONTENTSIZE_ERROR );
+        CHECK_EQ( ZSTD_decompressBound(frames, sizeof(frames)), ZSTD_CONTENTSIZE_ERROR );
+    }
+
+    /* Raw and RLE blocks larger than the block size maximum (6 KB window) :
+     * single-pass decoding accepts them (#3482), so the bound must cover them */
+    {   static const BYTE rleFrame[] = { 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x14, 0xEB, 0x25, 0x02, 'a' };
+        size_t const rawSize = 10000;
+        size_t const rawFrameSize = 9 + rawSize;
+        size_t const outCapacity = 1 << 15;
+        BYTE* const rawFrame = (BYTE*)malloc(rawFrameSize);
+        void* const outBuffer = malloc(outCapacity);
+        assert(rawFrame != NULL && outBuffer != NULL);
+        memcpy(rawFrame, rleFrame, 6);
+        MEM_writeLE24(rawFrame + 6, (U32)(1 /* last */ + (bt_raw << 1) + (rawSize << 3)));
+        memset(rawFrame + 9, 'a', rawSize);
+
+        CHECK_EQ( ZSTD_decompress(outBuffer, outCapacity, rleFrame, sizeof(rleFrame)), 17597 );
+        CHECK( ZSTD_decompressBound(rleFrame, sizeof(rleFrame)) >= 17597 );
+        CHECK_EQ( ZSTD_decompress(outBuffer, outCapacity, rawFrame, rawFrameSize), rawSize );
+        CHECK( ZSTD_decompressBound(rawFrame, rawFrameSize) >= rawSize );
+        free(rawFrame);
+        free(outBuffer);
+    }
+
     DISPLAYLEVEL(3, "OK \n");
 }