compression: buffered write stream was not being created

Author: Grok Compression

.vscode/launch.json

@@ -169,7 +169,38 @@
             ],
             "MIMode": "gdb",
             "miDebuggerPath": "/usr/bin/gdb"
-        },        
+        },
+        {
+            "name": "core compress",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/build/bin/core_compress",
+            "args": [],
+            "environment": [
+                { "name": "GRK_DEBUG", "value": "3" }
+            ],
+            "cwd": "${workspaceFolder}",
+            "MIMode": "gdb",
+            "miDebuggerPath": "/usr/bin/gdb"
+        },
+        {
+            "name": "encode TIFF",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/build/bin/grk_compress",
+            "args": [
+                "-i",
+                "$HOME/temp/test.tif",
+                "-o",
+                "$HOME/temp/test.tif.jp2",
+            ],
+            "environment": [
+                { "name": "GRK_DEBUG", "value": "3" }
+            ],
+            "cwd": "${workspaceFolder}",
+            "MIMode": "gdb",
+            "miDebuggerPath": "/usr/bin/gdb"
+        },
         {
             "name": "test",
             "type": "cppdbg",
@@ -407,6 +438,28 @@
             "miDebuggerPath": "/usr/bin/gdb"
         },
         {
+            "name": "Pleiades KDU",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "/usr/bin/time",
+            "args": [
+                "-v",
+                "$HOME/src/gorgon/gorgon/bin/Linux-x86-64-gcc/kdu_expand",
+                "-i",
+                "$HOME/temp/IMG_PHR1B_P_202406071720183_SEN_7038440101-1_R1C1.JP2",
+                "-region",
+                "\"{0.00000000000000000,0.00000000000000000},{0.10101109741060420,0.09905204101373573}\"",
+                "-o",
+                "/tmp/pleiades.tif"
+            ],
+            "cwd": "${workspaceFolder}",
+            "environment": [
+                { "name": "GRK_DEBUG", "value": "3" },
+            ],
+            "MIMode": "gdb",
+            "miDebuggerPath": "/usr/bin/gdb"
+        },
+                {
             "name": "Pleiades",
             "type": "cppdbg",
             "request": "launch",
@@ -416,10 +469,12 @@
                 "${workspaceFolder}/build/bin/core_decompress",
                 "-i",
                 "$HOME/temp/IMG_PHR1B_P_202406071720183_SEN_7038440101-1_R1C1.JP2",
+                "-d 0,0,2048,2048",
             ],
             "cwd": "${workspaceFolder}",
             "environment": [
                 { "name": "GRK_DEBUG", "value": "3" },
+                { "name": "GRK_WINDOWED_SCHEDULING", "value": "1" }
             ],
             "MIMode": "gdb",
             "miDebuggerPath": "/usr/bin/gdb"
@@ -705,9 +760,14 @@
             "program": "${workspaceFolder}/build/bin/grk_decompress",
             "args": [
                 "-i",
-                "$HOME/Downloads/clusterfuzz-testcase-minimized-grk_decompress_fuzzer-6466697174843392",
+                "$HOME/temp/clusterfuzz-testcase-minimized-grk_decompress_fuzzer-4734814600364032",
+                "-d 0,0,1024,1024",
                 "-o",
-                "$HOME/temp/fuzzer.tif"
+                "$HOME/temp/fuzz.tif"
+
+            ],
+            "environment": [
+                { "name": "GRK_DEBUG", "value": "3" },
             ],
             "cwd": "${workspaceFolder}",
             "MIMode": "gdb",

examples/core/core_compress.cpp

@@ -14,19 +14,22 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
-
 #include <cstdio>
 #include <cstring>
 #include <memory>
 #include <inttypes.h>
 
 #include "grok.h"
-
 #include "core.h"
 
-struct WriteStreamInfo
+/**
+ * @struct WriteCallbackInfo
+ * @brief used by write callbacks to get information about stream
+ *
+ */
+struct WriteCallbackInfo
 {
-  explicit WriteStreamInfo(grk_stream_params* streamParams)
+  explicit WriteCallbackInfo(grk_stream_params* streamParams)
       : streamParams_(streamParams), data_(nullptr), dataLen_(0), offset_(0)
   {}
   grk_stream_params* streamParams_;
@@ -35,17 +38,33 @@ struct WriteStreamInfo
   size_t offset_;
 };
 
+/**
+ * @brief stream write callback
+ *
+ * @param buffer //buffer of data to write
+ * @param numBytes  // number of bytes to write
+ * @param user_data  // user data
+ * @return size_t // number of bytes written
+ */
 size_t stream_write_fn(const uint8_t* buffer, size_t numBytes, void* user_data)
 {
-  auto sinfo = (WriteStreamInfo*)user_data;
+  auto sinfo = (WriteCallbackInfo*)user_data;
   if(sinfo->offset_ + numBytes <= sinfo->dataLen_)
     memcpy(sinfo->data_ + sinfo->offset_, buffer, numBytes);
 
   return numBytes;
 }
+
+/**
+ * @brief stream seek callback
+ *
+ * @param offset //offset to seek to
+ * @param user_data  // user dat
+ * @return true if seek successful, otherwise false
+ */
 bool stream_seek_fn(uint64_t offset, void* user_data)
 {
-  auto sinfo = (WriteStreamInfo*)user_data;
+  auto sinfo = (WriteCallbackInfo*)user_data;
   if(offset <= sinfo->dataLen_)
     sinfo->offset_ = offset;
   else
@@ -62,52 +81,58 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] const char** argv)
   const uint32_t precision = 8;
   const char* outFile = "test.jp2";
 
+  grk_object* codec = nullptr;
+  grk_image* inputImage = nullptr;
+  int32_t rc = EXIT_FAILURE;
+
   uint64_t compressedLength = 0;
 
-  // initialize compress parameters
+  // 1. initialize library
+  grk_initialize(nullptr, 0);
+
+  // 2. initialize compress parameters
   grk_cparameters compressParams;
   grk_compress_set_default_params(&compressParams);
   compressParams.cod_format = GRK_FMT_JP2;
   compressParams.verbose = true;
 
-  grk_object* codec = nullptr;
-  grk_image* image = nullptr;
-  int32_t rc = EXIT_FAILURE;
-
-  bool outputToBuffer = false;
-  bool useCallbacks = true;
-
-  // initialize library
-  grk_initialize(nullptr, 0);
+  // 3.initialize output stream
+  enum eStreamOutput
+  {
+    STREAM_OUTPUT_BUFFER, // output to buffer
+    STREAM_OUTPUT_CALLBACK, // output using user-defined callback
+    STREAM_OUTPUT_FILE // output to file
+  };
+  eStreamOutput output = STREAM_OUTPUT_BUFFER;
 
-  grk_stream_params streamParams = {};
-  WriteStreamInfo sinfo(&streamParams);
+  grk_stream_params outputStreamParams = {};
+  WriteCallbackInfo writeCallbackInfo(&outputStreamParams);
 
   std::unique_ptr<uint8_t[]> data;
   size_t bufLen = (size_t)numComps * ((precision + 7) / 8) * dimX * dimY;
-  if(useCallbacks || outputToBuffer)
+  if(output != STREAM_OUTPUT_FILE)
   {
     data = std::make_unique<uint8_t[]>(bufLen);
   }
-  if(useCallbacks)
+  if(output == STREAM_OUTPUT_CALLBACK)
   {
-    streamParams.seek_fn = stream_seek_fn;
-    streamParams.write_fn = stream_write_fn;
-    streamParams.user_data = &sinfo;
-    sinfo.data_ = data.get();
-    sinfo.dataLen_ = bufLen;
+    outputStreamParams.seek_fn = stream_seek_fn;
+    outputStreamParams.write_fn = stream_write_fn;
+    outputStreamParams.user_data = &writeCallbackInfo;
+    writeCallbackInfo.data_ = data.get();
+    writeCallbackInfo.dataLen_ = bufLen;
   }
-  else if(outputToBuffer)
+  else if(output == STREAM_OUTPUT_BUFFER)
   {
-    streamParams.buf = data.get();
-    streamParams.buf_len = bufLen;
+    outputStreamParams.buf = data.get();
+    outputStreamParams.buf_len = bufLen;
   }
   else
   {
-    safe_strcpy(streamParams.file, outFile);
+    safe_strcpy(outputStreamParams.file, outFile);
   }
 
-  // create blank image
+  // 4. create input image (blank)
   auto components = std::make_unique<grk_image_comp[]>(numComps);
   for(uint32_t i = 0; i < numComps; ++i)
   {
@@ -119,13 +144,12 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] const char** argv)
     c->prec = precision;
     c->sgnd = false;
   }
-  image = grk_image_new(numComps, components.get(), GRK_CLRSPC_SRGB, true);
+  inputImage = grk_image_new(numComps, components.get(), GRK_CLRSPC_SRGB, true);
 
-  // fill in component data
-  // see grok.h header for full details of image structure
-  for(uint16_t compno = 0; compno < image->numcomps; ++compno)
+  // fill in component data: see grok.h header for full details of image structure
+  for(uint16_t compno = 0; compno < inputImage->numcomps; ++compno)
   {
-    auto comp = image->comps + compno;
+    auto comp = inputImage->comps + compno;
     auto compWidth = comp->w;
     auto compHeight = comp->h;
     auto compData = (int32_t*)comp->data;
@@ -148,26 +172,25 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] const char** argv)
     delete[] srcData;
   }
 
-  // initialize compressor
-  codec = grk_compress_init(&streamParams, &compressParams, image);
+  // 5. initialize compressor
+  codec = grk_compress_init(&outputStreamParams, &compressParams, inputImage);
   if(!codec)
   {
     fprintf(stderr, "Failed to initialize compressor\n");
     goto beach;
   }
 
-  // compress
+  // 6. compress
   compressedLength = grk_compress(codec, nullptr);
   if(compressedLength == 0)
   {
     fprintf(stderr, "Failed to compress\n");
     goto beach;
   }
-
   printf("Compression succeeded: %" PRIu64 " bytes used.\n", compressedLength);
 
-  // write buffer to file
-  if(outputToBuffer)
+  // 7. write buffer to file if needed
+  if(output == STREAM_OUTPUT_FILE)
   {
     auto fp = fopen(outFile, "wb");
     if(!fp)
@@ -176,7 +199,7 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] const char** argv)
     }
     else
     {
-      size_t written = fwrite(streamParams.buf, 1, compressedLength, fp);
+      size_t written = fwrite(outputStreamParams.buf, 1, compressedLength, fp);
       if(written != compressedLength)
       {
         fprintf(stderr, "Buffer compress: only %" PRIu64 " bytes written out of %" PRIu64 " total",
@@ -188,9 +211,10 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] const char** argv)
 
   rc = EXIT_SUCCESS;
 beach:
+
   // cleanup
   grk_object_unref(codec);
-  grk_object_unref(&image->obj);
+  grk_object_unref(&inputImage->obj);
   grk_deinitialize();
 
   return rc;

src/lib/core/grok.h

@@ -556,15 +556,15 @@ typedef struct _grk_stream_params
   size_t initial_double_buffer_len; /* choose a larger initial length
                                        to read the main header in one go */
   bool from_network; /* indicates stream source is on network if true */
-  bool is_read_stream;
+  bool is_read_stream; /* true if read stream, otherwise false */
 
   /* 1. File Streaming */
   char file[GRK_PATH_LEN];
   bool use_stdio; /* use C file api - if false then use memory mapping */
 
   /* 2. Buffer Streaming */
-  uint8_t* buf;
-  size_t buf_len; /* buffer length */
+  uint8_t* buf; /* data buffer */
+  size_t buf_len; /* data buffer length */
   size_t buf_compressed_len; /* length of compressed stream (set by compressor, not client) */
 
   /* 3. Callback Streaming */

src/lib/core/stream/StreamGenerator.h

@@ -78,7 +78,7 @@ class StreamGenerator
   {
     if(streamParams_.buf && streamParams_.buf_len)
     {
-      return createBufferReadStream();
+      return createBufferStream(streamParams_.is_read_stream);
     }
 
     if(streamParams_.read_fn || streamParams_.write_fn)
@@ -125,10 +125,10 @@ class StreamGenerator
 
   IStream* createCallbackStream(void);
 
-  IStream* createBufferReadStream(void)
+  IStream* createBufferStream(bool isReadStream)
   {
     auto stream = memStreamCreate(streamParams_.buf, streamParams_.buf_len, false, nullptr,
-                                  GRK_CODEC_FORMAT::GRK_CODEC_UNK, true);
+                                  GRK_CODEC_FORMAT::GRK_CODEC_UNK, isReadStream);
     if(!stream)
     {
       grklog.error("Unable to create memory stream.");