Add support for partial bitstream decoding (#1407) (fixes #715)

Add a -allow-partial option to opj_decompress utility and a opj_decoder_set_strict_mode() option to the API

Co-authored-by: Chris Hafey <chafey@gmail.com>

src/bin/jp2/opj_decompress.c

@@ -153,6 +153,8 @@ typedef struct opj_decompress_params {
     int num_threads;
     /* Quiet */
     int quiet;
+    /* Allow partial decode */
+    int allow_partial;
     /** number of components to decode */
     OPJ_UINT32 numcomps;
     /** indices of components to decode */
@@ -246,6 +248,8 @@ static void decode_help_display(void)
         fprintf(stdout, "  -threads <num_threads|ALL_CPUS>\n"
                 "    Number of threads to use for decoding or ALL_CPUS for all available cores.\n");
     }
+    fprintf(stdout, "  -allow-partial\n"
+            "    Disable strict mode to allow decoding partial codestreams.\n");
     fprintf(stdout, "  -quiet\n"
             "    Disable output from the library and other output.\n");
     /* UniPG>> */
@@ -601,6 +605,7 @@ int parse_cmdline_decoder(int argc, char **argv,
         {"split-pnm", NO_ARG,  NULL, 1},
         {"threads",   REQ_ARG, NULL, 'T'},
         {"quiet", NO_ARG,  NULL, 1},
+        {"allow-partial", NO_ARG,  NULL, 1},
     };
 
     const char optlist[] = "i:o:r:l:x:d:t:p:c:"
@@ -616,6 +621,7 @@ int parse_cmdline_decoder(int argc, char **argv,
     long_option[3].flag = &(parameters->upsample);
     long_option[4].flag = &(parameters->split_pnm);
     long_option[6].flag = &(parameters->quiet);
+    long_option[7].flag = &(parameters->allow_partial);
     totlen = sizeof(long_option);
     opj_reset_options_reading();
     img_fol->set_out_format = 0;
@@ -1491,6 +1497,16 @@ int main(int argc, char **argv)
             goto fin;
         }
 
+        /* Disable strict mode if we want to decode partial codestreams. */
+        if (parameters.allow_partial &&
+                !opj_decoder_set_strict_mode(l_codec, OPJ_FALSE)) {
+            fprintf(stderr, "ERROR -> opj_decompress: failed to disable strict mode\n");
+            opj_stream_destroy(l_stream);
+            opj_destroy_codec(l_codec);
+            failed = 1;
+            goto fin;
+        }
+
         if (parameters.num_threads >= 1 &&
                 !opj_codec_set_threads(l_codec, parameters.num_threads)) {
             fprintf(stderr, "ERROR -> opj_decompress: failed to set number of threads\n");

src/lib/openjp2/j2k.c

@@ -4964,9 +4964,14 @@ static OPJ_BOOL opj_j2k_read_sod(opj_j2k_t *p_j2k,
         /* Check enough bytes left in stream before allocation */
         if ((OPJ_OFF_T)p_j2k->m_specific_param.m_decoder.m_sot_length >
                 opj_stream_get_number_byte_left(p_stream)) {
+            if (p_j2k->m_cp.strict) {
                 opj_event_msg(p_manager, EVT_ERROR,
                               "Tile part length size inconsistent with stream length\n");
                 return OPJ_FALSE;
+            } else {
+                opj_event_msg(p_manager, EVT_WARNING,
+                              "Tile part length size inconsistent with stream length\n");
+            }
         }
         if (p_j2k->m_specific_param.m_decoder.m_sot_length >
                 UINT_MAX - OPJ_COMMON_CBLK_DATA_EXTRA) {
@@ -6695,6 +6700,13 @@ void opj_j2k_setup_decoder(opj_j2k_t *j2k, opj_dparameters_t *parameters)
     }
 }
 
+void opj_j2k_decoder_set_strict_mode(opj_j2k_t *j2k, OPJ_BOOL strict)
+{
+    if (j2k) {
+        j2k->m_cp.strict = strict;
+    }
+}
+
 OPJ_BOOL opj_j2k_set_threads(opj_j2k_t *j2k, OPJ_UINT32 num_threads)
 {
     /* Currently we pass the thread-pool to the tcd, so we cannot re-set it */
@@ -10409,6 +10421,9 @@ opj_j2k_t* opj_j2k_create_decompress(void)
     /* per component is allowed */
     l_j2k->m_cp.allow_different_bit_depth_sign = 1;
 
+    /* Default to using strict mode. */
+    l_j2k->m_cp.strict = OPJ_TRUE;
+
 #ifdef OPJ_DISABLE_TPSOT_FIX
     l_j2k->m_specific_param.m_decoder.m_nb_tile_parts_correction_checked = 1;
 #endif

src/lib/openjp2/j2k.h

@@ -402,6 +402,8 @@ typedef struct opj_cp {
     }
     m_specific_param;
 
+    /** OPJ_TRUE if entire bit stream must be decoded, OPJ_FALSE if partial bitstream decoding allowed */
+    OPJ_BOOL strict;
 
     /* UniPG>> */
 #ifdef USE_JPWL
@@ -625,6 +627,8 @@ Decoding parameters are returned in j2k->cp.
 */
 void opj_j2k_setup_decoder(opj_j2k_t *j2k, opj_dparameters_t *parameters);
 
+void opj_j2k_decoder_set_strict_mode(opj_j2k_t *j2k, OPJ_BOOL strict);
+
 OPJ_BOOL opj_j2k_set_threads(opj_j2k_t *j2k, OPJ_UINT32 num_threads);
 
 /**

src/lib/openjp2/jp2.c

@@ -1901,6 +1901,11 @@ void opj_jp2_setup_decoder(opj_jp2_t *jp2, opj_dparameters_t *parameters)
                                  OPJ_DPARAMETERS_IGNORE_PCLR_CMAP_CDEF_FLAG;
 }
 
+void opj_jp2_decoder_set_strict_mode(opj_jp2_t *jp2, OPJ_BOOL strict)
+{
+    opj_j2k_decoder_set_strict_mode(jp2->j2k, strict);
+}
+
 OPJ_BOOL opj_jp2_set_threads(opj_jp2_t *jp2, OPJ_UINT32 num_threads)
 {
     return opj_j2k_set_threads(jp2->j2k, num_threads);

src/lib/openjp2/jp2.h

@@ -235,6 +235,15 @@ Decoding parameters are returned in jp2->j2k->cp.
 */
 void opj_jp2_setup_decoder(opj_jp2_t *jp2, opj_dparameters_t *parameters);
 
+/**
+Set the strict mode parameter.  When strict mode is enabled, the entire
+bitstream must be decoded or an error is returned.  When it is disabled,
+the decoder will decode partial bitstreams.
+@param jp2 JP2 decompressor handle
+@param strict OPJ_TRUE for strict mode
+*/
+void opj_jp2_decoder_set_strict_mode(opj_jp2_t *jp2, OPJ_BOOL strict);
+
 /** Allocates worker threads for the compressor/decompressor.
  *
  * @param jp2 JP2 decompressor handle

src/lib/openjp2/openjpeg.c

@@ -219,6 +219,10 @@ opj_codec_t* OPJ_CALLCONV opj_create_decompress(OPJ_CODEC_FORMAT p_format)
         l_codec->m_codec_data.m_decompression.opj_setup_decoder =
             (void (*)(void *, opj_dparameters_t *)) opj_j2k_setup_decoder;
 
+        l_codec->m_codec_data.m_decompression.opj_decoder_set_strict_mode =
+            (void (*)(void *, OPJ_BOOL)) opj_j2k_decoder_set_strict_mode;
+
+
         l_codec->m_codec_data.m_decompression.opj_read_tile_header =
             (OPJ_BOOL(*)(void *,
                          OPJ_UINT32*,
@@ -326,6 +330,9 @@ opj_codec_t* OPJ_CALLCONV opj_create_decompress(OPJ_CODEC_FORMAT p_format)
         l_codec->m_codec_data.m_decompression.opj_setup_decoder =
             (void (*)(void *, opj_dparameters_t *)) opj_jp2_setup_decoder;
 
+        l_codec->m_codec_data.m_decompression.opj_decoder_set_strict_mode =
+            (void (*)(void *, OPJ_BOOL)) opj_jp2_decoder_set_strict_mode;
+
         l_codec->m_codec_data.m_decompression.opj_set_decode_area =
             (OPJ_BOOL(*)(void *,
                          opj_image_t*,
@@ -426,6 +433,26 @@ OPJ_BOOL OPJ_CALLCONV opj_setup_decoder(opj_codec_t *p_codec,
     return OPJ_FALSE;
 }
 
+OPJ_API OPJ_BOOL OPJ_CALLCONV opj_decoder_set_strict_mode(opj_codec_t *p_codec,
+        OPJ_BOOL strict)
+{
+    if (p_codec) {
+        opj_codec_private_t * l_codec = (opj_codec_private_t *) p_codec;
+
+        if (! l_codec->is_decompressor) {
+            opj_event_msg(&(l_codec->m_event_mgr), EVT_ERROR,
+                          "Codec provided to the opj_decoder_set_strict_mode function is not a decompressor handler.\n");
+            return OPJ_FALSE;
+        }
+
+        l_codec->m_codec_data.m_decompression.opj_decoder_set_strict_mode(
+            l_codec->m_codec,
+            strict);
+        return OPJ_TRUE;
+    }
+    return OPJ_FALSE;
+}
+
 OPJ_BOOL OPJ_CALLCONV opj_read_header(opj_stream_t *p_stream,
                                       opj_codec_t *p_codec,
                                       opj_image_t **p_image)

src/lib/openjp2/openjpeg.h

@@ -1345,6 +1345,20 @@ OPJ_API void OPJ_CALLCONV opj_set_default_decoder_parameters(
 OPJ_API OPJ_BOOL OPJ_CALLCONV opj_setup_decoder(opj_codec_t *p_codec,
         opj_dparameters_t *parameters);
 
+/**
+ * Set strict decoding parameter for this decoder.  If strict decoding is enabled, partial bit
+ * streams will fail to decode.  If strict decoding is disabled, the decoder will decode partial
+ * bitstreams as much as possible without erroring
+ *
+ * @param p_codec       decompressor handler
+ * @param strict        OPJ_TRUE to enable strict decoding, OPJ_FALSE to disable
+ *
+ * @return true         if the decoder is correctly set
+ */
+
+OPJ_API OPJ_BOOL OPJ_CALLCONV opj_decoder_set_strict_mode(opj_codec_t *p_codec,
+        OPJ_BOOL strict);
+
 /**
  * Allocates worker threads for the compressor/decompressor.
  *

src/lib/openjp2/opj_codec.h

@@ -90,6 +90,9 @@ typedef struct opj_codec_private {
             /** Setup decoder function handler */
             void (*opj_setup_decoder)(void * p_codec, opj_dparameters_t * p_param);
 
+            /** Strict mode function handler */
+            void (*opj_decoder_set_strict_mode)(void * p_codec, OPJ_BOOL strict);
+
             /** Set decode area function handler */
             OPJ_BOOL(*opj_set_decode_area)(void * p_codec,
                                            opj_image_t * p_image,

src/lib/openjp2/t2.c

@@ -502,7 +502,6 @@ OPJ_BOOL opj_t2_decode_packets(opj_tcd_t* tcd,
                                     l_current_pi->precno, l_current_pi->layno, skip_packet ? "skipped" : "kept");
                 */
             }
-
             if (!skip_packet) {
                 l_nb_bytes_read = 0;
 
@@ -1378,6 +1377,7 @@ static OPJ_BOOL opj_t2_read_packet_data(opj_t2_t* p_t2,
     opj_tcd_cblk_dec_t* l_cblk = 00;
     opj_tcd_resolution_t* l_res =
         &p_tile->comps[p_pi->compno].resolutions[p_pi->resno];
+    OPJ_BOOL partial_buffer = OPJ_FALSE;
 
     OPJ_ARG_NOT_USED(p_t2);
     OPJ_ARG_NOT_USED(pack_info);
@@ -1397,6 +1397,12 @@ static OPJ_BOOL opj_t2_read_packet_data(opj_t2_t* p_t2,
         for (cblkno = 0; cblkno < l_nb_code_blocks; ++cblkno) {
             opj_tcd_seg_t *l_seg = 00;
 
+            // if we have a partial data stream, set numchunks to zero
+            // since we have no data to actually decode.
+            if (partial_buffer) {
+                l_cblk->numchunks = 0;
+            }
+
             if (!l_cblk->numnewpasses) {
                 /* nothing to do */
                 ++l_cblk;
@@ -1419,12 +1425,32 @@ static OPJ_BOOL opj_t2_read_packet_data(opj_t2_t* p_t2,
                 /* Check possible overflow (on l_current_data only, assumes input args already checked) then size */
                 if ((((OPJ_SIZE_T)l_current_data + (OPJ_SIZE_T)l_seg->newlen) <
                         (OPJ_SIZE_T)l_current_data) ||
-                        (l_current_data + l_seg->newlen > p_src_data + p_max_length)) {
+                        (l_current_data + l_seg->newlen > p_src_data + p_max_length) ||
+                        (partial_buffer)) {
+                    if (p_t2->cp->strict) {
                         opj_event_msg(p_manager, EVT_ERROR,
                                       "read: segment too long (%d) with max (%d) for codeblock %d (p=%d, b=%d, r=%d, c=%d)\n",
                                       l_seg->newlen, p_max_length, cblkno, p_pi->precno, bandno, p_pi->resno,
                                       p_pi->compno);
                         return OPJ_FALSE;
+                    } else {
+                        opj_event_msg(p_manager, EVT_WARNING,
+                                      "read: segment too long (%d) with max (%d) for codeblock %d (p=%d, b=%d, r=%d, c=%d)\n",
+                                      l_seg->newlen, p_max_length, cblkno, p_pi->precno, bandno, p_pi->resno,
+                                      p_pi->compno);
+                        // skip this codeblock since it is a partial read
+                        partial_buffer = OPJ_TRUE;
+                        l_cblk->numchunks = 0;
+
+                        l_seg->numpasses += l_seg->numnewpasses;
+                        l_cblk->numnewpasses -= l_seg->numnewpasses;
+                        if (l_cblk->numnewpasses > 0) {
+                            ++l_seg;
+                            ++l_cblk->numsegs;
+                            break;
+                        }
+                        continue;
+                    }
                 }
 
 #ifdef USE_JPWL
@@ -1486,8 +1512,12 @@ static OPJ_BOOL opj_t2_read_packet_data(opj_t2_t* p_t2,
         ++l_band;
     }
 
+    // return the number of bytes read
+    if (partial_buffer) {
+        *(p_data_read) = p_max_length;
+    } else {
         *(p_data_read) = (OPJ_UINT32)(l_current_data - p_src_data);
-
+    }
 
     return OPJ_TRUE;
 }
@@ -1549,11 +1579,18 @@ static OPJ_BOOL opj_t2_skip_packet_data(opj_t2_t* p_t2,
                 /* Check possible overflow then size */
                 if (((*p_data_read + l_seg->newlen) < (*p_data_read)) ||
                         ((*p_data_read + l_seg->newlen) > p_max_length)) {
+                    if (p_t2->cp->strict) {
                         opj_event_msg(p_manager, EVT_ERROR,
                                       "skip: segment too long (%d) with max (%d) for codeblock %d (p=%d, b=%d, r=%d, c=%d)\n",
                                       l_seg->newlen, p_max_length, cblkno, p_pi->precno, bandno, p_pi->resno,
                                       p_pi->compno);
                         return OPJ_FALSE;
+                    } else {
+                        opj_event_msg(p_manager, EVT_WARNING,
+                                      "skip: segment too long (%d) with max (%d) for codeblock %d (p=%d, b=%d, r=%d, c=%d)\n",
+                                      l_seg->newlen, p_max_length, cblkno, p_pi->precno, bandno, p_pi->resno,
+                                      p_pi->compno);
+                    }
                 }
 
 #ifdef USE_JPWL