Merge pull request #12 from magnostherobot/sysvinit/czinspect

czinspect: implement subsampling ratio scanning and filtering

src/czinspect/helptxt/extract.help

@@ -4,6 +4,11 @@ Options:
     -d <dir>    Output directory for extracted data files
     -a          Extract file attachments
     -e          Extract file metadata
+    -f <val>    Extract only those subblock components whose subsampling level
+                is equal to "val"
+    -g          If the subsampling level provided to '-f' is not found in the
+                input file, then round the value down to the nearest subsampling
+                level
     -s <str>    Extract the subblock components specified by the comma seperated
                 list of sections specified in "str"
 

src/czinspect/helptxt/main.help

@@ -13,7 +13,7 @@ Global options:
                 one time for each file. (This option should only be used on
                 32-bit systems.)
 
-**NOTE**: Only the '-E' operation is implemented at this time!
+**NOTE**: Only the '-E' and '-S' operations are implemented at this time!
 
 Pass '-h' with an operation to see usage information and options specific to
 that operation.

src/czinspect/helptxt/scan.help

@@ -1,3 +1,3 @@
 Usage: czinspect -S [options] <file>
 
-This operation does not have any options, and is currently unimplemented.
+This operation does not have any options.

src/czinspect/src/alloc.c

@@ -38,7 +38,7 @@ char *_xstrdup(const char *name, char *str) {
 static void* (*volatile lzmemset)(void *, int, size_t) = memset;
 
 lzstring _lzstr_new(const char *name, size_t num, size_t sz) {
-    lzstring ret = malloc(sizeof(lzstring));
+    lzstring ret = malloc(sizeof(struct lzstring_s));
     if (ret == NULL)
         nerr1(name, "could not allocate dynamic string");
 

src/czinspect/src/czinspect.c

@@ -32,7 +32,7 @@
 #define GETOPT_OPS    "SEDCh"
 
 #define GETOPT_S_STR  ""
-#define GETOPT_E_STR  "ad:es:"
+#define GETOPT_E_STR  "ad:ef:gs:"
 #define GETOPT_D_STR  "o:"
 #define GETOPT_C_STR  ""
 
@@ -151,6 +151,9 @@ static void parse_opt_scan(int opt) {
 
 /* process extraction options */
 static void parse_opt_extract(int opt) {
+    const char *errstr;
+    uint16_t filt;
+
     switch (opt) {
     case 'a':
         cfg.eflags |= EXT_F_ATTACH;
@@ -161,6 +164,16 @@ static void parse_opt_extract(int opt) {
     case 'e':
         cfg.eflags |= EXT_F_META;
         break;
+    case 'f':
+        filt = strtonum((const char *) optarg, 1, UINT16_MAX, &errstr);
+        if (errstr)
+            errx(1, "invalid filter level '%s': %s", optarg, errstr);
+        cfg.filter = filt;
+        cfg.eflags |= EXT_F_FILT;
+        break;
+    case 'g':
+        cfg.eflags |= EXT_F_FFUZZ;
+        break;
     case 's':
         cfg.eflags |= EXT_F_SBLK;
         cfg.esopts = xstrdup(optarg);

src/czinspect/src/extract.c

@@ -8,6 +8,7 @@
 #include <fcntl.h>
 #include <inttypes.h>
 #include <stdlib.h>
+#include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 
@@ -36,6 +37,7 @@ enum subopt_index {
 static int dirfd;
 static uint8_t sblk_opts;
 static uint8_t ext_opts;
+static uint32_t filterlevel;
 static lzstring fname;
 static lzstring suffix;
 static lzbuf dimensions;
@@ -91,8 +93,15 @@ static void parse_opts(struct config *cfg) {
         }
 
         xfree(cfg->esopts);
+    } else if (cfg->eflags & EXT_F_FILT) {
+        errx(1, "cannot perform subsampling level filtering when not extracting subblocks");
+    } else if (cfg->eflags & EXT_F_FFUZZ) {
+        errx(1, "cannot perform rounded subsampling level filtering when not extracting subblocks");
     }
 
+    if ((cfg->eflags & EXT_F_FFUZZ) == EXT_F_FFUZZ && (cfg->eflags & EXT_F_FILT) == 0)
+        errx(1, "cannot round subsampling level when not performing filtering");
+
     if (cfg->eflags == 0) {
         ext_opts = (EXT_F_META | EXT_F_SBLK | EXT_F_ATTACH);
         sblk_opts = EXT_S_ALL;
@@ -207,10 +216,13 @@ static void extract_attachments(struct map_ctx *c, uint64_t pos) {
     return;
 }
 
-static void make_suffix(lzstring str, struct czi_subblock_dimentry *dims, uint32_t count) {
+static void make_suffix(lzstring str, lzbuf diments, uint32_t count) {
     char dimname[5];
     lzstring tmp;
+    struct czi_subblock_dimentry *dims;
 
+    dims = &lzbuf_get(struct czi_subblock_dimentry, diments, 0);
+
     /* null termination, for sanity */
     memcpy(&dimname, dims->dimension, 4);
     dimname[4] = '\0';
@@ -224,7 +236,7 @@ static void make_suffix(lzstring str, struct czi_subblock_dimentry *dims, uint32
     tmp = lzstr_new();
 
     for (uint32_t i = 1; i < count; i++) {
-        dims += sizeof(struct czi_subblock_dimentry);
+        dims = &lzbuf_get(struct czi_subblock_dimentry, diments, i);
 
         lzstr_zero(tmp);
 
@@ -245,6 +257,7 @@ static void extract_subblock(struct map_ctx *c) {
     struct czi_subblock sblk;
     struct czi_subblock_dimentry *entry;
     size_t offset;
+    uint32_t level;
 
     if (czi_read_sh(c, &head) == -1)
         ferrx1("could not read segment header");
@@ -266,20 +279,28 @@ static void extract_subblock(struct map_ctx *c) {
         lzbuf_grow(struct czi_subblock_dimentry, dimensions);
 
     uint32_t i;
-    for (i = 0, entry = &lzbuf_get(struct czi_subblock_dimentry, dimensions, 0);
-         i < sblk.dir_entry.dimension_count;
-         i++, entry += sizeof(struct czi_subblock_dimentry))
-        if (map_read(c, entry, sizeof(struct czi_subblock_dimentry)) == -1)
+    for (i = 0; i < sblk.dir_entry.dimension_count; i++) {
+        entry = &lzbuf_get(struct czi_subblock_dimentry, dimensions, i);
+        if (czi_read_sblk_dimentry(c, entry) == -1)
             ferrx1("could not read subblock dimension entries");
+    }
 
+    if (filterlevel != 0) {
+        level = get_subsample_level(dimensions, sblk.dir_entry.dimension_count);
+        if (level != filterlevel) {
+            lzbuf_zero(dimensions);
+            return;
+        }
+    }
+
     /* fix file offset */
     offset = map_file_offset(c) - offset;
     offset = 256 - offset;
     if (offset > 0)
         if (map_seek(c, offset, MAP_FORW) == -1)
             ferrx("cannot seek forwards %" PRIu64 " bytes", offset);
 
-    make_suffix(suffix, (struct czi_subblock_dimentry *) dimensions->data, sblk.dir_entry.dimension_count);
+    make_suffix(suffix, dimensions, sblk.dir_entry.dimension_count);
 
     if (sblk.metadata_size != 0) {
         if (sblk_opts & EXT_S_META) {
@@ -314,6 +335,8 @@ static void extract_subblock(struct map_ctx *c) {
         }
     }
 
+    lzbuf_zero(dimensions);
+
     return;
 }
 
@@ -340,7 +363,8 @@ static void extract_sblk_directory(struct map_ctx *c, uint64_t pos) {
         ferrx1("could not read directory segment");
 
     if (cdir.entry_count == 0) {
-        fwarnx1("directory entry segment");
+        /* in this case, we fall past the loop */
+        fwarnx1("directory entry segment contains no entries");
     }
 
     for (uint32_t i = 0; i < cdir.entry_count; i ++) {
@@ -373,7 +397,35 @@ static void extract_sblk_directory(struct map_ctx *c, uint64_t pos) {
 
     return;
 }
-
+
+static void setup_filterlevel(struct config *cfg, struct map_ctx *c, uint64_t dirpos) {
+    lzbuf reslist;
+    uint32_t rnum = 0;
+
+    if ((cfg->eflags & EXT_F_FILT) == 0)
+        return;
+
+    if (map_seek(c, dirpos, MAP_SET) == -1)
+        ferrx("could not seek to offset %" PRIu64 " to read subblock directory for subsampling scan",
+              dirpos);
+
+    reslist = lzbuf_new(uint32_t);
+
+    if (make_reslist(c, reslist, &rnum) == -1)
+        ferrx1("could not scan for subsampling levels in input file");
+
+    for (uint32_t i = 0; i < rnum; i++)
+        if ((cfg->eflags & EXT_F_FFUZZ) ?
+            (lzbuf_get(uint32_t, reslist, i) <= cfg->filter) :
+            (lzbuf_get(uint32_t, reslist, i) == cfg->filter))
+            filterlevel = lzbuf_get(uint32_t, reslist, i);
+
+    if (filterlevel == 0)
+        errx(1, "invalid filter level: %" PRIu32, cfg->filter);
+
+    return;
+}
+
 void do_extract(struct config *cfg) {
     struct czi_seg_header head;
     struct czi_zrf zisrf;
@@ -420,6 +472,7 @@ void do_extract(struct config *cfg) {
             /* as above */
             fwarnx1("file subblock extraction requested but none found, continuing...");
         } else {
+            setup_filterlevel(cfg, c, zisrf.directory_position);
             extract_sblk_directory(c, zisrf.directory_position);
         }
     }

src/czinspect/src/operations.c

@@ -3,10 +3,6 @@
 
 #include "types.h"
 
-void do_scan(struct config *cfg) {
-    return;
-}
-
 void do_dump(struct config *cfg) {
     return;
 }

src/czinspect/src/operations.h

@@ -7,6 +7,8 @@
 #define EXT_F_META   0x01
 #define EXT_F_ATTACH 0x02
 #define EXT_F_SBLK   0x04
+#define EXT_F_FILT   0x08
+#define EXT_F_FFUZZ  0x10 /* round the extraction level */
 
 void do_scan(struct config *);
 void do_extract(struct config *);

src/czinspect/src/scan.c

@@ -0,0 +1,65 @@
+
+#include "config.h"
+
+#include <err.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "types.h"
+#include "alloc.h"
+#include "macros.h"
+#include "mapfile.h"
+#include "util.h"
+#include "zeiss.h"
+#include "zeissio.h"
+
+static void parse_opts(struct config *cfg) {
+    /* currently a nop */
+    (void)cfg;
+    return;
+}
+
+void do_scan(struct config *cfg) {
+    struct czi_seg_header head;
+    struct czi_zrf zisrf;
+    struct map_ctx *c;
+    lzbuf reslist;
+    uint32_t rnum = 0;
+
+    parse_opts(cfg);
+
+    c = cfg->inctx;
+
+        /* read initial segment */
+    if (czi_read_sh(c, &head) == -1)
+        ferrx1("could not read initial segment header from input file");
+
+    if (czi_getsegid(&head) != ZISRAWFILE)
+        ferrx1("input file does not start with a ZISRAWFILE segment");
+
+    if (czi_read_zrf(c, &zisrf) == -1)
+        ferrx1("could not read ZISRAWFILE segment");
+
+    /* XXX this should be changed when segment-local scanning is implemented */
+    if (zisrf.directory_position == 0)
+        ferrx1("file contains no subblock directory, cannot scan for subsampling levels");
+
+    if (map_seek(c, zisrf.directory_position, MAP_SET) == -1)
+        ferrx("could not seek to offset %" PRIu64 " to read subblock directory", zisrf.directory_position);
+
+    reslist = lzbuf_new(uint32_t);
+
+    if (make_reslist(c, reslist, &rnum) == -1)
+        ferrx1("could not scan for subsampling levels in input file");
+
+    printf("Available subsampling levels (smaller number indicates higher resolution):\n\n");
+
+    for (uint32_t i = 0; i < rnum; i++)
+        printf("\t%" PRIu32 "\n", lzbuf_get(uint32_t, reslist, i));
+
+    printf("\n");
+    lzbuf_free(reslist);
+
+    return;
+}

src/czinspect/src/types.h

@@ -32,6 +32,7 @@ struct config {
     /* extraction options */
     char *outdir;
     uint8_t eflags;    /* whole-file extraction flags */
+    uint32_t filter;   /* filter level */
     char *esopts;      /* option string for extracting data from subblocks */
 
     /* dumping options */