From 470877ffe5b714c8afa6cc70ada50daecced05ae Mon Sep 17 00:00:00 2001 From: David Miller Date: Wed, 1 Aug 2018 15:03:09 +0100 Subject: [PATCH 1/3] Multiple fixes to compatibility code: - Big endian compat code now works - configtest code now defines _DEFAULT_SOURCE on non-Apple platforms - Fix format specifiers for size_t values --- src/czinspect/src/endian.h | 27 -------- src/czinspect/src/extract.c | 4 +- src/czinspect/src/mapfile.c | 14 ++-- src/czinspect/src/zeissio.c | 108 +++++++++++++++++++------------ src/czinspect/tools/configtest.c | 16 ++--- 5 files changed, 80 insertions(+), 89 deletions(-) delete mode 100644 src/czinspect/src/endian.h diff --git a/src/czinspect/src/endian.h b/src/czinspect/src/endian.h deleted file mode 100644 index 6bd6259..0000000 --- a/src/czinspect/src/endian.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef _ENDIAN_H -#define _ENDIAN_H - -#include - -#include "config.h" - -/* magic endianness switching for platforms that require it */ - -#ifdef IS_BIG_ENDIAN -# define SWITCH_ENDS(v) switch_endianness((char *) v, sizeof(*v)) - -static inline void switch_endianness(char *val, size_t sz) { - static char tmp; - - for (size_t i = 0; i < sz / 2; i++) { - tmp = val[sz - i]; - val[sz - i] = val[i]; - val[i] = tmp; - } -} - -#else -# define SWITCH_ENDS(v) -#endif - -#endif /* _ENDIAN_H */ diff --git a/src/czinspect/src/extract.c b/src/czinspect/src/extract.c index 4eb0979..940a7be 100644 --- a/src/czinspect/src/extract.c +++ b/src/czinspect/src/extract.c @@ -298,7 +298,7 @@ static void extract_subblock(struct map_ctx *c) { offset = 256 - offset; if (offset > 0) if (map_seek(c, offset, MAP_FORW) == -1) - ferrx("cannot seek forwards %" PRIu64 " bytes", offset); + ferrx("cannot seek forwards %zu bytes", offset); make_suffix(suffix, dimensions, sblk.dir_entry.dimension_count); @@ -388,7 +388,7 @@ static void extract_sblk_directory(struct map_ctx *c, uint64_t pos) { extract_subblock(c); if (map_seek(c, offset, MAP_SET) == -1) - ferrx("could not seek to previous offset %" PRIu64 " to resume scanning subblocks", offset); + ferrx("could not seek to previous offset %zu to resume scanning subblocks", offset); } lzstr_free(fname); diff --git a/src/czinspect/src/mapfile.c b/src/czinspect/src/mapfile.c index a4eaa56..643564c 100644 --- a/src/czinspect/src/mapfile.c +++ b/src/czinspect/src/mapfile.c @@ -138,17 +138,17 @@ int map_seek(struct map_ctx *ctx, size_t offset, int whence) { switch (whence) { case MAP_SET: if (offset > ctx->fsize) - return fwarnx("invalid seek offset of %" PRIi64, offset), -1; + return fwarnx("invalid seek offset of %zu", offset), -1; target = offset; break; case MAP_FORW: if (map_file_remaining(ctx) + offset > ctx->fsize) - return fwarnx("invalid seek offset of %" PRIi64, offset), -1; + return fwarnx("invalid seek offset of %zu", offset), -1; target = map_file_offset(ctx) + offset; break; case MAP_BACK: if (offset > map_file_offset(ctx)) - return fwarnx("invalid seek offset of %" PRIi64, offset), -1; + return fwarnx("invalid seek offset of %zu", offset), -1; target = map_file_offset(ctx) - offset; break; default: @@ -173,7 +173,7 @@ int map_seek(struct map_ctx *ctx, size_t offset, int whence) { int map_read(struct map_ctx *ctx, void *buf, size_t len) { if (len > map_file_remaining(ctx)) - return fwarnx("cannot read %" PRIu64 " bytes from \"%s\"", len, ctx->fname), -1; + return fwarnx("cannot read %zu bytes from \"%s\"", len, ctx->fname), -1; while ((ctx->offset + len) > ctx->chunklen) { memcpy(buf, map_chunk_ptr(ctx), map_chunk_remain(ctx)); @@ -200,7 +200,7 @@ static int write_loop(int fd, void *buf, size_t len) { if (errno == EINTR || errno == EAGAIN) continue; - return fwarn("cannot write %" PRIu64" bytes to file descriptor %u", + return fwarn("cannot write %zu bytes to file descriptor %u", len, fd), -1; } @@ -215,7 +215,7 @@ static int write_loop(int fd, void *buf, size_t len) { int map_dwrite(struct map_ctx *ctx, int fd, size_t len) { if (len > map_file_remaining(ctx)) - return fwarnx("cannot read %" PRIu64 " bytes from \"%s\"", len, ctx->fname), -1; + return fwarnx("cannot read %zu bytes from \"%s\"", len, ctx->fname), -1; while ((ctx->offset + len) > ctx->chunklen) { if (write_loop(fd, map_chunk_ptr(ctx), map_chunk_remain(ctx)) == -1) @@ -240,7 +240,7 @@ int map_splice(struct map_ctx *from, struct map_ctx *to, size_t len) { size_t adv; /* how many bytes to advance in the loop */ if (len > map_file_remaining(from) || len > map_file_remaining(to)) - return fwarnx("cannot copy %" PRIu64 " bytes from \"%s\" to \"%s\"", + return fwarnx("cannot copy %zu bytes from \"%s\" to \"%s\"", len, from->fname, to->fname), -1; while ((from->offset + len) > from->chunklen || (to->offset + len) > to->chunklen) { diff --git a/src/czinspect/src/zeissio.c b/src/czinspect/src/zeissio.c index 0a10818..8649560 100644 --- a/src/czinspect/src/zeissio.c +++ b/src/czinspect/src/zeissio.c @@ -6,7 +6,6 @@ #include #include "zeiss.h" -#include "endian.h" #include "mapfile.h" #include "types.h" @@ -23,71 +22,94 @@ } while (0) #ifdef IS_BIG_ENDIAN -# define SWE(t, m) do { \ - t tmp = data->m; \ - SWITCH_ENDS(&tmp); \ - data->m = tmp; \ - } while (0) +# define SW32(v) data->v = sw32(data->v) +# define SW64(v) data->v = sw64(data->v) +# define SWF(f) data->f = swf(data->f) + +static uint32_t sw32(uint32_t v) { + return ((v & 0xff) << 24) | (((v >> 8) & 0xff) << 16) | (((v >> 16) & 0xff) << 8) | ((v >> 24) & 0xff); +} + +static uint64_t sw64(uint64_t v) { + return ((v & 0xff) << 56) | (((v >> 8) & 0xff) << 48) | (((v >> 16) & 0xff) << 40) | (((v >> 24) & 0xff) << 32) | + (((v >> 32) & 0xff) << 24) | (((v >> 40) & 0xff) << 16) | (((v >> 48) & 0xff) << 8) | ((v >> 56) & 0xff); +} + +static float swf(float f) { + float ret; + unsigned char *in = (char *) &f; + unsigned char *out = (char *) &ret; + + out[0] = in[3]; + out[1] = in[2]; + out[2] = in[1]; + out[3] = in[0]; + + return ret; +} + #else -# define SWE(t, n) +# define SW32(v) +# define SW64(v) +# define SWF(v) #endif /* read in a segment header */ F(sh, czi_seg_header) { READM(); - SWE(uint64_t, allocated_size); - SWE(uint64_t, used_size); - + SW64(allocated_size); + SW64(used_size); + return 0; } F(zrf, czi_zrf) { READM(); - SWE(uint32_t, major); - SWE(uint32_t, minor); - SWE(uint32_t, reserved1); - SWE(uint32_t, reserved2); - - SWE(uint32_t, file_part); - SWE(uint64_t, directory_position); - SWE(uint32_t, metadata_position); - SWE(uint32_t, update_pending); - SWE(uint64_t, attachment_directory_position); - + SW32(major); + SW32(minor); + SW32(reserved1); + SW32(reserved2); + + SW32(file_part); + SW64(directory_position); + SW32(metadata_position); + SW32(update_pending); + SW64(attachment_directory_position); + return 0; } F(metadata, czi_metadata) { READM(); - SWE(uint32_t, xml_size); - SWE(uint32_t, attachment_size); - + SW32(xml_size); + SW32(attachment_size); + return 0; } F(sblk_dimentry, czi_subblock_dimentry) { READM(); - SWE(int32_t, start); - SWE(uint32_t, size); - SWE(float, start_coordinate); - SWE(uint32_t, stored_size); - + SW32(start); + SW32(size); + SWF(start_coordinate); + SW32(stored_size); + return 0; } F(sblk_direntry, czi_subblock_direntry) { READM(); - SWE(uint32_t, pixel_type); - SWE(uint64_t, file_position); - SWE(uint32_t, file_part); - SWE(uint32_t, compression); - SWE(uint32_t, dimension_count); - + SW32(pixel_type); + SW64(file_position); + SW32(file_part); + SW32(compression); + SW32(dimension_count); + return 0; } @@ -97,9 +119,9 @@ F(subblock, czi_subblock) { READM2(attachment_size); READM2(data_size); - SWE(uint32_t, metadata_size); - SWE(uint32_t, attachment_size); - SWE(uint64_t, data_size); + SW32(metadata_size); + SW32(attachment_size); + SW64(data_size); return czi_read_sblk_direntry(ctx, &data->dir_entry); } @@ -107,7 +129,7 @@ F(subblock, czi_subblock) { F(directory, czi_directory) { READM(); - SWE(uint32_t, entry_count); + SW32(entry_count); return 0; } @@ -115,8 +137,8 @@ F(directory, czi_directory) { F(atmt_entry, czi_attach_entry) { READM(); - SWE(uint64_t, file_position); - SWE(int32_t, file_part); + SW32(file_position); + SW32(file_part); return 0; } @@ -131,7 +153,7 @@ F(attach, czi_attach) { READM2(reserved2); - SWE(uint32_t, data_size); + SW32(data_size); return 0; } @@ -139,7 +161,7 @@ F(attach, czi_attach) { F(atmt_dir, czi_attach_dir) { READM(); - SWE(uint32_t, entry_count); + SW32(entry_count); return 0; } diff --git a/src/czinspect/tools/configtest.c b/src/czinspect/tools/configtest.c index 00f6d90..51803dc 100644 --- a/src/czinspect/tools/configtest.c +++ b/src/czinspect/tools/configtest.c @@ -12,19 +12,11 @@ * POSIX functions. */ -const char posixmacro[] = -#if defined(__APPLE__) || defined(__MACH__) - "_DARWIN_C_SOURCE" -#else - "_POSIX_C_SOURCE" -#endif - ; int main() { uint32_t u32 = 0x44332211; uint32_t *u32ptr = &u32; unsigned char *cptr = (unsigned char *) u32ptr; - unsigned long posixver = 200809L; printf("#ifndef _CONFIG_H\n"); printf("#define _CONFIG_H\n"); @@ -41,8 +33,12 @@ int main() { errx(1, "could not determine machine endianness"); } - printf("#define %s %luL\n", posixmacro, posixver); - +#if defined(__APPLE__) + printf("#define _DARWIN_C_SOURCE 200809L\n"); +#else + printf("#define _DEFAULT_SOURCE\n"); +#endif + printf("#endif /* _CONFIG_H */\n"); exit(0); From e46367260e93489c1f8e7fd99d31796b167d4459 Mon Sep 17 00:00:00 2001 From: David Miller Date: Wed, 1 Aug 2018 15:37:05 +0100 Subject: [PATCH 2/3] Correct integer widths in calls to endian-swapping code --- src/czinspect/src/zeissio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/czinspect/src/zeissio.c b/src/czinspect/src/zeissio.c index 8649560..3b477ea 100644 --- a/src/czinspect/src/zeissio.c +++ b/src/czinspect/src/zeissio.c @@ -74,7 +74,7 @@ F(zrf, czi_zrf) { SW32(file_part); SW64(directory_position); - SW32(metadata_position); + SW64(metadata_position); SW32(update_pending); SW64(attachment_directory_position); @@ -137,7 +137,7 @@ F(directory, czi_directory) { F(atmt_entry, czi_attach_entry) { READM(); - SW32(file_position); + SW64(file_position); SW32(file_part); return 0; From b15b19e66b4ad483006b377f103d9b5a09e6c080 Mon Sep 17 00:00:00 2001 From: David Miller Date: Wed, 1 Aug 2018 16:09:56 +0100 Subject: [PATCH 3/3] Refactor subblock extraction ratio filtering code --- src/czinspect/src/czinspect.c | 4 +-- src/czinspect/src/extract.c | 46 +++++++++++++++++++++++----------- src/czinspect/src/operations.h | 6 +++-- src/czinspect/src/types.h | 1 + 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/czinspect/src/czinspect.c b/src/czinspect/src/czinspect.c index f26c42b..9d66d40 100644 --- a/src/czinspect/src/czinspect.c +++ b/src/czinspect/src/czinspect.c @@ -169,10 +169,10 @@ static void parse_opt_extract(int opt) { if (errstr) errx(1, "invalid filter level '%s': %s", optarg, errstr); cfg.filter = filt; - cfg.eflags |= EXT_F_FILT; + cfg.filtflags |= EXT_FI_FILT; break; case 'g': - cfg.eflags |= EXT_F_FFUZZ; + cfg.filtflags |= EXT_FI_FFUZZ; break; case 's': cfg.eflags |= EXT_F_SBLK; diff --git a/src/czinspect/src/extract.c b/src/czinspect/src/extract.c index 940a7be..f12d3d6 100644 --- a/src/czinspect/src/extract.c +++ b/src/czinspect/src/extract.c @@ -37,6 +37,7 @@ enum subopt_index { static int dirfd; static uint8_t sblk_opts; static uint8_t ext_opts; +static uint8_t filt_opts; static uint32_t filterlevel; static lzstring fname; static lzstring suffix; @@ -93,20 +94,23 @@ static void parse_opts(struct config *cfg) { } xfree(cfg->esopts); - } else if (cfg->eflags & EXT_F_FILT) { - errx(1, "cannot perform subsampling level filtering when not extracting subblocks"); - } else if (cfg->eflags & EXT_F_FFUZZ) { - errx(1, "cannot perform rounded subsampling level filtering when not extracting subblocks"); } - if ((cfg->eflags & EXT_F_FFUZZ) == EXT_F_FFUZZ && (cfg->eflags & EXT_F_FILT) == 0) - errx(1, "cannot round subsampling level when not performing filtering"); - if (cfg->eflags == 0) { ext_opts = (EXT_F_META | EXT_F_SBLK | EXT_F_ATTACH); sblk_opts = EXT_S_ALL; - } else + } else { ext_opts = cfg->eflags; + } + + if (((ext_opts & EXT_F_SBLK) == 0) && (cfg->filtflags & (EXT_FI_FILT | EXT_FI_FFUZZ))) { + errx(1, "cannot perform subsampling level filtering when not extracting subblocks"); + } + + if ((cfg->filtflags & EXT_FI_FFUZZ) == EXT_FI_FFUZZ && (cfg->filtflags & EXT_FI_FILT) == 0) + errx(1, "cannot round subsampling level when not performing filtering"); + + filt_opts = cfg->filtflags; return; } @@ -401,8 +405,10 @@ static void extract_sblk_directory(struct map_ctx *c, uint64_t pos) { static void setup_filterlevel(struct config *cfg, struct map_ctx *c, uint64_t dirpos) { lzbuf reslist; uint32_t rnum = 0; + uint32_t max = 0; + uint32_t l; - if ((cfg->eflags & EXT_F_FILT) == 0) + if ((filt_opts & EXT_FI_FILT) == 0) return; if (map_seek(c, dirpos, MAP_SET) == -1) @@ -414,12 +420,22 @@ static void setup_filterlevel(struct config *cfg, struct map_ctx *c, uint64_t di if (make_reslist(c, reslist, &rnum) == -1) ferrx1("could not scan for subsampling levels in input file"); - for (uint32_t i = 0; i < rnum; i++) - if ((cfg->eflags & EXT_F_FFUZZ) ? - (lzbuf_get(uint32_t, reslist, i) <= cfg->filter) : - (lzbuf_get(uint32_t, reslist, i) == cfg->filter)) - filterlevel = lzbuf_get(uint32_t, reslist, i); - + + if (filt_opts & EXT_FI_FFUZZ) { + for (uint32_t i = 0; i < rnum; i++) { + l = lzbuf_get(uint32_t, reslist, i); + + if (l <= cfg->filter && l > max) { + filterlevel = l; + max = l; + } + } + } else { + for (uint32_t i = 0; i < rnum; i++) + if (lzbuf_get(uint32_t, reslist, i) == cfg->filter) + filterlevel = cfg->filter; + } + if (filterlevel == 0) errx(1, "invalid filter level: %" PRIu32, cfg->filter); diff --git a/src/czinspect/src/operations.h b/src/czinspect/src/operations.h index 32b64be..c964433 100644 --- a/src/czinspect/src/operations.h +++ b/src/czinspect/src/operations.h @@ -7,8 +7,10 @@ #define EXT_F_META 0x01 #define EXT_F_ATTACH 0x02 #define EXT_F_SBLK 0x04 -#define EXT_F_FILT 0x08 -#define EXT_F_FFUZZ 0x10 /* round the extraction level */ + +/* filtering flags */ +#define EXT_FI_FILT 0x01 +#define EXT_FI_FFUZZ 0x02 /* round the extraction level */ void do_scan(struct config *); void do_extract(struct config *); diff --git a/src/czinspect/src/types.h b/src/czinspect/src/types.h index f72f1ff..6e0b8d8 100644 --- a/src/czinspect/src/types.h +++ b/src/czinspect/src/types.h @@ -32,6 +32,7 @@ struct config { /* extraction options */ char *outdir; uint8_t eflags; /* whole-file extraction flags */ + uint8_t filtflags; /* extraction filtering flags */ uint32_t filter; /* filter level */ char *esopts; /* option string for extracting data from subblocks */