diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index eb942fd..f6ebc89 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -15,23 +15,20 @@ jobs: include: - description: "default" - compiler: gcc - configure: "--disable-c-pkgconfig --disable-c-pages --disable-c-fsindex --disable-c-readerchannel --disable-c-writerchannel --disable-c-vfs" - description: "no pkgcfg, pages, fsindex, readedchannel, writerchannel, vfs" + configure: "--disable-c-pkgconfig --disable-c-pages --disable-c-fsindex --disable-c-readerchannel --disable-c-writer --disable-c-writerchannel --disable-c-vfs" + description: "no pkgcfg, pages, fsindex, readedchannel, writer, writerchannel, vfs" - compiler: gcc - configure: "--disable-c-pages --disable-c-fsindex --disable-c-readerchannel --disable-c-writerchannel --disable-c-vfs" - description: "no pages, fsindex, readedchannel, writerchannel, vfs" + configure: "--disable-c-pages --disable-c-fsindex --disable-c-readerchannel --disable-c-writer --disable-c-writerchannel --disable-c-vfs" + description: "no pages, fsindex, readedchannel, writer, writerchannel, vfs" - compiler: gcc - configure: "--disable-c-fsindex --disable-c-readerchannel --disable-c-writerchannel --disable-c-vfs" - description: "no fsindex, readedchannel, writerchannel, vfs" + configure: "--disable-c-fsindex --disable-c-readerchannel --disable-c-writer --disable-c-writerchannel --disable-c-vfs" + description: "no fsindex, readedchannel, writer, writerchannel, vfs" - compiler: gcc - configure: "--disable-c-readerchannel --disable-c-writerchannel --disable-c-vfs" - description: "no readedchannel, writerchannel, vfs" + configure: "--disable-c-readerchannel --disable-c-writer --disable-c-writerchannel --disable-c-vfs" + description: "no readedchannel, writer, writerchannel, vfs" - compiler: gcc - configure: "--disable-c-writerchannel --disable-c-vfs" - description: "no writerchannel, vfs" - - compiler: gcc - configure: "--disable-c-vfs" - description: "no vfs" + configure: " --disable-c-writer --disable-c-writerchannel --disable-c-vfs" + description: "no writer, writerchannel, vfs" name: build (${{ matrix.compiler }}, ${{ matrix.description }}) steps: - name: Checkout diff --git a/ChangeLog b/ChangeLog index 41c2327..4d37d63 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +2024-05-25 Konstantin Kushnir + * Add support for writer in C + 2024-05-21 Konstantin Kushnir * Add support for vfs in C * Make the writer object available for C functions diff --git a/configure b/configure index c49ca1a..20fa538 100755 --- a/configure +++ b/configure @@ -657,6 +657,7 @@ MAKE_LIB EGREP GREP COOKFS_PKGCONFIG_USECPKGCONFIG +COOKFS_PKGCONFIG_USECWRITER COOKFS_PKGCONFIG_USECVFS COOKFS_PKGCONFIG_FEATURE_METADATA COOKFS_PKGCONFIG_USECFSINDEX @@ -793,6 +794,7 @@ enable_c_readerchannel enable_c_writerchannel enable_c_pkgconfig enable_c_vfs +enable_c_writer ' ac_precious_vars='build_alias host_alias @@ -1439,6 +1441,7 @@ Optional Features: --disable-c-writerchannel Use writer handler written in C. Defaults to true --disable-c-pkgconfig Use pkgconfig written in C. Defaults to true --disable-c-vfs Use VFS support written in C. Defaults to true + --disable-c-writer Use writer handler written in C. Defaults to true Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] @@ -8650,6 +8653,14 @@ else $as_nop USECVFS=yes fi +# Check whether --enable-c-writer was given. +if test ${enable_c_writer+y} +then : + enableval=$enable_c_writer; USECWRITER=${enableval} +else $as_nop + USECWRITER=yes +fi + if test ${INTERNALDEBUG} = yes; then printf "%s\n" "#define COOKFS_INTERNAL_DEBUG 1" >>confdefs.h @@ -8935,6 +8946,8 @@ else USECREADERCHAN="no" # c-writerchannel requires c-pages USECWRITERCHAN="no" + # c-writer requires c-pages + USECWRITER="no" fi if test ${USECFSINDEX} = yes; then @@ -8984,6 +8997,8 @@ else USECREADERCHAN="no" # c-writerchannel requires c-fsindex USECWRITERCHAN="no" + # c-writer requires c-pages + USECWRITER="no" fi COOKFS_PKGCONFIG_FEATURE_METADATA=1 @@ -9033,7 +9048,51 @@ else USECVFS="no" fi -if test ${USECWRITERCHAN} = yes; then +# c-writer / c-writerchan / c-vfs must be enabled together +if test ${USECWRITER} = yes \ + && test ${USECWRITERCHAN} = yes \ + && test ${USECVFS} = yes +then + + printf "%s\n" "#define COOKFS_USECWRITER 1" >>confdefs.h + + COOKFS_PKGCONFIG_USECWRITER=1 + + vars="writer.c writerCmd.c" + for i in $vars; do + case $i in + \$*) + # allow $-var names + PKG_SOURCES="$PKG_SOURCES $i" + PKG_OBJECTS="$PKG_OBJECTS $i" + ;; + *) + # check for existence - allows for generic/win/unix VPATH + # To add more dirs here (like 'src'), you have to update VPATH + # in Makefile.in as well + if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \ + -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \ + -a ! -f "${srcdir}/macosx/$i" \ + ; then + as_fn_error $? "could not find source file '$i'" "$LINENO" 5 + fi + PKG_SOURCES="$PKG_SOURCES $i" + # this assumes it is in a VPATH dir + i=`basename $i` + # handle user calling this before or after TEA_SETUP_COMPILER + if test x"${OBJEXT}" != x ; then + j="`echo $i | sed -e 's/\.[^.]*$//'`.${OBJEXT}" + else + j="`echo $i | sed -e 's/\.[^.]*$//'`.\${OBJEXT}" + fi + PKG_OBJECTS="$PKG_OBJECTS $j" + ;; + esac + done + + + + printf "%s\n" "#define COOKFS_USECWRITERCHAN 1" >>confdefs.h COOKFS_PKGCONFIG_USECWRITERCHAN=1 @@ -9072,18 +9131,12 @@ if test ${USECWRITERCHAN} = yes; then -else - COOKFS_PKGCONFIG_USECWRITERCHAN=0 - # c-vfs requires c-writerchannel - USECVFS="no" -fi -if test ${USECVFS} = yes; then printf "%s\n" "#define COOKFS_USECVFS 1" >>confdefs.h COOKFS_PKGCONFIG_USECVFS=1 - vars="writer.c vfsDriver.c vfsVfs.c vfs.c vfsCmd.c" + vars="vfsDriver.c vfsVfs.c vfs.c vfsCmd.c" for i in $vars; do case $i in \$*) @@ -9117,7 +9170,10 @@ if test ${USECVFS} = yes; then + else + COOKFS_PKGCONFIG_USECWRITER=0 + COOKFS_PKGCONFIG_USECWRITERCHAN=0 COOKFS_PKGCONFIG_USECVFS=0 fi @@ -9131,6 +9187,7 @@ fi + if test ${USECPKGCONFIG} = yes; then COOKFS_PKGCONFIG_USECPKGCONFIG=1 printf "%s\n" "#define COOKFS_PKGCONFIG_FEATURE_ASIDE $COOKFS_PKGCONFIG_FEATURE_ASIDE" >>confdefs.h @@ -9153,8 +9210,10 @@ if test ${USECPKGCONFIG} = yes; then printf "%s\n" "#define COOKFS_PKGCONFIG_USECVFS $COOKFS_PKGCONFIG_USECVFS" >>confdefs.h + printf "%s\n" "#define COOKFS_PKGCONFIG_USECWRITER $COOKFS_PKGCONFIG_USECWRITER" >>confdefs.h + - printf "%s\n" "#define COOKFS_USEPKGCONFIG 1" >>confdefs.h + printf "%s\n" "#define COOKFS_USECPKGCONFIG 1" >>confdefs.h else COOKFS_PKGCONFIG_USECPKGCONFIG=0 diff --git a/configure.in b/configure.in index da34adb..1505439 100644 --- a/configure.in +++ b/configure.in @@ -123,6 +123,7 @@ AC_ARG_ENABLE(c-readerchannel, [ --disable-c-readerchannel Use reader handler w AC_ARG_ENABLE(c-writerchannel, [ --disable-c-writerchannel Use writer handler written in C. Defaults to true], USECWRITERCHAN=${enableval}, USECWRITERCHAN=yes) AC_ARG_ENABLE(c-pkgconfig, [ --disable-c-pkgconfig Use pkgconfig written in C. Defaults to true], USECPKGCONFIG=${enableval}, USECPKGCONFIG=yes) AC_ARG_ENABLE(c-vfs, [ --disable-c-vfs Use VFS support written in C. Defaults to true], USECVFS=${enableval}, USECVFS=yes) +AC_ARG_ENABLE(c-writer, [ --disable-c-writer Use writer handler written in C. Defaults to true], USECWRITER=${enableval}, USECWRITER=yes) if test ${INTERNALDEBUG} = yes; then AC_DEFINE(COOKFS_INTERNAL_DEBUG) @@ -190,6 +191,8 @@ else USECREADERCHAN="no" # c-writerchannel requires c-pages USECWRITERCHAN="no" + # c-writer requires c-pages + USECWRITER="no" fi if test ${USECFSINDEX} = yes; then @@ -204,6 +207,8 @@ else USECREADERCHAN="no" # c-writerchannel requires c-fsindex USECWRITERCHAN="no" + # c-writer requires c-pages + USECWRITER="no" fi COOKFS_PKGCONFIG_FEATURE_METADATA=1 @@ -218,21 +223,27 @@ else USECVFS="no" fi -if test ${USECWRITERCHAN} = yes; then +# c-writer / c-writerchan / c-vfs must be enabled together +if test ${USECWRITER} = yes \ + && test ${USECWRITERCHAN} = yes \ + && test ${USECVFS} = yes +then + + AC_DEFINE(COOKFS_USECWRITER) + COOKFS_PKGCONFIG_USECWRITER=1 + TEA_ADD_SOURCES([writer.c writerCmd.c]) + AC_DEFINE(COOKFS_USECWRITERCHAN) COOKFS_PKGCONFIG_USECWRITERCHAN=1 TEA_ADD_SOURCES([writerchannel.c writerchannelIO.c]) -else - COOKFS_PKGCONFIG_USECWRITERCHAN=0 - # c-vfs requires c-writerchannel - USECVFS="no" -fi -if test ${USECVFS} = yes; then AC_DEFINE(COOKFS_USECVFS) COOKFS_PKGCONFIG_USECVFS=1 - TEA_ADD_SOURCES([writer.c vfsDriver.c vfsVfs.c vfs.c vfsCmd.c]) + TEA_ADD_SOURCES([vfsDriver.c vfsVfs.c vfs.c vfsCmd.c]) + else + COOKFS_PKGCONFIG_USECWRITER=0 + COOKFS_PKGCONFIG_USECWRITERCHAN=0 COOKFS_PKGCONFIG_USECVFS=0 fi @@ -245,6 +256,7 @@ AC_SUBST(COOKFS_PKGCONFIG_USEXZ) AC_SUBST(COOKFS_PKGCONFIG_USECFSINDEX) AC_SUBST(COOKFS_PKGCONFIG_FEATURE_METADATA) AC_SUBST(COOKFS_PKGCONFIG_USECVFS) +AC_SUBST(COOKFS_PKGCONFIG_USECWRITER) if test ${USECPKGCONFIG} = yes; then COOKFS_PKGCONFIG_USECPKGCONFIG=1 @@ -258,8 +270,9 @@ if test ${USECPKGCONFIG} = yes; then AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USECFSINDEX, $COOKFS_PKGCONFIG_USECFSINDEX) AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_FEATURE_METADATA, $COOKFS_PKGCONFIG_FEATURE_METADATA) AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USECVFS, $COOKFS_PKGCONFIG_USECVFS) + AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USECWRITER, $COOKFS_PKGCONFIG_USECWRITER) - AC_DEFINE(COOKFS_USEPKGCONFIG) + AC_DEFINE(COOKFS_USECPKGCONFIG) else COOKFS_PKGCONFIG_USECPKGCONFIG=0 fi diff --git a/generic/cookfs.c b/generic/cookfs.c index a977acc..34e56c2 100644 --- a/generic/cookfs.c +++ b/generic/cookfs.c @@ -56,6 +56,12 @@ Cookfs_Init(Tcl_Interp *interp) } #endif +#ifdef COOKFS_USECWRITER + if (Cookfs_InitWriterCmd(interp) != TCL_OK) { + return TCL_ERROR; + } +#endif + #ifdef COOKFS_USECWRITERCHAN if (Cookfs_InitWriterchannelCmd(interp) != TCL_OK) { return TCL_ERROR; @@ -76,7 +82,7 @@ Cookfs_Init(Tcl_Interp *interp) } #endif -#ifdef COOKFS_USEPKGCONFIG +#ifdef COOKFS_USECPKGCONFIG Tcl_RegisterConfig(interp, PACKAGE_NAME, cookfs_pkgconfig, "iso8859-1"); if (Tcl_PkgProvide(interp, PACKAGE_NAME "::pkgconfig", PACKAGE_VERSION) != TCL_OK) { diff --git a/generic/cookfs.h b/generic/cookfs.h index 9adf4fa..f690f6e 100644 --- a/generic/cookfs.h +++ b/generic/cookfs.h @@ -52,6 +52,11 @@ #include "readerchannelIO.h" #endif /* COOKFS_USECREADERCHAN */ +#ifdef COOKFS_USECWRITER +#include "writer.h" +#include "writerCmd.h" +#endif /* COOKFS_USECWRITER */ + #ifdef COOKFS_USECVFS #include "writer.h" #include "vfs.h" @@ -65,7 +70,7 @@ #include "writerchannelIO.h" #endif /* COOKFS_USECWRITERCHAN */ -#ifdef COOKFS_USEPKGCONFIG +#ifdef COOKFS_USECPKGCONFIG static Tcl_Config const cookfs_pkgconfig[] = { {"package-version", PACKAGE_VERSION}, {"c-pages", STRINGIFY(COOKFS_PKGCONFIG_USECPAGES)}, @@ -73,12 +78,13 @@ static Tcl_Config const cookfs_pkgconfig[] = { {"c-readerchannel", STRINGIFY(COOKFS_PKGCONFIG_USECREADERCHAN)}, {"c-writerchannel", STRINGIFY(COOKFS_PKGCONFIG_USECWRITERCHAN)}, {"c-vfs", STRINGIFY(COOKFS_PKGCONFIG_USECVFS)}, + {"c-writer", STRINGIFY(COOKFS_PKGCONFIG_USECWRITER)}, {"feature-aside", STRINGIFY(COOKFS_PKGCONFIG_FEATURE_ASIDE)}, {"feature-bzip2", STRINGIFY(COOKFS_PKGCONFIG_USEBZ2)}, {"feature-xz", STRINGIFY(COOKFS_PKGCONFIG_USEXZ)}, {"feature-metadata", STRINGIFY(COOKFS_PKGCONFIG_FEATURE_METADATA)}, {NULL, NULL} }; -#endif /* COOKFS_USEPKGCONFIG */ +#endif /* COOKFS_USECPKGCONFIG */ #endif /* COOKFS_H */ diff --git a/generic/fsindex.c b/generic/fsindex.c index 9df2ff7..ca257f4 100644 --- a/generic/fsindex.c +++ b/generic/fsindex.c @@ -20,6 +20,87 @@ static Cookfs_FsindexEntry *CookfsFsindexFind(Cookfs_Fsindex *i, Cookfs_FsindexE static Cookfs_FsindexEntry *CookfsFsindexFindInDirectory(Cookfs_FsindexEntry *currentNode, char *pathTailStr, int command, Cookfs_FsindexEntry *newFileNode); static void CookfsFsindexChildtableToHash(Cookfs_FsindexEntry *e); +/* + *---------------------------------------------------------------------- + * + * Cookfs_FsindexUpdateEntryFileSize -- + * + * Updates the filesize field for specified entry + * + * Results: + * None + * + * Side effects: + * None + * + *---------------------------------------------------------------------- + */ + +void Cookfs_FsindexUpdateEntryFileSize(Cookfs_FsindexEntry *e, + Tcl_WideInt fileSize) +{ + e->data.fileInfo.fileSize = fileSize; + return ; +} + +/* + *---------------------------------------------------------------------- + * + * Cookfs_FsindexUpdateEntryBlock -- + * + * Updates the block data for specified entry + * + * Results: + * None + * + * Side effects: + * None + * + *---------------------------------------------------------------------- + */ + +void Cookfs_FsindexUpdateEntryBlock(Cookfs_Fsindex *i, Cookfs_FsindexEntry *e, + int blockNumber, int blockIndex, int blockOffset, int blockSize) +{ + int blockIndexOffset = blockNumber * 3 + 0; + // Reducing block utilization for an already set block index + Cookfs_FsindexModifyBlockUsage(i, e->data.fileInfo.fileBlockOffsetSize[blockIndexOffset + 0], -1); + e->data.fileInfo.fileBlockOffsetSize[blockIndexOffset + 0] = blockIndex; + e->data.fileInfo.fileBlockOffsetSize[blockIndexOffset + 1] = blockOffset; + if (blockSize >= 0) { + e->data.fileInfo.fileBlockOffsetSize[blockIndexOffset + 2] = blockSize; + } + // Increase block utilization for the new block index + Cookfs_FsindexModifyBlockUsage(i, blockIndex, 1); + // Register new change in the change counter + Cookfs_FsindexIncrChangeCount(i, 1); + return ; +} + +/* + *---------------------------------------------------------------------- + * + * Cookfs_FsindexUpdatePendingEntry -- + * + * Updates the block index on an entry that was previously registered + * as being in the small file buffer + * + * Results: + * None + * + * Side effects: + * None + * + *---------------------------------------------------------------------- + */ + +void Cookfs_FsindexUpdatePendingEntry(Cookfs_Fsindex *i, Cookfs_FsindexEntry *e, + int blockIndex, int blockOffset) +{ + Cookfs_FsindexUpdateEntryBlock(i, e, 0, blockIndex, blockOffset, -1); + return ; +} + /* *---------------------------------------------------------------------- * @@ -729,6 +810,12 @@ Cookfs_FsindexEntry *Cookfs_FsindexEntryAlloc(int fileNameLength, int numBlocks, e->data.dirInfo.childCount = 0; } else { /* for files, block information is filled by caller */ + + // Set block index to dummy value -1 so that its block usage counter is + // not touched when the entry is updated. + for (int i = 0; i < numBlocks; i++) { + e->data.fileInfo.fileBlockOffsetSize[i*3 + 0] = -1; + } } //CookfsLog(printf("Cookfs_FsindexEntryAlloc: allocated entry %p", e)); diff --git a/generic/fsindex.h b/generic/fsindex.h index ee713c8..665d59d 100644 --- a/generic/fsindex.h +++ b/generic/fsindex.h @@ -85,6 +85,14 @@ void Cookfs_FsindexResetChangeCount(Cookfs_Fsindex *i); int Cookfs_FsindexEntryIsPending(Cookfs_FsindexEntry *e); int Cookfs_FsindexEntryIsDirectory(Cookfs_FsindexEntry *e); +void Cookfs_FsindexUpdateEntryBlock(Cookfs_Fsindex *i, Cookfs_FsindexEntry *e, + int blockNumber, int blockIndex, int blockOffset, int blockSize); +void Cookfs_FsindexUpdateEntryFileSize(Cookfs_FsindexEntry *e, + Tcl_WideInt fileSize); + +void Cookfs_FsindexUpdatePendingEntry(Cookfs_Fsindex *i, Cookfs_FsindexEntry *e, + int blockIndex, int blockOffset); + #endif /* COOKFS_USECFSINDEX */ #endif /* COOKFS_FSINDEX_H */ diff --git a/generic/pages.c b/generic/pages.c index 2e76aea..d3595cf 100644 --- a/generic/pages.c +++ b/generic/pages.c @@ -437,7 +437,7 @@ Tcl_WideInt Cookfs_PagesClose(Cookfs_Pages *p) { } /* write index */ - indexSize = Cookfs_WritePage(p, -1, p->dataIndex, NULL); + indexSize = Cookfs_WritePageObj(p, -1, p->dataIndex, NULL); if (indexSize < 0) { /* TODO: handle index writing issues better */ Tcl_Panic("Unable to compress index"); @@ -561,12 +561,34 @@ void Cookfs_PagesFini(Cookfs_Pages *p) { Tcl_Free((void *) p); } - /* *---------------------------------------------------------------------- * * Cookfs_PageAdd -- * + * Same as Cookfs_PageAddRaw, but uses Tcl_Obj as the page data source. + * + * Results: + * Index that can be used in subsequent calls to Cookfs_PageGet() + * + * Side effects: + * None + * + *---------------------------------------------------------------------- + */ + +int Cookfs_PageAdd(Cookfs_Pages *p, Tcl_Obj *dataObj) { + int objLength; + unsigned char *bytes = Tcl_GetByteArrayFromObj(dataObj, &objLength); + return Cookfs_PageAddRaw(p, bytes, objLength); +} + + +/* + *---------------------------------------------------------------------- + * + * Cookfs_PageAddRaw -- + * * Add new page or return index of existing page, if page with * same content already exists * @@ -579,16 +601,12 @@ void Cookfs_PagesFini(Cookfs_Pages *p) { *---------------------------------------------------------------------- */ -int Cookfs_PageAdd(Cookfs_Pages *p, Tcl_Obj *dataObj) { +int Cookfs_PageAddRaw(Cookfs_Pages *p, unsigned char *bytes, int objLength) { int idx; int dataSize; unsigned char md5sum[16]; - unsigned char *bytes; - int objLength; unsigned char *pageMD5 = p->dataPagesMD5; - bytes = Tcl_GetByteArrayFromObj(dataObj, &objLength); - CookfsLog(printf("Cookfs_PageAdd: new page with [%d] bytes", objLength)) if (p->pageHash == COOKFS_HASH_CRC32) { @@ -597,6 +615,7 @@ int Cookfs_PageAdd(Cookfs_Pages *p, Tcl_Obj *dataObj) { Tcl_Obj *data; Tcl_Obj *prevResult; + Tcl_Obj *dataObj = Tcl_NewByteArrayObj(bytes, objLength); Tcl_IncrRefCount(dataObj); p->zipCmdCrc[1] = dataObj; @@ -672,7 +691,7 @@ int Cookfs_PageAdd(Cookfs_Pages *p, Tcl_Obj *dataObj) { /* if this page has an aside page set up, ask it to add new page */ if (p->dataAsidePages != NULL) { CookfsLog(printf("Cookfs_PageAdd: Sending add command to asidePages")) - return Cookfs_PageAdd(p->dataAsidePages, dataObj); + return Cookfs_PageAddRaw(p->dataAsidePages, bytes, objLength); } /* if file is read only, return page can't be added */ @@ -691,11 +710,11 @@ int Cookfs_PageAdd(Cookfs_Pages *p, Tcl_Obj *dataObj) { CookfsLog(printf("MD5sum is %08x%08x%08x%08x\n", ((int *) md5sum)[0], ((int *) md5sum)[1], ((int *) md5sum)[2], ((int *) md5sum)[3])) - if (Cookfs_AsyncPageAdd(p, idx, dataObj)) { + if (Cookfs_AsyncPageAdd(p, idx, bytes, objLength)) { p->pagesUptodate = 0; p->dataPagesSize[idx] = -1; } else { - dataSize = Cookfs_WritePage(p, idx, dataObj, NULL); + dataSize = Cookfs_WritePage(p, idx, bytes, objLength, NULL); if (dataSize < 0) { /* TODO: if writing failed, we can't be certain of archive state - need to handle this at vfs layer somehow */ CookfsLog(printf("Unable to compress page")) diff --git a/generic/pages.h b/generic/pages.h index f154bca..e35847c 100644 --- a/generic/pages.h +++ b/generic/pages.h @@ -141,6 +141,7 @@ Cookfs_Pages *Cookfs_PagesInit(Tcl_Interp *interp, Tcl_Obj *fileName, int fileRe Tcl_WideInt Cookfs_PagesClose(Cookfs_Pages *p); Tcl_WideInt Cookfs_GetFilesize(Cookfs_Pages *p); void Cookfs_PagesFini(Cookfs_Pages *p); +int Cookfs_PageAddRaw(Cookfs_Pages *p, unsigned char *bytes, int objLength); int Cookfs_PageAdd(Cookfs_Pages *p, Tcl_Obj *dataObj); Tcl_Obj *Cookfs_PageGet(Cookfs_Pages *p, int index, int weight); Tcl_Obj *Cookfs_PageCacheGet(Cookfs_Pages *p, int index, int update, int weight); diff --git a/generic/pagesCompr.c b/generic/pagesCompr.c index 5784e83..d30a074 100644 --- a/generic/pagesCompr.c +++ b/generic/pagesCompr.c @@ -27,13 +27,13 @@ static Tcl_Obj **CookfsCreateCompressionCommand(Tcl_Interp *interp, Tcl_Obj *cmd, int *lenPtr, int additionalElements); static void CookfsWriteCompression(Cookfs_Pages *p, int compression); static Tcl_Obj *CookfsReadPageZlib(Cookfs_Pages *p, int size); -static int CookfsWritePageZlib(Cookfs_Pages *p, Tcl_Obj *data, int origSize); +static int CookfsWritePageZlib(Cookfs_Pages *p, unsigned char *bytes, int origSize); static Tcl_Obj *CookfsReadPageBz2(Cookfs_Pages *p, int size); -static int CookfsWritePageBz2(Cookfs_Pages *p, Tcl_Obj *data, int origSize); +static int CookfsWritePageBz2(Cookfs_Pages *p, unsigned char *bytes, int origSize); static Tcl_Obj *CookfsReadPageXz(Cookfs_Pages *p, int size); -static int CookfsWritePageXz(Cookfs_Pages *p, Tcl_Obj *data, int origSize); +static int CookfsWritePageXz(Cookfs_Pages *p, unsigned char *bytes, int origSize); static Tcl_Obj *CookfsReadPageCustom(Cookfs_Pages *p, int size); -static int CookfsWritePageCustom(Cookfs_Pages *p, Tcl_Obj *data, int origSize); +static int CookfsWritePageCustom(Cookfs_Pages *p, unsigned char *bytes, int origSize); #ifdef USE_VFS_COMMANDS_FOR_ZIP static int CookfsCheckCommandExists(Tcl_Interp *interp, const char *commandName); #endif @@ -515,14 +515,9 @@ void Cookfs_SeekToPage(Cookfs_Pages *p, int idx) { *---------------------------------------------------------------------- */ -int Cookfs_WritePage(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compressedData) { - int origSize; +int Cookfs_WritePage(Cookfs_Pages *p, int idx, unsigned char *bytes, int origSize, Tcl_Obj *compressedData) { int size = -1; - /* get binary data */ - Tcl_GetByteArrayFromObj(data, &origSize); - Tcl_IncrRefCount(data); - /* if last operation was not write, we need to seek * to make sure we're at location where we should be writing */ if ((idx >= 0) && (p->fileLastOp != COOKFS_LASTOP_WRITE)) { @@ -540,26 +535,26 @@ int Cookfs_WritePage(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compresse size += 1; } else { CookfsWriteCompression(p, COOKFS_COMPRESSION_NONE); - Tcl_WriteObj(p->fileChannel, data); + Tcl_Write(p->fileChannel, (const char *)bytes, origSize); size = origSize + 1; } } else { /* try to write compressed if compression was enabled */ switch (p->fileCompression) { case COOKFS_COMPRESSION_ZLIB: - size = CookfsWritePageZlib(p, data, origSize); + size = CookfsWritePageZlib(p, bytes, origSize); break; case COOKFS_COMPRESSION_CUSTOM: - size = CookfsWritePageCustom(p, data, origSize); + size = CookfsWritePageCustom(p, bytes, origSize); break; case COOKFS_COMPRESSION_BZ2: - size = CookfsWritePageBz2(p, data, origSize); + size = CookfsWritePageBz2(p, bytes, origSize); break; case COOKFS_COMPRESSION_XZ: - size = CookfsWritePageXz(p, data, origSize); + size = CookfsWritePageXz(p, bytes, origSize); break; } @@ -567,7 +562,7 @@ int Cookfs_WritePage(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compresse * be written as raw data, write it uncompressed */ if (size == -1) { CookfsWriteCompression(p, COOKFS_COMPRESSION_NONE); - Tcl_WriteObj(p->fileChannel, data); + Tcl_Write(p->fileChannel, (const char *)bytes, origSize); size = origSize + 1; } else { /* otherwise add 1 byte for compression */ @@ -577,11 +572,16 @@ int Cookfs_WritePage(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compresse } else { size = 0; } - Tcl_DecrRefCount(data); return size; } +int Cookfs_WritePageObj(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compressedData) { + int dataSize; + unsigned char *bytes = Tcl_GetByteArrayFromObj(data, &dataSize); + return Cookfs_WritePage(p, idx, bytes, dataSize, compressedData); +} + /* *---------------------------------------------------------------------- * @@ -643,11 +643,9 @@ Tcl_Obj *Cookfs_AsyncPageGet(Cookfs_Pages *p, int idx) { *---------------------------------------------------------------------- */ -int Cookfs_AsyncPageAdd(Cookfs_Pages *p, int idx, Tcl_Obj *data) { +int Cookfs_AsyncPageAdd(Cookfs_Pages *p, int idx, unsigned char *bytes, int dataSize) { if ((p->fileCompression == COOKFS_COMPRESSION_CUSTOM) && (p->asyncCompressCommandPtr != NULL) && (p->asyncCompressCommandLen > 3)) { Tcl_Obj *newObjObj; - int newObjSize; - unsigned char *newObjData; int asyncIdx; // retrieve any already processed queues while (Cookfs_AsyncCompressWait(p, 0)) {} @@ -655,15 +653,12 @@ int Cookfs_AsyncPageAdd(Cookfs_Pages *p, int idx, Tcl_Obj *data) { Cookfs_AsyncCompressWait(p, 0); } asyncIdx = p->asyncPageSize++; - newObjData = Tcl_GetByteArrayFromObj(data, &newObjSize); - newObjObj = Tcl_NewByteArrayObj(newObjData, newObjSize); + newObjObj = Tcl_NewByteArrayObj(bytes, dataSize); // copy the object to avoid increased memory usage Tcl_IncrRefCount(newObjObj); - Tcl_IncrRefCount(data); - Tcl_DecrRefCount(data); p->asyncPage[asyncIdx].pageIdx = idx; p->asyncPage[asyncIdx].pageContents = newObjObj; - CookfsRunAsyncCompressCommand(p, p->asyncCommandProcess, idx, data); + CookfsRunAsyncCompressCommand(p, p->asyncCommandProcess, idx, newObjObj); return 1; } else { return 0; @@ -729,7 +724,7 @@ int Cookfs_AsyncCompressWait(Cookfs_Pages *p, int require) { } Tcl_IncrRefCount(resObj); - size = Cookfs_WritePage(p, idx, p->asyncPage[0].pageContents, resObj); + size = Cookfs_WritePageObj(p, idx, p->asyncPage[0].pageContents, resObj); p->dataPagesSize[idx] = size; Tcl_DecrRefCount(resObj); Tcl_DecrRefCount(result); @@ -1111,7 +1106,7 @@ static Tcl_Obj *CookfsReadPageZlib(Cookfs_Pages *p, int size) { *---------------------------------------------------------------------- */ -static int CookfsWritePageZlib(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { +static int CookfsWritePageZlib(Cookfs_Pages *p, unsigned char *bytes, int origSize) { #ifdef USE_ZLIB_TCL86 int size = origSize; /* use Tcl 8.6 API for zlib compression */ @@ -1123,6 +1118,7 @@ static int CookfsWritePageZlib(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { return -1; } + Tcl_Obj *data = Tcl_NewByteArrayObj(bytes, origSize); Tcl_IncrRefCount(data); if (Tcl_ZlibStreamPut(zshandle, data, TCL_ZLIB_FINALIZE) != TCL_OK) { Tcl_DecrRefCount(data); @@ -1157,6 +1153,7 @@ static int CookfsWritePageZlib(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { Tcl_Obj *prevResult; Tcl_Obj *compressed; + Tcl_Obj data = Tcl_NewByteArrayObj(bytes, origSize); Tcl_IncrRefCount(data); p->zipCmdCompress[p->zipCmdOffset] = data; prevResult = Tcl_GetObjResult(p->interp); @@ -1314,7 +1311,7 @@ XzOutStreamWrite(const struct ISeqOutStream_ *p, const void *buf, size_t size) *---------------------------------------------------------------------- */ -static int CookfsWritePageXz(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { +static int CookfsWritePageXz(Cookfs_Pages *p, unsigned char *bytes, int origSize) { #ifdef COOKFS_USEXZ XzInStream istream; XzOutStream ostream; @@ -1331,11 +1328,7 @@ static int CookfsWritePageXz(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { CookfsLog(printf("CookfsWritePageXz: start. Want to write %d bytes.", origSize)); - istream.source = Tcl_GetByteArrayFromObj(data, NULL); - if (istream.source == NULL) { - CookfsLog(printf("CookfsWritePageXz: Tcl_GetByteArrayFromObj failed")); - return -1; - } + istream.source = bytes; bufObj = Tcl_NewByteArrayObj(NULL, 0); Tcl_IncrRefCount(bufObj); @@ -1521,19 +1514,14 @@ static Tcl_Obj *CookfsReadPageBz2(Cookfs_Pages *p, int size) { *---------------------------------------------------------------------- */ -static int CookfsWritePageBz2(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { +static int CookfsWritePageBz2(Cookfs_Pages *p, unsigned char *bytes, int origSize) { #ifdef COOKFS_USEBZ2 int size; unsigned char *source; unsigned char *dest; Tcl_Obj *destObj; - source = Tcl_GetByteArrayFromObj(data, NULL); - if (source == NULL) { - CookfsLog(printf("Cookfs_WritePage: Tcl_GetByteArrayFromObj failed")) - return -1; - } - + source = bytes; destObj = Tcl_NewByteArrayObj(NULL, 0); size = origSize * 2 + 1024; Tcl_SetByteArrayLength(destObj, size + 4); @@ -1561,7 +1549,7 @@ static int CookfsWritePageBz2(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { return size; #else UNUSED(p); - UNUSED(data); + UNUSED(bytes); UNUSED(origSize); return -1; #endif @@ -1644,7 +1632,7 @@ static Tcl_Obj *CookfsReadPageCustom(Cookfs_Pages *p, int size) { *---------------------------------------------------------------------- */ -static int CookfsWritePageCustom(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { +static int CookfsWritePageCustom(Cookfs_Pages *p, unsigned char *bytes, int origSize) { int size = origSize; /* use vfs::zip command for compression */ Tcl_Obj *prevResult; @@ -1654,6 +1642,7 @@ static int CookfsWritePageCustom(Cookfs_Pages *p, Tcl_Obj *data, int origSize) { return -1; } + Tcl_Obj *data = Tcl_NewByteArrayObj(bytes, origSize); Tcl_IncrRefCount(data); p->compressCommandPtr[p->compressCommandLen - 1] = data; prevResult = Tcl_GetObjResult(p->interp); diff --git a/generic/pagesCompr.h b/generic/pagesCompr.h index f42fe61..2a7dd55 100644 --- a/generic/pagesCompr.h +++ b/generic/pagesCompr.h @@ -42,10 +42,11 @@ void Cookfs_PagesFiniCompr(Cookfs_Pages *rc); int Cookfs_SetCompressCommands(Cookfs_Pages *p, Tcl_Obj *compressCommand, Tcl_Obj *decompressCommand, Tcl_Obj *asyncCompressCommand, Tcl_Obj *asyncDecompressCommand); void Cookfs_SeekToPage(Cookfs_Pages *p, int idx); -int Cookfs_WritePage(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compressedData); +int Cookfs_WritePage(Cookfs_Pages *p, int idx, unsigned char *bytes, int origSize, Tcl_Obj *compressedData); +int Cookfs_WritePageObj(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compressedData); Tcl_Obj *Cookfs_ReadPage(Cookfs_Pages *p, int idx, int size, int decompress, int compressionType); Tcl_Obj *Cookfs_AsyncPageGet(Cookfs_Pages *p, int idx); -int Cookfs_AsyncPageAdd(Cookfs_Pages *p, int idx, Tcl_Obj *data); +int Cookfs_AsyncPageAdd(Cookfs_Pages *p, int idx, unsigned char *bytes, int dataSize); void Cookfs_AsyncDecompressWaitIfLoading(Cookfs_Pages *p, int idx); int Cookfs_AsyncCompressWait(Cookfs_Pages *p, int require); void Cookfs_AsyncCompressFinalize(Cookfs_Pages *p); diff --git a/generic/vfs.c b/generic/vfs.c index eb0d2b6..2664b9e 100644 --- a/generic/vfs.c +++ b/generic/vfs.c @@ -90,8 +90,6 @@ int Cookfs_VfsFini(Tcl_Interp *interp, Cookfs_Vfs *vfs, return TCL_OK; } - int ret; - // Let's purge writer first CookfsLog(printf("Cookfs_VfsFini: purge writer...")); if (Cookfs_WriterPurge(vfs->writer) != TCL_OK) { @@ -113,21 +111,11 @@ int Cookfs_VfsFini(Tcl_Interp *interp, Cookfs_Vfs *vfs, CookfsLog(printf("Cookfs_VfsFini: pages readonly: false")); } - int writetomemory; - CookfsLog(printf("Cookfs_VfsFini: check writetomemory" - " in writer...")); - ret = Cookfs_WriterGetWritetomemory(vfs->writer, &writetomemory); - if (ret != TCL_OK) { - CookfsLog(printf("Cookfs_VfsFini: failed to get integer result" - " from writer")); - Tcl_SetObjResult(interp, Tcl_NewStringObj("unable to get writetomemory" - " status from writer", -1)); - return TCL_ERROR; - } - CookfsLog(printf("Cookfs_VfsFini: writer writetomemory: %d", - writetomemory)); - if (writetomemory) { + if (Cookfs_WriterGetWritetomemory(vfs->writer)) { + CookfsLog(printf("Cookfs_VfsFini: writer writetomemory: true")); goto skipSavingIndex; + } else { + CookfsLog(printf("Cookfs_VfsFini: writer writetomemory: false")); } // If we are here, then we need to store index diff --git a/generic/vfsCmd.c b/generic/vfsCmd.c index 845d842..7c5c656 100644 --- a/generic/vfsCmd.c +++ b/generic/vfsCmd.c @@ -11,11 +11,11 @@ typedef int (Cookfs_MountHandleCommandProc)(Cookfs_Vfs *vfs, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); - // Forward declarations static Tcl_ObjCmdProc CookfsMountCmd; static Tcl_ObjCmdProc CookfsUnmountCmd; static Tcl_ObjCmdProc CookfsMountHandleCmd; +static Tcl_CmdDeleteProc CookfsMountHandleCmdDeleteProc; int Cookfs_InitVfsMountCmd(Tcl_Interp *interp) { @@ -231,6 +231,14 @@ static int CookfsMountCmd(ClientData clientData, Tcl_Interp *interp, return TCL_ERROR; } + if (smallfilesize > pagesize) { + CookfsLog(printf("CookfsMountCmd: ERROR: smallfilesize [%ld]" + " > pagesize [%ld]", smallfilesize, pagesize)); + Tcl_SetObjResult(interp, Tcl_NewStringObj("smallfilesize cannot be" + " larger than pagesize", -1)); + return TCL_ERROR; + } + Cookfs_Vfs *vfs = NULL; Cookfs_Pages *pages = NULL; Cookfs_Fsindex *index = NULL; @@ -536,7 +544,7 @@ static int CookfsMountCmd(ClientData clientData, Tcl_Interp *interp, if (!nocommand) { CookfsLog(printf("CookfsMountCmd: creating vfs command handler...")); vfs->commandToken = Tcl_CreateObjCommand(interp, cmd, - CookfsMountHandleCmd, vfs, NULL); + CookfsMountHandleCmd, vfs, CookfsMountHandleCmdDeleteProc); } else { CookfsLog(printf("CookfsMountCmd: no need to create vfs" " command handler")); @@ -712,8 +720,15 @@ static int CookfsUnmountCmd(ClientData clientData, Tcl_Interp *interp, return TCL_OK; } + +static void CookfsMountHandleCmdDeleteProc(ClientData clientData) { + Cookfs_Vfs *vfs = (Cookfs_Vfs *)clientData; + vfs->commandToken = NULL; +} + static Cookfs_MountHandleCommandProc CookfsMountHandleCommandGetpages; static Cookfs_MountHandleCommandProc CookfsMountHandleCommandGetindex; +static Cookfs_MountHandleCommandProc CookfsMountHandleCommandGetwriter; static Cookfs_MountHandleCommandProc CookfsMountHandleCommandGetmetadata; static Cookfs_MountHandleCommandProc CookfsMountHandleCommandSetmetadata; static Cookfs_MountHandleCommandProc CookfsMountHandleCommandAside; @@ -721,6 +736,7 @@ static Cookfs_MountHandleCommandProc CookfsMountHandleCommandWritetomemory; static Cookfs_MountHandleCommandProc CookfsMountHandleCommandFilesize; static Cookfs_MountHandleCommandProc CookfsMountHandleCommandSmallfilebuffersize; static Cookfs_MountHandleCommandProc CookfsMountHandleCommandCompression; +static Cookfs_MountHandleCommandProc CookfsMountHandleCommandWritefiles; static int CookfsMountHandleCmd(ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) @@ -729,13 +745,15 @@ static int CookfsMountHandleCmd(ClientData clientData, Tcl_Interp *interp, Cookfs_Vfs *vfs = (Cookfs_Vfs *)clientData; static const char *const commands[] = { - "getpages", "getindex", "getmetadata", "setmetadata", "aside", - "writetomemory", "filesize", "smallfilebuffersize", "compression", + "getpages", "getindex", "getwriter", "getmetadata", "setmetadata", + "aside", "writetomemory", "filesize", "smallfilebuffersize", + "compression", "writeFiles", NULL }; enum commands { - cmdGetpages, cmdGetindex, cmdGetmetadata, cmdSetmetadata, cmdAside, - cmdWritetomemory, cmdFilesize, cmdSmallfilebuffersize, cmdCompression + cmdGetpages, cmdGetindex, cmdGetwriter, cmdGetmetadata, cmdSetmetadata, + cmdAside, cmdWritetomemory, cmdFilesize, cmdSmallfilebuffersize, + cmdCompression, cmdWritefiles }; if (objc < 2) { @@ -755,6 +773,8 @@ static int CookfsMountHandleCmd(ClientData clientData, Tcl_Interp *interp, return CookfsMountHandleCommandGetpages(vfs, interp, objc, objv); case cmdGetindex: return CookfsMountHandleCommandGetindex(vfs, interp, objc, objv); + case cmdGetwriter: + return CookfsMountHandleCommandGetwriter(vfs, interp, objc, objv); case cmdGetmetadata: return CookfsMountHandleCommandGetmetadata(vfs, interp, objc, objv); case cmdSetmetadata: @@ -769,6 +789,8 @@ static int CookfsMountHandleCmd(ClientData clientData, Tcl_Interp *interp, return CookfsMountHandleCommandSmallfilebuffersize(vfs, interp, objc, objv); case cmdCompression: return CookfsMountHandleCommandCompression(vfs, interp, objc, objv); + case cmdWritefiles: + return CookfsMountHandleCommandWritefiles(vfs, interp, objc, objv); } return TCL_OK; @@ -796,6 +818,17 @@ static int CookfsMountHandleCommandGetindex(Cookfs_Vfs *vfs, Tcl_Interp *interp, return TCL_OK; } +static int CookfsMountHandleCommandGetwriter(Cookfs_Vfs *vfs, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]) +{ + if (objc != 2) { + Tcl_WrongNumArgs(interp, 2, objv, NULL); + return TCL_ERROR; + } + Tcl_SetObjResult(interp, CookfsGetWriterObjectCmd(interp, vfs->writer)); + return TCL_OK; +} + static int CookfsMountHandleCommandGetmetadata(Cookfs_Vfs *vfs, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) { @@ -824,16 +857,15 @@ static int CookfsMountHandleCommandAside(Cookfs_Vfs *vfs, Tcl_Interp *interp, return TCL_ERROR; } - CookfsLog(printf("CookfsMountHandleCommandAside: check writetomemory in" - " writer...")); - int writetomemory = 0; - Cookfs_WriterGetWritetomemory(vfs->writer, &writetomemory); - if (writetomemory) { - CookfsLog(printf("CookfsMountHandleCommandAside: ERROR: rite to memory" + if (Cookfs_WriterGetWritetomemory(vfs->writer)) { + CookfsLog(printf("CookfsMountHandleCommandAside: ERROR: write to memory" " option enabled")); Tcl_SetObjResult(interp, Tcl_NewStringObj("Write to memory option" " enabled; not creating add-aside archive", -1)); return TCL_ERROR; + } else { + CookfsLog(printf("CookfsMountHandleCommandAside: writer" + " writetomemory: false")); } CookfsLog(printf("CookfsMountHandleCommandAside: purge writer...")); @@ -865,11 +897,9 @@ static int CookfsMountHandleCommandWritetomemory(Cookfs_Vfs *vfs, Tcl_WrongNumArgs(interp, 2, objv, NULL); return TCL_ERROR; } - int ret = Cookfs_WriterSetWritetomemory(vfs->writer, 1); - if (ret == TCL_OK) { - Cookfs_VfsSetReadonly(vfs, 0); - } - return ret; + Cookfs_WriterSetWritetomemory(vfs->writer, 1); + Cookfs_VfsSetReadonly(vfs, 0); + return TCL_OK; } static int CookfsMountHandleCommandFilesize(Cookfs_Vfs *vfs, @@ -891,12 +921,9 @@ static int CookfsMountHandleCommandSmallfilebuffersize(Cookfs_Vfs *vfs, Tcl_WrongNumArgs(interp, 2, objv, NULL); return TCL_ERROR; } - Tcl_WideInt size; - int ret = Cookfs_WriterGetSmallfilebuffersize(vfs->writer, &size); - if (ret == TCL_OK) { - Tcl_SetObjResult(interp, Tcl_NewWideIntObj(size)); - } - return ret; + Tcl_WideInt size = Cookfs_WriterGetSmallfilebuffersize(vfs->writer); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(size)); + return TCL_OK; } static int CookfsMountHandleCommandCompression(Cookfs_Vfs *vfs, @@ -917,3 +944,9 @@ static int CookfsMountHandleCommandCompression(Cookfs_Vfs *vfs, return CookfsPagesCmdCompression(vfs->pages, interp, objc, objv); } + +static int CookfsMountHandleCommandWritefiles(Cookfs_Vfs *vfs, + Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) +{ + return CookfsWriterHandleCommandWrite(vfs->writer, interp, objc, objv); +} diff --git a/generic/vfsDriver.c b/generic/vfsDriver.c index 73afd7c..fa07e6c 100644 --- a/generic/vfsDriver.c +++ b/generic/vfsDriver.c @@ -348,7 +348,7 @@ static Tcl_Channel CookfsOpenFileChannel(Tcl_Interp *interp, Tcl_Obj *pathPtr, CookfsLog(printf("CookfsOpenFileChannel: the file is in" " a pending state, open it using writerchannel")); channel = Cookfs_CreateWriterchannel(pages, index, - vfs->writer->commandObj, NULL, 0, entry, interp); + vfs->writer, NULL, 0, entry, interp); } else { CookfsLog(printf("CookfsOpenFileChannel: the file is NOT in" " a pending state, open it using readerchannel")); @@ -412,9 +412,9 @@ static Tcl_Channel CookfsOpenFileChannel(Tcl_Interp *interp, Tcl_Obj *pathPtr, entry = NULL; } - channel = Cookfs_CreateWriterchannel(pages, index, - vfs->writer->commandObj, internalRep->relativePathObj, - internalRep->relativePathObjLen, entry, interp); + channel = Cookfs_CreateWriterchannel(pages, index, vfs->writer, + internalRep->relativePathObj, internalRep->relativePathObjLen, entry, + interp); if (channel == NULL) { CookfsLog(printf("CookfsOpenFileChannel: got NULL from" diff --git a/generic/writer.c b/generic/writer.c index 2847e38..5ed76b8 100644 --- a/generic/writer.c +++ b/generic/writer.c @@ -8,6 +8,60 @@ #include "cookfs.h" +static void Cookfs_WriterSetLastErrorObj(Cookfs_Writer *w, Tcl_Obj *msg) { + if (w->lastErrorObj != NULL) { + Tcl_DecrRefCount(w->lastErrorObj); + } + if (msg == NULL) { + w->lastErrorObj = NULL; + } else { + w->lastErrorObj = msg; + Tcl_IncrRefCount(w->lastErrorObj); + } +} + +static void Cookfs_WriterSetLastError(Cookfs_Writer *w, const char *msg) { + Cookfs_WriterSetLastErrorObj(w, Tcl_NewStringObj(msg, -1)); +} + +Tcl_Obj *Cookfs_WriterGetLastError(Cookfs_Writer *w) { + return w->lastErrorObj; +} + +static Cookfs_WriterBuffer *Cookfs_WriterWriterBufferAlloc(Tcl_Obj *pathObj, + Tcl_WideInt mtime) +{ + Cookfs_WriterBuffer *wb = ckalloc(sizeof(Cookfs_WriterBuffer)); + if (wb != NULL) { + wb->buffer = NULL; + wb->bufferSize = 0; + wb->mtime = mtime; + wb->pathObj = pathObj; + Tcl_IncrRefCount(pathObj); + wb->entry = NULL; + wb->sortKey = NULL; + wb->next = NULL; + } + CookfsLog(printf("Cookfs_WriterWriterBufferAlloc: buffer [%p]", + (void *)wb)); + return wb; +} + +static void Cookfs_WriterWriterBufferFree(Cookfs_WriterBuffer *wb) { + CookfsLog(printf("Cookfs_WriterWriterBufferFree: buffer [%p]", + (void *)wb)); + if (wb->buffer != NULL) { + ckfree(wb->buffer); + } + if (wb->pathObj != NULL) { + Tcl_DecrRefCount(wb->pathObj); + } + if (wb->sortKey != NULL) { + Tcl_DecrRefCount(wb->sortKey); + } + ckfree(wb); +} + Cookfs_Writer *Cookfs_WriterInit(Tcl_Interp* interp, Cookfs_Pages *pages, Cookfs_Fsindex *index, Tcl_WideInt smallfilebuffer, Tcl_WideInt smallfilesize, @@ -19,197 +73,990 @@ Cookfs_Writer *Cookfs_WriterInit(Tcl_Interp* interp, (void *)interp, (void *)pages, (void *)index, smallfilebuffer, smallfilesize, pagesize, writetomemory)); - Cookfs_Writer *w = NULL; - if (interp == NULL || pages == NULL || index == NULL) { CookfsLog(printf("Cookfs_WriterInit: failed, something is NULL")); return NULL; } - // Initialize writer - - Tcl_Obj *objv[13]; - objv[0] = Tcl_NewStringObj("::cookfs::tcl::writer", -1); - objv[1] = Tcl_NewStringObj("-pages", -1); - objv[2] = CookfsGetPagesObjectCmd(interp, pages); - objv[3] = Tcl_NewStringObj("-index", -1); - objv[4] = CookfsGetFsindexObjectCmd(interp, index); - objv[5] = Tcl_NewStringObj("-smallfilebuffersize", -1); - objv[6] = Tcl_NewWideIntObj(smallfilebuffer); - objv[7] = Tcl_NewStringObj("-smallfilesize", -1); - objv[8] = Tcl_NewWideIntObj(smallfilesize); - objv[9] = Tcl_NewStringObj("-pagesize", -1); - objv[10] = Tcl_NewWideIntObj(pagesize); - objv[11] = Tcl_NewStringObj("-writetomemory", -1); - objv[12] = Tcl_NewIntObj(writetomemory); - - Tcl_Obj *cmd = Tcl_NewListObj(13, objv); - Tcl_IncrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterInit: init Tcl writer...")); - int ret = Tcl_EvalObjEx(interp, cmd, TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - Tcl_DecrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterInit: Tcl writer: %d", ret)); - if (ret != TCL_OK) { + // Double check that smallfilesize is less than pagesize + if (smallfilesize > pagesize) { + CookfsLog(printf("Cookfs_WriterInit: failed," + " smallfilesize > pagesize")); return NULL; } - w = (Cookfs_Writer *)ckalloc(sizeof(Cookfs_Writer)); + Cookfs_Writer *w = (Cookfs_Writer *)ckalloc(sizeof(Cookfs_Writer)); if (w == NULL) { CookfsLog(printf("Cookfs_WriterInit: failed, OOM")); return NULL; } + w->lastErrorObj = NULL; + w->commandToken = NULL; w->interp = interp; + w->fatalError = 0; + w->isDead = 0; + + w->pages = pages; + w->index = index; + + w->isWriteToMemory = writetomemory; + w->smallFileSize = smallfilesize; + w->maxBufferSize = smallfilebuffer; + w->pageSize = pagesize; - // Use interp result from above initialization as a writer command - w->commandObj = Tcl_GetObjResult(interp); - Tcl_IncrRefCount(w->commandObj); - CookfsLog(printf("Cookfs_WriterInit: created writer [%s]", - Tcl_GetString(w->commandObj))); + w->bufferFirst = NULL; + w->bufferLast = NULL; + w->bufferSize = 0; + w->bufferCount = 0; + + CookfsLog(printf("Cookfs_WriterInit: ok [%p]", (void *)w)); return w; } void Cookfs_WriterFini(Cookfs_Writer *w) { + if (w == NULL) { CookfsLog(printf("Cookfs_WriterFini: ERROR: writer is NULL")); return; } - Tcl_Obj *objv[2]; - objv[0] = w->commandObj; - objv[1] = Tcl_NewStringObj("delete", -1); - Tcl_Obj *cmd = Tcl_NewListObj(2, objv); - Tcl_IncrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterFini: free Tcl writer...")); - // Ignore result here - int ret = Tcl_EvalObjEx(w->interp, cmd, TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - if (ret != TCL_OK) { - Tcl_ResetResult(w->interp); - } - Tcl_DecrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterFini: Tcl writer: %d", ret)); + + if (w->isDead) { + return; + } + w->isDead = 1; + + CookfsLog(printf("Cookfs_WriterFini: enter [%p]", (void *)w)); + + if (w->commandToken != NULL) { + CookfsLog(printf("Cookfs_WriterFini: Cleaning tcl command")); + Tcl_DeleteCommandFromToken(w->interp, w->commandToken); + } else { + CookfsLog(printf("Cookfs_WriterFini: No tcl command")); + } + + CookfsLog(printf("Cookfs_WriterFini: free buffers")); + Cookfs_WriterBuffer *wb = w->bufferFirst; + while (wb != NULL) { + Cookfs_WriterBuffer *next = wb->next; + Cookfs_WriterWriterBufferFree(wb); + wb = next; + } + + CookfsLog(printf("Cookfs_WriterFini: free all")); + + if (w->lastErrorObj != NULL) { + Tcl_DecrRefCount(w->lastErrorObj); + } + + ckfree(w); + return; } -int Cookfs_WriterPurge(Cookfs_Writer *w) { - if (w == NULL) { - CookfsLog(printf("Cookfs_WriterPurge: ERROR: writer is NULL")); - return 0; - } - Tcl_Obj *objv[2]; - objv[0] = w->commandObj; - objv[1] = Tcl_NewStringObj("purge", -1); - Tcl_Obj *cmd = Tcl_NewListObj(2, objv); - Tcl_IncrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterPurge: call Tcl writer...")); - int ret = Tcl_EvalObjEx(w->interp, cmd, TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - Tcl_DecrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterPurge: Tcl writer: %d", ret)); - return ret; -} - -int Cookfs_WriterGetWritetomemory(Cookfs_Writer *w, int *status) { - // Set status before anything else to return known value - // in case of error and avoid undefined behaviour - *status = 0; - if (w == NULL) { - CookfsLog(printf("Cookfs_WriterGetWritetomemory: ERROR: writer is" - " NULL")); +int Cookfs_WriterAddBufferToSmallFiles(Cookfs_Writer *w, Tcl_Obj *pathObj, + Tcl_WideInt mtime, void *buffer, Tcl_WideInt bufferSize) +{ + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: add buf [%p]," + " size: %ld", buffer, bufferSize)); + + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: alloc" + " WriterBuffer")); + Cookfs_WriterBuffer *wb = Cookfs_WriterWriterBufferAlloc(pathObj, mtime); + if (wb == NULL) { + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: failed to" + " alloc")); + Cookfs_WriterSetLastError(w, "failed to alloc WriterBuffer"); return TCL_ERROR; } - Tcl_Obj *objv[2]; - objv[0] = w->commandObj; - objv[1] = Tcl_NewStringObj("writetomemory", -1); - Tcl_Obj *cmd = Tcl_NewListObj(2, objv); - Tcl_IncrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterGetWritetomemory: call Tcl writer...")); - int ret = Tcl_EvalObjEx(w->interp, cmd, TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - Tcl_DecrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterGetWritetomemory: Tcl writer: %d", ret)); - if (ret == TCL_OK) { - if (Tcl_GetIntFromObj(w->interp, Tcl_GetObjResult(w->interp), - status) != TCL_OK) - { - CookfsLog(printf("Cookfs_WriterGetWritetomemory: unable to get" - " an integer from writer result")); - ret = TCL_ERROR; - } else { - CookfsLog(printf("Cookfs_WriterGetWritetomemory: got value from" - " writer: %d", *status)); + + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: create an entry" + " in fsindex...")); + wb->entry = Cookfs_FsindexSet(w->index, pathObj, 1); + if (wb->entry == NULL) { + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: failed to create" + " the entry")); + Cookfs_WriterSetLastError(w, "unable to create entry"); + Cookfs_WriterWriterBufferFree(wb); + return TCL_ERROR; + } + + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: set fsindex" + " entry values")); + wb->entry->data.fileInfo.fileBlockOffsetSize[0] = -w->bufferCount - 1; + wb->entry->data.fileInfo.fileBlockOffsetSize[1] = 0; + wb->entry->data.fileInfo.fileBlockOffsetSize[2] = bufferSize; + wb->entry->data.fileInfo.fileSize = bufferSize; + wb->entry->fileTime = mtime; + + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: set WritterBuffer" + " values and add to the chain")); + wb->buffer = buffer; + wb->bufferSize = bufferSize; + + if (w->bufferFirst == NULL) { + w->bufferFirst = wb; + } else { + w->bufferLast->next = wb; + } + + w->bufferLast = wb; + + w->bufferCount++; + w->bufferSize += bufferSize; + + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: currently have" + " %d buffers, total size: %ld", w->bufferCount, w->bufferSize)); + CookfsLog(printf("Cookfs_WriterAddBufferToSmallFiles: ok")); + return TCL_OK; +} + +static Tcl_WideInt Cookfs_WriterReadChannel(char *buffer, + Tcl_WideInt bufferSize, Tcl_Channel channel) +{ + + CookfsLog(printf("Cookfs_WriterReadChannel: want to read %ld bytes from" + " channel %p", bufferSize, (void *)channel)); + + Tcl_WideInt readSize = 0; + + while (readSize < bufferSize) { + if (Tcl_Eof(channel)) { + CookfsLog(printf("Cookfs_WriterReadChannel: EOF reached")); + break; } + CookfsLog(printf("Cookfs_WriterReadChannel: read bytes from" + " the channel")); + readSize += Tcl_Read(channel, buffer + readSize, + bufferSize - readSize); + CookfsLog(printf("Cookfs_WriterReadChannel: got %ld bytes from" + " the channel", readSize)); } - return ret; + CookfsLog(printf("Cookfs_WriterReadChannel: return %ld bytes from" + " the channel", readSize)); + + return readSize; + } -int Cookfs_WriterSetWritetomemory(Cookfs_Writer *w, int status) { - if (w == NULL) { - CookfsLog(printf("Cookfs_WriterSetWritetomemory: ERROR: writer is" - " NULL")); +#define DATA_FILE (Tcl_Obj *)data +#define DATA_CHANNEL (Tcl_Channel)data +#define DATA_OBJECT (Tcl_Obj *)data + +int Cookfs_WriterAddFile(Cookfs_Writer *w, Tcl_Obj *pathObj, + Cookfs_WriterDataSource dataType, void *data, Tcl_WideInt dataSize) +{ + CookfsLog(printf("Cookfs_WriterAddFile: enter [%p] [%s] size: %ld", data, + (dataType == COOKFS_WRITER_SOURCE_BUFFER ? "buffer" : + (dataType == COOKFS_WRITER_SOURCE_FILE ? "file" : + (dataType == COOKFS_WRITER_SOURCE_CHANNEL ? "channel" : + (dataType == COOKFS_WRITER_SOURCE_OBJECT ? "object" : "unknown")))), + dataSize)); + + // Check if a fatal error has occurred previously + if (w->fatalError) { + CookfsLog(printf("Cookfs_WriterAddFile: ERROR: writer in a fatal" + " error state: [%s]", + Tcl_GetString(Cookfs_WriterGetLastError(w)))); return TCL_ERROR; } - Tcl_Obj *objv[3]; - objv[0] = w->commandObj; - objv[1] = Tcl_NewStringObj("writetomemory", -1); - objv[2] = Tcl_NewIntObj(status); - Tcl_Obj *cmd = Tcl_NewListObj(3, objv); - Tcl_IncrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterSetWritetomemory: call Tcl writer with" - " arg [%d]...", status)); - int ret = Tcl_EvalObjEx(w->interp, cmd, TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - Tcl_DecrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterSetWritetomemory: Tcl writer: %d", ret)); - return ret; -} - -int Cookfs_WriterGetSmallfilebuffersize(Cookfs_Writer *w, Tcl_WideInt *size) { - // Set size before anything else to return known value - // in case of error and avoid undefined behaviour - *size = 0; - if (w == NULL) { - CookfsLog(printf("Cookfs_WriterGetSmallfilebuffersize: ERROR:" - " writer is NULL")); - return TCL_ERROR; + + int result = TCL_OK; + void *readBuffer = NULL; + Tcl_WideInt mtime = -1; + Tcl_DString chanTranslation, chanEncoding; + Cookfs_FsindexEntry *entry = NULL; + + switch (dataType) { + + case COOKFS_WRITER_SOURCE_BUFFER: + + // No need to do anything. It is here to simply avoid the compiler warning. + + break; + + case COOKFS_WRITER_SOURCE_FILE: + + CookfsLog(printf("Cookfs_WriterAddFile: alloc statbuf")); + + Tcl_StatBuf *sb = Tcl_AllocStatBuf(); + if (sb == NULL) { + Cookfs_WriterSetLastError(w, "could not alloc statbuf"); + return TCL_ERROR; + } + + CookfsLog(printf("Cookfs_WriterAddFile: get file stat for [%s]", + Tcl_GetString(DATA_FILE))); + if (Tcl_FSStat(DATA_FILE, sb) != TCL_OK) { + CookfsLog(printf("Cookfs_WriterAddFile: failed, return error")); + ckfree(sb); + Cookfs_WriterSetLastError(w, "could get stat for the file"); + return TCL_ERROR; + } + + if (dataSize < 0) { + dataSize = Tcl_GetSizeFromStat(sb); + CookfsLog(printf("Cookfs_WriterAddFile: got file size: %ld", + dataSize)); + } else { + CookfsLog(printf("Cookfs_WriterAddFile: use specified size")); + } + + mtime = Tcl_GetModificationTimeFromStat(sb); + CookfsLog(printf("Cookfs_WriterAddFile: got mtime from the file: %ld", + mtime)); + + ckfree(sb); + + CookfsLog(printf("Cookfs_WriterAddFile: open the file")); + data = (void *)Tcl_FSOpenFileChannel(NULL, DATA_FILE, "rb", 0); + if (data == NULL) { + CookfsLog(printf("Cookfs_WriterAddFile: failed to open the file")); + Cookfs_WriterSetLastError(w, "could not open the file"); + return TCL_ERROR; + } + + // Let's believe that "rb" for file open mode worked + //Tcl_SetChannelOption(NULL, DATA_CHANNEL, "-translation", "binary"); + + break; + + case COOKFS_WRITER_SOURCE_CHANNEL: + + if (dataSize < 0) { + CookfsLog(printf("Cookfs_WriterAddFile: get datasize from" + " the channel")); + Tcl_WideInt pos = Tcl_Tell(DATA_CHANNEL); + dataSize = Tcl_Seek(DATA_CHANNEL, 0, SEEK_END); + Tcl_Seek(DATA_CHANNEL, pos, SEEK_SET); + CookfsLog(printf("Cookfs_WriterAddFile: got data size: %ld", + dataSize)); + } else { + CookfsLog(printf("Cookfs_WriterAddFile: use specified size")); + } + + Tcl_DStringInit(&chanTranslation); + Tcl_DStringInit(&chanEncoding); + Tcl_GetChannelOption(NULL, DATA_CHANNEL, "-encoding", &chanEncoding); + Tcl_GetChannelOption(NULL, DATA_CHANNEL, "-translation", + &chanTranslation); + Tcl_SetChannelOption(NULL, DATA_CHANNEL, "-translation", "binary"); + + break; + + case COOKFS_WRITER_SOURCE_OBJECT: ; // an empty statement + + int length; + data = (void *)Tcl_GetStringFromObj(DATA_OBJECT, &length); + + if (dataSize < 0) { + CookfsLog(printf("Cookfs_WriterAddFile: get datasize from" + " the object")); + dataSize = length; + CookfsLog(printf("Cookfs_WriterAddFile: got data size: %ld", + dataSize)); + } else if (dataSize > length) { + dataSize = length; + CookfsLog(printf("Cookfs_WriterAddFile: WARNING: data size was" + " corrected to %ld avoid overflow", dataSize)); + } else { + CookfsLog(printf("Cookfs_WriterAddFile: use specified size")); + } + + break; + } + + if (mtime == -1) { + Tcl_Time now; + Tcl_GetTime(&now); + mtime = now.sec; + CookfsLog(printf("Cookfs_WriterAddFile: use current time for" + " mtime: %ld", mtime)); + } + + // If the file is empty, then just add it to the index and skip + // everything else + if (dataSize == 0) { + // Create an entry + CookfsLog(printf("Cookfs_WriterAddFile: create an entry" + " in fsindex for empty file with 1 block...")); + entry = Cookfs_FsindexSet(w->index, pathObj, 1); + if (entry == NULL) { + CookfsLog(printf("Cookfs_WriterAddFile: failed to create" + " the entry")); + Cookfs_WriterSetLastError(w, "unable to create entry"); + goto error; + } + // Set entry block information + Cookfs_FsindexUpdateEntryBlock(w->index, entry, 0, -1, 0, 0); + Cookfs_FsindexUpdateEntryFileSize(entry, 0); + // Unset entry to avoid releasing it in the final part of this function + entry = NULL; + goto done; + } + + if (((dataSize <= w->smallFileSize) && (dataSize <= w->pageSize)) || + w->isWriteToMemory) + { + + CookfsLog(printf("Cookfs_WriterAddFile: write file to small file" + " buffer")); + + if (dataType != COOKFS_WRITER_SOURCE_BUFFER) { + + CookfsLog(printf("Cookfs_WriterAddFile: alloc buffer")); + readBuffer = ckalloc(dataSize); + if (readBuffer == NULL) { + CookfsLog(printf("Cookfs_WriterAddFile: failed to alloc" + " buffer")); + Cookfs_WriterSetLastError(w, "failed to alloc buffer"); + goto error; + } + + if (dataType == COOKFS_WRITER_SOURCE_OBJECT) { + CookfsLog(printf("Cookfs_WriterAddFile: copy object's bytes" + " to the buffer")); + memcpy(readBuffer, data, dataSize); + } else { + + CookfsLog(printf("Cookfs_WriterAddFile: read bytes from" + " the channel")); + Tcl_WideInt readSize = Cookfs_WriterReadChannel( + (char *)readBuffer, dataSize, DATA_CHANNEL); + + if (readSize < dataSize) { + CookfsLog(printf("Cookfs_WriterAddFile: ERROR: got less" + " bytes than required")); + Cookfs_WriterSetLastError(w, "could not read specified" + " amount of bytes from the file"); + goto error; + } + + } + + } + + CookfsLog(printf("Cookfs_WriterAddFile: add to small file buf...")); + int ret = Cookfs_WriterAddBufferToSmallFiles(w, pathObj, mtime, + (dataType == COOKFS_WRITER_SOURCE_BUFFER ? data : readBuffer), + dataSize); + if (ret != TCL_OK) { + goto error; + } + + // readBuffer now owned by small file buffer. Set it to NULL to + // avoid releasing it. + readBuffer = NULL; + + if (!w->isWriteToMemory && (w->bufferSize >= w->maxBufferSize)) { + CookfsLog(printf("Cookfs_WriterAddFile: need to purge")); + result = Cookfs_WriterPurge(w); + } else { + CookfsLog(printf("Cookfs_WriterAddFile: no need to purge")); + } + + } else { + + CookfsLog(printf("Cookfs_WriterAddFile: write big file")); + + if (dataType == COOKFS_WRITER_SOURCE_CHANNEL || + dataType == COOKFS_WRITER_SOURCE_FILE) + { + CookfsLog(printf("Cookfs_WriterAddFile: alloc page buffer")); + readBuffer = ckalloc(w->pageSize); + if (readBuffer == NULL) { + CookfsLog(printf("Cookfs_WriterAddFile: failed to alloc")); + Cookfs_WriterSetLastError(w, "failed to alloc buffer"); + goto error; + } + } + + // Calculate number of blocks + int numBlocks = (dataSize / w->pageSize) + 1; + + // Create an entry + CookfsLog(printf("Cookfs_WriterAddFile: create an entry" + " in fsindex with %d blocks...", numBlocks)); + entry = Cookfs_FsindexSet(w->index, pathObj, numBlocks); + if (entry == NULL) { + CookfsLog(printf("Cookfs_WriterAddFile: failed to create" + " the entry")); + Cookfs_WriterSetLastError(w, "unable to create entry"); + goto error; + } + Cookfs_FsindexUpdateEntryFileSize(entry, dataSize); + entry->fileTime = mtime; + + Tcl_WideInt currentOffset = 0; + int currentBlockNumber = 0; + Tcl_WideInt bytesLeft = dataSize; + + while (bytesLeft) { + + Tcl_WideInt bytesToWrite = (bytesLeft > w->pageSize ? + w->pageSize : bytesLeft); + + CookfsLog(printf("Cookfs_WriterAddFile: want to write %ld bytes...", + bytesToWrite)); + + if (dataType == COOKFS_WRITER_SOURCE_CHANNEL || + dataType == COOKFS_WRITER_SOURCE_FILE) + { + CookfsLog(printf("Cookfs_WriterAddFile: read bytes from" + " the channel")); + Tcl_WideInt readSize = Cookfs_WriterReadChannel( + (char *)readBuffer, bytesToWrite, DATA_CHANNEL); + + if (readSize < bytesToWrite) { + CookfsLog(printf("Cookfs_WriterAddFile: ERROR: got less" + " bytes than required")); + Cookfs_WriterSetLastError(w, "could not read specified" + " amount of bytes from the file"); + goto error; + } + } + + // Try to add page + CookfsLog(printf("Cookfs_WriterPurge: add page...")); + int block = Cookfs_PageAddRaw(w->pages, + (readBuffer == NULL ? + (char *)data + currentOffset : readBuffer), bytesToWrite); + CookfsLog(printf("Cookfs_WriterPurge: got block index: %d", block)); + + if (block < 0) { + Tcl_Obj *err = Cookfs_PagesGetLastError(w->pages); + Cookfs_WriterSetLastErrorObj(w, Tcl_ObjPrintf("error while adding" + " page: %s", + (err == NULL ? "unknown error" : Tcl_GetString(err)))); + w->fatalError = 1; + goto error; + } + + CookfsLog(printf("Cookfs_WriterPurge: update block number %d" + " of fsindex entry...", currentBlockNumber)); + Cookfs_FsindexUpdateEntryBlock(w->index, entry, currentBlockNumber, + block, 0, bytesToWrite); + + currentBlockNumber++; + currentOffset += bytesToWrite; + bytesLeft -= bytesToWrite; + + } + + // Unset entry to avoid releasing it at the end + entry = NULL; + + } + + goto done; + +error: + + result = TCL_ERROR; + +done: + + if (readBuffer != NULL) { + CookfsLog(printf("Cookfs_WriterAddFile: free readBuffer")); + ckfree(readBuffer); + } + + // Unset entry for the file if an error occurred while adding it + if (entry != NULL) { + CookfsLog(printf("Cookfs_WriterAddFile: unset fsindex entry")); + Cookfs_FsindexUnset(w->index, pathObj); + } + + if (dataType == COOKFS_WRITER_SOURCE_CHANNEL) { + CookfsLog(printf("Cookfs_WriterAddFile: restore chan" + " translation/encoding")); + Tcl_SetChannelOption(NULL, DATA_CHANNEL, "-translation", + Tcl_DStringValue(&chanTranslation)); + Tcl_DStringFree(&chanTranslation); + Tcl_SetChannelOption(NULL, DATA_CHANNEL, "-encoding", + Tcl_DStringValue(&chanEncoding)); + Tcl_DStringFree(&chanEncoding); + } else if (dataType == COOKFS_WRITER_SOURCE_FILE) { + CookfsLog(printf("Cookfs_WriterAddFile: close channel")); + Tcl_Close(NULL, DATA_CHANNEL); } - Tcl_Obj *objv[2]; - objv[0] = w->commandObj; - objv[1] = Tcl_NewStringObj("smallfilebuffersize", -1); - Tcl_Obj *cmd = Tcl_NewListObj(2, objv); - Tcl_IncrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterGetSmallfilebuffersize: call Tcl writer...")); - int ret = Tcl_EvalObjEx(w->interp, cmd, TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - Tcl_DecrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterGetSmallfilebuffersize: Tcl writer: %d", ret)); - if (ret == TCL_OK) { - if (Tcl_GetWideIntFromObj(w->interp, Tcl_GetObjResult(w->interp), - size) != TCL_OK) + + if (result == TCL_ERROR) { + CookfsLog(printf("Cookfs_WriterAddFile: return ERROR")); + } else { + CookfsLog(printf("Cookfs_WriterAddFile: ok")); + } + + return result; + +} + +static int Cookfs_WriterPurgeSortFunc(const void *a, const void *b) { + Cookfs_WriterBuffer *wba = *(Cookfs_WriterBuffer **)a; + Cookfs_WriterBuffer *wbb = *(Cookfs_WriterBuffer **)b; + return strcmp(wba->sortKeyStr, wbb->sortKeyStr); +} + +int Cookfs_WriterPurge(Cookfs_Writer *w) { + + CookfsLog(printf("Cookfs_WriterPurge: enter [%p]", (void *)w)); + if (w->bufferCount == 0) { + CookfsLog(printf("Cookfs_WriterPurge: nothing to purge")); + return TCL_OK; + } + + int result = TCL_OK; + unsigned char *pageBuffer = NULL; + Cookfs_WriterBuffer **sortedWB = NULL; + Cookfs_WriterBuffer *wb; + int i; + + // Sort our files + + // A few words about sorting: + // + // Below we use the qsort() function to sort the buffers according to + // the sort key. + // + // A sort key is "file extension + '.' + file name + full path". Thus, + // the files in the archive will be sorted first by extension, then + // by name, then by full path. + // + // However, we also want to consider files with the same content. + // When adding files to the page, we will compare the contents of + // the current buffer with the previous buffer. If they are the same, + // then duplicate data will not be added. In order for this logic to work, + // the same files must come one after the other. This means that when + // sorting, we must first consider the sameness of the files and then + // the sort key. + // + // Quicksort does not work in this case. There may be a pivot whose buffer + // does not match any file. In this case, identical buffers will be + // placed to different partitions and will never be next to each other in + // the sort results. + // + // To solve this problem, we will check the buffers and if they are + // identical, then we will use the same sort key for those buffers. + + CookfsLog(printf("Cookfs_WriterPurge: sort %d entries", w->bufferCount)); + // Create array for all our entries + sortedWB = ckalloc(w->bufferCount * sizeof(Cookfs_WriterBuffer *)); + if (sortedWB == NULL) { + CookfsLog(printf("Cookfs_WriterPurge: unable to alloc sortedWB")); + Cookfs_WriterSetLastError(w, "failed to alloc sortedWB"); + goto fatalError; + } + + char *justDot = "."; + + // Fill the buffer + for (i = 0, wb = w->bufferFirst; wb != NULL; i++, wb = wb->next) { + + CookfsLog(printf("Cookfs_WriterPurge: add buffer [%p] size %ld" + " to sort buffer at #%d", (void *)wb->buffer, + wb->bufferSize, i)); + sortedWB[i] = wb; + + // If we have less than 3 buffers, then we will not sort them + if (w->bufferCount < 3) { + continue; + } + + // First we check previously processed buffers, trying to find out + // if there is already exactly the same buffer in the queue. If such + // a buffer is found, then we will use its sorting key. + if (i > 0) { + int j; + for (j = 0; j < i; j++) { + Cookfs_WriterBuffer *wbc = sortedWB[j]; + CookfsLog(printf("Cookfs_WriterPurge: check if [%p] equals" + " to [%p]", (void *)wb->buffer, + (void *)wbc->buffer)); + if ((wb->bufferSize == wbc->bufferSize) + && memcmp(wb->buffer, wbc->buffer, wb->bufferSize) == 0) + { + // We found the same buffer + CookfsLog(printf("Cookfs_WriterPurge: the same buffer" + " has been found")); + // Let's use its sort key + wb->sortKey = wbc->sortKey; + Tcl_IncrRefCount(wb->sortKey); + wb->sortKeyStr = wbc->sortKeyStr; + wb->sortKeyLen = wbc->sortKeyLen; + // Stop processing + break; + } + } + // If 'j' did not reach 'i', then we have found a matching buffer. + // No need to anything else with this buffer. Let's continue with + // the next buffer. + if (j != i) { + continue; + } + } + + // Let's generate a sort key for the file + int pathLength; + if (Tcl_ListObjLength(NULL, wb->pathObj, &pathLength) != TCL_OK || + pathLength < 1 + ) { + goto fatalErrorCantHappened; + } + + // Get file name + Tcl_Obj *pathTail; + if (Tcl_ListObjIndex(NULL, wb->pathObj, pathLength - 1, &pathTail) + != TCL_OK || pathTail == NULL) + { + goto fatalErrorCantHappened; + } + + Tcl_Obj *sortPrefix; + pathTail = Tcl_DuplicateObj(pathTail); + Tcl_IncrRefCount(pathTail); + + // Let's move extension to front of filename + int pathTailLength; + char *pathTailStr = Tcl_GetStringFromObj(pathTail, &pathTailLength); + // Check to see if we have an empty filename, this shouldn't + // happen, but who knows? + if (!pathTailLength) { + goto skipExtension; + } + char *dotPosition = strrchr(pathTailStr, '.'); + // No dot or dot at the first position? then skip + if (dotPosition == NULL || dotPosition == pathTailStr) { + goto skipExtension; + } + int nameLength = dotPosition - pathTailStr; + // Create a new object with extension only + sortPrefix = Tcl_NewStringObj(++dotPosition, + pathTailLength - nameLength - 1); + Tcl_IncrRefCount(sortPrefix); + // Append dot symbol + Tcl_AppendToObj(sortPrefix, justDot, 1); + // Append file name without extension + Tcl_AppendToObj(sortPrefix, pathTailStr, nameLength); + + // pathTail is not needed anymore + Tcl_DecrRefCount(pathTail); + + goto generateSortKey; + +skipExtension: + + // If we have no extension, than just use the filename as + // the sortPrefix. We don't create a new object and we don't decrement + // refcount on pathTail. Later, we will reduce the refcount for + // sortPrefix, which essentially means reducing the refcount + // for pathTail. + sortPrefix = pathTail; + +generateSortKey: ; // empty statement + + // Let's generate our sort key. First, use the split path + // as the source. + Tcl_Obj *sortKeySplit = Tcl_DuplicateObj(wb->pathObj); + Tcl_IncrRefCount(sortKeySplit); + + // Add sortPrefix to the beggining + if (Tcl_ListObjReplace(NULL, sortKeySplit, 0, 0, 1, &sortPrefix) + != TCL_OK) { - CookfsLog(printf("Cookfs_WriterGetSmallfilebuffersize: unable to" - " get a wide integer from writer result")); - ret = TCL_ERROR; + // Don't forget to cleanup + Tcl_DecrRefCount(sortPrefix); + Tcl_DecrRefCount(sortKeySplit); + goto fatalErrorCantHappened; + } + + // sortPrefix is not needed anymore + Tcl_DecrRefCount(sortPrefix); + + // Generate the sort key by compining split sort key + wb->sortKey = Tcl_FSJoinPath(sortKeySplit, -1); + Tcl_IncrRefCount(wb->sortKey); + + // We don't need split sort key anymore + Tcl_DecrRefCount(sortKeySplit); + + // Optimize a bit + wb->sortKeyStr = Tcl_GetStringFromObj(wb->sortKey, &wb->sortKeyLen); + + CookfsLog(printf("Cookfs_WriterPurge: generated the sort key [%s]", + Tcl_GetString(wb->sortKey))); + + } + + // If we have more than 2 buffers, then sort them + if (w->bufferCount > 2) { + CookfsLog( \ + printf("Cookfs_WriterPurge: == entries ===========> \n"); \ + for (i = 0; i < w->bufferCount; i++) { \ + printf("Cookfs_WriterPurge: %p %s\n", \ + (void *)sortedWB[i]->buffer, \ + (char *)sortedWB[i]->sortKeyStr); \ + }; \ + printf("Cookfs_WriterPurge: <======================") + ); + CookfsLog(printf("Cookfs_WriterPurge: sort buffers...")); + qsort(sortedWB, w->bufferCount, sizeof(Cookfs_WriterBuffer *), + Cookfs_WriterPurgeSortFunc); + CookfsLog( \ + printf("Cookfs_WriterPurge: == entries ===========> \n"); \ + for (i = 0; i < w->bufferCount; i++) { \ + printf("Cookfs_WriterPurge: %p %s\n", \ + (void *)sortedWB[i]->buffer, \ + (char *)sortedWB[i]->sortKeyStr); \ + }; \ + printf("Cookfs_WriterPurge: <======================") + ); + } else { + CookfsLog(printf("Cookfs_WriterPurge: no need to sort buffers")); + } + + // If our small buffer has fewer bytes than the page size, we will + // allocate in the page buffer only what is needed to store all + // the small buffer files. + CookfsLog(printf("Cookfs_WriterPurge: alloc page buffer for %ld bytes", + w->bufferSize < w->pageSize ? w->bufferSize : w->pageSize)); + pageBuffer = ckalloc(w->bufferSize < w->pageSize ? + w->bufferSize : w->pageSize); + + for (int bufferIdx = 0; bufferIdx < w->bufferCount;) { + + int firstBufferIdx = bufferIdx; + // The first stage. Let's go through the buffers, fill pageBuffer + // and determine which buffer we have reached the page size limit. + Tcl_WideInt pageBufferSize = 0; + wb = sortedWB[bufferIdx]; + while (1) { + + CookfsLog(printf("Cookfs_WriterPurge: add buffer [%p] size %ld" + " to page buffer", (void *)wb->buffer, wb->bufferSize)); + + // Check to see if the exact same block has already been added + // before + if (bufferIdx != 0) { + int found = 0; + Cookfs_WriterBuffer *prevWB = sortedWB[bufferIdx - 1]; + if ((wb->bufferSize == prevWB->bufferSize) + && memcmp(wb->buffer, prevWB->buffer, wb->bufferSize) == 0) + { + CookfsLog(printf("Cookfs_WriterPurge: this buffer is equal" + " to the previous buffer [%p]", + (void *)prevWB->buffer)); + // Copy block-offset from the equal buffer + wb->pageBlock = prevWB->pageBlock; + wb->pageOffset = prevWB->pageOffset; + found = 1; + } + // We don't need the previous buffer and free it now + // to optimize memory usage. + CookfsLog(printf("Cookfs_WriterPurge: free data from" + " the previous buffer [%p] as it is no longer needed", + (void *)prevWB->buffer)); + ckfree(prevWB->buffer); + prevWB->buffer = NULL; + prevWB->bufferSize = 0; + // Skip adding data from this buffer to the page, since we have + // already found the same buffer + if (found) { + goto next; + } + } + + wb->pageBlock = -1; + wb->pageOffset = pageBufferSize; + + memcpy((void *)(pageBuffer + pageBufferSize), wb->buffer, + wb->bufferSize); + pageBufferSize += wb->bufferSize; + +next: + + if (++bufferIdx >= w->bufferCount) { + CookfsLog(printf("Cookfs_WriterPurge: reached the end of" + " buffers")); + break; + } + wb = sortedWB[bufferIdx]; + if ((pageBufferSize + wb->bufferSize) > w->pageSize) { + CookfsLog(printf("Cookfs_WriterPurge: the next buffer will" + " cause a page buffer overflow, the page buffer must" + " be flushed")); + break; + } + + } + + int pageBlock; + + // Try to add page if we need to save something from pageBuffer + if (pageBufferSize) { + CookfsLog(printf("Cookfs_WriterPurge: add page...")); + pageBlock = Cookfs_PageAddRaw(w->pages, pageBuffer, pageBufferSize); + CookfsLog(printf("Cookfs_WriterPurge: got block index: %d", + pageBlock)); + + if (pageBlock < 0) { + Tcl_Obj *err = Cookfs_PagesGetLastError(w->pages); + Cookfs_WriterSetLastErrorObj(w, Tcl_ObjPrintf("error while adding" + " page of small files: %s", + (err == NULL ? "unknown error" : Tcl_GetString(err)))); + goto fatalError; + } + // Reduce size by already saved buffers size + w->bufferSize -= pageBufferSize; } else { - CookfsLog(printf("Cookfs_WriterGetSmallfilebuffersize: got value" - " from writer: %ld", *size)); + pageBlock = -1; + } + + // The second state. Update entries in fsindex. The bufferIdx variable + // corresponds to the last not saved buffer. + + CookfsLog(printf("Cookfs_WriterPurge: modify %d files", + bufferIdx - firstBufferIdx)); + for (i = firstBufferIdx; i < bufferIdx; i++) { + + wb = sortedWB[i]; + + // Update pageBlock to the new index of the saved page. It is + // possible that pageBlock is already set. This can happen if we + // add some file that uses a previously added page. Thus, update + // pageBlock only if it is not defined yet, i.e. equal to -1. + if (wb->pageBlock == -1) { + wb->pageBlock = pageBlock; + } + + // Update entry + CookfsLog(printf("Cookfs_WriterPurge: update fsindex entry for" + " buffer %p: pageBlock:%d pageOffset:%d", (void *)wb, + wb->pageBlock, wb->pageOffset)); + + Cookfs_FsindexUpdatePendingEntry(w->index, wb->entry, + wb->pageBlock, wb->pageOffset); + } + } - return ret; + + // Cleanup small file buffer + CookfsLog(printf("Cookfs_WriterPurge: cleanup small file buffer")); + for (i = 0; i < w->bufferCount; i++) { + Cookfs_WriterWriterBufferFree(sortedWB[i]); + } + + w->bufferFirst = NULL; + w->bufferLast = NULL; + w->bufferSize = 0; + w->bufferCount = 0; + + goto done; + +fatalErrorCantHappened: + + Cookfs_WriterSetLastError(w, "this case doesn't have to happen"); + +fatalError: + + + CookfsLog(printf("Cookfs_WriterPurge: !!! SET FATAL ERROR STATE !!!")); + w->fatalError = 1; + result = TCL_ERROR; + +done: + + if (sortedWB != NULL) { + CookfsLog(printf("Cookfs_WriterPurge: free sortedWB")); + ckfree(sortedWB); + } + + if (pageBuffer != NULL) { + CookfsLog(printf("Cookfs_WriterPurge: free pageBuffer")); + ckfree(pageBuffer); + } + + if (result == TCL_ERROR) { + CookfsLog(printf("Cookfs_WriterPurge: return ERROR")); + } else { + CookfsLog(printf("Cookfs_WriterPurge: ok")); + } + + return result; } -int Cookfs_WriterSetIndex(Cookfs_Writer *w, Cookfs_Fsindex *index) { - if (w == NULL) { - CookfsLog(printf("Cookfs_WriterSetIndex: ERROR: writer is" - " NULL")); - return TCL_ERROR; +// IMPLEMENTED + +const void *Cookfs_WriterGetBuffer(Cookfs_Writer *w, int blockNumber, + Tcl_WideInt *blockSize) +{ + CookfsLog(printf("Cookfs_WriterGetBuffer: enter [%p] block: %d", + (void *)w, blockNumber)); + + if (blockNumber < 0) { + blockNumber = -blockNumber - 1; + } + CookfsLog(printf("Cookfs_WriterGetBuffer: real block number: %d;" + " current number of blocks: %d", blockNumber, w->bufferCount)); + + Cookfs_WriterBuffer *wb = w->bufferFirst; + while (blockNumber && wb != NULL) { + wb = wb->next; + blockNumber--; + } + + if (wb == NULL) { + CookfsLog(printf("Cookfs_WriterGetBuffer: ERROR: block number" + " is incorrect")); + return NULL; + } + + CookfsLog(printf("Cookfs_WriterGetBuffer: the block has been found [%p]" + " data [%p] size [%ld]", (void *)wb, (void *)wb->buffer, + wb->bufferSize)); + + *blockSize = wb->bufferSize; + return wb->buffer; +} + +Tcl_Obj *Cookfs_WriterGetBufferObj(Cookfs_Writer *w, int blockNumber) +{ + CookfsLog(printf("Cookfs_WriterGetBufferObj: enter [%p] block: %d", + (void *)w, blockNumber)); + + Tcl_WideInt blockSize; + const void *blockData = Cookfs_WriterGetBuffer(w, blockNumber, &blockSize); + if (blockData == NULL) { + CookfsLog(printf("Cookfs_WriterGetBufferObj: ERROR: block number" + " is incorrect")); + return NULL; + } + Tcl_Obj *rc = Tcl_NewByteArrayObj(blockData, blockSize); + CookfsLog(printf("Cookfs_WriterGetBuffer: return obj [%p]", (void *)rc)); + return rc; +} + +int Cookfs_WriterGetWritetomemory(Cookfs_Writer *w) { + return w->isWriteToMemory; +} + +void Cookfs_WriterSetWritetomemory(Cookfs_Writer *w, int status) { + w->isWriteToMemory = status; + return; +} + +Tcl_WideInt Cookfs_WriterGetSmallfilebuffersize(Cookfs_Writer *w) { + return w->bufferSize; +} + +Cookfs_Writer *Cookfs_WriterGetHandle(Tcl_Interp *interp, const char *cmdName) { + Tcl_CmdInfo cmdInfo; + CookfsLog(printf("Cookfs_WriterGetHandle: get handle from cmd [%s]", cmdName)); + if (!Tcl_GetCommandInfo(interp, cmdName, &cmdInfo)) { + return NULL; } - Tcl_Obj *objv[3]; - objv[0] = w->commandObj; - objv[1] = Tcl_NewStringObj("index", -1); - objv[2] = CookfsGetFsindexObjectCmd(w->interp, index);; - Tcl_Obj *cmd = Tcl_NewListObj(3, objv); - Tcl_IncrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterSetIndex: call Tcl writer...")); - int ret = Tcl_EvalObjEx(w->interp, cmd, TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - Tcl_DecrRefCount(cmd); - CookfsLog(printf("Cookfs_WriterSetIndex: Tcl writer: %d", ret)); - return ret; + CookfsLog(printf("Cookfs_WriterGetHandle: return [%p]", cmdInfo.objClientData)); + return (Cookfs_Writer *)(cmdInfo.objClientData); } diff --git a/generic/writer.h b/generic/writer.h index 7ea6739..154c92f 100644 --- a/generic/writer.h +++ b/generic/writer.h @@ -5,11 +5,55 @@ /* Tcl public API */ +typedef enum { + COOKFS_WRITER_SOURCE_FILE, + COOKFS_WRITER_SOURCE_CHANNEL, + COOKFS_WRITER_SOURCE_OBJECT, + COOKFS_WRITER_SOURCE_BUFFER +} Cookfs_WriterDataSource; + +typedef struct Cookfs_WriterBuffer { + void *buffer; + Tcl_WideInt bufferSize; + Tcl_Obj *pathObj; + Tcl_WideInt mtime; + Cookfs_FsindexEntry *entry; + + Tcl_Obj *sortKey; + const void *sortKeyStr; + int sortKeyLen; + + int pageBlock; + int pageOffset; + + struct Cookfs_WriterBuffer *next; +} Cookfs_WriterBuffer; + typedef struct Cookfs_Writer { + Tcl_Interp *interp; - Tcl_Obj *commandObj; + Tcl_Command commandToken; + Tcl_Obj *lastErrorObj; + int fatalError; + int isDead; + + Cookfs_Pages *pages; + Cookfs_Fsindex *index; + + int isWriteToMemory; + Tcl_WideInt smallFileSize; + Tcl_WideInt maxBufferSize; + Tcl_WideInt pageSize; + + Cookfs_WriterBuffer *bufferFirst; + Cookfs_WriterBuffer *bufferLast; + Tcl_WideInt bufferSize; + int bufferCount; + } Cookfs_Writer; +Cookfs_Writer *Cookfs_WriterGetHandle(Tcl_Interp *interp, const char *cmdName); + Cookfs_Writer *Cookfs_WriterInit(Tcl_Interp* interp, Cookfs_Pages *pages, Cookfs_Fsindex *index, Tcl_WideInt smallfilebuffer, Tcl_WideInt smallfilesize, @@ -17,14 +61,20 @@ Cookfs_Writer *Cookfs_WriterInit(Tcl_Interp* interp, void Cookfs_WriterFini(Cookfs_Writer *w); +Tcl_Obj *Cookfs_WriterGetLastError(Cookfs_Writer *w); + int Cookfs_WriterPurge(Cookfs_Writer *w); -int Cookfs_WriterGetWritetomemory(Cookfs_Writer *w, int *status); -int Cookfs_WriterSetWritetomemory(Cookfs_Writer *w, int status); +Tcl_Obj *Cookfs_WriterGetBufferObj(Cookfs_Writer *w, int blockNumber); +const void *Cookfs_WriterGetBuffer(Cookfs_Writer *w, int blockNumber, + Tcl_WideInt *blockSize); + +int Cookfs_WriterAddFile(Cookfs_Writer *w, Tcl_Obj *pathObj, + Cookfs_WriterDataSource dataType, void *data, Tcl_WideInt dataSize); -int Cookfs_WriterGetSmallfilebuffersize(Cookfs_Writer *w, - Tcl_WideInt *size); +int Cookfs_WriterGetWritetomemory(Cookfs_Writer *w); +void Cookfs_WriterSetWritetomemory(Cookfs_Writer *w, int status); -int Cookfs_WriterSetIndex(Cookfs_Writer *w, Cookfs_Fsindex *index); +Tcl_WideInt Cookfs_WriterGetSmallfilebuffersize(Cookfs_Writer *w); #endif /* COOKFS_WRITER_H */ diff --git a/generic/writerCmd.c b/generic/writerCmd.c new file mode 100644 index 0000000..9f5550d --- /dev/null +++ b/generic/writerCmd.c @@ -0,0 +1,266 @@ +/* + * vfs.c + * + * Provides implementation for writer + * + * (c) 2024 Konstantin Kushnir + */ + +#include "cookfs.h" + +// Creating a writer object from Tcl is not supported as for now +// static Tcl_ObjCmdProc CookfsWriterCmd; + +static Tcl_ObjCmdProc CookfsWriterHandlerCmd; +static Tcl_CmdDeleteProc CookfsWriterHandlerCmdDeleteProc; + +int Cookfs_InitWriterCmd(Tcl_Interp *interp) { + + Tcl_CreateNamespace(interp, "::cookfs::c::writer", NULL, NULL); + + // Creating a writer object from Tcl is not supported as for now + + /* + Tcl_CreateObjCommand(interp, "::cookfs::c::writer", CookfsWriterCmd, + (ClientData) NULL, NULL); + + Tcl_CreateAlias(interp, "::cookfs::writer", interp, "::cookfs::c::writer", + 0, NULL); + + */ + + return TCL_OK; + +} + +static void CookfsRegisterExistingWriterObjectCmd(Tcl_Interp *interp, Cookfs_Writer *w) { + if (w->commandToken != NULL) { + return; + } + char buf[128]; + /* create Tcl command and return its name */ + sprintf(buf, "::cookfs::c::writer::handle%p", (void *)w); + w->commandToken = Tcl_CreateObjCommand(interp, buf, CookfsWriterHandlerCmd, + (ClientData) w, CookfsWriterHandlerCmdDeleteProc); + w->interp = interp; + Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); +} + + +Tcl_Obj *CookfsGetWriterObjectCmd(Tcl_Interp *interp, Cookfs_Writer *w) { + CookfsRegisterExistingWriterObjectCmd(interp, w); + Tcl_Obj *rc = Tcl_NewObj(); + Tcl_GetCommandFullName(interp, w->commandToken, rc); + return rc; +} + +// Creating a writer object from Tcl is not supported as for now +/* +static int CookfsWriterCmd(ClientData clientData, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]) +{ + + UNUSED(clientData); + UNUSED(objc); + UNUSED(objv); + + CookfsLog(printf("CookfsWriterCmd: ENTER")); + + Tcl_SetObjResult(interp, Tcl_ObjPrintf("%s is not implemented." + " A new writer object should be created by cookfs::Mount command.", + Tcl_GetString(objv[0]))); + + CookfsLog(printf("CookfsWriterCmd: return ERROR")); + + return TCL_ERROR; +} +*/ + +static void CookfsWriterHandlerCmdDeleteProc(ClientData clientData) { + Cookfs_Writer *w = (Cookfs_Writer *)clientData; + w->commandToken = NULL; +} + +static Cookfs_WriterHandleCommandProc CookfsWriterHandleCommandGetbuf; +// In headers: +// Cookfs_WriterHandleCommandProc CookfsWriterHandleCommandWrite; + +static int CookfsWriterHandlerCmd(ClientData clientData, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]) +{ + + Cookfs_Writer *w = (Cookfs_Writer *)clientData; + + static const char *const commands[] = { + "getbuf", "write", + NULL + }; + enum commands { + cmdGetbuf, cmdWrite + }; + + if (objc < 2) { + Tcl_WrongNumArgs(interp, 1, objv, "command ?args?"); + return TCL_ERROR; + } + + int command; + if (Tcl_GetIndexFromObj(interp, objv[1], commands, "command", 0, + &command) != TCL_OK) + { + CookfsLog(printf("CookfsWriterHandlerCmd: ERROR: unknown command [%s]", + Tcl_GetString(objv[1]))); + return TCL_ERROR; + } + + switch ((enum commands) command) { + case cmdGetbuf: + return CookfsWriterHandleCommandGetbuf(w, interp, objc, objv); + case cmdWrite: + return CookfsWriterHandleCommandWrite(w, interp, objc, objv); + } + + return TCL_OK; + +} + +static int CookfsWriterHandleCommandGetbuf(Cookfs_Writer *w, + Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) +{ + + CookfsLog(printf("CookfsWriterHandleCommandGetbuf: enter")); + + if (objc != 3) { + CookfsLog(printf("CookfsWriterHandleCommandGetbuf: ERR: wrong # args")); + Tcl_WrongNumArgs(interp, 2, objv, "index"); + return TCL_ERROR; + } + + int bufNumber; + if (Tcl_GetIntFromObj(interp, objv[2], &bufNumber) != TCL_OK) { + CookfsLog(printf("CookfsWriterHandleCommandGetbuf: ERROR: wrong buf #" + " [%s]", Tcl_GetString(objv[2]))); + Tcl_SetObjResult(interp, Tcl_ObjPrintf("integer index is expected," + " but got \"%s\"", Tcl_GetString(objv[2]))); + return TCL_ERROR; + } + + Tcl_Obj *rc = Cookfs_WriterGetBufferObj(w, bufNumber); + if (rc == NULL) { + CookfsLog(printf("CookfsWriterHandleCommandGetbuf: ERROR: got NULL")); + Tcl_SetObjResult(interp, Tcl_ObjPrintf("unable to get buf index %d", + bufNumber)); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, rc); + CookfsLog(printf("CookfsWriterHandleCommandGetbuf: ok")); + return TCL_OK; + +} + +int CookfsWriterHandleCommandWrite(Cookfs_Writer *w, + Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) +{ + + CookfsLog(printf("CookfsWriterHandleCommandWrite: enter")); + + if (objc < 3 || ((objc - 2) % 4) != 0) { + CookfsLog(printf("CookfsWriterHandleCommandWrite: ERR: wrong # args" + " (%d)", objc)); + Tcl_WrongNumArgs(interp, 2, objv, "path datatype data size ?path" + " datatype data size...?"); + return TCL_ERROR; + } + + static const char *const dataTypes[] = { + "file", "channel", "data", NULL + }; + + for (int i = 2; i < objc;) { + + Tcl_Obj *path = objv[i++]; + Tcl_Obj *splitPath = Tcl_FSSplitPath(path, NULL); + if (path == NULL) { + CookfsLog(printf("CookfsWriterHandleCommandWrite: unable to split" + " path [%s]", Tcl_GetString(path))); + Tcl_SetObjResult(interp, Tcl_ObjPrintf("error while splitting" + " path")); + goto error; + } + + int dataTypeInt; + if (Tcl_GetIndexFromObj(interp, objv[i++], dataTypes, "datatype", + TCL_EXACT, &dataTypeInt) != TCL_OK) + { + CookfsLog(printf("CookfsWriterHandleCommandWrite: ERR: unknown" + " datatype [%s]", Tcl_GetString(objv[i - 1]))); + goto error; + } + + Tcl_Channel channel = NULL; + Tcl_Obj *data = objv[i++]; + if ((Cookfs_WriterDataSource)dataTypeInt == COOKFS_WRITER_SOURCE_CHANNEL) { + int mode; + channel = Tcl_GetChannel(interp, Tcl_GetString(data), &mode); + if (channel == NULL) { + CookfsLog(printf("CookfsWriterHandleCommandWrite: ERR: is not a" + " channel [%s]", Tcl_GetString(data))); + Tcl_SetObjResult(interp, Tcl_ObjPrintf("channel name expected," + " but got \"%s\"", Tcl_GetString(data))); + goto error; + } + if (!(mode & TCL_READABLE)) { + CookfsLog(printf("CookfsWriterHandleCommandWrite: ERR: channel" + " [%s] is not readable", Tcl_GetString(data))); + Tcl_SetObjResult(interp, Tcl_ObjPrintf("channel \"%s\" is not" + " readable", Tcl_GetString(data))); + goto error; + } + } + + Tcl_Obj *dataSizeObj = objv[i++]; + Tcl_WideInt dataSize = -1; + if (Tcl_GetCharLength(dataSizeObj) != 0) { + if (Tcl_GetWideIntFromObj(interp, dataSizeObj, &dataSize) + != TCL_OK) + { + CookfsLog(printf("CookfsWriterHandleCommandWrite: ERR: datasize" + " [%s] is not a wide int", Tcl_GetString(dataSizeObj))); + goto error; + } + } + + int ret = Cookfs_WriterAddFile(w, splitPath, + (Cookfs_WriterDataSource)dataTypeInt, ( + (Cookfs_WriterDataSource)dataTypeInt == COOKFS_WRITER_SOURCE_CHANNEL + ? (void *)channel : (void *)data), dataSize); + if (ret != TCL_OK) { + Tcl_Obj *err = Cookfs_WriterGetLastError(w); + if (err == NULL) { + CookfsLog(printf("CookfsWriterHandleCommandWrite: got error" + " and unknown message from Cookfs_WriterAddFile()")); + Tcl_SetObjResult(interp, Tcl_ObjPrintf("unknown error")); + } else { + CookfsLog(printf("CookfsWriterHandleCommandWrite: got error" + " from Cookfs_WriterAddFile(): %s", Tcl_GetString(err))); + Tcl_SetObjResult(interp, err); + } + goto error; + } + + continue; + +error: + + Tcl_SetObjResult(interp, Tcl_ObjPrintf("unable to add \"%s\": %s", + Tcl_GetString(path), Tcl_GetString(Tcl_GetObjResult(interp)))); + CookfsLog(printf("CookfsWriterHandleCommandWrite: ERROR while adding" + " [%s]", Tcl_GetString(path))); + return TCL_ERROR; + + } + + return TCL_OK; + +} diff --git a/generic/writerCmd.h b/generic/writerCmd.h new file mode 100644 index 0000000..99e06c7 --- /dev/null +++ b/generic/writerCmd.h @@ -0,0 +1,16 @@ +/* (c) 2024 Konstantin Kushnir */ + +#ifndef COOKFS_WRITER_CMD_H +#define COOKFS_WRITER_CMD_H 1 + +/* Tcl public API */ + +typedef int (Cookfs_WriterHandleCommandProc)(Cookfs_Writer *w, + Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); + +int Cookfs_InitWriterCmd(Tcl_Interp *interp); + +Tcl_Obj *CookfsGetWriterObjectCmd(Tcl_Interp *interp, Cookfs_Writer *w); +Cookfs_WriterHandleCommandProc CookfsWriterHandleCommandWrite; + +#endif /* COOKFS_WRITER_CMD_H */ diff --git a/generic/writerchannel.c b/generic/writerchannel.c index dfb058b..9cb10f4 100644 --- a/generic/writerchannel.c +++ b/generic/writerchannel.c @@ -32,7 +32,7 @@ static int Cookfs_CreateWriterchannelCreate(Cookfs_WriterChannelInstData } static Cookfs_WriterChannelInstData *Cookfs_CreateWriterchannelAlloc( - Cookfs_Pages *pages, Cookfs_Fsindex *index, Tcl_Obj *writerObjCmd, + Cookfs_Pages *pages, Cookfs_Fsindex *index, Cookfs_Writer *writer, Tcl_Obj *pathObj, int pathObjLen, Tcl_Interp *interp, Tcl_WideInt initialBufferSize) { @@ -70,8 +70,8 @@ static Cookfs_WriterChannelInstData *Cookfs_CreateWriterchannelAlloc( instData->pages = pages; instData->index = index; - instData->writerObjCmd = writerObjCmd; - Tcl_IncrRefCount(writerObjCmd); + instData->writer = writer; + instData->pathObjLen = pathObjLen; instData->pathObj = pathObj; if (pathObj != NULL) { @@ -108,7 +108,6 @@ void Cookfs_CreateWriterchannelFree(Cookfs_WriterChannelInstData *instData) { if (instData->pathObj != NULL) { Tcl_DecrRefCount(instData->pathObj); } - Tcl_DecrRefCount(instData->writerObjCmd); ckfree((void *)instData); CookfsLog(printf("Cookfs_CreateWriterchannelFree: ok")) @@ -116,20 +115,22 @@ void Cookfs_CreateWriterchannelFree(Cookfs_WriterChannelInstData *instData) { } Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, - Cookfs_Fsindex *index, Tcl_Obj *writerObjCmd, Tcl_Obj *pathObj, + Cookfs_Fsindex *index, Cookfs_Writer *writer, Tcl_Obj *pathObj, int pathObjLen, Cookfs_FsindexEntry *entry, Tcl_Interp *interp) { CookfsLog(printf("Cookfs_CreateWriterchannel: start")); Cookfs_WriterChannelInstData *instData = Cookfs_CreateWriterchannelAlloc( - pages, index, writerObjCmd, pathObj, pathObjLen, interp, + pages, index, writer, pathObj, pathObjLen, interp, (entry == NULL ? 0 : entry->data.fileInfo.fileSize)); if (instData == NULL) { return NULL; } + Tcl_Obj *blockObj = NULL; + if (Cookfs_CreateWriterchannelCreate(instData, interp) != TCL_OK) { CookfsLog(printf("Cookfs_CreateWriterchannel: " "Cookfs_CreateWriterchannelCreate failed")) @@ -155,47 +156,26 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, continue; } - Tcl_Obj *blockObj; + int blockSize; + const char *blockBuffer; - // If block is < 0, this block is in the writer's smallfilebuffer if (block < 0) { - // Create the writer command to retrieve the required block - Tcl_Obj *writerCmd = Tcl_NewListObj(0, NULL); - Tcl_IncrRefCount(writerCmd); - - Tcl_ListObjAppendElement(interp, writerCmd, writerObjCmd); - Tcl_ListObjAppendElement(interp, writerCmd, - Tcl_NewStringObj("getbuf", -1)); - Tcl_ListObjAppendElement(interp, writerCmd, - Tcl_NewIntObj(block)); - - // Exec the writer command - CookfsLog(printf("Cookfs_CreateWriterchannel: exec writer to get" - " block [%d]", block)); - int ret = Tcl_EvalObjEx(instData->interp, writerCmd, - TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - CookfsLog(printf("Cookfs_CreateWriterchannel: writer" - " returned: %d", ret)); - - // We don't need the writer command anymore - Tcl_DecrRefCount(writerCmd); - - // Returns an error if the writer was unable to retrieve the block - if (ret != TCL_OK) { + // If block is < 0, this block is in the writer's smallfilebuffer + CookfsLog(printf("Cookfs_CreateWriterchannel: reading the block" + " from writer")); + + Tcl_WideInt blockSizeWide; + blockBuffer = Cookfs_WriterGetBuffer(writer, block, &blockSizeWide); + + // Returns an error if the pages was unable to retrieve the block + if (blockBuffer == NULL) { CookfsLog(printf("Cookfs_CreateWriterchannel: return an error" " as writer failed")); goto error; } - blockObj = Tcl_GetObjResult(interp); - if (blockObj == NULL) { - CookfsLog(printf("Cookfs_CreateWriterchannel: failed to get" - " result from Tcl")); - goto error; - } - - Tcl_IncrRefCount(blockObj); + blockSize = blockSizeWide; } else { @@ -222,13 +202,10 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, } Tcl_IncrRefCount(blockObj); + blockBuffer = (char *)Tcl_GetByteArrayFromObj(blockObj, &blockSize); } - int blockSize; - const char *blockBuffer = (char *)Tcl_GetByteArrayFromObj(blockObj, - &blockSize); - CookfsLog(printf("Cookfs_CreateWriterchannel: got block size [%d]", blockSize)); @@ -236,7 +213,6 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, if ((offset + size) > blockSize) { CookfsLog(printf("Cookfs_Writerchannel_CloseHandler: not enough" " bytes in block, return an error")); - Tcl_DecrRefCount(blockObj); goto error; } @@ -247,9 +223,6 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, int writtenBytes = Tcl_ChannelOutputProc(CookfsWriterChannel())( (ClientData)instData, blockBuffer + offset, size, &errorCode); - // We don't need the blockObj object anymore - Tcl_DecrRefCount(blockObj); - if (writtenBytes != size) { CookfsLog(printf("Cookfs_CreateWriterchannel: only [%d]" " bytes were written to the channel, consider this" @@ -257,6 +230,12 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, goto error; } + // We don't need the blockObj object anymore + if (blockObj != NULL) { + Tcl_DecrRefCount(blockObj); + blockObj = NULL; + } + } // Set current position to the start of file instData->currentOffset = 0; @@ -272,6 +251,10 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, error: + if (blockObj != NULL) { + Tcl_DecrRefCount(blockObj); + } + if (instData != NULL) { Cookfs_CreateWriterchannelFree(instData); } @@ -318,9 +301,16 @@ static int CookfsCreateWriterchannelCmd(ClientData clientData, return TCL_ERROR; } - // TODO: Validate somehow the specified writer object? May be try to execute - // some known method? - Tcl_Obj *writerObjCmd = objv[3]; + Cookfs_Writer *writer = Cookfs_WriterGetHandle(interp, + Tcl_GetStringFromObj(objv[3], NULL)); + CookfsLog(printf("CookfsCreateWriterchannelCmd: writer [%p]", + (void *)writer)); + + if (writer == NULL) { + Tcl_SetObjResult(interp, Tcl_NewStringObj("Unable to find" + " writer object", -1)); + return TCL_ERROR; + } int pathObjLen; Tcl_Obj *pathObj = Tcl_FSSplitPath(objv[4], &pathObjLen); @@ -345,9 +335,7 @@ static int CookfsCreateWriterchannelCmd(ClientData clientData, // If entry exists, make sure it is not a directory if (entry != NULL) { - int isDirectory = entry->fileBlocks == COOKFS_NUMBLOCKS_DIRECTORY ? - 1 : 0; - if (isDirectory) { + if (Cookfs_FsindexEntryIsDirectory(entry)) { Tcl_SetObjResult(interp, Tcl_ObjPrintf("file \"%s\" exists" " and it is a directory", Tcl_GetString(objv[4]))); goto error; @@ -364,7 +352,7 @@ static int CookfsCreateWriterchannelCmd(ClientData clientData, } // Check if parent is a directory - if (entryParent->fileBlocks != COOKFS_NUMBLOCKS_DIRECTORY) { + if (!Cookfs_FsindexEntryIsDirectory(entryParent)) { Tcl_SetObjResult(interp, Tcl_ObjPrintf("Unable to open" " file \"%s\" for writing, since its parent" " is not a directory", Tcl_GetString(objv[4]))); @@ -376,9 +364,8 @@ static int CookfsCreateWriterchannelCmd(ClientData clientData, // to read previous data from the file and it should be created // from scratch, then pass NULL as entry. - Tcl_Channel channel = Cookfs_CreateWriterchannel(pages, index, - writerObjCmd, pathObj, pathObjLen, (wantToRead ? entry : NULL), - interp); + Tcl_Channel channel = Cookfs_CreateWriterchannel(pages, index, writer, + pathObj, pathObjLen, (wantToRead ? entry : NULL), interp); if (channel == NULL) { goto error; diff --git a/generic/writerchannel.h b/generic/writerchannel.h index f80d259..05013f4 100644 --- a/generic/writerchannel.h +++ b/generic/writerchannel.h @@ -19,7 +19,8 @@ typedef struct Cookfs_WriterChannelInstData { Cookfs_Pages *pages; Cookfs_Fsindex *index; - Tcl_Obj *writerObjCmd; + Cookfs_Writer *writer; + Tcl_Obj *pathObj; int pathObjLen; @@ -33,7 +34,7 @@ typedef struct Cookfs_WriterChannelInstData { int Cookfs_InitWriterchannelCmd(Tcl_Interp *interp); Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, - Cookfs_Fsindex *fsindex, Tcl_Obj *writerObjCmd, Tcl_Obj *pathObj, + Cookfs_Fsindex *fsindex, Cookfs_Writer *writer, Tcl_Obj *pathObj, int pathObjLen, Cookfs_FsindexEntry *entry, Tcl_Interp *interp); void Cookfs_CreateWriterchannelFree(Cookfs_WriterChannelInstData *instData); diff --git a/generic/writerchannelIO.c b/generic/writerchannelIO.c index f9c12d3..64974b7 100644 --- a/generic/writerchannelIO.c +++ b/generic/writerchannelIO.c @@ -136,24 +136,6 @@ static int Cookfs_Writerchannel_Realloc(Cookfs_WriterChannelInstData *instData, return 1; } -/* - * - * Warning from tclvfs: - * - * IMPORTANT: This procedure must *not* modify the interpreter's result - * this leads to the objResultPtr being corrupted (somehow), and curious - * crashes in the future (which are very hard to debug ;-). - * - * This is particularly important since we are evaluating arbitrary - * Tcl code in the callback. - * - * Also note we are relying on the close-callback to occur just before - * the channel is about to be properly closed, but after all output - * has been flushed. That way we can, in the callback, read in the - * entire contents of the channel and, say, compress it for storage - * into a tclkit or zip archive. - */ - void Cookfs_Writerchannel_CloseHandler(ClientData clientData) { Cookfs_WriterChannelInstData *instData = (Cookfs_WriterChannelInstData *)clientData; @@ -170,83 +152,30 @@ void Cookfs_Writerchannel_CloseHandler(ClientData clientData) { CookfsLog(printf("Cookfs_Writerchannel_CloseHandler: channel [%s] at [%p]", Tcl_GetChannelName(instData->channel), (void *)instData)); - // Move current offset to the start of channel. instData->currentOffset - // should not be used here. The channel can be buffered, and the Tcl core - // must reset the internal buffer before we change the current position. - // If we do it hacky way by directly changing the position, the Tcl core - // will push data from the buffer into the channel via Output(), and this - // will break the current position. - CookfsLog(printf("Cookfs_Writerchannel_CloseHandler: move current offset" - " to the start")); - Tcl_Seek(instData->channel, 0, SEEK_SET); - - // Save interp result, see the notice before this function - Tcl_SavedResult savedResult; - Tcl_SaveResult(instData->interp, &savedResult); - - // Construct the writer command - Tcl_Obj *writerCmd = Tcl_NewListObj(0, NULL); - Tcl_IncrRefCount(writerCmd); - - Tcl_ListObjAppendElement(instData->interp, writerCmd, - instData->writerObjCmd); - Tcl_ListObjAppendElement(instData->interp, writerCmd, - Tcl_NewStringObj("write", -1)); - Tcl_ListObjAppendElement(instData->interp, writerCmd, - Tcl_FSJoinPath(instData->pathObj, instData->pathObjLen)); - Tcl_ListObjAppendElement(instData->interp, writerCmd, - Tcl_NewStringObj("channel", -1)); - Tcl_ListObjAppendElement(instData->interp, writerCmd, - Tcl_NewStringObj(Tcl_GetChannelName(instData->channel), -1)); - Tcl_ListObjAppendElement(instData->interp, writerCmd, Tcl_NewObj()); - - /* - * Another magic from tclvfs: - * - * The interpreter needs to know about the channel, else the Tcl - * callback will fail, so we register the channel (this allows - * the Tcl code to use the channel's string-name). - */ - Tcl_RegisterChannel(instData->interp, instData->channel); - - // The channel must be in raw mode before saving - Tcl_SetChannelOption(instData->interp, instData->channel, - "-translation", "binary"); - - // Exec the writer command - CookfsLog(printf("Cookfs_Writerchannel_CloseHandler: exec writer...")); - int ret = Tcl_EvalObjEx(instData->interp, writerCmd, - TCL_EVAL_GLOBAL | TCL_EVAL_DIRECT); - CookfsLog(printf("Cookfs_Writerchannel_CloseHandler: writer" - " returned: %d", ret)); - - /* - * One more magic from tclvfs: - * - * More complications; we can't just unregister the channel, - * because it is in the middle of being cleaned up, and the cleanup - * code doesn't like a channel to be closed again while it is - * already being closed. So, we do the same trick as above to - * unregister it without cleanup. - */ - Tcl_DetachChannel(instData->interp, instData->channel); + CookfsLog(printf("Cookfs_Writerchannel_CloseHandler: flush channel")); + Tcl_Flush(instData->channel); + // Release the closeResult if it exists if (instData->closeResult != NULL) { Tcl_DecrRefCount(instData->closeResult); + instData->closeResult = NULL; } - if (ret != TCL_OK) { - instData->closeResult = Tcl_GetObjResult(instData->interp); - Tcl_IncrRefCount(instData->closeResult); + CookfsLog(printf("Cookfs_Writerchannel_CloseHandler: write file...")); + if (Cookfs_WriterAddFile(instData->writer, instData->pathObj, + COOKFS_WRITER_SOURCE_BUFFER, instData->buffer, instData->currentSize) + == TCL_OK) + { + // buffer is owned by writer now. Set to null to avoid releasing it. + instData->buffer = NULL; + instData->bufferSize = 0; } else { - instData->closeResult = NULL; + instData->closeResult = Cookfs_WriterGetLastError(instData->writer); + if (instData->closeResult != NULL) { + Tcl_IncrRefCount(instData->closeResult); + } } - // Restore interp result - Tcl_RestoreResult(instData->interp, &savedResult); - - // We don't need the writer command anymore - Tcl_DecrRefCount(writerCmd); } static int Cookfs_Writerchannel_Close(ClientData instanceData, diff --git a/pkgconfig.tcl.in b/pkgconfig.tcl.in index a9e9804..473815e 100644 --- a/pkgconfig.tcl.in +++ b/pkgconfig.tcl.in @@ -13,6 +13,7 @@ proc cookfs::pkgconfig {{command ""} {value ""} {newValue ""}} { c-fsindex "@COOKFS_PKGCONFIG_USECFSINDEX@" c-readerchannel "@COOKFS_PKGCONFIG_USECREADERCHAN@" c-writerchannel "@COOKFS_PKGCONFIG_USECWRITERCHAN@" + c-writer "@COOKFS_PKGCONFIG_USECWRITER@" feature-aside "@COOKFS_PKGCONFIG_FEATURE_ASIDE@" feature-bzip2 "@COOKFS_PKGCONFIG_USEBZ2@" diff --git a/scripts/vfs.tcl b/scripts/vfs.tcl index a0e24db..6b6e372 100644 --- a/scripts/vfs.tcl +++ b/scripts/vfs.tcl @@ -28,7 +28,6 @@ proc cookfs::tcl::Mount {args} { {noregister {Do not register this filesystem (for temporary writes)}} {nocommand {Do not create command for managing cookfs archive}} {bootstrap.arg "" {Bootstrap code to add in the beginning}} - {compression.arg "zlib" {Compression type to use}} {alwayscompress {Always compress}} {compresscommand.arg "" {Command to use for custom compression}} {asynccompresscommand.arg "" {Command to use for custom, async compression}} @@ -53,6 +52,12 @@ proc cookfs::tcl::Mount {args} { {nodirectorymtime {Do not initialize mtime for new directories to current date and time}} } + if { [::cookfs::pkgconfig get feature-xz] } { + lappend options {compression.arg "xz" {Compression type to use}} + } else { + lappend options {compression.arg "zlib" {Compression type to use}} + } + if { [::cookfs::pkgconfig get c-pages] || ![catch { package require md5 2 }] } { lappend options {pagehash.arg "md5" {Hash algorithm to use for page hashing}} } else { @@ -439,6 +444,13 @@ proc cookfs::tcl::vfs::mountHandler {fsid args} { } return $fs(index) } + getwriter { + if {[llength $args] != 1} { + set ei "wrong # args: should be \"$fsid getwriter\"" + error $ei $ei + } + return $fs(writer) + } default { set ei "unknown subcommand \"[lindex $args 0]\": must be one of aside, compression, filesize, flush, getmetadata, optimizelist, setmetadata, smallfilebuffersize, writeFiles or writetomemory." error $ei $ei diff --git a/scripts/writer.tcl b/scripts/writer.tcl index 7f045f5..80ca137 100644 --- a/scripts/writer.tcl +++ b/scripts/writer.tcl @@ -7,12 +7,12 @@ namespace eval cookfs {} namespace eval cookfs::tcl {} -namespace eval cookfs::tcl::writer {} - -set ::cookfs::writerhandleIdx 0 +namespace eval cookfs::tcl::writer { + variable writerhandleIdx 0 +} proc cookfs::tcl::writer {args} { - set name ::cookfs::tcl::writer::handle[incr ::cookfs::writerhandleIdx] + set name ::cookfs::tcl::writer::handle[incr writer::writerhandleIdx] upvar #0 $name c array set c { pages "" diff --git a/tests/all.tcl b/tests/all.tcl index 50f0928..01e5cd7 100644 --- a/tests/all.tcl +++ b/tests/all.tcl @@ -17,6 +17,10 @@ if {[info tclversion] == "8.5"} { source [file join [tcltest::testsDirectory] common.tcl] +if {[info exists ::env(MEMDEBUG)]} { + source [file join [tcltest::testsDirectory] memdebug.tcl] +} + set constraints [tcltest::configure -constraints] if {[file exists [file join [tcltest::testsDirectory] .. cookfswriter cookfswriter.tcl]]} { diff --git a/tests/dedup.test b/tests/dedup.test index a6a0f1f..82cc7ee 100644 --- a/tests/dedup.test +++ b/tests/dedup.test @@ -27,7 +27,7 @@ tcltest::test cookfsDedup-1.2 "Small file deduplication for same resulting pages -pagesize 16384 -smallfilesize 16384 -smallfilebuffer 16384] set i 0 foreach d [concat $datas $datas $datas $datas] { - set fh [open $file/d[incr i] w] + set fh [open $file/d[format "%02i" [incr i]] w] fconfigure $fh -translation binary puts -nonewline $fh $d close $fh diff --git a/tests/memdebug.tcl b/tests/memdebug.tcl new file mode 100644 index 0000000..3c8847f --- /dev/null +++ b/tests/memdebug.tcl @@ -0,0 +1,35 @@ +# (c) 2024 Konstantin Kushnir + +rename tcltest::test tcltest::test_memdebug + +proc current_mem {} { + lindex [split [lindex [lmap x [split [memory info] "\n"] { if { ![string match {current bytes allocated*} $x] } { continue } { set x } }] 0] " "] end +} + +proc tcltest::test { args } { + set id [lindex $args 0] + + set mem [current_mem] + + catch { + memory tag $id + uplevel 1 [list tcltest::test_memdebug {*}$args] + memory tag "" + } res opts + + puts "$id leak [expr { [current_mem] - $mem }]" + + catch { file delete -force memdump } + memory active memdump + set fi [open memdump r] + set fo [open "${id}.dump" w] + while { [gets $fi line] != -1 } { + if { [string match "* $id" $line] } { + puts $fo $line + } + } + close $fi + close $fo + file delete -force memdump + +} \ No newline at end of file diff --git a/tests/pkgconfig.test b/tests/pkgconfig.test index 6c9fa89..a9ab389 100644 --- a/tests/pkgconfig.test +++ b/tests/pkgconfig.test @@ -19,6 +19,7 @@ c-fsindex bool c-pages bool c-readerchannel bool c-vfs bool +c-writer bool c-writerchannel bool feature-aside bool feature-bzip2 bool diff --git a/tests/vfs.test b/tests/vfs.test index 99cdf98..4aef5b6 100644 --- a/tests/vfs.test +++ b/tests/vfs.test @@ -139,16 +139,16 @@ tcltest::test cookfsVfs-1.9 "Test storing same files" -setup { set h [vfs::cookfs::Mount $file $file -readonly] if {![fsindexEqual [[$h getindex] get test4a] [[$h getindex] get test4b]]} { - error "Index for test4 differs: [[set ${h}(index)] get test4a] != [[set ${h}(index)] get test4b]" + error "Index for test4 differs: [[$h getindex] get test4a] != [[$h getindex] get test4b]" } if {![fsindexEqual [[$h getindex] get test3a] [[$h getindex] get test3b]]} { - error "Index for test3 differs: [[set ${h}(index)] get test3a] != [[set ${h}(index)] get test3b]" + error "Index for test3 differs: [[$h getindex] get test3a] != [[$h getindex] get test3b]" } if {![fsindexEqual [[$h getindex] get test2a] [[$h getindex] get test2b]]} { - error "Index for test2 differs: [[set ${h}(index)] get test2a] != [[set ${h}(index)] get test2b]" + error "Index for test2 differs: [[$h getindex] get test2a] != [[$h getindex] get test2b]" } if {![fsindexEqual [[$h getindex] get test1a] [[$h getindex] get test1b]]} { - error "Index for test1 differs: [[set ${h}(index)] get test1a] != [[set ${h}(index)] get test1b]" + error "Index for test1 differs: [[$h getindex] get test1a] != [[$h getindex] get test1b]" } } -cleanup { catch {vfs::unmount $file}