From 590143b49578830377ed355f63b3184fea0df7eb Mon Sep 17 00:00:00 2001 From: Konstantin Kushnir Date: Fri, 7 Jun 2024 05:54:43 +0000 Subject: [PATCH 1/2] Eliminate Tcl_Obj usage by pages object --- ChangeLog | 3 + configure | 5 +- configure.in | 5 +- generic/cookfs.h | 1 + generic/fsindexCmd.c | 4 +- generic/fsindexIO.c | 72 ++++++++++++----- generic/fsindexIO.h | 5 +- generic/pageObj.c | 38 +++++++++ generic/pageObj.h | 59 ++++++++++++++ generic/pages.c | 165 ++++++++++++++++++++------------------ generic/pages.h | 17 ++-- generic/pagesCmd.c | 40 +++++++-- generic/pagesCompr.c | 158 +++++++++++++++++++++--------------- generic/pagesCompr.h | 7 +- generic/readerchannel.c | 2 +- generic/readerchannel.h | 2 +- generic/readerchannelIO.c | 15 ++-- generic/vfs.c | 6 +- generic/vfsCmd.c | 2 +- generic/writerchannel.c | 11 +-- 20 files changed, 409 insertions(+), 208 deletions(-) create mode 100644 generic/pageObj.c create mode 100644 generic/pageObj.h diff --git a/ChangeLog b/ChangeLog index 6725c42..6d5617d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +2024-06-07 Konstantin Kushnir + * Thread support preparation: eliminate Tcl_Obj usage by pages object + 2024-06-06 Konstantin Kushnir * Bumped version to 1.7.0 diff --git a/configure b/configure index 82435be..c7b83c0 100755 --- a/configure +++ b/configure @@ -8729,7 +8729,7 @@ if test ${USECPAGES} = yes; then COOKFS_PKGCONFIG_USECPAGES=1 COOKFS_PKGCONFIG_FEATURE_ASIDE=1 - vars="pages.c pagesCompr.c pagesCmd.c" + vars="pageObj.c pages.c pagesCompr.c pagesCmd.c" for i in $vars; do case $i in \$*) @@ -8805,6 +8805,9 @@ if test ${USECPAGES} = yes; then printf "%s\n" "#define COOKFS_USEBZ2 1" >>confdefs.h COOKFS_PKGCONFIG_USEBZ2=1 + # This is required to compile bz2 sources. + # See: https://github.com/Perl/perl5/issues/17528 + #TEA_ADD_CFLAGS([-Wno-implicit-fallthrough]) else COOKFS_PKGCONFIG_USEBZ2=0 fi diff --git a/configure.in b/configure.in index a8f44e6..4057693 100644 --- a/configure.in +++ b/configure.in @@ -146,7 +146,7 @@ if test ${USECPAGES} = yes; then AC_DEFINE(COOKFS_USECPAGES) COOKFS_PKGCONFIG_USECPAGES=1 COOKFS_PKGCONFIG_FEATURE_ASIDE=1 - TEA_ADD_SOURCES([pages.c pagesCompr.c pagesCmd.c]) + TEA_ADD_SOURCES([pageObj.c pages.c pagesCompr.c pagesCmd.c]) # enable bz2 files only if pages are handled using C if test ${USEBZ2} = yes; then @@ -154,6 +154,9 @@ if test ${USECPAGES} = yes; then TEA_ADD_SOURCES([bzip2/blocksort.c bzip2/huffman.c bzip2/crctable.c bzip2/randtable.c bzip2/compress.c bzip2/decompress.c bzip2/bzlib.c]) AC_DEFINE(COOKFS_USEBZ2) COOKFS_PKGCONFIG_USEBZ2=1 + # This is required to compile bz2 sources. + # See: https://github.com/Perl/perl5/issues/17528 + #TEA_ADD_CFLAGS([-Wno-implicit-fallthrough]) else COOKFS_PKGCONFIG_USEBZ2=0 fi diff --git a/generic/cookfs.h b/generic/cookfs.h index 5f6e1dd..9d0cf1f 100644 --- a/generic/cookfs.h +++ b/generic/cookfs.h @@ -53,6 +53,7 @@ #include "hashes.h" #ifdef COOKFS_USECPAGES +#include "pageObj.h" #include "pages.h" #include "pagesCmd.h" #include "pagesCompr.h" diff --git a/generic/fsindexCmd.c b/generic/fsindexCmd.c index d559e8d..f1c511a 100644 --- a/generic/fsindexCmd.c +++ b/generic/fsindexCmd.c @@ -167,7 +167,7 @@ static int CookfsRegisterFsindexObjectCmd(ClientData clientData, Tcl_Interp *int /* import fsindex from specified data if specified, otherwise create new fsindex */ if (objc == 2) { - i = Cookfs_FsindexFromObject(interp, NULL, objv[1]); + i = Cookfs_FsindexFromTclObj(interp, NULL, objv[1]); CookfsLog(printf("CookfsRegisterFsindexObjectCmd: created fsindex from obj [%p]", (void *)i)); } else { i = Cookfs_FsindexInit(interp, NULL); @@ -420,7 +420,7 @@ static int CookfsFsindexCmdImport(Cookfs_Fsindex *fsIndex, Tcl_Interp *interp, i return TCL_ERROR; } - Cookfs_Fsindex *result = Cookfs_FsindexFromObject(interp, fsIndex, objv[2]); + Cookfs_Fsindex *result = Cookfs_FsindexFromTclObj(interp, fsIndex, objv[2]); return (result == NULL ? TCL_ERROR : TCL_OK); } diff --git a/generic/fsindexIO.c b/generic/fsindexIO.c index 0dc0d1c..f69c7b9 100644 --- a/generic/fsindexIO.c +++ b/generic/fsindexIO.c @@ -35,6 +35,18 @@ static int CookfsFsindexImportMetadata(Cookfs_Fsindex *fsIndex, unsigned char *b * subdirectories are inlined completely and recursively */ +Cookfs_PageObj Cookfs_FsindexToPageObj(Cookfs_Fsindex *fsindex) { + Cookfs_PageObj rc; + Tcl_Obj *obj = Cookfs_FsindexToObject(fsindex); + if (obj != NULL) { + Tcl_IncrRefCount(obj); + rc = Cookfs_PageObjNewFromByteArray(obj); + Tcl_DecrRefCount(obj); + } else { + rc = NULL; + } + return rc; +} /* *---------------------------------------------------------------------- @@ -102,17 +114,28 @@ Cookfs_Fsindex *Cookfs_FsindexFromPages(Tcl_Interp *interp, Cookfs_Fsindex *fsin Cookfs_Fsindex *rc; CookfsLog(printf("Cookfs_FsindexFromPages: get index data from pages...")); - Tcl_Obj *indexDataObj = Cookfs_PagesGetIndex(pages); - Tcl_IncrRefCount(indexDataObj); + Cookfs_PageObj indexDataObj = Cookfs_PagesGetIndex(pages); Tcl_Size indexDataLen; - Tcl_GetByteArrayFromObj(indexDataObj, &indexDataLen); - CookfsLog(printf("Cookfs_FsindexFromPages: got index data %" - TCL_SIZE_MODIFIER "d bytes", indexDataLen)); + if (indexDataObj == NULL) { + CookfsLog(printf("Cookfs_FsindexFromPages: got NULL as index data")); + indexDataLen = 0; + } else { + Cookfs_PageObjIncrRefCount(indexDataObj); + indexDataLen = Cookfs_PageObjSize(indexDataObj); + CookfsLog(printf("Cookfs_FsindexFromPages: got index data %" + TCL_SIZE_MODIFIER "d bytes", indexDataLen)); + // If data is empty, then free the object now as at the end we + // will release the object only if indexDataLen > 0 + if (!indexDataLen) { + Cookfs_PageObjDecrRefCount(indexDataObj); + } + } if (indexDataLen) { CookfsLog(printf("Cookfs_FsindexFromPages: import from the object...")); - rc = Cookfs_FsindexFromObject(interp, fsindex, indexDataObj); + rc = Cookfs_FsindexFromPageObj(interp, fsindex, indexDataObj); + Cookfs_PageObjDecrRefCount(indexDataObj); } else { if (fsindex == NULL) { CookfsLog(printf("Cookfs_FsindexFromPages: create a new clean index")); @@ -124,20 +147,30 @@ Cookfs_Fsindex *Cookfs_FsindexFromPages(Tcl_Interp *interp, Cookfs_Fsindex *fsin } } - Tcl_DecrRefCount(indexDataObj); CookfsLog(printf("Cookfs_FsindexFromPages: return [%p]", (void *)rc)); return rc; } #endif /* COOKFS_USECPAGES */ +Cookfs_Fsindex *Cookfs_FsindexFromTclObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Tcl_Obj *o) { + Tcl_Size size; + unsigned char *bytes = Tcl_GetByteArrayFromObj(o, &size); + return Cookfs_FsindexFromBytes(interp, fsindex, bytes, size); + +} + +Cookfs_Fsindex *Cookfs_FsindexFromPageObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Cookfs_PageObj o) { + return Cookfs_FsindexFromBytes(interp, fsindex, o, Cookfs_PageObjSize(o)); +} + /* *---------------------------------------------------------------------- * - * Cookfs_FsindexFromObject -- + * Cookfs_FsindexFromBytes -- * - * Imports fsindex information from byte array object - * storing platform-independant binary data + * Imports fsindex information from raw buffer storing platform-independant + * binary data * * Results: * Pointer to imported Cookfs_Fsindex; NULL in case of import error @@ -148,10 +181,8 @@ Cookfs_Fsindex *Cookfs_FsindexFromPages(Tcl_Interp *interp, Cookfs_Fsindex *fsin *---------------------------------------------------------------------- */ -Cookfs_Fsindex *Cookfs_FsindexFromObject(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Tcl_Obj *o) { - Tcl_Size objLength; +Cookfs_Fsindex *Cookfs_FsindexFromBytes(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, unsigned char *bytes, Tcl_Size size) { int i; - unsigned char *bytes; Cookfs_Fsindex *result; CookfsLog(printf("Cookfs_FsindexFromObject - BEGIN")) @@ -169,10 +200,7 @@ Cookfs_Fsindex *Cookfs_FsindexFromObject(Tcl_Interp *interp, Cookfs_Fsindex *fsi return NULL; } - /* get bytes from binary data and check if they contain proper header */ - bytes = Tcl_GetByteArrayFromObj(o, &objLength); - - if (objLength < COOKFS_FSINDEX_HEADERLENGTH) { + if (size < COOKFS_FSINDEX_HEADERLENGTH) { CookfsLog(printf("Cookfs_FsindexFromObject - unable to compare header 1")) return NULL; } @@ -182,17 +210,17 @@ Cookfs_Fsindex *Cookfs_FsindexFromObject(Tcl_Interp *interp, Cookfs_Fsindex *fsi } /* import root entry; import of subdirectories happens recursively */ - i = CookfsFsindexImportDirectory(result, result->rootItem, bytes, objLength, 8); + i = CookfsFsindexImportDirectory(result, result->rootItem, bytes, size, 8); CookfsLog(printf("Cookfs_FsindexFromObject - Import directory done -" - " %d vs %" TCL_SIZE_MODIFIER "d", i, objLength)) - if (i < objLength) { + " %d vs %" TCL_SIZE_MODIFIER "d", i, size)) + if (i < size) { // cppcheck-suppress unreadVariable symbolName=i - i = CookfsFsindexImportMetadata(result, bytes, objLength, i); + i = CookfsFsindexImportMetadata(result, bytes, size, i); } CookfsLog(printf("Cookfs_FsindexFromObject - Import metadata done -" - " %d vs %" TCL_SIZE_MODIFIER "d", i, objLength)) + " %d vs %" TCL_SIZE_MODIFIER "d", i, size)) Cookfs_FsindexResetChangeCount(result); diff --git a/generic/fsindexIO.h b/generic/fsindexIO.h index c1ba390..97b837f 100644 --- a/generic/fsindexIO.h +++ b/generic/fsindexIO.h @@ -7,7 +7,10 @@ #ifdef COOKFS_USECFSINDEX Tcl_Obj *Cookfs_FsindexToObject(Cookfs_Fsindex *fsindex); -Cookfs_Fsindex *Cookfs_FsindexFromObject(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Tcl_Obj *o); +Cookfs_PageObj Cookfs_FsindexToPageObj(Cookfs_Fsindex *fsindex); +Cookfs_Fsindex *Cookfs_FsindexFromBytes(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, unsigned char *bytes, Tcl_Size size); +Cookfs_Fsindex *Cookfs_FsindexFromTclObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Tcl_Obj *o); +Cookfs_Fsindex *Cookfs_FsindexFromPageObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Cookfs_PageObj o); #ifdef COOKFS_USECPAGES Cookfs_Fsindex *Cookfs_FsindexFromPages(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Cookfs_Pages *pages); #endif /* COOKFS_USECPAGES */ diff --git a/generic/pageObj.c b/generic/pageObj.c new file mode 100644 index 0000000..5b873ec --- /dev/null +++ b/generic/pageObj.c @@ -0,0 +1,38 @@ +/* + * pageObj.c + * + * Provides functions creating and managing a page object + * + * (c) 2024 Konstantin Kushnir + */ + +#include "cookfs.h" + +Cookfs_PageObj Cookfs_PageObjAlloc(Tcl_Size size) { + // Align physical page size to 16 bytes. This will be usefull for + // AES encryption. + Tcl_Size bufferSize = size + (16 - (size % 16)); + CookfsLog(printf("Cookfs_PageObjAlloc: want bytes %" TCL_SIZE_MODIFIER "d" + " alloc %" TCL_SIZE_MODIFIER "d bytes", size, bufferSize)); + Cookfs_PageObj p = ckalloc(bufferSize + sizeof(Cookfs_PageObjStruct)); + if (p != NULL) { + Cookfs_PageObjStruct *s = (Cookfs_PageObjStruct *)p; + s->bufferSize = bufferSize; + s->effectiveSize = size; + s->refCount = 0; + p += sizeof(Cookfs_PageObjStruct); + } + CookfsLog(printf("Cookfs_PageObjAlloc: return %p", (void *)p)); + return p; +} + +Cookfs_PageObj Cookfs_PageObjNewFromByteArray(Tcl_Obj *obj) { + Tcl_Size size; + unsigned char *bytes = Tcl_GetByteArrayFromObj(obj, &size); + Cookfs_PageObj rc = Cookfs_PageObjAlloc(size); + if (rc != NULL) { + memcpy(rc, bytes, size); + } + return rc; +} + diff --git a/generic/pageObj.h b/generic/pageObj.h new file mode 100644 index 0000000..b8c1fcb --- /dev/null +++ b/generic/pageObj.h @@ -0,0 +1,59 @@ +/* + (c) 2024 Konstantin Kushnir +*/ + +#ifndef COOKFS_PAGEOBJ_H +#define COOKFS_PAGEOBJ_H 1 + +typedef unsigned char* Cookfs_PageObj; + +typedef struct Cookfs_PageObjStruct { + Tcl_Size bufferSize; + Tcl_Size effectiveSize; + int refCount; +} Cookfs_PageObjStruct; + +/* +#define Cookfs_PageObjIncrRefCount(p) \ + ((Cookfs_PageObjStruct *)((Cookfs_PageObj)(p) - \ + sizeof(Cookfs_PageObjStruct)))->refCount++ +*/ + +#define Cookfs_PageObjIncrRefCount(p) \ + { \ + ((Cookfs_PageObjStruct *)((Cookfs_PageObj)(p) - \ + sizeof(Cookfs_PageObjStruct)))->refCount++; \ + CookfsLog(printf("Cookfs_PageObjIncrRefCount: %p", (void *)p)); \ + } + +/* +#define Cookfs_PageObjDecrRefCount(p) \ + { \ + Cookfs_PageObjStruct *tmp = (Cookfs_PageObjStruct *)( \ + (Cookfs_PageObj)(p) - sizeof(Cookfs_PageObjStruct)); \ + if (!(--tmp->refCount)) { ckfree(tmp); } \ + } +*/ + +#define Cookfs_PageObjDecrRefCount(p) \ + { \ + CookfsLog(printf("Cookfs_PageObjDecrRefCount: %p", (void *)p)); \ + Cookfs_PageObjStruct *tmp = (Cookfs_PageObjStruct *)( \ + (Cookfs_PageObj)(p) - sizeof(Cookfs_PageObjStruct)); \ + if (!(--tmp->refCount)) { \ + ckfree(tmp); \ + CookfsLog(printf("Cookfs_PageObjDecrRefCount: release %p", (void *)p)); \ + } \ + } + +#define Cookfs_PageObjSize(p) \ + (((Cookfs_PageObjStruct *)((Cookfs_PageObj)(p) - \ + sizeof(Cookfs_PageObjStruct)))->effectiveSize) + +#define Cookfs_PageObjCopyAsByteArray(p) \ + Tcl_NewByteArrayObj(p, Cookfs_PageObjSize(p)) + +Cookfs_PageObj Cookfs_PageObjAlloc(Tcl_Size size); +Cookfs_PageObj Cookfs_PageObjNewFromByteArray(Tcl_Obj *obj); + +#endif /* COOKFS_PAGEOBJ_H */ diff --git a/generic/pages.c b/generic/pages.c index e2e26e1..31af93d 100644 --- a/generic/pages.c +++ b/generic/pages.c @@ -19,7 +19,7 @@ #define COOKFS_PAGES_ERRORMSG "Unable to create Cookfs object" /* declarations of static and/or internal functions */ -static Tcl_Obj *CookfsPagesPageGetInt(Cookfs_Pages *p, int index); +static Cookfs_PageObj CookfsPagesPageGetInt(Cookfs_Pages *p, int index); static void CookfsPagesPageCacheMoveToTop(Cookfs_Pages *p, int index); static int CookfsReadIndex(Tcl_Interp *interp, Cookfs_Pages *p); static void CookfsPagesPageExtendIfNeeded(Cookfs_Pages *p, int count); @@ -294,7 +294,7 @@ Cookfs_Pages *Cookfs_PagesInit(Tcl_Interp *interp, Tcl_Obj *fileName, int fileRe rc->dataAsidePages = NULL; rc->dataPagesIsAside = isAside; - rc->dataIndex = Tcl_NewStringObj("", 0); + rc->dataIndex = NULL; rc->asyncPageSize = 0; rc->asyncDecompressQueue = 0; // TODO: un-hardcode! @@ -319,8 +319,6 @@ Cookfs_Pages *Cookfs_PagesInit(Tcl_Interp *interp, Tcl_Obj *fileName, int fileRe Tcl_IncrRefCount(rc->zipCmdCrc[0]); #endif - Tcl_IncrRefCount(rc->dataIndex); - /* initialize cache */ for (i = 0; i < COOKFS_MAX_CACHE_PAGES; i++) { rc->cache[i].pageObj = NULL; @@ -456,7 +454,7 @@ Tcl_WideInt Cookfs_PagesClose(Cookfs_Pages *p) { CookfsLog(printf("Cookfs_PagesClose - Pages up to date = %d, Index changed = %d", p->pagesUptodate, p->indexChanged)) /* if changes were made, save them to disk */ if ((!p->pagesUptodate) || (p->indexChanged)) { - int indexSize; + int indexSize = 0; unsigned char buf[COOKFS_SUFFIX_BYTES]; Tcl_Obj *obj; @@ -491,10 +489,12 @@ Tcl_WideInt Cookfs_PagesClose(Cookfs_Pages *p) { } /* write index */ - indexSize = Cookfs_WritePageObj(p, -1, p->dataIndex, NULL); - if (indexSize < 0) { - /* TODO: handle index writing issues better */ - Tcl_Panic("Unable to compress index"); + if (p->dataIndex != NULL) { + indexSize = Cookfs_WritePageObj(p, -1, p->dataIndex, NULL); + if (indexSize < 0) { + /* TODO: handle index writing issues better */ + Tcl_Panic("Unable to compress index"); + } } CookfsLog(printf("Cookfs_PagesClose - Offset write: %d", (int) Tcl_Seek(p->fileChannel, 0, SEEK_CUR))) @@ -577,7 +577,7 @@ void Cookfs_PagesFini(Cookfs_Pages *p) { CookfsLog(printf("Cleaning up cache")) for (i = 0; i < p->cacheSize; i++) { if (p->cache[i].pageObj != NULL) { - Tcl_DecrRefCount(p->cache[i].pageObj); + Cookfs_PageObjDecrRefCount(p->cache[i].pageObj); } } @@ -601,7 +601,9 @@ void Cookfs_PagesFini(Cookfs_Pages *p) { /* clean up index */ CookfsLog(printf("Cleaning up index data")) - Tcl_DecrRefCount(p->dataIndex); + if (p->dataIndex != NULL) { + Cookfs_PageObjDecrRefCount(p->dataIndex); + } /* clean up pages data */ CookfsLog(printf("Cleaning up pages MD5/size")) @@ -820,7 +822,7 @@ int Cookfs_PageAddStamp(Cookfs_Pages *p, Tcl_WideInt size) { * * Cookfs_PageAdd -- * - * Same as Cookfs_PageAddRaw, but uses Tcl_Obj as the page data source. + * Same as Cookfs_PageAddRaw, but uses Cookfs_PageObj as the page data source. * * Results: * Index that can be used in subsequent calls to Cookfs_PageGet() @@ -831,12 +833,31 @@ int Cookfs_PageAddStamp(Cookfs_Pages *p, Tcl_WideInt size) { *---------------------------------------------------------------------- */ -int Cookfs_PageAdd(Cookfs_Pages *p, Tcl_Obj *dataObj) { - Tcl_Size objLength; - unsigned char *bytes = Tcl_GetByteArrayFromObj(dataObj, &objLength); - return Cookfs_PageAddRaw(p, bytes, objLength); +int Cookfs_PageAdd(Cookfs_Pages *p, Cookfs_PageObj dataObj) { + return Cookfs_PageAddRaw(p, dataObj, Cookfs_PageObjSize(dataObj)); } +/* + *---------------------------------------------------------------------- + * + * Cookfs_PageAddTclObj -- + * + * Same as Cookfs_PageAddRaw, but uses Tcl_Obj as the page data source. + * + * Results: + * Index that can be used in subsequent calls to Cookfs_PageGet() + * + * Side effects: + * None + * + *---------------------------------------------------------------------- + */ + +int Cookfs_PageAddTclObj(Cookfs_Pages *p, Tcl_Obj *dataObj) { + Tcl_Size size; + unsigned char *bytes = Tcl_GetByteArrayFromObj(dataObj, &size); + return Cookfs_PageAddRaw(p, bytes, size); +} /* *---------------------------------------------------------------------- @@ -902,38 +923,33 @@ int Cookfs_PageAddRaw(Cookfs_Pages *p, unsigned char *bytes, int objLength) { for (idx = 0; idx < p->dataNumPages; idx++, pageMD5 += 16) { if (memcmp(pageMD5, md5sum, 16) == 0) { /* even if MD5 checksums are the same, we still need to validate contents of the page */ - Tcl_Obj *otherPageData; - unsigned char *otherBytes; - Tcl_Size otherObjLength; - int isMatched = 1; + Cookfs_PageObj otherPageData; + int isMatched = 0; CookfsLog(printf("Cookfs_PageAdd: Comparing page %d", idx)) /* use -1000 weight as it is temporary page and we don't really need it in cache */ otherPageData = Cookfs_PageGet(p, idx, -1000); - // No need to incr refcounter on otherPageData because Cookfs_PageGet() - // always returns pages with refcount=1. - - /* fail in case when decompression is not available */ - if (otherPageData == NULL) { - CookfsLog(printf("Cookfs_PageAdd: Unable to verify page with same MD5 checksum")) - return -1; - } - otherBytes = Tcl_GetByteArrayFromObj(otherPageData, &otherObjLength); - - /* if page with same checksum was found, verify its contents as we + /* fail in case when decompression is not available + * + * if page with same checksum was found, verify its contents as we * do not rely on MD5 checksum - this avoids issue with MD5 collissions */ - if (otherObjLength != objLength) { - isMatched = 0; - } else { - if (memcmp(bytes, otherBytes, objLength) != 0) { - isMatched = 0; - } + if (otherPageData == NULL) { + CookfsLog(printf("Cookfs_PageAdd: Unable to verify page with same MD5 checksum")); + return -1; + } else { + Cookfs_PageObjIncrRefCount(otherPageData); + if (Cookfs_PageObjSize(otherPageData) != objLength) { + CookfsLog(printf("Cookfs_PageAdd: the length doesn't match")) + } else if (memcmp(bytes, otherPageData, objLength) != 0) { + CookfsLog(printf("Cookfs_PageAdd: the data doesn't match")) + } else { + isMatched = 1; + } + Cookfs_PageObjDecrRefCount(otherPageData); } - Tcl_DecrRefCount(otherPageData); - if (isMatched) { CookfsLog(printf("Cookfs_PageAdd: Matched page (size=%d bytes) as %d", objLength, idx)) if (p->dataPagesIsAside) { @@ -996,8 +1012,8 @@ int Cookfs_PageAddRaw(Cookfs_Pages *p, unsigned char *bytes, int objLength) { * Gets contents of a page at specified index and sets its weight in cache * * Results: - * Tcl_Obj with page data as bytearray object - * This Tcl_Obj may also be used by pages cache therefore it is + * Cookfs_PageObj with page data + * This Cookfs_PageObj may also be used by pages cache therefore it is * needed to Tcl_IncrRefCount() / Tcl_DecrRefCount() to properly * handle memory management * @@ -1009,8 +1025,8 @@ int Cookfs_PageAddRaw(Cookfs_Pages *p, unsigned char *bytes, int objLength) { *---------------------------------------------------------------------- */ -Tcl_Obj *Cookfs_PageGet(Cookfs_Pages *p, int index, int weight) { - Tcl_Obj *rc; +Cookfs_PageObj Cookfs_PageGet(Cookfs_Pages *p, int index, int weight) { + Cookfs_PageObj rc; int preloadIndex = index + 1; CookfsLog(printf("Cookfs_PageGet: index [%d] with weight [%d]", index, weight)) @@ -1041,10 +1057,6 @@ Tcl_Obj *Cookfs_PageGet(Cookfs_Pages *p, int index, int weight) { rc = Cookfs_PageCacheGet(p, index, 1, weight); if (rc != NULL) { CookfsLog(printf("Cookfs_PageGet: Returning from cache [%p]", (void *)rc)); - // Increase refcount by 1 for pages from cache because - // CookfsPagesPageGetInt()->Cookfs_ReadPage() returns pages with - // refcount=1 - Tcl_IncrRefCount(rc); return rc; } @@ -1069,7 +1081,7 @@ Tcl_Obj *Cookfs_PageGet(Cookfs_Pages *p, int index, int weight) { * its weight if the argument update is true * * Results: - * Tcl_Obj with page data as bytearray object + * Cookfs_PageObj with page data * NULL if not cached * * Side effects: @@ -1080,8 +1092,8 @@ Tcl_Obj *Cookfs_PageGet(Cookfs_Pages *p, int index, int weight) { *---------------------------------------------------------------------- */ -Tcl_Obj *Cookfs_PageCacheGet(Cookfs_Pages *p, int index, int update, int weight) { - Tcl_Obj *rc; +Cookfs_PageObj Cookfs_PageCacheGet(Cookfs_Pages *p, int index, int update, int weight) { + Cookfs_PageObj rc; int i; /* if page is disabled, immediately get page */ @@ -1124,7 +1136,7 @@ Tcl_Obj *Cookfs_PageCacheGet(Cookfs_Pages *p, int index, int update, int weight) *---------------------------------------------------------------------- */ -void Cookfs_PageCacheSet(Cookfs_Pages *p, int idx, Tcl_Obj *obj, int weight) { +void Cookfs_PageCacheSet(Cookfs_Pages *p, int idx, Cookfs_PageObj obj, int weight) { if (p->cacheSize <= 0) { return; @@ -1187,13 +1199,13 @@ void Cookfs_PageCacheSet(Cookfs_Pages *p, int idx, Tcl_Obj *obj, int weight) { } /* release the previous entry */ - Tcl_DecrRefCount(p->cache[newIdx].pageObj); + Cookfs_PageObjDecrRefCount(p->cache[newIdx].pageObj); found_empty: p->cache[newIdx].pageIdx = idx; p->cache[newIdx].pageObj = obj; p->cache[newIdx].weight = weight; - Tcl_IncrRefCount(obj); + Cookfs_PageObjIncrRefCount(obj); CookfsLog(printf("Cookfs_PageCacheSet: replace entry [%d]", newIdx)); /* age will be set by CookfsPagesPageCacheMoveToTop */ CookfsPagesPageCacheMoveToTop(p, newIdx); @@ -1454,11 +1466,9 @@ void Cookfs_PagesSetAside(Cookfs_Pages *p, Cookfs_Pages *aside) { p->dataAsidePages = aside; if (aside != NULL) { CookfsLog(printf("Cookfs_PagesSetAside: Checking if index in add-aside archive should be overwritten.")) - Tcl_Obj *asideIndex; - Tcl_Size asideIndexLength; + Cookfs_PageObj asideIndex; asideIndex = Cookfs_PagesGetIndex(aside); - Tcl_GetByteArrayFromObj(asideIndex, &asideIndexLength); - if (asideIndexLength == 0) { + if (asideIndex == NULL) { CookfsLog(printf("Cookfs_PagesSetAside: Copying index from main archive to add-aside archive.")) Cookfs_PagesSetIndex(aside, p->dataIndex); CookfsLog(printf("Cookfs_PagesSetAside: done copying index.")) @@ -1481,21 +1491,23 @@ void Cookfs_PagesSetAside(Cookfs_Pages *p, Cookfs_Pages *aside) { * None * * Side effects: - * Reference counter for Tcl_Obj storing previous index is + * Reference counter for Cookfs_PageObj storing previous index is * decremented; improper handling of ref count for indexes * might cause crashes * *---------------------------------------------------------------------- */ -void Cookfs_PagesSetIndex(Cookfs_Pages *p, Tcl_Obj *dataIndex) { +void Cookfs_PagesSetIndex(Cookfs_Pages *p, Cookfs_PageObj dataIndex) { if (p->dataAsidePages != NULL) { - Cookfs_PagesSetIndex(p->dataAsidePages, dataIndex); + Cookfs_PagesSetIndex(p->dataAsidePages, dataIndex); } else { - Tcl_DecrRefCount(p->dataIndex); - p->dataIndex = dataIndex; + if (p->dataIndex != NULL) { + Cookfs_PageObjDecrRefCount(p->dataIndex); + } + p->dataIndex = dataIndex; p->indexChanged = 1; - Tcl_IncrRefCount(p->dataIndex); + Cookfs_PageObjIncrRefCount(p->dataIndex); } } @@ -1509,7 +1521,7 @@ void Cookfs_PagesSetIndex(Cookfs_Pages *p, Tcl_Obj *dataIndex) { * metadata * * Results: - * Tcl_Obj storing index as bytearray + * Cookfs_PageObj storing index * * This is passed to Cookfs_FsindexFromObject() * @@ -1519,7 +1531,7 @@ void Cookfs_PagesSetIndex(Cookfs_Pages *p, Tcl_Obj *dataIndex) { *---------------------------------------------------------------------- */ -Tcl_Obj *Cookfs_PagesGetIndex(Cookfs_Pages *p) { +Cookfs_PageObj Cookfs_PagesGetIndex(Cookfs_Pages *p) { if (p->dataAsidePages != NULL) { return Cookfs_PagesGetIndex(p->dataAsidePages); } else { @@ -1560,7 +1572,7 @@ void Cookfs_PagesSetCacheSize(Cookfs_Pages *p, int size) { p->cache[i].weight = 0; p->cache[i].pageIdx = -1; if (p->cache[i].pageObj != NULL) { - Tcl_DecrRefCount(p->cache[i].pageObj); + Cookfs_PageObjDecrRefCount(p->cache[i].pageObj); p->cache[i].pageObj = NULL; } } @@ -1724,9 +1736,7 @@ Tcl_WideInt Cookfs_PagesGetPageOffset(Cookfs_Pages *p, int idx) { * also manages caching of pages. * * Results: - * Tcl_Obj with page information; NULL otherwise - * If non-NULL, returned object already has its reference counter - * incremented by 1 (i.e. should not eb incremented again if cached) + * Cookfs_PageObj with page information; NULL otherwise * * Side effects: * None @@ -1734,8 +1744,8 @@ Tcl_WideInt Cookfs_PagesGetPageOffset(Cookfs_Pages *p, int idx) { *---------------------------------------------------------------------- */ -static Tcl_Obj *CookfsPagesPageGetInt(Cookfs_Pages *p, int index) { - Tcl_Obj *buffer; +static Cookfs_PageObj CookfsPagesPageGetInt(Cookfs_Pages *p, int index) { + Cookfs_PageObj buffer; CookfsLog(printf("Cookfs_PageGetInt: index [%d]", index)) /* if specified index is an aside-index */ @@ -1919,9 +1929,13 @@ static int CookfsReadIndex(Tcl_Interp *interp, Cookfs_Pages *p) { } CookfsLog(printf("IndexOffset Read = %d", (int) seekOffset)) - buffer = Cookfs_ReadPage(p, -1, indexLength, 1, COOKFS_COMPRESSION_ANY); + if (p->dataIndex != NULL) { + Cookfs_PageObjDecrRefCount(p->dataIndex); + } - if (buffer != NULL) { + p->dataIndex = Cookfs_ReadPage(p, -1, indexLength, 1, COOKFS_COMPRESSION_ANY); + + if (p->dataIndex != NULL) { goto indexReadOk; } @@ -1935,11 +1949,8 @@ static int CookfsReadIndex(Tcl_Interp *interp, Cookfs_Pages *p) { indexReadOk: + Cookfs_PageObjIncrRefCount(p->dataIndex); /* read page MD5 checksums and pages */ - Tcl_DecrRefCount(p->dataIndex); - p->dataIndex = buffer; - // Do not increase refcount for p->dataIndex because Cookfs_ReadPage returns Tcl_Obj with refcount=1 - /* seek to beginning of data, depending on if foffset was specified */ Tcl_Seek(p->fileChannel, p->foffset, SEEK_SET); seekOffset = Tcl_Seek(p->fileChannel, -COOKFS_SUFFIX_BYTES - (pageCount * 20) - indexLength, SEEK_CUR); diff --git a/generic/pages.h b/generic/pages.h index 1d17716..9e5d31c 100644 --- a/generic/pages.h +++ b/generic/pages.h @@ -51,7 +51,7 @@ typedef struct Cookfs_CacheEntry { int pageIdx; int weight; int age; - Tcl_Obj *pageObj; + Cookfs_PageObj pageObj; } Cookfs_CacheEntry; typedef struct Cookfs_Pages { @@ -95,7 +95,7 @@ typedef struct Cookfs_Pages { int dataPagesDataSize; int *dataPagesSize; unsigned char *dataPagesMD5; - Tcl_Obj *dataIndex; + Cookfs_PageObj dataIndex; int dataPagesIsAside; struct Cookfs_Pages *dataAsidePages; @@ -135,10 +135,11 @@ Tcl_WideInt Cookfs_PagesClose(Cookfs_Pages *p); Tcl_WideInt Cookfs_GetFilesize(Cookfs_Pages *p); void Cookfs_PagesFini(Cookfs_Pages *p); int Cookfs_PageAddRaw(Cookfs_Pages *p, unsigned char *bytes, int objLength); -int Cookfs_PageAdd(Cookfs_Pages *p, Tcl_Obj *dataObj); -Tcl_Obj *Cookfs_PageGet(Cookfs_Pages *p, int index, int weight); -Tcl_Obj *Cookfs_PageCacheGet(Cookfs_Pages *p, int index, int update, int weight); -void Cookfs_PageCacheSet(Cookfs_Pages *p, int idx, Tcl_Obj *obj, int weight); +int Cookfs_PageAdd(Cookfs_Pages *p, Cookfs_PageObj dataObj); +int Cookfs_PageAddTclObj(Cookfs_Pages *p, Tcl_Obj *dataObj); +Cookfs_PageObj Cookfs_PageGet(Cookfs_Pages *p, int index, int weight); +Cookfs_PageObj Cookfs_PageCacheGet(Cookfs_Pages *p, int index, int update, int weight); +void Cookfs_PageCacheSet(Cookfs_Pages *p, int idx, Cookfs_PageObj obj, int weight); Tcl_Obj *Cookfs_PageGetHead(Cookfs_Pages *p); Tcl_Obj *Cookfs_PageGetHeadMD5(Cookfs_Pages *p); Tcl_Obj *Cookfs_PageGetTail(Cookfs_Pages *p); @@ -157,8 +158,8 @@ int Cookfs_PagesIsReadonly(Cookfs_Pages *p); void Cookfs_PagesSetAside(Cookfs_Pages *p, Cookfs_Pages *aside); -void Cookfs_PagesSetIndex(Cookfs_Pages *p, Tcl_Obj *dataIndex); -Tcl_Obj *Cookfs_PagesGetIndex(Cookfs_Pages *p); +void Cookfs_PagesSetIndex(Cookfs_Pages *p, Cookfs_PageObj dataIndex); +Cookfs_PageObj Cookfs_PagesGetIndex(Cookfs_Pages *p); Tcl_WideInt Cookfs_PagesGetPageOffset(Cookfs_Pages *p, int idx); diff --git a/generic/pagesCmd.c b/generic/pagesCmd.c index db5d5ce..19cb065 100644 --- a/generic/pagesCmd.c +++ b/generic/pagesCmd.c @@ -375,7 +375,9 @@ static int CookfsPagesCmd(ClientData clientData, Tcl_Interp *interp, int objc, T Tcl_WrongNumArgs(interp, 2, objv, "data"); return TCL_ERROR; } - idx = Cookfs_PageAdd(p, objv[2]); + Tcl_Size size; + unsigned char *bytes = Tcl_GetByteArrayFromObj(objv[2], &size); + idx = Cookfs_PageAddRaw(p, bytes, size); if (idx < 0) { Tcl_Obj *err = Cookfs_PagesGetLastError(p); if (err == NULL) { @@ -410,16 +412,23 @@ static int CookfsPagesCmd(ClientData clientData, Tcl_Interp *interp, int objc, T if (Tcl_GetIntFromObj(interp, objv[objc-1], &idx) != TCL_OK) { return TCL_ERROR; } - rc = Cookfs_PageGet(p, idx, weight); + Cookfs_PageObj data = Cookfs_PageGet(p, idx, weight); + CookfsLog(printf("cmdGet data [%s]", data == NULL ? "NULL" : "SET")) + if (data != NULL) { + Cookfs_PageObjIncrRefCount(data); + rc = Cookfs_PageObjCopyAsByteArray(data); + Cookfs_PageObjDecrRefCount(data); + } else { + rc = NULL; + } + // Check rc here again as conversion PageObj->ByteArray may fail + // due to OOM. CookfsLog(printf("cmdGet [%s]", rc == NULL ? "NULL" : "SET")) if (rc == NULL) { Tcl_SetObjResult(interp, Tcl_NewStringObj("Unable to retrieve chunk", -1)); return TCL_ERROR; } else { Tcl_SetObjResult(interp, rc); - // Cookfs_PageGet always returns a page with refcount=1. We need - // to decrease refcount now. - Tcl_DecrRefCount(rc); } break; } @@ -497,10 +506,27 @@ static int CookfsPagesCmd(ClientData clientData, Tcl_Interp *interp, int objc, T } case cmdIndex: { + Cookfs_PageObj data; if (objc == 3) { - Cookfs_PagesSetIndex(p, objv[2]); + data = Cookfs_PageObjNewFromByteArray(objv[2]); + Cookfs_PageObjIncrRefCount(data); + Cookfs_PagesSetIndex(p, data); + Cookfs_PageObjDecrRefCount(data); + } + data = Cookfs_PagesGetIndex(p); + if (data == NULL) { + Tcl_SetObjResult(interp, Tcl_NewObj()); + } else { + Cookfs_PageObjIncrRefCount(data); + Tcl_Obj *rc = Cookfs_PageObjCopyAsByteArray(data); + Cookfs_PageObjDecrRefCount(data); + if (rc == NULL) { + Tcl_SetObjResult(interp, Tcl_NewStringObj("Unable to" + " convert from PageObj", -1)); + } else { + Tcl_SetObjResult(interp, rc); + } } - Tcl_SetObjResult(interp, Cookfs_PagesGetIndex(p)); break; } case cmdLength: diff --git a/generic/pagesCompr.c b/generic/pagesCompr.c index 075be18..a2b60d1 100644 --- a/generic/pagesCompr.c +++ b/generic/pagesCompr.c @@ -26,13 +26,13 @@ /* declarations of static and/or internal functions */ static Tcl_Obj **CookfsCreateCompressionCommand(Tcl_Interp *interp, Tcl_Obj *cmd, int *lenPtr, int additionalElements); static void CookfsWriteCompression(Cookfs_Pages *p, int compression); -static Tcl_Obj *CookfsReadPageZlib(Cookfs_Pages *p, int size); +static Cookfs_PageObj CookfsReadPageZlib(Cookfs_Pages *p, int size); static int CookfsWritePageZlib(Cookfs_Pages *p, unsigned char *bytes, int origSize); -static Tcl_Obj *CookfsReadPageBz2(Cookfs_Pages *p, int size); +static Cookfs_PageObj CookfsReadPageBz2(Cookfs_Pages *p, int size); static int CookfsWritePageBz2(Cookfs_Pages *p, unsigned char *bytes, int origSize); -static Tcl_Obj *CookfsReadPageLzma(Cookfs_Pages *p, int size); +static Cookfs_PageObj CookfsReadPageLzma(Cookfs_Pages *p, int size); static int CookfsWritePageLzma(Cookfs_Pages *p, unsigned char *bytes, int origSize); -static Tcl_Obj *CookfsReadPageCustom(Cookfs_Pages *p, int size); +static Cookfs_PageObj CookfsReadPageCustom(Cookfs_Pages *p, int size); static int CookfsWritePageCustom(Cookfs_Pages *p, unsigned char *bytes, int origSize); #ifdef USE_VFS_COMMANDS_FOR_ZIP static int CookfsCheckCommandExists(Tcl_Interp *interp, const char *commandName); @@ -365,7 +365,7 @@ int Cookfs_SetCompressCommands(Cookfs_Pages *p, Tcl_Obj *compressCommand, Tcl_Ob *---------------------------------------------------------------------- */ -Tcl_Obj *Cookfs_ReadPage(Cookfs_Pages *p, int idx, int size, int decompress, int compressionType) { +Cookfs_PageObj Cookfs_ReadPage(Cookfs_Pages *p, int idx, int size, int decompress, int compressionType) { if (idx >= p->dataNumPages) { return NULL; @@ -376,9 +376,7 @@ Tcl_Obj *Cookfs_ReadPage(Cookfs_Pages *p, int idx, int size, int decompress, int CookfsLog(printf("Cookfs_ReadPage I=%d S=%d C=%d", idx, size, p->fileCompression)) if (size == 0) { /* if page was empty, no need to read anything */ - Tcl_Obj *obj = Tcl_NewByteArrayObj((unsigned char *) "", 0); - Tcl_IncrRefCount(obj); - return obj; + return Cookfs_PageObjAlloc(0); } else { /* read compression algorithm first */ Tcl_Obj *byteObj; @@ -418,20 +416,17 @@ Tcl_Obj *Cookfs_ReadPage(Cookfs_Pages *p, int idx, int size, int decompress, int switch (compression) { case COOKFS_COMPRESSION_NONE: { /* simply read raw data */ - Tcl_Obj *data; - data = Tcl_NewObj(); - int count = Tcl_ReadChars(p->fileChannel, data, size, 0); + Cookfs_PageObj data = Cookfs_PageObjAlloc(size); + int count = Tcl_Read(p->fileChannel, (char *)data, size); if (count != size) { CookfsLog(printf("Unable to read - %d != %d", count, size)) - Tcl_IncrRefCount(data); - Tcl_DecrRefCount(data); + Cookfs_PageObjIncrRefCount(data); + Cookfs_PageObjDecrRefCount(data); return NULL; } - if (!decompress) { CookfsLog(printf("Cookfs_ReadPage retrieved chunk %d", idx)); } - Tcl_IncrRefCount(data); return data; } /* run proper reading functions */ @@ -555,10 +550,15 @@ int Cookfs_WritePage(Cookfs_Pages *p, int idx, unsigned char *bytes, int origSiz } -int Cookfs_WritePageObj(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compressedData) { - Tcl_Size dataSize; - unsigned char *bytes = Tcl_GetByteArrayFromObj(data, &dataSize); - return Cookfs_WritePage(p, idx, bytes, dataSize, compressedData); +int Cookfs_WritePageObj(Cookfs_Pages *p, int idx, Cookfs_PageObj data, Tcl_Obj *compressedData) { + CookfsLog(printf("Cookfs_WritePageObj: data: %p", (void *)data)); + return Cookfs_WritePage(p, idx, data, Cookfs_PageObjSize(data), compressedData); +} + +int Cookfs_WriteTclObj(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compressedData) { + Tcl_Size size; + unsigned char *bytes = Tcl_GetByteArrayFromObj(data, &size); + return Cookfs_WritePage(p, idx, bytes, size, compressedData); } /* @@ -579,13 +579,12 @@ int Cookfs_WritePageObj(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compre *---------------------------------------------------------------------- */ -Tcl_Obj *Cookfs_AsyncPageGet(Cookfs_Pages *p, int idx) { +Cookfs_PageObj Cookfs_AsyncPageGet(Cookfs_Pages *p, int idx) { if ((p->fileCompression == COOKFS_COMPRESSION_CUSTOM) && (p->asyncCompressCommandPtr != NULL) && (p->asyncCompressCommandLen > 3)) { int i; for (i = 0 ; i < p->asyncPageSize ; i++) { if (p->asyncPage[i].pageIdx == idx) { - Tcl_IncrRefCount(p->asyncPage[i].pageContents); - return p->asyncPage[i].pageContents; + return Cookfs_PageObjNewFromByteArray(p->asyncPage[i].pageContents); } } } @@ -710,7 +709,7 @@ int Cookfs_AsyncCompressWait(Cookfs_Pages *p, int require) { } Tcl_IncrRefCount(resObj); - int size = Cookfs_WritePageObj(p, idx, p->asyncPage[0].pageContents, resObj); + int size = Cookfs_WriteTclObj(p, idx, p->asyncPage[0].pageContents, resObj); p->dataPagesSize[idx] = size; Tcl_DecrRefCount(resObj); Tcl_DecrRefCount(result); @@ -788,16 +787,28 @@ int Cookfs_AsyncPagePreload(Cookfs_Pages *p, int idx) { } CookfsLog(printf("Cookfs_AsyncPagePreload: Reading page %d for async decompress", idx)) - dataObj = Cookfs_ReadPage(p, idx, -1, 0, COOKFS_COMPRESSION_CUSTOM); - - if (dataObj != NULL) { - p->asyncDecompressIdx[p->asyncDecompressQueue] = idx; - p->asyncDecompressQueue++; - CookfsLog(printf("Adding page %d for async decompress", idx)) - CookfsRunAsyncDecompressCommand(p, p->asyncCommandProcess, idx, dataObj); - Tcl_DecrRefCount(dataObj); + Cookfs_PageObj dataPageObj = Cookfs_ReadPage(p, idx, -1, 0, COOKFS_COMPRESSION_CUSTOM); + if (dataPageObj == NULL) { + CookfsLog(printf("Cookfs_AsyncPagePreload: ERROR: Cookfs_ReadPage returned NULL, return 1")); + return 1; + } + + Cookfs_PageObjIncrRefCount(dataPageObj); + dataObj = Cookfs_PageObjCopyAsByteArray(dataPageObj); + Cookfs_PageObjDecrRefCount(dataPageObj); + + if (dataObj == NULL) { + CookfsLog(printf("Cookfs_AsyncPagePreload: ERROR: failed to convert Tcl_Obj->PageObj, return 1")); + return 1; } + Tcl_IncrRefCount(dataObj); + p->asyncDecompressIdx[p->asyncDecompressQueue] = idx; + p->asyncDecompressQueue++; + CookfsLog(printf("Adding page %d for async decompress", idx)) + CookfsRunAsyncDecompressCommand(p, p->asyncCommandProcess, idx, dataObj); + Tcl_DecrRefCount(dataObj); + CookfsLog(printf("Cookfs_AsyncPagePreload: return 1")) return 1; } @@ -857,12 +868,18 @@ int Cookfs_AsyncDecompressWait(Cookfs_Pages *p, int idx, int require) { CookfsLog(printf("Cookfs_AsyncDecompressWait: callback returned data for %d", i)) Tcl_IncrRefCount(resObj); - /* - Set the page weight to 1000 because it should be cached and used further. - If it will be displaced by other weighty pages, then preloading makes no sense. - Real page weight will be set by Cookfs_PageGet - */ - Cookfs_PageCacheSet(p, i, resObj, 1000); + Cookfs_PageObj pageObj = Cookfs_PageObjNewFromByteArray(resObj); + Tcl_DecrRefCount(resObj); + if (pageObj != NULL) { + Cookfs_PageObjIncrRefCount(pageObj); + /* + Set the page weight to 1000 because it should be cached and used further. + If it will be displaced by other weighty pages, then preloading makes no sense. + Real page weight will be set by Cookfs_PageGet + */ + Cookfs_PageCacheSet(p, i, pageObj, 1000); + Cookfs_PageObjDecrRefCount(pageObj); + } Tcl_DecrRefCount(result); @@ -990,7 +1007,7 @@ static Tcl_Obj **CookfsCreateCompressionCommand(Tcl_Interp *interp, Tcl_Obj *cmd *---------------------------------------------------------------------- */ -static Tcl_Obj *CookfsReadPageZlib(Cookfs_Pages *p, int size) { +static Cookfs_PageObj CookfsReadPageZlib(Cookfs_Pages *p, int size) { #ifdef USE_ZLIB_TCL86 /* use Tcl 8.6 API for decompression */ Tcl_Obj *data; @@ -1026,19 +1043,20 @@ static Tcl_Obj *CookfsReadPageZlib(Cookfs_Pages *p, int size) { CookfsLog(printf("Reading")) /* read resulting object */ cobj = Tcl_NewObj(); + Tcl_IncrRefCount(cobj); while (!Tcl_ZlibStreamEof(zshandle)) { if (Tcl_ZlibStreamGet(zshandle, cobj, -1) != TCL_OK) { - Tcl_IncrRefCount(cobj); Tcl_DecrRefCount(cobj); Tcl_ZlibStreamClose(zshandle); CookfsLog(printf("Unable to decompress - reading")) return NULL; } } - Tcl_IncrRefCount(cobj); - CookfsLog(printf("Returning = [%s]", cobj == NULL ? "NULL" : "SET")) + Cookfs_PageObj rc = Cookfs_PageObjNewFromByteArray(cobj); + Tcl_DecrRefCount(cobj); Tcl_ZlibStreamClose(zshandle); - return cobj; + CookfsLog(printf("Returning = [%s]", rc == NULL ? "NULL" : "SET")) + return rc; #else /* use vfs::zip command for decompression */ Tcl_Obj *prevResult; @@ -1069,7 +1087,9 @@ static Tcl_Obj *CookfsReadPageZlib(Cookfs_Pages *p, int size) { Tcl_IncrRefCount(data); Tcl_SetObjResult(p->interp, prevResult); Tcl_DecrRefCount(prevResult); - return data; + Cookfs_PageObj rc = Cookfs_PageObjNewFromByteArray(data); + Tcl_DecrRefCount(data); + return rc; #endif } @@ -1302,7 +1322,7 @@ static int CookfsWritePageLzma(Cookfs_Pages *p, unsigned char *bytes, int origSi *---------------------------------------------------------------------- */ -static Tcl_Obj *CookfsReadPageLzma(Cookfs_Pages *p, int size) { +static Cookfs_PageObj CookfsReadPageLzma(Cookfs_Pages *p, int size) { #ifdef COOKFS_USELZMA CookfsLog(printf("CookfsReadPageLzma: start. Want to read %d bytes.", size)); @@ -1326,20 +1346,24 @@ static Tcl_Obj *CookfsReadPageLzma(Cookfs_Pages *p, int size) { } int destSize; - Tcl_Obj *destObj = Tcl_NewByteArrayObj(NULL, 0); - Tcl_IncrRefCount(destObj); Cookfs_Binary2Int(source, &destSize, 1); + + Cookfs_PageObj destObj = Cookfs_PageObjAlloc(destSize); + if (destObj == NULL) { + CookfsLog(printf("CookfsReadPageLzma: ERROR: failed to alloc")); + Tcl_DecrRefCount(data); + return NULL; + } + CookfsLog(printf("CookfsReadPageLzma: uncompressed size=%d from %d", destSize, size)); - Tcl_SetByteArrayLength(destObj, destSize); - unsigned char *dest = Tcl_GetByteArrayFromObj(destObj, NULL); CookfsLog(printf("CookfsReadPageLzma: call LzmaDecode() ...")); SizeT destSizeResult = destSize; ELzmaStatus status; // Source buffer also contains the original size and lzma properties SizeT srcLen = size - 4 - LZMA_PROPS_SIZE; - SRes res = LzmaDecode(dest, &destSizeResult, + SRes res = LzmaDecode(destObj, &destSizeResult, &source[4 + LZMA_PROPS_SIZE], &srcLen, &source[4], LZMA_PROPS_SIZE, LZMA_FINISH_END, &status, &g_CookfsLzmaAlloc); CookfsLog(printf("CookfsReadPageLzma: result: %s; status: %s", @@ -1365,7 +1389,8 @@ static Tcl_Obj *CookfsReadPageLzma(Cookfs_Pages *p, int size) { (status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK))) { CookfsLog(printf("CookfsReadPageLzma: failed")) - Tcl_DecrRefCount(destObj); + Cookfs_PageObjIncrRefCount(destObj); + Cookfs_PageObjDecrRefCount(destObj); return NULL; } @@ -1394,13 +1419,11 @@ static Tcl_Obj *CookfsReadPageLzma(Cookfs_Pages *p, int size) { *---------------------------------------------------------------------- */ -static Tcl_Obj *CookfsReadPageBz2(Cookfs_Pages *p, int size) { +static Cookfs_PageObj CookfsReadPageBz2(Cookfs_Pages *p, int size) { #ifdef COOKFS_USEBZ2 int destSize; int count; unsigned char *source; - unsigned char *dest; - Tcl_Obj *destObj; Tcl_Obj *data; data = Tcl_NewObj(); @@ -1414,23 +1437,26 @@ static Tcl_Obj *CookfsReadPageBz2(Cookfs_Pages *p, int size) { source = Tcl_GetByteArrayFromObj(data, NULL); if (source == NULL) { - CookfsLog(printf("Cookfs_ReadPage: Tcl_GetByteArrayFromObj failed")) + Tcl_DecrRefCount(data); + CookfsLog(printf("CookfsReadPageBz2: Tcl_GetByteArrayFromObj failed")) return NULL; } - destObj = Tcl_NewByteArrayObj(NULL, 0); - Tcl_IncrRefCount(destObj); Cookfs_Binary2Int(source, &destSize, 1); + Cookfs_PageObj destObj = Cookfs_PageObjAlloc(destSize); + if (destObj == NULL) { + Tcl_DecrRefCount(data); + CookfsLog(printf("CookfsReadPageBz2: failed to alloc")) + return NULL; + } - CookfsLog(printf("Cookfs_ReadPage: uncompressed size=%d from %d", destSize, size)) - Tcl_SetByteArrayLength(destObj, destSize); - dest = Tcl_GetByteArrayFromObj(destObj, NULL); + CookfsLog(printf("CookfsReadPageBz2: uncompressed size=%d from %d", destSize, size)) - if (BZ2_bzBuffToBuffDecompress((char *) dest, (unsigned int *) &destSize, (char *) source + 4, (unsigned int) size - 4, 0, 0) != BZ_OK) { + if (BZ2_bzBuffToBuffDecompress((char *) destObj, (unsigned int *) &destSize, (char *) source + 4, (unsigned int) size - 4, 0, 0) != BZ_OK) { Tcl_DecrRefCount(data); - Tcl_IncrRefCount(destObj); - Tcl_DecrRefCount(destObj); - CookfsLog(printf("Cookfs_ReadPage: BZ2_bzBuffToBuffDecompress failed")) + Cookfs_PageObjIncrRefCount(destObj); + Cookfs_PageObjDecrRefCount(destObj); + CookfsLog(printf("CookfsReadPageBz2: BZ2_bzBuffToBuffDecompress failed")) return NULL; } @@ -1519,7 +1545,7 @@ static int CookfsWritePageBz2(Cookfs_Pages *p, unsigned char *bytes, int origSiz *---------------------------------------------------------------------- */ -static Tcl_Obj *CookfsReadPageCustom(Cookfs_Pages *p, int size) { +static Cookfs_PageObj CookfsReadPageCustom(Cookfs_Pages *p, int size) { /* use vfs::zip command for decompression */ Tcl_Obj *prevResult; Tcl_Obj *compressed; @@ -1558,7 +1584,9 @@ static Tcl_Obj *CookfsReadPageCustom(Cookfs_Pages *p, int size) { Tcl_IncrRefCount(data); Tcl_SetObjResult(p->interp, prevResult); Tcl_DecrRefCount(prevResult); - return data; + Cookfs_PageObj rc = Cookfs_PageObjNewFromByteArray(data); + Tcl_DecrRefCount(data); + return rc; } diff --git a/generic/pagesCompr.h b/generic/pagesCompr.h index 3a7749b..e3da9dc 100644 --- a/generic/pagesCompr.h +++ b/generic/pagesCompr.h @@ -43,9 +43,10 @@ int Cookfs_SetCompressCommands(Cookfs_Pages *p, Tcl_Obj *compressCommand, Tcl_Ob void Cookfs_SeekToPage(Cookfs_Pages *p, int idx); int Cookfs_WritePage(Cookfs_Pages *p, int idx, unsigned char *bytes, int origSize, Tcl_Obj *compressedData); -int Cookfs_WritePageObj(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compressedData); -Tcl_Obj *Cookfs_ReadPage(Cookfs_Pages *p, int idx, int size, int decompress, int compressionType); -Tcl_Obj *Cookfs_AsyncPageGet(Cookfs_Pages *p, int idx); +int Cookfs_WritePageObj(Cookfs_Pages *p, int idx, Cookfs_PageObj data, Tcl_Obj *compressedData); +int Cookfs_WriteTclObj(Cookfs_Pages *p, int idx, Tcl_Obj *data, Tcl_Obj *compressedData); +Cookfs_PageObj Cookfs_ReadPage(Cookfs_Pages *p, int idx, int size, int decompress, int compressionType); +Cookfs_PageObj Cookfs_AsyncPageGet(Cookfs_Pages *p, int idx); int Cookfs_AsyncPageAdd(Cookfs_Pages *p, int idx, unsigned char *bytes, int dataSize); void Cookfs_AsyncDecompressWaitIfLoading(Cookfs_Pages *p, int idx); int Cookfs_AsyncCompressWait(Cookfs_Pages *p, int require); diff --git a/generic/readerchannel.c b/generic/readerchannel.c index c4fcd91..049aaa4 100644 --- a/generic/readerchannel.c +++ b/generic/readerchannel.c @@ -170,7 +170,7 @@ void Cookfs_CreateReaderchannelFree(Cookfs_ReaderChannelInstData *instData) { instData->event = NULL; } if (instData->cachedPageObj != NULL) { - Tcl_DecrRefCount(instData->cachedPageObj); + Cookfs_PageObjDecrRefCount(instData->cachedPageObj); } ckfree((void *) instData); } diff --git a/generic/readerchannel.h b/generic/readerchannel.h index 6f0803f..5200c40 100644 --- a/generic/readerchannel.h +++ b/generic/readerchannel.h @@ -27,7 +27,7 @@ typedef struct Cookfs_ReaderChannelInstData { int currentBlockOffset; int firstTimeRead; - Tcl_Obj *cachedPageObj; + Cookfs_PageObj cachedPageObj; int cachedPageNum; int bufSize; diff --git a/generic/readerchannelIO.c b/generic/readerchannelIO.c index 0ce92bf..1308703 100644 --- a/generic/readerchannelIO.c +++ b/generic/readerchannelIO.c @@ -32,8 +32,6 @@ int Cookfs_Readerchannel_Input(ClientData instanceData, char *buf, int bufSize, int bytesLeft; int blockLeft; int blockRead; - char *pageBuf; - Tcl_Size pageBufSize; CookfsLog(printf("Cookfs_Readerchannel_Input: ===> read %d, current offset: %" TCL_LL_MODIFIER "d", bufSize, instData->currentOffset)) @@ -72,7 +70,7 @@ int Cookfs_Readerchannel_Input(ClientData instanceData, char *buf, int bufSize, " the previously retrieved page index#%d", pageIndex)); goto gotPage; } - Tcl_DecrRefCount(instData->cachedPageObj); + Cookfs_PageObjDecrRefCount(instData->cachedPageObj); } CookfsLog(printf("Cookfs_Readerchannel_Input: reading page index#%d", pageIndex)); @@ -96,8 +94,7 @@ int Cookfs_Readerchannel_Input(ClientData instanceData, char *buf, int bufSize, instData->firstTimeRead = 0; } instData->cachedPageObj = Cookfs_PageGet(instData->pages, pageIndex, pageWeight); - // No need to increase refcount on retrieved page, it already has refcount=1 - // after Cookfs_PageGet(). + Cookfs_PageObjIncrRefCount(instData->cachedPageObj); CookfsLog(printf("Cookfs_Readerchannel_Input: got the page: %p", (void *)instData->cachedPageObj)) if (instData->cachedPageObj == NULL) { @@ -107,14 +104,12 @@ int Cookfs_Readerchannel_Input(ClientData instanceData, char *buf, int bufSize, gotPage: - pageBuf = (char *) Tcl_GetByteArrayFromObj(instData->cachedPageObj, &pageBufSize); - CookfsLog(printf("Cookfs_Readerchannel_Input: copying %d+%d", instData->buf[instData->currentBlock + 1], instData->currentBlockOffset)) // validate enough data is available in the buffer - if (pageBufSize < (instData->buf[instData->currentBlock + 1] + instData->currentBlockOffset + blockRead)) { + if (Cookfs_PageObjSize(instData->cachedPageObj) < (instData->buf[instData->currentBlock + 1] + instData->currentBlockOffset + blockRead)) { goto error; } - memcpy(buf + bytesRead, pageBuf + instData->buf[instData->currentBlock + 1] + instData->currentBlockOffset, blockRead); + memcpy(buf + bytesRead, instData->cachedPageObj + instData->buf[instData->currentBlock + 1] + instData->currentBlockOffset, blockRead); instData->currentBlockOffset += blockRead; bytesRead += blockRead; instData->currentOffset += blockRead; @@ -124,7 +119,7 @@ int Cookfs_Readerchannel_Input(ClientData instanceData, char *buf, int bufSize, // it anymore and can release it. if (instData->currentBlockOffset == instData->buf[instData->currentBlock + 2]) { CookfsLog(printf("Cookfs_Readerchannel_Input: release the page")); - Tcl_DecrRefCount(instData->cachedPageObj); + Cookfs_PageObjDecrRefCount(instData->cachedPageObj); instData->cachedPageObj = NULL; } else { CookfsLog(printf("Cookfs_Readerchannel_Input: keep the page")); diff --git a/generic/vfs.c b/generic/vfs.c index 312ea10..5d32305 100644 --- a/generic/vfs.c +++ b/generic/vfs.c @@ -122,18 +122,18 @@ int Cookfs_VfsFini(Tcl_Interp *interp, Cookfs_Vfs *vfs, // If we are here, then we need to store index CookfsLog(printf("Cookfs_VfsFini: dump index...")); - Tcl_Obj *exportObj = Cookfs_FsindexToObject(vfs->index); + Cookfs_PageObj exportObj = Cookfs_FsindexToPageObj(vfs->index); if (exportObj == NULL) { CookfsLog(printf("Cookfs_VfsFini: failed to get index dump")); Tcl_SetObjResult(interp, Tcl_NewStringObj("unable to get index" " dump", -1)); return TCL_ERROR; } - Tcl_IncrRefCount(exportObj); + Cookfs_PageObjIncrRefCount(exportObj); CookfsLog(printf("Cookfs_VfsFini: store index...")); Cookfs_PagesSetIndex(vfs->pages, exportObj); - Tcl_DecrRefCount(exportObj); + Cookfs_PageObjDecrRefCount(exportObj); skipSavingIndex: diff --git a/generic/vfsCmd.c b/generic/vfsCmd.c index 482ef67..e516268 100644 --- a/generic/vfsCmd.c +++ b/generic/vfsCmd.c @@ -532,7 +532,7 @@ int Cookfs_Mount(Tcl_Interp *interp, Tcl_Obj *archive, Tcl_Obj *local, CookfsLog(printf("Cookfs_Mount: bootstrap is empty")); } else { CookfsLog(printf("Cookfs_Mount: add bootstrap")); - int idx = Cookfs_PageAdd(pages, props->bootstrap); + int idx = Cookfs_PageAddTclObj(pages, props->bootstrap); if (idx < 0) { Tcl_Obj *err = Cookfs_PagesGetLastError(pages); Tcl_SetObjResult(interp, Tcl_ObjPrintf("Unable to add" diff --git a/generic/writerchannel.c b/generic/writerchannel.c index a475538..99ae325 100644 --- a/generic/writerchannel.c +++ b/generic/writerchannel.c @@ -129,7 +129,7 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, return NULL; } - Tcl_Obj *blockObj = NULL; + Cookfs_PageObj blockObj = NULL; if (Cookfs_CreateWriterchannelCreate(instData, interp) != TCL_OK) { CookfsLog(printf("Cookfs_CreateWriterchannel: " @@ -201,8 +201,9 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, goto error; } - Tcl_IncrRefCount(blockObj); - blockBuffer = (char *)Tcl_GetByteArrayFromObj(blockObj, &blockSize); + Cookfs_PageObjIncrRefCount(blockObj); + blockSize = Cookfs_PageObjSize(blockObj); + blockBuffer = (const char *)blockObj; } @@ -232,7 +233,7 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, // We don't need the blockObj object anymore if (blockObj != NULL) { - Tcl_DecrRefCount(blockObj); + Cookfs_PageObjDecrRefCount(blockObj); blockObj = NULL; } @@ -252,7 +253,7 @@ Tcl_Channel Cookfs_CreateWriterchannel(Cookfs_Pages *pages, error: if (blockObj != NULL) { - Tcl_DecrRefCount(blockObj); + Cookfs_PageObjDecrRefCount(blockObj); } if (instData != NULL) { From 4427dcc6c4609ac40c8d29f4448abf81ef6edd22 Mon Sep 17 00:00:00 2001 From: Konstantin Kushnir Date: Fri, 7 Jun 2024 06:03:42 +0000 Subject: [PATCH 2/2] Fix the build without c-pages --- generic/fsindexIO.c | 12 ++++++++---- generic/fsindexIO.h | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/generic/fsindexIO.c b/generic/fsindexIO.c index f69c7b9..76da078 100644 --- a/generic/fsindexIO.c +++ b/generic/fsindexIO.c @@ -35,6 +35,8 @@ static int CookfsFsindexImportMetadata(Cookfs_Fsindex *fsIndex, unsigned char *b * subdirectories are inlined completely and recursively */ +#ifdef COOKFS_USECPAGES + Cookfs_PageObj Cookfs_FsindexToPageObj(Cookfs_Fsindex *fsindex) { Cookfs_PageObj rc; Tcl_Obj *obj = Cookfs_FsindexToObject(fsindex); @@ -48,6 +50,8 @@ Cookfs_PageObj Cookfs_FsindexToPageObj(Cookfs_Fsindex *fsindex) { return rc; } +#endif /* COOKFS_USECPAGES */ + /* *---------------------------------------------------------------------- * @@ -151,6 +155,10 @@ Cookfs_Fsindex *Cookfs_FsindexFromPages(Tcl_Interp *interp, Cookfs_Fsindex *fsin return rc; } +Cookfs_Fsindex *Cookfs_FsindexFromPageObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Cookfs_PageObj o) { + return Cookfs_FsindexFromBytes(interp, fsindex, o, Cookfs_PageObjSize(o)); +} + #endif /* COOKFS_USECPAGES */ Cookfs_Fsindex *Cookfs_FsindexFromTclObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Tcl_Obj *o) { @@ -160,10 +168,6 @@ Cookfs_Fsindex *Cookfs_FsindexFromTclObj(Tcl_Interp *interp, Cookfs_Fsindex *fsi } -Cookfs_Fsindex *Cookfs_FsindexFromPageObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Cookfs_PageObj o) { - return Cookfs_FsindexFromBytes(interp, fsindex, o, Cookfs_PageObjSize(o)); -} - /* *---------------------------------------------------------------------- * diff --git a/generic/fsindexIO.h b/generic/fsindexIO.h index 97b837f..d71b286 100644 --- a/generic/fsindexIO.h +++ b/generic/fsindexIO.h @@ -7,11 +7,11 @@ #ifdef COOKFS_USECFSINDEX Tcl_Obj *Cookfs_FsindexToObject(Cookfs_Fsindex *fsindex); -Cookfs_PageObj Cookfs_FsindexToPageObj(Cookfs_Fsindex *fsindex); Cookfs_Fsindex *Cookfs_FsindexFromBytes(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, unsigned char *bytes, Tcl_Size size); Cookfs_Fsindex *Cookfs_FsindexFromTclObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Tcl_Obj *o); -Cookfs_Fsindex *Cookfs_FsindexFromPageObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Cookfs_PageObj o); #ifdef COOKFS_USECPAGES +Cookfs_PageObj Cookfs_FsindexToPageObj(Cookfs_Fsindex *fsindex); +Cookfs_Fsindex *Cookfs_FsindexFromPageObj(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Cookfs_PageObj o); Cookfs_Fsindex *Cookfs_FsindexFromPages(Tcl_Interp *interp, Cookfs_Fsindex *fsindex, Cookfs_Pages *pages); #endif /* COOKFS_USECPAGES */