Skip to content

Commit

Permalink
Add support for optimize list command for c-vfs
Browse files Browse the repository at this point in the history
  • Loading branch information
@chpock
chpock committed May 27, 2024
1 parent 6bd85db commit a8c4787
Show file tree
Hide file tree
Showing 4 changed files with 281 additions and 5 deletions.
1 change: 1 addition & 0 deletions ChangeLog
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
2024-05-27 Konstantin Kushnir <[email protected]>
* Fix a bug when deleting the last entry from small file buffer
* Add support for optimize list command for c-vfs

2024-05-26 Konstantin Kushnir <[email protected]>
* Add tests for writer
Expand Down
156 changes: 154 additions & 2 deletions generic/vfsCmd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -737,6 +737,7 @@ static Cookfs_MountHandleCommandProc CookfsMountHandleCommandFilesize;
static Cookfs_MountHandleCommandProc CookfsMountHandleCommandSmallfilebuffersize;
static Cookfs_MountHandleCommandProc CookfsMountHandleCommandCompression;
static Cookfs_MountHandleCommandProc CookfsMountHandleCommandWritefiles;
static Cookfs_MountHandleCommandProc CookfsMountHandleCommandOptimizelist;

static int CookfsMountHandleCmd(ClientData clientData, Tcl_Interp *interp,
int objc, Tcl_Obj *const objv[])
Expand All @@ -747,13 +748,13 @@ static int CookfsMountHandleCmd(ClientData clientData, Tcl_Interp *interp,
static const char *const commands[] = {
"getpages", "getindex", "getwriter", "getmetadata", "setmetadata",
"aside", "writetomemory", "filesize", "smallfilebuffersize",
"compression", "writeFiles",
"compression", "writeFiles", "optimizelist",
NULL
};
enum commands {
cmdGetpages, cmdGetindex, cmdGetwriter, cmdGetmetadata, cmdSetmetadata,
cmdAside, cmdWritetomemory, cmdFilesize, cmdSmallfilebuffersize,
cmdCompression, cmdWritefiles
cmdCompression, cmdWritefiles, cmdOptimizelist
};

if (objc < 2) {
Expand Down Expand Up @@ -791,6 +792,8 @@ static int CookfsMountHandleCmd(ClientData clientData, Tcl_Interp *interp,
return CookfsMountHandleCommandCompression(vfs, interp, objc, objv);
case cmdWritefiles:
return CookfsMountHandleCommandWritefiles(vfs, interp, objc, objv);
case cmdOptimizelist:
return CookfsMountHandleCommandOptimizelist(vfs, interp, objc, objv);
}

return TCL_OK;
Expand Down Expand Up @@ -950,3 +953,152 @@ static int CookfsMountHandleCommandWritefiles(Cookfs_Vfs *vfs,
{
return CookfsWriterHandleCommandWrite(vfs->writer, interp, objc, objv);
}

static int CookfsMountHandleCommandOptimizelist(Cookfs_Vfs *vfs,
Tcl_Interp *interp, int objc, Tcl_Obj *const objv[])
{

CookfsLog(printf("CookfsMountHandleCommandOptimizelist: enter;"
" objc: %d", objc));

if (objc != 4) {
Tcl_WrongNumArgs(interp, 2, objv, "base filelist");
return TCL_ERROR;
}

int i;

Tcl_Obj **fileTails;
int fileCount;
if (Tcl_ListObjGetElements(interp, objv[3], &fileCount, &fileTails)
!= TCL_OK)
{
return TCL_ERROR;
}

Cookfs_Pages *pages = vfs->pages;

if (!pages->dataNumPages) {
CookfsLog(printf("CookfsMountHandleCommandOptimizelist: there is"
" no pages, return the list as is"));
Tcl_SetObjResult(interp, objv[2]);
return TCL_OK;
}

CookfsLog(printf("CookfsMountHandleCommandOptimizelist: alloc pageFiles"));
Tcl_Obj **pageFiles = ckalloc(sizeof(Tcl_Obj *) * pages->dataNumPages);
if (pageFiles == NULL) {
Tcl_SetObjResult(interp, Tcl_NewStringObj("failed to alloc"
" pageFiles", -1));
return TCL_ERROR;
}

for (i = 0; i < pages->dataNumPages; i++) {
pageFiles[i] = NULL;
}

Tcl_Obj *largeFiles = Tcl_NewListObj(0, NULL);
Tcl_IncrRefCount(largeFiles);

Tcl_Obj *baseTemplate = Tcl_NewListObj(1, &objv[2]);
Tcl_IncrRefCount(baseTemplate);

Cookfs_Fsindex *index = vfs->index;

CookfsLog(printf("CookfsMountHandleCommandOptimizelist: checking %d files",
fileCount));
for (i = 0; i < fileCount; i++) {

Tcl_Obj *fileTail = fileTails[i];
CookfsLog(printf("CookfsMountHandleCommandOptimizelist: checking"
" file [%s]", Tcl_GetString(fileTail)));

// Construct full path
Tcl_Obj *fullName = Tcl_DuplicateObj(baseTemplate);
Tcl_IncrRefCount(fullName);

Tcl_ListObjAppendElement(NULL, fullName, fileTail);

Tcl_Obj *fullNameJoined = Tcl_FSJoinPath(fullName, -1);
Tcl_IncrRefCount(fullNameJoined);

CookfsLog(printf("CookfsMountHandleCommandOptimizelist: full path:"
" [%s]", Tcl_GetString(fullNameJoined)));

Tcl_Obj *fullNameSplit = Tcl_FSSplitPath(fullNameJoined, NULL);
Tcl_IncrRefCount(fullNameSplit);

Cookfs_FsindexEntry *entry = Cookfs_FsindexGet(index, fullNameSplit);

Tcl_Obj *listToAdd = NULL;
int pageNum = -1;

if (entry == NULL) {
CookfsLog(printf("CookfsMountHandleCommandOptimizelist: got NULL"
" entry"));
listToAdd = largeFiles;
} else if (entry->fileBlocks != 1) {
CookfsLog(printf("CookfsMountHandleCommandOptimizelist: fileBlocks"
" [%d] is not 1", entry->fileBlocks));
listToAdd = largeFiles;
} else {
// Check if the file has correct page number
pageNum = entry->data.fileInfo.fileBlockOffsetSize[0];
if (pageNum < 0 || pageNum >= pages->dataNumPages) {
CookfsLog(printf("CookfsMountHandleCommandOptimizelist:"
" incorrect page number: %d", pageNum));
listToAdd = largeFiles;
}
}

if (listToAdd == NULL) {
if (pageFiles[pageNum] == NULL) {
pageFiles[pageNum] = Tcl_NewListObj(0, NULL);
Tcl_IncrRefCount(pageFiles[pageNum]);
}
listToAdd = pageFiles[pageNum];
CookfsLog(printf("CookfsMountHandleCommandOptimizelist: add to"
" small file list, page: %d", pageNum));
} else {
CookfsLog(printf("CookfsMountHandleCommandOptimizelist: add to"
" large file list"));
}

Tcl_ListObjAppendElement(NULL, listToAdd, fileTail);

// Cleanup
Tcl_DecrRefCount(fullNameSplit);
Tcl_DecrRefCount(fullNameJoined);
Tcl_DecrRefCount(fullName);

}

Tcl_DecrRefCount(baseTemplate);

CookfsLog(printf("CookfsMountHandleCommandOptimizelist: create a small"
" file list"));
Tcl_Obj *smallFiles = Tcl_NewListObj(0, NULL);
Tcl_IncrRefCount(smallFiles);

for (i = 0; i < pages->dataNumPages; i++) {
if (pageFiles[i] != NULL) {
CookfsLog(printf("CookfsMountHandleCommandOptimizelist: add files"
" from page %d to small file list", i));
Tcl_ListObjAppendList(interp, smallFiles, pageFiles[i]);
Tcl_DecrRefCount(pageFiles[i]);
}
}

CookfsLog(printf("CookfsMountHandleCommandOptimizelist: add the large"
" files to the small files"));
Tcl_ListObjAppendList(interp, smallFiles, largeFiles);
Tcl_DecrRefCount(largeFiles);

Tcl_SetObjResult(interp, smallFiles);
CookfsLog(printf("CookfsMountHandleCommandOptimizelist: ok [%s]",
Tcl_GetString(smallFiles)));
Tcl_DecrRefCount(smallFiles);

return TCL_OK;

}
94 changes: 91 additions & 3 deletions tests/common.tcl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,15 @@ proc makeTree { dir tree } {
foreach { type name data } $tree {
switch -glob -- $type {
f* {
if { [string index $name end] eq "%" } {
set name [string range $name 0 end-1]
set filler [randomData $data]
} else {
set filler [string repeat "a" $data]
}
set fp [open [file join $dir $name] w]
fconfigure $fp -translation binary
puts -nonewline $fp [string repeat "a" $data]
puts -nonewline $fp $filler
close $fp
}
d* {
Expand Down Expand Up @@ -51,10 +57,53 @@ proc makeSimpleTree { dir } {
}
}

# expects -smallfilesize 0x10 -smallfilebuffer 0x30 -pagesize 0x10
proc makeSimpleTree2 { dir } {
makeTree $dir {
file big-two-pages.a% 0x20
file medium-one-page.a% 0x10
file rootfile 3
file small-half-page.a% 0x8
file very-bigfile% 0x100
dir onedir {
file empty-file 0
file big-two-pages.b% 0x20
dir twodir {
file very-small-one% 1
file very-small-two% 2
file "with spaces.txt" 2
dir "with spaces" {
file null 0
file very-small-one.f% 1
}
dir emptydir {
}
file very-small-three% 3
file small-half-page.c% 0x8
}
file small-half-page.b% 0x8
}
file medium-one-page.b% 0x10
dir anotherdir {
file foobar 1
}
file emptyfile 0
file medium-one-page.c% 0x10
}
}

proc randomData {bytes} {
set rc {}
for {set i 0} {$i < $bytes} {incr i 4} {
append rc [binary format I [expr {wide(rand()*0x100000000)}]]
# speed up random data generation by duplicating one random fragment
if { $bytes > 512 } {
set fragment [randomData 508]
for {set i 0} {$i < $bytes} {incr i 512} {
append rc $fragment [binary format I [expr {wide(rand()*0x100000000)}]]
}
} else {
for {set i 0} {$i < $bytes} {incr i 4} {
append rc [binary format I [expr {wide(rand()*0x100000000)}]]
}
}
incr bytes -1
return [string range $rc 0 $bytes]
Expand All @@ -68,6 +117,45 @@ proc randomDatas {count bytes} {
return $rc
}

proc getFilesRelative { dir } {
set files [lsort [glob -nocomplain -tails -type f -directory $dir *]]
foreach subdir [lsort [glob -nocomplain -type d -directory $dir *]] {
foreach f [getFilesRelative $subdir] {
lappend files [file join [file tail $subdir] $f]
}
}
return $files
}

proc testOptimizedList { base files fsindex } {
# We expect small files with increasing block number to go first.
# Then the large files will follow.
set result [list]
set lastPage 0
set lastSmall 1

foreach { file } $files {
set file [file join $base $file]
set chunklist [lindex [$fsindex get $file] 2]
set isSmall [expr { [llength $chunklist] == 3 }]
set onPage [lindex $chunklist 0]
if { $isSmall } {
# puts "small $file on $onPage"
if { $onPage < $lastPage } {
lappend result "Unexpected file \"$file\" on page #$onPage when previous page was $lastPage"
}
if { !$lastSmall } {
lappend result "Unexpected small file \"$file\" when lage files are expected"
}
set lastPage $onPage
} else {
# puts "large $file on $onPage"
set lastSmall 0
}
}

}

proc testIfEqual {a b} {
if {(![file exists $a]) || (![file exists $b])} {
return "file exists $a: [file exists $a]\nfile exists $b: [file exists $b]"
Expand Down
35 changes: 35 additions & 0 deletions tests/vfs.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -999,3 +999,38 @@ tcltest::test cookfsVfs-14.8 "Test unknown hash value" -setup {
tcltest::removeFile $file
} -returnCodes error -result {bad hash "foo": must be md5 or crc32}

tcltest::test cookfsVfs-15.1 "Test optimizelist with empty base" -setup {
set file [tcltest::makeFile {} cookfs.cfs]
set fsid [vfs::cookfs::Mount $file $file -compression none \
-smallfilesize 0x10 -smallfilebuffer 0x30 -pagesize 0x10]
makeSimpleTree2 $file
makeSimpleTree2 $file/subtree1
tcltest::makeBinFile "ABCD" diff $file/subtree1
vfs::unmount $file
} -body {
set h [vfs::cookfs::Mount $file $file]
set files [$h optimizelist "" [getFilesRelative $file]]
# Verify the result.
join [testOptimizedList "" $files [$h getindex]] \n
} -cleanup {
catch { vfs::unmount $file }
tcltest::removeFile $file
} -result {}

tcltest::test cookfsVfs-15.2 "Test optimizelist with non-empty base" -setup {
set file [tcltest::makeFile {} cookfs.cfs]
set fsid [vfs::cookfs::Mount $file $file -compression none \
-smallfilesize 0x10 -smallfilebuffer 0x30 -pagesize 0x10]
makeSimpleTree2 $file
makeSimpleTree2 $file/subtree1
tcltest::makeBinFile "ABCD" diff $file/subtree1
vfs::unmount $file
} -body {
set h [vfs::cookfs::Mount $file $file]
set files [$h optimizelist "subtree1" [getFilesRelative $file/subtree1]]
# Verify the result.
join [testOptimizedList "subtree1" $files [$h getindex]] \n
} -cleanup {
catch { vfs::unmount $file }
tcltest::removeFile $file
} -result {}

0 comments on commit a8c4787

Please sign in to comment.