fix: list images api (#5)

- time parse err
- d.Walk now contains folders

Author: kites262

registry/storage/driver/cos/cos.go

@@ -5,11 +5,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/distribution/distribution/v3/internal/dcontext"
-	storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
-	"github.com/distribution/distribution/v3/registry/storage/driver/base"
-	"github.com/distribution/distribution/v3/registry/storage/driver/factory"
-	"github.com/tencentyun/cos-go-sdk-v5"
 	"io"
 	"net/http"
 	"net/url"
@@ -18,6 +13,12 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/distribution/distribution/v3/internal/dcontext"
+	storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
+	"github.com/distribution/distribution/v3/registry/storage/driver/base"
+	"github.com/distribution/distribution/v3/registry/storage/driver/factory"
+	"github.com/tencentyun/cos-go-sdk-v5"
 )
 
 const (
@@ -286,7 +287,10 @@ func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo,
 	}
 	// file info
 	size, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
-	modTime, err := time.Parse(time.RFC1123, res.Header.Get("Last-Modified"))
+	if err != nil {
+		return nil, err
+	}
+	modTime, err := ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		return nil, err
 	}
@@ -374,7 +378,7 @@ func (d *driver) copy(ctx context.Context, sourcePath, destPath string) error {
 		return nil
 	}
 	// file size is larger than 32MB
-	v, _, err := d.cosClient.Object.InitiateMultipartUpload(ctx, key, &cos.InitiateMultipartUploadOptions{
+	v, _, _ := d.cosClient.Object.InitiateMultipartUpload(ctx, key, &cos.InitiateMultipartUploadOptions{
 		ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{
 			ContentType: contentType,
 		},
@@ -524,6 +528,8 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from, sta
 	var retError error
 	// the most recent skip directory to avoid walking over undesirable files
 	var prevSkipDir string
+	// the most recent directory walked for de-duplication
+	var prevDir string
 
 	key := d.pathToKey(from)
 	opt := &cos.BucketGetOptions{
@@ -532,7 +538,7 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from, sta
 		Marker:  d.pathToKey(startAfter),
 	}
 	ctx, done := dcontext.WithTrace(parentCtx)
-	defer done("cos.Bucket.Get(%s)", opt)
+	defer done("cos.Bucket.Get(%+v)", opt)
 	isTruncated := true
 	for isTruncated {
 		// list all the objects
@@ -541,30 +547,45 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from, sta
 			return err
 		}
 		walkInfos := make([]storagedriver.FileInfoInternal, 0, len(res.Contents))
+		prevDir = from
 		for _, content := range res.Contents {
-			if strings.HasSuffix(content.Key, "/") { // directory
+			// COS returns only objects, without directories
+			path := d.keyToPath(content.Key)
+
+			// get a list of all directories between the previous
+			dirs := DirectoryDiff(prevDir, path)
+			for _, dir := range dirs {
 				walkInfos = append(walkInfos, storagedriver.FileInfoInternal{
 					FileInfoFields: storagedriver.FileInfoFields{
 						IsDir: true,
-						Path:  strings.TrimRight(content.Key, "/"),
-					},
-				})
-			} else { // file object
-				// last modification time
-				modTime, err := time.Parse(time.RFC1123, content.LastModified)
-				if err != nil {
-					return err
-				}
-				walkInfos = append(walkInfos, storagedriver.FileInfoInternal{
-					FileInfoFields: storagedriver.FileInfoFields{
-						IsDir:   false,
-						Size:    content.Size,
-						ModTime: modTime,
-						Path:    content.Key,
+						Path:  dir,
 					},
 				})
+				prevDir = dir
+			}
+
+			// corner case
+			if strings.HasSuffix(path, "/") {
+				continue
 			}
+
+			modTime, err := ParseTime(content.LastModified)
+			if err != nil {
+				return err
+			}
+
+			walkInfos = append(walkInfos, storagedriver.FileInfoInternal{
+				FileInfoFields: storagedriver.FileInfoFields{
+					IsDir:   false,
+					Size:    content.Size,
+					ModTime: modTime,
+					Path:    path,
+				},
+			})
 		}
+
+		// according to the files, append directories
+
 		isTruncated = res.IsTruncated
 		opt.Marker = res.NextMarker
 		// iterative
@@ -600,14 +621,12 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from, sta
 
 // add the prefix
 func (d *driver) pathToKey(path string) string {
-	// Important! delete the root prefix
-	newPath := strings.TrimPrefix(path, d.rootDirectory)
-	return strings.TrimLeft(strings.TrimRight(d.rootDirectory, "/")+newPath, "/")
+	return PathToKey(d.rootDirectory, path)
 }
 
 // remove the prefix
 func (d *driver) keyToPath(key string) string {
-	return "/" + strings.TrimRight(strings.TrimPrefix(key, d.rootDirectory), "/")
+	return KeyToPath(d.rootDirectory, key)
 }
 
 var _ storagedriver.FileWriter = &writer{}

registry/storage/driver/cos/parser.go

@@ -2,6 +2,7 @@ package cos
 
 import (
 	"errors"
+
 	"github.com/mitchellh/mapstructure"
 )
 

registry/storage/driver/cos/utils.go

@@ -0,0 +1,86 @@
+package cos
+
+import (
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+)
+
+func ParseTime(timeStr string) (time.Time, error) {
+	var timeObj time.Time
+	timeObj, err := time.Parse(time.RFC3339, timeStr)
+	if err != nil {
+		timeObj, err = time.Parse(time.RFC1123, timeStr)
+		if err != nil {
+			return time.Time{}, err
+		}
+	}
+	return timeObj, nil
+}
+
+// DirectoryDiff finds all directories that are not in common between
+// the previous and current paths in sorted order.
+//
+// # Examples
+//
+//	DirectoryDiff("/path/to/folder", "/path/to/folder/folder/file")
+//	// => [ "/path/to/folder/folder" ]
+//
+//	DirectoryDiff("/path/to/folder/folder1", "/path/to/folder/folder2/file")
+//	// => [ "/path/to/folder/folder2" ]
+//
+//	DirectoryDiff("/path/to/folder/folder1/file", "/path/to/folder/folder2/file")
+//	// => [ "/path/to/folder/folder2" ]
+//
+//	DirectoryDiff("/path/to/folder/folder1/file", "/path/to/folder/folder2/folder1/file")
+//	// => [ "/path/to/folder/folder2", "/path/to/folder/folder2/folder1" ]
+//
+//	DirectoryDiff("/", "/path/to/folder/folder/file")
+//	// => [ "/path", "/path/to", "/path/to/folder", "/path/to/folder/folder" ]
+func DirectoryDiff(prev, current string) []string {
+	var paths []string
+
+	if prev == "" || current == "" {
+		return paths
+	}
+
+	parent := current
+	for {
+		parent = filepath.Dir(parent)
+		if parent == "/" || parent == prev || strings.HasPrefix(prev+"/", parent+"/") {
+			break
+		}
+		paths = append(paths, parent)
+	}
+	slices.Reverse(paths)
+	return paths
+}
+
+// PathToKey 将路径转换为存储键
+func PathToKey(rootDir, path string) string {
+	rootDir = strings.Trim(rootDir, "/")
+	path = strings.Trim(path, "/")
+	// Important! delete the root prefix if existed
+	path = strings.TrimLeft(strings.TrimPrefix(path, rootDir), "/")
+	if rootDir == "" {
+		return path
+	}
+	if path == "" {
+		return rootDir
+	}
+	return rootDir + "/" + path
+}
+
+// KeyToPath 将存储键转换为路径
+func KeyToPath(rootDir, key string) string {
+	rootDir = strings.Trim(rootDir, "/")
+	key = strings.Trim(key, "/")
+	if rootDir == "" {
+		return "/" + key
+	}
+	if key == "" {
+		return ""
+	}
+	return "/" + strings.TrimLeft(strings.TrimPrefix(key, rootDir), "/")
+}

registry/storage/driver/cos/utils_test.go

@@ -0,0 +1,51 @@
+package cos
+
+import "testing"
+
+func TestPathToKey(t *testing.T) {
+	cases := []struct {
+		root string
+		path string
+		key  string
+	}{
+		{root: "", path: "/a/b/c.txt", key: "a/b/c.txt"},
+		{root: "", path: "a/b/c.txt", key: "a/b/c.txt"},
+		{root: "", path: "/", key: ""},
+		{root: "rootdir", path: "/a/b/c.txt", key: "rootdir/a/b/c.txt"},
+		{root: "rootdir", path: "a/b/c.txt", key: "rootdir/a/b/c.txt"},
+		{root: "rootdir", path: "/", key: "rootdir"},
+		{root: "dup-rootdir", path: "/dup-rootdir/a/b/c.txt", key: "dup-rootdir/a/b/c.txt"},
+		{root: "dup-rootdir", path: "dup-rootdir/a/b/c.txt", key: "dup-rootdir/a/b/c.txt"},
+	}
+
+	for _, c := range cases {
+		key := PathToKey(c.root, c.path)
+		if key != c.key {
+			t.Errorf("PathToKey(%q, %q) = %q; want %q", c.root, c.path, key, c.key)
+		}
+	}
+}
+
+func TestKeyToPath(t *testing.T) {
+	cases := []struct {
+		root string
+		key  string
+		path string
+	}{
+		{root: "", key: "a/b/c.txt", path: "/a/b/c.txt"},
+		{root: "", key: "a/b/c.txt", path: "/a/b/c.txt"},
+		{root: "", key: "/", path: "/"},
+		{root: "rootdir", key: "rootdir/a/b/c.txt", path: "/a/b/c.txt"},
+		{root: "rootdir", key: "rootdir/a/b/c.txt", path: "/a/b/c.txt"},
+		{root: "rootdir", key: "rootdir", path: "/"},
+		{root: "dup-rootdir", key: "dup-rootdir/a/b/c.txt", path: "/a/b/c.txt"},
+		{root: "dup-rootdir", key: "dup-rootdir/a/b/c.txt", path: "/a/b/c.txt"},
+	}
+
+	for _, c := range cases {
+		path := KeyToPath(c.root, c.key)
+		if path != c.path {
+			t.Errorf("KeyToPath(%q, %q) = %q; want %q", c.root, c.key, path, c.path)
+		}
+	}
+}