Merge pull request #65 from 0chain/sprint-1.10

sprint-1.10

.dockerignore

@@ -6,5 +6,6 @@ default.etcd
 *.tar.gz
 *.bzip2
 *.zip
+*.md
 browser/node_modules
 node_modules

.github/workflows/build-push-client-docker-image.yaml

@@ -20,7 +20,7 @@ env:
   MINIO_CLIENT_REGISTRY: ${{ secrets.MINIO_CLIENT_REGISTRY }}
 
 jobs:
-    dockerize_minio:
+    dockerize_client:
       runs-on: [self-hosted, arc-runner]
 
       outputs:

.github/workflows/build-push-logsearchapi-docker-image.yaml

@@ -20,7 +20,7 @@ env:
   MINIO_LOGSEARCHAPI_REGISTRY: ${{ secrets.MINIO_LOGSEARCHAPI_REGISTRY }}
 
 jobs:
-    dockerize_minio:
+    dockerize_logsearchapi:
       runs-on: [self-hosted, arc-runner]
 
       outputs:

Dockerfile

@@ -5,17 +5,12 @@ ENV GO111MODULE=on
 
 # Download the dependencies:
 # Will be cached if we don't change mod/sum files
+WORKDIR $SRC_DIR
+COPY . .
 
-COPY ./go.mod $SRC_DIR/
-COPY ./go.sum $SRC_DIR/
-
-RUN cd $SRC_DIR && go mod download -x
-
-COPY . $SRC_DIR/
-
-WORKDIR /minio
-
-RUN go build -o minio -buildvcs=false
+RUN go mod download -x && \
+    go build -o minio -buildvcs=false && \
+    ls /usr/local/lib/
 
 # Copy the build artifact into a minimal runtime image:
 FROM alpine:3.18

cmd/api-router.go

@@ -302,12 +302,12 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc(
 			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectExtractHandler)))))
 
-		// PutObject
-		router.Methods(http.MethodPut).Path("/{object}").HandlerFunc(
-			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler)))))
 		// PutMultipleObjects
 		router.Methods(http.MethodPut).HandlerFunc(
 			collectAPIStats("putmultipleobjects", maxClients(gz(httpTraceHdrs(api.PutMultipleObjectsHandler))))).Queries("multiupload", "true")
+		// PutObject
+		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler)))))
 
 		// DeleteObject
 		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(

cmd/gateway/zcn/dStorage.go

@@ -77,43 +77,50 @@ func listRegularRefs(alloc *sdk.Allocation, remotePath, marker, fileType string,
 	var markedPath string
 
 	remotePath = filepath.Clean(remotePath)
-	commonPrefix := getCommonPrefix(remotePath)
-	offsetPath := filepath.Join(remotePath, marker)
-	for {
-		oResult, err := getRegularRefs(alloc, remotePath, offsetPath, fileType, pageLimit)
-		if err != nil {
-			return nil, true, "", nil, err
-		}
-		if len(oResult.Refs) == 0 {
-			break
-		}
 
-		for i := 0; i < len(oResult.Refs); i++ {
-			ref := oResult.Refs[i]
-			trimmedPath := strings.TrimPrefix(ref.Path, remotePath+"/")
-			if isDelimited {
+	directories := []string{remotePath}
+	var currentRemotePath string
+	for len(directories) > 0 && !isTruncated {
+		currentRemotePath = directories[0]
+		directories = directories[1:] // dequeue from the directories queue
+		commonPrefix := getCommonPrefix(currentRemotePath)
+		offsetPath := filepath.Join(currentRemotePath, marker)
+		for {
+			oResult, err := getRegularRefs(alloc, currentRemotePath, offsetPath, fileType, pageLimit)
+			if err != nil {
+				return nil, true, "", nil, err
+			}
+			if len(oResult.Refs) == 0 {
+				break
+			}
+
+			for i := 0; i < len(oResult.Refs); i++ {
+				ref := oResult.Refs[i]
+				trimmedPath := strings.TrimPrefix(ref.Path, currentRemotePath+"/")
 				if ref.Type == dirType {
-					dirPrefix := filepath.Join(commonPrefix, trimmedPath) + "/"
-					prefixes = append(prefixes, dirPrefix)
-					continue
+					if isDelimited {
+						dirPrefix := filepath.Join(commonPrefix, trimmedPath) + "/"
+						prefixes = append(prefixes, dirPrefix)
+						continue
+					} else {
+						directories = append(directories, ref.Path)
+					}
 				}
-			}
 
-			ref.Name = filepath.Join(commonPrefix, trimmedPath)
+				ref.Name = filepath.Join(commonPrefix, trimmedPath)
 
-			refs = append(refs, ref)
-			if maxRefs != 0 && len(refs) >= maxRefs {
-				markedPath = ref.Path
-				isTruncated = true
-				break
+				refs = append(refs, ref)
+				if maxRefs != 0 && len(refs) >= maxRefs {
+					markedPath = ref.Path
+					isTruncated = true
+					break
+				}
 			}
+			offsetPath = oResult.OffsetPath
 		}
-
-		offsetPath = oResult.OffsetPath
-
 	}
 	if isTruncated {
-		marker = strings.TrimPrefix(markedPath, remotePath+"/")
+		marker = strings.TrimPrefix(markedPath, currentRemotePath+"/")
 	} else {
 		marker = ""
 	}
@@ -123,12 +130,14 @@ func listRegularRefs(alloc *sdk.Allocation, remotePath, marker, fileType string,
 
 func getRegularRefs(alloc *sdk.Allocation, remotePath, offsetPath, fileType string, pageLimit int) (oResult *sdk.ObjectTreeResult, err error) {
 	level := len(strings.Split(strings.TrimSuffix(remotePath, "/"), "/")) + 1
+	remotePath = filepath.Clean(remotePath)
 	oResult, err = alloc.GetRefs(remotePath, offsetPath, "", "", fileType, "regular", level, pageLimit)
 	return
 }
 
 func getSingleRegularRef(alloc *sdk.Allocation, remotePath string) (*sdk.ORef, error) {
 	level := len(strings.Split(strings.TrimSuffix(remotePath, "/"), "/"))
+	remotePath = filepath.Clean(remotePath)
 	oREsult, err := alloc.GetRefs(remotePath, "", "", "", "", "regular", level, 1)
 	if err != nil {
 		logger.Error("error with GetRefs", err.Error(), " this is the error")

cmd/gateway/zcn/gateway-zcn.go

@@ -245,6 +245,10 @@ func (zob *zcnObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min
 		return
 	}
 
+	if ref.Type != dirType {
+		return bi, minio.BucketNotFound{Bucket: bucket}
+	}
+
 	return minio.BucketInfo{Name: ref.Name, Created: ref.CreatedAt.ToTime()}, nil
 }
 
@@ -258,17 +262,21 @@ func (zob *zcnObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 	}
 
 	var ref *sdk.ORef
-	ref, err = getSingleRegularRef(zob.alloc, remotePath)
+	ref, err = getSingleRegularRef(zob.alloc, filepath.Clean(remotePath))
 	if err != nil {
 		if isPathNoExistError(err) {
 			return objInfo, minio.ObjectNotFound{Bucket: bucket, Object: object}
 		}
 		return
 	}
 
+	if ref.Type == dirType {
+		return minio.ObjectInfo{}, minio.ObjectNotFound{Bucket: bucket, Object: object}
+	}
+
 	return minio.ObjectInfo{
 		Bucket:      bucket,
-		Name:        getCommonPrefix(remotePath),
+		Name:        getRelativePathOfObj(ref.Path, bucket),
 		ModTime:     ref.UpdatedAt.ToTime(),
 		Size:        ref.ActualFileSize,
 		IsDir:       ref.Type == dirType,
@@ -392,12 +400,6 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 		remotePath = filepath.Join(rootPath, bucket, prefix)
 	}
 
-	var isSuffix bool
-	if strings.HasSuffix(prefix, "/") {
-		remotePath = filepath.Clean(remotePath) + "/"
-		isSuffix = true
-	}
-
 	var ref *sdk.ORef
 	ref, err = getSingleRegularRef(zob.alloc, remotePath)
 	if err != nil {
@@ -408,23 +410,20 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 	}
 
 	if ref.Type == fileType {
-		if isSuffix {
+		if strings.HasSuffix(prefix, "/") {
 			return minio.ListObjectsInfo{
 					IsTruncated: false,
 					Objects:     []minio.ObjectInfo{},
 					Prefixes:    []string{},
 				},
 				nil
 		}
-		parentPath, fileName := filepath.Split(ref.Path)
-		commonPrefix := getCommonPrefix(parentPath)
-		objName := filepath.Join(commonPrefix, fileName)
 		return minio.ListObjectsInfo{
 				IsTruncated: false,
 				Objects: []minio.ObjectInfo{
 					{
 						Bucket:       bucket,
-						Name:         objName,
+						Name:         getRelativePathOfObj(ref.Path, bucket),
 						Size:         ref.ActualFileSize,
 						IsDir:        false,
 						ModTime:      ref.UpdatedAt.ToTime(),
@@ -460,7 +459,7 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 
 		objects = append(objects, minio.ObjectInfo{
 			Bucket:       bucket,
-			Name:         ref.Name,
+			Name:         getRelativePathOfObj(ref.Path, bucket),
 			ModTime:      ref.UpdatedAt.ToTime(),
 			Size:         ref.ActualFileSize,
 			IsDir:        false,
@@ -477,6 +476,20 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 	return
 }
 
+// getRelativePathOfObj returns the relative path of a file without the leading slash and without the name of the bucket
+func getRelativePathOfObj(refPath, bucketName string) string {
+	//eg: refPath = "/myFile.txt" bucketName = "/", return value = "myFile.txt"
+	//eg: refPath = "/buck1/myFile.txt" bucketName = anything other than "/" or "root", return value = "myFile.txt"
+	//eg: refPath = "/myFile.txt" bucketName = "abc", return value = "myFile.txt"
+	//remotePath = "/xyz/abc/def", return value = "abc/def"
+
+	if bucketName == rootPath || bucketName == rootBucketName {
+		return strings.TrimPrefix(refPath, rootPath)
+	}
+
+	return getCommonPrefix(refPath)
+}
+
 func (zob *zcnObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
 	// Create a directory; ignore opts
 	remotePath := filepath.Join(rootPath, bucket)

cmd/object-handlers.go

@@ -1910,7 +1910,9 @@ func (api objectAPIHandlers) PutMultipleObjectsHandler(w http.ResponseWriter, r
 
 	// parses the multipart form to retrieve the file data. This parses in 32 MB memory.
 	if err := r.ParseMultipartForm(32 << 20); err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		e := fmt.Errorf("for PUT `/:bucket/:object?multiupload=true` api, body should be sent in a "+
+			"multipart/form-data type with key as objectKey and value as file object, error: %s", err.Error())
+		http.Error(w, e.Error(), http.StatusBadRequest)
 		return
 	}
 	var objectKeys []string

go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	cloud.google.com/go/storage v1.27.0
 	github.com/0chain/errors v1.0.3
-	github.com/0chain/gosdk v1.8.17-0.20230807135703-eb69fb614b1d
+	github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9
 	github.com/Azure/azure-pipeline-go v0.2.2
 	github.com/Azure/azure-storage-blob-go v0.10.0
 	github.com/Shopify/sarama v1.28.0

go.sum

@@ -62,8 +62,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs
 github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E=
 github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
 github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
-github.com/0chain/gosdk v1.8.17-0.20230807135703-eb69fb614b1d h1:EyYzKcfpZCsLVFDqouoFwFOcmzlHjuDpI2GxjF3LBwc=
-github.com/0chain/gosdk v1.8.17-0.20230807135703-eb69fb614b1d/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4=
+github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9 h1:GHTdYTmhNY9genBkNWLXdn3Z1yCtcbSNkcIFaKrqBRU=
+github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4=
 github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=