diff --git a/.dockerignore b/.dockerignore index f614e0154..c72ffd5dc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,5 +6,6 @@ default.etcd *.tar.gz *.bzip2 *.zip +*.md browser/node_modules node_modules \ No newline at end of file diff --git a/.github/workflows/build-push-client-docker-image.yaml b/.github/workflows/build-push-client-docker-image.yaml index eb1726637..e9839060e 100644 --- a/.github/workflows/build-push-client-docker-image.yaml +++ b/.github/workflows/build-push-client-docker-image.yaml @@ -20,7 +20,7 @@ env: MINIO_CLIENT_REGISTRY: ${{ secrets.MINIO_CLIENT_REGISTRY }} jobs: - dockerize_minio: + dockerize_client: runs-on: [self-hosted, arc-runner] outputs: diff --git a/.github/workflows/build-push-logsearchapi-docker-image.yaml b/.github/workflows/build-push-logsearchapi-docker-image.yaml index 5b0d11da2..ff4343bae 100644 --- a/.github/workflows/build-push-logsearchapi-docker-image.yaml +++ b/.github/workflows/build-push-logsearchapi-docker-image.yaml @@ -20,7 +20,7 @@ env: MINIO_LOGSEARCHAPI_REGISTRY: ${{ secrets.MINIO_LOGSEARCHAPI_REGISTRY }} jobs: - dockerize_minio: + dockerize_logsearchapi: runs-on: [self-hosted, arc-runner] outputs: diff --git a/Dockerfile b/Dockerfile index 6f9fdd430..81f52729a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,17 +5,12 @@ ENV GO111MODULE=on # Download the dependencies: # Will be cached if we don't change mod/sum files +WORKDIR $SRC_DIR +COPY . . -COPY ./go.mod $SRC_DIR/ -COPY ./go.sum $SRC_DIR/ - -RUN cd $SRC_DIR && go mod download -x - -COPY . $SRC_DIR/ - -WORKDIR /minio - -RUN go build -o minio -buildvcs=false +RUN go mod download -x && \ + go build -o minio -buildvcs=false && \ + ls /usr/local/lib/ # Copy the build artifact into a minimal runtime image: FROM alpine:3.18 diff --git a/cmd/api-router.go b/cmd/api-router.go index 2ebf89ef4..d0ba330e9 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -302,12 +302,12 @@ func registerAPIRouter(router *mux.Router) { router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc( collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectExtractHandler))))) - // PutObject - router.Methods(http.MethodPut).Path("/{object}").HandlerFunc( - collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler))))) // PutMultipleObjects router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putmultipleobjects", maxClients(gz(httpTraceHdrs(api.PutMultipleObjectsHandler))))).Queries("multiupload", "true") + // PutObject + router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( + collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler))))) // DeleteObject router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( diff --git a/cmd/gateway/zcn/dStorage.go b/cmd/gateway/zcn/dStorage.go index ce370e29c..75ff45f14 100644 --- a/cmd/gateway/zcn/dStorage.go +++ b/cmd/gateway/zcn/dStorage.go @@ -77,43 +77,50 @@ func listRegularRefs(alloc *sdk.Allocation, remotePath, marker, fileType string, var markedPath string remotePath = filepath.Clean(remotePath) - commonPrefix := getCommonPrefix(remotePath) - offsetPath := filepath.Join(remotePath, marker) - for { - oResult, err := getRegularRefs(alloc, remotePath, offsetPath, fileType, pageLimit) - if err != nil { - return nil, true, "", nil, err - } - if len(oResult.Refs) == 0 { - break - } - for i := 0; i < len(oResult.Refs); i++ { - ref := oResult.Refs[i] - trimmedPath := strings.TrimPrefix(ref.Path, remotePath+"/") - if isDelimited { + directories := []string{remotePath} + var currentRemotePath string + for len(directories) > 0 && !isTruncated { + currentRemotePath = directories[0] + directories = directories[1:] // dequeue from the directories queue + commonPrefix := getCommonPrefix(currentRemotePath) + offsetPath := filepath.Join(currentRemotePath, marker) + for { + oResult, err := getRegularRefs(alloc, currentRemotePath, offsetPath, fileType, pageLimit) + if err != nil { + return nil, true, "", nil, err + } + if len(oResult.Refs) == 0 { + break + } + + for i := 0; i < len(oResult.Refs); i++ { + ref := oResult.Refs[i] + trimmedPath := strings.TrimPrefix(ref.Path, currentRemotePath+"/") if ref.Type == dirType { - dirPrefix := filepath.Join(commonPrefix, trimmedPath) + "/" - prefixes = append(prefixes, dirPrefix) - continue + if isDelimited { + dirPrefix := filepath.Join(commonPrefix, trimmedPath) + "/" + prefixes = append(prefixes, dirPrefix) + continue + } else { + directories = append(directories, ref.Path) + } } - } - ref.Name = filepath.Join(commonPrefix, trimmedPath) + ref.Name = filepath.Join(commonPrefix, trimmedPath) - refs = append(refs, ref) - if maxRefs != 0 && len(refs) >= maxRefs { - markedPath = ref.Path - isTruncated = true - break + refs = append(refs, ref) + if maxRefs != 0 && len(refs) >= maxRefs { + markedPath = ref.Path + isTruncated = true + break + } } + offsetPath = oResult.OffsetPath } - - offsetPath = oResult.OffsetPath - } if isTruncated { - marker = strings.TrimPrefix(markedPath, remotePath+"/") + marker = strings.TrimPrefix(markedPath, currentRemotePath+"/") } else { marker = "" } @@ -123,12 +130,14 @@ func listRegularRefs(alloc *sdk.Allocation, remotePath, marker, fileType string, func getRegularRefs(alloc *sdk.Allocation, remotePath, offsetPath, fileType string, pageLimit int) (oResult *sdk.ObjectTreeResult, err error) { level := len(strings.Split(strings.TrimSuffix(remotePath, "/"), "/")) + 1 + remotePath = filepath.Clean(remotePath) oResult, err = alloc.GetRefs(remotePath, offsetPath, "", "", fileType, "regular", level, pageLimit) return } func getSingleRegularRef(alloc *sdk.Allocation, remotePath string) (*sdk.ORef, error) { level := len(strings.Split(strings.TrimSuffix(remotePath, "/"), "/")) + remotePath = filepath.Clean(remotePath) oREsult, err := alloc.GetRefs(remotePath, "", "", "", "", "regular", level, 1) if err != nil { logger.Error("error with GetRefs", err.Error(), " this is the error") diff --git a/cmd/gateway/zcn/gateway-zcn.go b/cmd/gateway/zcn/gateway-zcn.go index 3c96c6a7d..176c50744 100644 --- a/cmd/gateway/zcn/gateway-zcn.go +++ b/cmd/gateway/zcn/gateway-zcn.go @@ -245,6 +245,10 @@ func (zob *zcnObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min return } + if ref.Type != dirType { + return bi, minio.BucketNotFound{Bucket: bucket} + } + return minio.BucketInfo{Name: ref.Name, Created: ref.CreatedAt.ToTime()}, nil } @@ -258,7 +262,7 @@ func (zob *zcnObjects) GetObjectInfo(ctx context.Context, bucket, object string, } var ref *sdk.ORef - ref, err = getSingleRegularRef(zob.alloc, remotePath) + ref, err = getSingleRegularRef(zob.alloc, filepath.Clean(remotePath)) if err != nil { if isPathNoExistError(err) { return objInfo, minio.ObjectNotFound{Bucket: bucket, Object: object} @@ -266,9 +270,13 @@ func (zob *zcnObjects) GetObjectInfo(ctx context.Context, bucket, object string, return } + if ref.Type == dirType { + return minio.ObjectInfo{}, minio.ObjectNotFound{Bucket: bucket, Object: object} + } + return minio.ObjectInfo{ Bucket: bucket, - Name: getCommonPrefix(remotePath), + Name: getRelativePathOfObj(ref.Path, bucket), ModTime: ref.UpdatedAt.ToTime(), Size: ref.ActualFileSize, IsDir: ref.Type == dirType, @@ -392,12 +400,6 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker, remotePath = filepath.Join(rootPath, bucket, prefix) } - var isSuffix bool - if strings.HasSuffix(prefix, "/") { - remotePath = filepath.Clean(remotePath) + "/" - isSuffix = true - } - var ref *sdk.ORef ref, err = getSingleRegularRef(zob.alloc, remotePath) if err != nil { @@ -408,7 +410,7 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker, } if ref.Type == fileType { - if isSuffix { + if strings.HasSuffix(prefix, "/") { return minio.ListObjectsInfo{ IsTruncated: false, Objects: []minio.ObjectInfo{}, @@ -416,15 +418,12 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker, }, nil } - parentPath, fileName := filepath.Split(ref.Path) - commonPrefix := getCommonPrefix(parentPath) - objName := filepath.Join(commonPrefix, fileName) return minio.ListObjectsInfo{ IsTruncated: false, Objects: []minio.ObjectInfo{ { Bucket: bucket, - Name: objName, + Name: getRelativePathOfObj(ref.Path, bucket), Size: ref.ActualFileSize, IsDir: false, ModTime: ref.UpdatedAt.ToTime(), @@ -460,7 +459,7 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker, objects = append(objects, minio.ObjectInfo{ Bucket: bucket, - Name: ref.Name, + Name: getRelativePathOfObj(ref.Path, bucket), ModTime: ref.UpdatedAt.ToTime(), Size: ref.ActualFileSize, IsDir: false, @@ -477,6 +476,20 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker, return } +// getRelativePathOfObj returns the relative path of a file without the leading slash and without the name of the bucket +func getRelativePathOfObj(refPath, bucketName string) string { + //eg: refPath = "/myFile.txt" bucketName = "/", return value = "myFile.txt" + //eg: refPath = "/buck1/myFile.txt" bucketName = anything other than "/" or "root", return value = "myFile.txt" + //eg: refPath = "/myFile.txt" bucketName = "abc", return value = "myFile.txt" + //remotePath = "/xyz/abc/def", return value = "abc/def" + + if bucketName == rootPath || bucketName == rootBucketName { + return strings.TrimPrefix(refPath, rootPath) + } + + return getCommonPrefix(refPath) +} + func (zob *zcnObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error { // Create a directory; ignore opts remotePath := filepath.Join(rootPath, bucket) diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 94fd668df..a565cf138 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1910,7 +1910,9 @@ func (api objectAPIHandlers) PutMultipleObjectsHandler(w http.ResponseWriter, r // parses the multipart form to retrieve the file data. This parses in 32 MB memory. if err := r.ParseMultipartForm(32 << 20); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + e := fmt.Errorf("for PUT `/:bucket/:object?multiupload=true` api, body should be sent in a "+ + "multipart/form-data type with key as objectKey and value as file object, error: %s", err.Error()) + http.Error(w, e.Error(), http.StatusBadRequest) return } var objectKeys []string diff --git a/go.mod b/go.mod index 33fa3037e..fabe02ac4 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( cloud.google.com/go/storage v1.27.0 github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.8.17-0.20230807135703-eb69fb614b1d + github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9 github.com/Azure/azure-pipeline-go v0.2.2 github.com/Azure/azure-storage-blob-go v0.10.0 github.com/Shopify/sarama v1.28.0 diff --git a/go.sum b/go.sum index 36b716387..0ea79d120 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.8.17-0.20230807135703-eb69fb614b1d h1:EyYzKcfpZCsLVFDqouoFwFOcmzlHjuDpI2GxjF3LBwc= -github.com/0chain/gosdk v1.8.17-0.20230807135703-eb69fb614b1d/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= +github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9 h1:GHTdYTmhNY9genBkNWLXdn3Z1yCtcbSNkcIFaKrqBRU= +github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=