Merge pull request #67 from volcengine/release_release_v2.7.8

release_v2.7.8

CHANGELOG.md

@@ -1,4 +1,9 @@
 # ChangeLog of TOS SDK for Go
+## 版本号 v2.7.8 日期：2024-11-20
+- ListObject V1 接口支持 CRC64
+- 修复 Restore ExpiryDate 时间错误
+- 修复 UploadFile 上传失败错误处理
+
 ## 版本号 v2.7.7 日期：2024-09-22
 - 优化 HeadBucket
 

tos/consts.go

@@ -7,7 +7,7 @@ import (
 
 const (
 	// Version tos-go-sdk version
-	Version = "v2.7.7"
+	Version = "v2.7.8"
 )
 
 const TempFileSuffix = ".temp"

tos/metadata.go

@@ -85,7 +85,7 @@ func parseRestoreInfo(res *Response) *RestoreInfo {
 		return nil
 	}
 	resp := &RestoreInfo{}
-	param := parseParams(restore)
+	param := parseRestoreParams(restore)
 	if result, ok := param["ongoing-request"]; ok && result == "true" {
 		resp.RestoreStatus.OngoingRequest = true
 	}
@@ -150,9 +150,9 @@ func (om *ObjectMeta) fromResponse(res *Response, disableEncodingMeta bool) {
 	om.IsDirectory = res.Header.Get(HeaderDirectory) == "true"
 
 }
-func parseParams(params string) map[string]string {
+func parseRestoreParams(params string) map[string]string {
 	result := make(map[string]string)
-	parts := strings.Split(params, ",") // 按逗号分割参数
+	parts := strings.SplitAfterN(params, ",", 2) // 按逗号分割参数
 
 	for _, part := range parts {
 		keyValue := strings.Split(part, "=") // 按等号分割键和值

tos/mime_test.go

@@ -21,3 +21,11 @@ func TestMime(t *testing.T) {
 	typ = me.ContentType("a.json")
 	require.Equal(t, "", typ)
 }
+
+func TestRestoreInfo(t *testing.T) {
+	res := &Response{Header: map[string][]string{}}
+	res.Header.Set(HeaderRestore, "ongoing-request=\"false\", expiry-date=\"Mon, 02 Dec 2024 00:00:00 GMT\"")
+	restoreInfo := parseRestoreInfo(res)
+	require.Equal(t, false, restoreInfo.RestoreStatus.OngoingRequest)
+	require.Equal(t, true, restoreInfo.RestoreStatus.ExpiryDate.String() == "2024-12-02 00:00:00 +0000 UTC")
+}

tos/object.go

@@ -915,16 +915,30 @@ func (bkt *Bucket) ListObjects(ctx context.Context, input *ListObjectsInput, opt
 	}
 	contents := make([]ListedObject, 0, len(internalOutput.Contents))
 	for _, content := range internalOutput.Contents {
+
+		var hashCrc uint64
+		if len(content.HashCrc64ecma) == 0 {
+			hashCrc = 0
+		} else {
+			hashCrc, err = strconv.ParseUint(content.HashCrc64ecma, 10, 64)
+			if err != nil {
+				return nil, &TosServerError{
+					TosError:    newTosErr("tos: server returned invalid HashCrc64Ecma", res.RequestUrl, res.RequestInfo().EcCode, res.RequestInfo().RequestID),
+					RequestInfo: RequestInfo{RequestID: res.RequestInfo().RequestID},
+				}
+			}
+		}
 		contents = append(contents, ListedObject{
-			Key:          content.Key,
-			LastModified: content.LastModified,
-			ETag:         content.ETag,
-			Size:         content.Size,
-			Owner:        content.Owner,
-			StorageClass: content.StorageClass,
-			Type:         content.Type,
-			Meta:         parseUserMetaData(content.Meta),
-			ObjectType:   content.ObjectType,
+			Key:           content.Key,
+			LastModified:  content.LastModified,
+			ETag:          content.ETag,
+			Size:          content.Size,
+			Owner:         content.Owner,
+			StorageClass:  content.StorageClass,
+			Type:          content.Type,
+			Meta:          parseUserMetaData(content.Meta),
+			HashCrc64ecma: hashCrc,
+			ObjectType:    content.ObjectType,
 		})
 	}
 	output.Contents = contents
@@ -1132,18 +1146,31 @@ func (bkt *Bucket) ListObjectVersions(ctx context.Context, input *ListObjectVers
 
 	contents := make([]ListedObjectVersion, 0, len(interOutput.Versions))
 	for _, content := range interOutput.Versions {
+		var hashCrc uint64
+		if len(content.HashCrc64ecma) == 0 {
+			hashCrc = 0
+		} else {
+			hashCrc, err = strconv.ParseUint(content.HashCrc64ecma, 10, 64)
+			if err != nil {
+				return nil, &TosServerError{
+					TosError:    newTosErr("tos: server returned invalid HashCrc64Ecma", res.RequestUrl, res.RequestInfo().EcCode, res.RequestInfo().RequestID),
+					RequestInfo: RequestInfo{RequestID: res.RequestInfo().RequestID},
+				}
+			}
+		}
 		contents = append(contents, ListedObjectVersion{
-			Key:          content.Key,
-			IsLatest:     content.IsLatest,
-			LastModified: content.LastModified,
-			ETag:         content.ETag,
-			Size:         content.Size,
-			Owner:        content.Owner,
-			StorageClass: content.StorageClass,
-			Type:         content.Type,
-			VersionID:    content.VersionID,
-			Meta:         parseUserMetaData(content.Meta),
-			ObjectType:   content.ObjectType,
+			Key:           content.Key,
+			IsLatest:      content.IsLatest,
+			LastModified:  content.LastModified,
+			ETag:          content.ETag,
+			Size:          content.Size,
+			Owner:         content.Owner,
+			StorageClass:  content.StorageClass,
+			Type:          content.Type,
+			VersionID:     content.VersionID,
+			Meta:          parseUserMetaData(content.Meta),
+			ObjectType:    content.ObjectType,
+			HashCrc64ecma: hashCrc,
 		})
 	}
 	output.Versions = contents

tos/tests/list_object_test.go

@@ -90,6 +90,8 @@ func TestListObjWithMeta(t *testing.T) {
 			mValue, ok := obj.Meta.Get(metaKey)
 			require.True(t, ok)
 			require.Equal(t, mValue, metaValue)
+			require.True(t, obj.HashCrc64ecma > 0)
+
 		}
 	}
 
@@ -107,6 +109,7 @@ func TestListObjWithMeta(t *testing.T) {
 			mValue, ok := obj.Meta.Get(metaKey)
 			require.True(t, ok)
 			require.Equal(t, mValue, metaValue)
+			require.True(t, obj.HashCrc64ecma > 0)
 		}
 	}
 

tos/tests/object_test.go

@@ -2258,7 +2258,6 @@ func TestRestoreObject(t *testing.T) {
 		Content: strings.NewReader(body),
 	})
 	require.Nil(t, err)
-
 	out, err := client.RestoreObject(ctx, &tos.RestoreObjectInput{
 		Bucket:               bucket,
 		Key:                  key,
@@ -2267,15 +2266,28 @@ func TestRestoreObject(t *testing.T) {
 	})
 	require.Nil(t, err)
 	require.Equal(t, out.StatusCode, http.StatusAccepted)
-
-	headResp, err := client.HeadObjectV2(ctx, &tos.HeadObjectV2Input{Bucket: bucket, Key: key})
+	var headResp *tos.HeadObjectV2Output
+	start := time.Now()
+	headResp, err = client.HeadObjectV2(ctx, &tos.HeadObjectV2Input{Bucket: bucket, Key: key})
 	require.Nil(t, err)
 	require.Equal(t, headResp.RestoreInfo.RestoreStatus.OngoingRequest, true)
 	require.Equal(t, headResp.RestoreInfo.RestoreParam.ExpiryDays, 10)
+	require.False(t, headResp.RestoreInfo.RestoreParam.RequestDate.IsZero())
 	require.Equal(t, headResp.RestoreInfo.RestoreParam.Tier, enum.TierExpedited)
+	for {
+		require.True(t, time.Now().Sub(start) < 5*time.Minute)
+		headResp, err = client.HeadObjectV2(ctx, &tos.HeadObjectV2Input{Bucket: bucket, Key: key})
+		require.Nil(t, err)
+		if headResp.RestoreInfo.RestoreStatus.OngoingRequest {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		require.False(t, headResp.RestoreInfo.RestoreStatus.ExpiryDate.IsZero())
+		break
+	}
 
 	_, err = client.GetObjectV2(ctx, &tos.GetObjectV2Input{Bucket: bucket, Key: key})
-	require.NotNil(t, err)
+	require.Nil(t, err)
 
 }
 

tos/tests/presigned_test.go

@@ -621,14 +621,6 @@ func TestPreSignedPolicyURLWithExpires(t *testing.T) {
 	res, err = client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, 200, res.StatusCode)
-	// Unmarshal ListObjectsOutput
-	data, err := ioutil.ReadAll(res.Body)
-	require.Nil(t, err)
-	data = bytes.TrimSpace(data)
-	jsonOut := tos.ListObjectsOutput{}
-	err = json.Unmarshal(data, &jsonOut)
-	require.Nil(t, err)
-	require.Equal(t, len(jsonOut.Contents), 2)
 
 	// head test based policy url for key2
 	getUrl2 := output.GetSignedURLForGetOrHead(key2, nil)

tos/type.go

@@ -660,27 +660,29 @@ type ListObjectsInput struct {
 }
 
 type ListedObject struct {
-	Key          string   `json:"Key,omitempty"`
-	LastModified string   `json:"LastModified,omitempty"`
-	ETag         string   `json:"ETag,omitempty"`
-	Size         int64    `json:"Size,omitempty"`
-	Owner        Owner    `json:"Owner,omitempty"`
-	StorageClass string   `json:"StorageClass,omitempty"`
-	Type         string   `json:"Type,omitempty"`
-	Meta         Metadata `json:"UserMeta,omitempty"`
-	ObjectType   string   `json:"Type,omitempty"`
+	Key           string   `json:"Key,omitempty"`
+	LastModified  string   `json:"LastModified,omitempty"`
+	ETag          string   `json:"ETag,omitempty"`
+	Size          int64    `json:"Size,omitempty"`
+	Owner         Owner    `json:"Owner,omitempty"`
+	StorageClass  string   `json:"StorageClass,omitempty"`
+	Type          string   `json:"Type,omitempty"`
+	Meta          Metadata `json:"UserMeta,omitempty"`
+	HashCrc64ecma uint64   `json:"HashCrc64Ecma,omitempty"`
+	ObjectType    string   `json:"Type,omitempty"`
 }
 
 type listedObject struct {
-	Key          string     `json:"Key,omitempty"`
-	LastModified string     `json:"LastModified,omitempty"`
-	ETag         string     `json:"ETag,omitempty"`
-	Size         int64      `json:"Size,omitempty"`
-	Owner        Owner      `json:"Owner,omitempty"`
-	StorageClass string     `json:"StorageClass,omitempty"`
-	Type         string     `json:"Type,omitempty"`
-	Meta         []userMeta `json:"UserMeta,omitempty"`
-	ObjectType   string     `json:"Type,omitempty"`
+	Key           string     `json:"Key,omitempty"`
+	LastModified  string     `json:"LastModified,omitempty"`
+	ETag          string     `json:"ETag,omitempty"`
+	Size          int64      `json:"Size,omitempty"`
+	Owner         Owner      `json:"Owner,omitempty"`
+	StorageClass  string     `json:"StorageClass,omitempty"`
+	Type          string     `json:"Type,omitempty"`
+	Meta          []userMeta `json:"UserMeta,omitempty"`
+	HashCrc64ecma string     `json:"HashCrc64Ecma,omitempty"`
+	ObjectType    string     `json:"Type,omitempty"`
 }
 
 type ListedObjectV2 struct {
@@ -802,17 +804,18 @@ type ListObjectVersionsV2Input struct {
 }
 
 type ListedObjectVersion struct {
-	ETag         string   `json:"ETag,omitempty"`
-	IsLatest     bool     `json:"IsLatest,omitempty"`
-	Key          string   `json:"Key,omitempty"`
-	LastModified string   `json:"LastModified,omitempty"`
-	Owner        Owner    `json:"Owner,omitempty"`
-	Size         int64    `json:"Size,omitempty"`
-	StorageClass string   `json:"StorageClass,omitempty"`
-	Type         string   `json:"Type,omitempty"`
-	VersionID    string   `json:"VersionId,omitempty"`
-	Meta         Metadata `json:"UserMeta,omitempty"`
-	ObjectType   string   `json:"Type,omitempty"`
+	ETag          string   `json:"ETag,omitempty"`
+	IsLatest      bool     `json:"IsLatest,omitempty"`
+	Key           string   `json:"Key,omitempty"`
+	LastModified  string   `json:"LastModified,omitempty"`
+	Owner         Owner    `json:"Owner,omitempty"`
+	Size          int64    `json:"Size,omitempty"`
+	StorageClass  string   `json:"StorageClass,omitempty"`
+	Type          string   `json:"Type,omitempty"`
+	VersionID     string   `json:"VersionId,omitempty"`
+	Meta          Metadata `json:"UserMeta,omitempty"`
+	ObjectType    string   `json:"Type,omitempty"`
+	HashCrc64ecma uint64   `json:"HashCrc64Ecma,omitempty"`
 }
 
 type listedObjectVersionV2 struct {
@@ -894,17 +897,18 @@ type ListObjectVersionsV2Output struct {
 }
 
 type listedObjectVersion struct {
-	ETag         string     `json:"ETag,omitempty"`
-	IsLatest     bool       `json:"IsLatest,omitempty"`
-	Key          string     `json:"Key,omitempty"`
-	LastModified string     `json:"LastModified,omitempty"`
-	Owner        Owner      `json:"Owner,omitempty"`
-	Size         int64      `json:"Size,omitempty"`
-	StorageClass string     `json:"StorageClass,omitempty"`
-	Type         string     `json:"Type,omitempty"`
-	VersionID    string     `json:"VersionId,omitempty"`
-	Meta         []userMeta `json:"UserMeta,omitempty"`
-	ObjectType   string     `json:"Type,omitempty"`
+	ETag          string     `json:"ETag,omitempty"`
+	IsLatest      bool       `json:"IsLatest,omitempty"`
+	Key           string     `json:"Key,omitempty"`
+	LastModified  string     `json:"LastModified,omitempty"`
+	Owner         Owner      `json:"Owner,omitempty"`
+	Size          int64      `json:"Size,omitempty"`
+	StorageClass  string     `json:"StorageClass,omitempty"`
+	Type          string     `json:"Type,omitempty"`
+	VersionID     string     `json:"VersionId,omitempty"`
+	Meta          []userMeta `json:"UserMeta,omitempty"`
+	ObjectType    string     `json:"Type,omitempty"`
+	HashCrc64ecma string     `json:"HashCrc64Ecma,omitempty"`
 }
 
 type listObjectVersionsOutput struct {

tos/type_internal.go

@@ -321,7 +321,7 @@ func (u *uploadCheckpoint) UpdatePartsInfo(result interface{}) {
 }
 
 func (u *uploadCheckpoint) GetCheckPointFilePath() string {
-	return u.FilePath
+	return u.checkpointPath
 }
 
 func (u *uploadCheckpoint) Valid(uploadFileStat os.FileInfo, bucketName, key, uploadFile string) bool {

tos/util.go

@@ -76,7 +76,9 @@ Loop:
 		case taskErr := <-t.errCh:
 			if StatusCode(taskErr) == 403 || StatusCode(taskErr) == 404 || StatusCode(taskErr) == 405 {
 				close(t.abortHandle)
-				_ = os.Remove(t.checkPoint.GetCheckPointFilePath())
+				if t.checkPoint.GetCheckPointFilePath() != "" {
+					_ = os.Remove(t.checkPoint.GetCheckPointFilePath())
+				}
 				t.postEvent.PostEvent(EventPartAborted, nil, taskErr)
 
 				return successNum, fmt.Errorf("status code not service error, err:%s. ", taskErr.Error())