Skip to content

testing 64MB chunk size #230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 6, 2024
Merged

testing 64MB chunk size #230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
65f9f4f
chore: Refactor chunk size for file upload in multipart request to im…
ShocOne Jun 6, 2024
25b692d
Refactor chunk size for file upload in multipart request to 64MB for …
ShocOne Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 0 additions & 114 deletions httpclient/multipartrequest.go.back
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package httpclient

import (
"bytes"
"context"
"encoding/base64"
"fmt"
Expand All @@ -11,7 +10,6 @@ import (
"net/textproto"
"os"
"path/filepath"
"strings"
"time"

"github.com/deploymenttheory/go-api-http-client/authenticationhandler"
Expand Down Expand Up @@ -85,19 +83,12 @@ func (c *Client) DoMultiPartRequest(method, endpoint string, files map[string][]

log.Info("Executing multipart file upload request", zap.String("method", method), zap.String("endpoint", endpoint))

// body, contentType, err := createMultipartRequestBody(files, formDataFields, fileContentTypes, formDataPartHeaders, log)
// if err != nil {
// return nil, err
// }

// Call the helper function to create a streaming multipart request body
body, contentType, err := createStreamingMultipartRequestBody(files, formDataFields, fileContentTypes, formDataPartHeaders, log)
if err != nil {
return nil, err
}

//logMultiPartRequestBody(body, log)

url := c.APIHandler.ConstructAPIResourceEndpoint(endpoint, log)

// Create a context with timeout
Expand Down Expand Up @@ -176,54 +167,6 @@ func createStreamingMultipartRequestBody(files map[string][]string, formDataFiel
return pr, writer.FormDataContentType(), nil
}

// createMultipartRequestBody creates a multipart request body with the provided files and form fields, supporting custom content types and headers.
// This function constructs the body of a multipart/form-data request by adding each file and form field to the multipart writer,
// setting custom content types and headers for each part as specified.

// Parameters:
// - files: A map where the key is the field name and the value is a slice of file paths to be included in the request. Each file path
// corresponds to a file that will be included in the multipart request.
// - formDataFields: A map of additional form fields to be included in the multipart request, where the key is the field name
// and the value is the field value. These are regular form fields that accompany the file uploads.
// - fileContentTypes: A map specifying the content type for each file part. The key is the field name and the value is the
// content type (e.g., "image/jpeg"). If a content type is not specified for a field, "application/octet-stream" will be used as default.
// - formDataPartHeaders: A map specifying custom headers for each part of the multipart form data. The key is the field name
// and the value is an http.Header containing the headers for that part. These headers are added to the multipart parts individually.
// - log: An instance of a logger implementing the logger.Logger interface, used to log informational messages, warnings,
// and errors encountered during the construction of the multipart request body.

// Returns:
// - *bytes.Buffer: The constructed multipart request body. This buffer contains the full multipart form data payload ready to be sent.
// - string: The content type of the multipart request body. This includes the boundary string used by the multipart writer.
// - error: An error object indicating failure during the construction of the multipart request body. This could be due to issues
// such as file reading errors or multipart writer errors.

// func createMultipartRequestBody(files map[string][]string, formDataFields map[string]string, fileContentTypes map[string]string, formDataPartHeaders map[string]http.Header, log logger.Logger) (*bytes.Buffer, string, error) {
// body := &bytes.Buffer{}
// writer := multipart.NewWriter(body)

// for fieldName, filePaths := range files {
// for _, filePath := range filePaths {
// if err := addFilePart(writer, fieldName, filePath, fileContentTypes, formDataPartHeaders, log); err != nil {
// return nil, "", err
// }
// }
// }

// for key, val := range formDataFields {
// if err := addFormField(writer, key, val, log); err != nil {
// return nil, "", err
// }
// }

// if err := writer.Close(); err != nil {
// log.Error("Failed to close writer", zap.Error(err))
// return nil, "", err
// }

// return body, writer.FormDataContentType(), nil
// }

// addFilePart adds a base64 encoded file part to the multipart writer with the provided field name and file path.
// This function opens the specified file, sets the appropriate content type and headers, and adds it to the multipart writer.

Expand Down Expand Up @@ -427,60 +370,3 @@ func logUploadProgress(file *os.File, fileSize int64, log logger.Logger) func(in
}
}
}

// logMultiPartRequestBody logs the constructed request body for debugging purposes.
func logMultiPartRequestBody(body *bytes.Buffer, log logger.Logger) {
bodyBytes := body.Bytes()
bodyStr := string(bodyBytes)

// Find the boundary string
boundaryIndex := strings.Index(bodyStr, "\r\n")
if boundaryIndex == -1 {
log.Warn("No boundary found in request body")
return
}
boundary := bodyStr[:boundaryIndex]

// Split the body by boundaries
parts := strings.Split(bodyStr, boundary)

var loggedParts []string
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "--" || part == "" {
continue // Skip the last boundary marker or empty parts
}

headersEndIndex := strings.Index(part, "\r\n\r\n")
if headersEndIndex != -1 {
headers := part[:headersEndIndex]
bodyContent := part[headersEndIndex+4:]

encoding := "none"
if strings.Contains(headers, "base64") || strings.Contains(bodyContent, "base64,") {
encoding = "base64"
}

// Log headers and indicate content is omitted
if strings.Contains(headers, "Content-Disposition: form-data; name=\"file\"") {
log.Info("Multipart section",
zap.String("content_disposition", headers),
zap.String("encoding", encoding))
loggedParts = append(loggedParts, headers+"\r\n\r\n<file content omitted>")
} else {
log.Info("Multipart section",
zap.String("content_disposition", headers),
zap.String("encoding", encoding))
// Log the entire part if it's not a file
loggedParts = append(loggedParts, headers+"\r\n\r\n"+bodyContent)
}
} else {
loggedParts = append(loggedParts, part)
}
}

// Join the logged parts back together with the boundary
loggedBody := boundary + "\r\n" + strings.Join(loggedParts, "\r\n"+boundary+"\r\n") + "\r\n" + boundary + "--"

log.Info("Request body preview", zap.String("body", loggedBody))
}
132 changes: 3 additions & 129 deletions httpclient/multipartrequest.go → httpclient/multipartrequestwithretry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package httpclient

import (
"bytes"
"context"
"encoding/base64"
"fmt"
Expand All @@ -11,7 +10,6 @@ import (
"net/textproto"
"os"
"path/filepath"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -101,7 +99,7 @@ func (c *Client) DoMultiPartRequest(method, endpoint string, files map[string][]
// Retry logic
maxRetries := 3
for attempt := 1; attempt <= maxRetries; attempt++ {
// Create a context with a longer timeout
// Create a context with timeout based on the custom timeout duration
ctx, cancel := context.WithTimeout(context.Background(), c.clientConfig.ClientOptions.Timeout.CustomTimeout.Duration())
defer cancel()

Expand Down Expand Up @@ -200,73 +198,6 @@ func createStreamingMultipartRequestBody(files map[string][]string, formDataFiel
return pr, writer.FormDataContentType(), nil
}

// createMultipartRequestBody creates a multipart request body with the provided files and form fields, supporting custom content types and headers.
// This function constructs the body of a multipart/form-data request by adding each file and form field to the multipart writer,
// setting custom content types and headers for each part as specified.

// Parameters:
// - files: A map where the key is the field name and the value is a slice of file paths to be included in the request. Each file path
// corresponds to a file that will be included in the multipart request.
// - formDataFields: A map of additional form fields to be included in the multipart request, where the key is the field name
// and the value is the field value. These are regular form fields that accompany the file uploads.
// - fileContentTypes: A map specifying the content type for each file part. The key is the field name and the value is the
// content type (e.g., "image/jpeg"). If a content type is not specified for a field, "application/octet-stream" will be used as default.
// - formDataPartHeaders: A map specifying custom headers for each part of the multipart form data. The key is the field name
// and the value is an http.Header containing the headers for that part. These headers are added to the multipart parts individually.
// - log: An instance of a logger implementing the logger.Logger interface, used to log informational messages, warnings,
// and errors encountered during the construction of the multipart request body.

// Returns:
// - *bytes.Buffer: The constructed multipart request body. This buffer contains the full multipart form data payload ready to be sent.
// - string: The content type of the multipart request body. This includes the boundary string used by the multipart writer.
// - error: An error object indicating failure during the construction of the multipart request body. This could be due to issues
// such as file reading errors or multipart writer errors.

// func createMultipartRequestBody(files map[string][]string, formDataFields map[string]string, fileContentTypes map[string]string, formDataPartHeaders map[string]http.Header, log logger.Logger) (*bytes.Buffer, string, error) {
// body := &bytes.Buffer{}
// writer := multipart.NewWriter(body)

// for fieldName, filePaths := range files {
// for _, filePath := range filePaths {
// if err := addFilePart(writer, fieldName, filePath, fileContentTypes, formDataPartHeaders, log); err != nil {
// return nil, "", err
// }
// }
// }

// for key, val := range formDataFields {
// if err := addFormField(writer, key, val, log); err != nil {
// return nil, "", err
// }
// }

// if err := writer.Close(); err != nil {
// log.Error("Failed to close writer", zap.Error(err))
// return nil, "", err
// }

// return body, writer.FormDataContentType(), nil
// }

// addFilePart adds a base64 encoded file part to the multipart writer with the provided field name and file path.
// This function opens the specified file, sets the appropriate content type and headers, and adds it to the multipart writer.

// Parameters:
// - writer: The multipart writer used to construct the multipart request body.
// - fieldName: The field name for the file part.
// - filePath: The path to the file to be included in the request.
// - fileContentTypes: A map specifying the content type for each file part. The key is the field name and the value is the
// content type (e.g., "image/jpeg").
// - formDataPartHeaders: A map specifying custom headers for each part of the multipart form data. The key is the field name
// and the value is an http.Header containing the headers for that part.
// - log: An instance of a logger implementing the logger.Logger interface, used to log informational messages, warnings,
// and errors encountered during the addition of the file part.

// Returns:
// - error: An error object indicating failure during the addition of the file part. This could be due to issues such as
// file reading errors or multipart writer errors.
//
// addFilePart adds a base64 encoded file part to the multipart writer with the provided field name and file path.
// addFilePart adds a base64 encoded file part to the multipart writer with the provided field name and file path.
func addFilePart(writer *multipart.Writer, fieldName, filePath string, fileContentTypes map[string]string, formDataPartHeaders map[string]http.Header, log logger.Logger) error {
file, err := os.Open(filePath)
Expand Down Expand Up @@ -363,7 +294,7 @@ func setFormDataPartHeader(fieldname, filename, contentType string, customHeader

// chunkFileUpload reads the file upload into chunks and writes it to the writer.
// This function reads the file in chunks and writes it to the provided writer, allowing for progress logging during the upload.
// chunk size is set to 1024 KB (1 MB) by default.
// chunk size is set to 65536 KB (64 MB) by default.

// Parameters:
// - file: The file to be uploaded.
Expand All @@ -378,7 +309,7 @@ func setFormDataPartHeader(fieldname, filename, contentType string, customHeader
//
// chunkFileUpload reads the file upload into chunks and writes it to the writer.
func chunkFileUpload(file *os.File, writer io.Writer, log logger.Logger, updateProgress func(int64), uploadState *UploadState) error {
const chunkSize = 1024 * 1024 // 1024 bytes * 1024 (1 MB)
const chunkSize = 1024 * 65536 // 1024 bytes * 65536 (64 MB)
buffer := make([]byte, chunkSize)
totalWritten := int64(0)
chunkWritten := int64(0)
Expand Down Expand Up @@ -465,60 +396,3 @@ func logUploadProgress(file *os.File, fileSize int64, log logger.Logger) func(in
}
}
}

// logMultiPartRequestBody logs the constructed request body for debugging purposes.
func logMultiPartRequestBody(body *bytes.Buffer, log logger.Logger) {
bodyBytes := body.Bytes()
bodyStr := string(bodyBytes)

// Find the boundary string
boundaryIndex := strings.Index(bodyStr, "\r\n")
if boundaryIndex == -1 {
log.Warn("No boundary found in request body")
return
}
boundary := bodyStr[:boundaryIndex]

// Split the body by boundaries
parts := strings.Split(bodyStr, boundary)

var loggedParts []string
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "--" || part == "" {
continue // Skip the last boundary marker or empty parts
}

headersEndIndex := strings.Index(part, "\r\n\r\n")
if headersEndIndex != -1 {
headers := part[:headersEndIndex]
bodyContent := part[headersEndIndex+4:]

encoding := "none"
if strings.Contains(headers, "base64") || strings.Contains(bodyContent, "base64,") {
encoding = "base64"
}

// Log headers and indicate content is omitted
if strings.Contains(headers, "Content-Disposition: form-data; name=\"file\"") {
log.Info("Multipart section",
zap.String("content_disposition", headers),
zap.String("encoding", encoding))
loggedParts = append(loggedParts, headers+"\r\n\r\n<file content omitted>")
} else {
log.Info("Multipart section",
zap.String("content_disposition", headers),
zap.String("encoding", encoding))
// Log the entire part if it's not a file
loggedParts = append(loggedParts, headers+"\r\n\r\n"+bodyContent)
}
} else {
loggedParts = append(loggedParts, part)
}
}

// Join the logged parts back together with the boundary
loggedBody := boundary + "\r\n" + strings.Join(loggedParts, "\r\n"+boundary+"\r\n") + "\r\n" + boundary + "--"

log.Info("Request body preview", zap.String("body", loggedBody))
}
Loading