From 71742bd5b6feb403ced341d6eda7ee595f80b1ca Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 12:22:20 +0200 Subject: [PATCH 01/14] feat: add composite upload option for large file writes Co-Authored-By: Claude Opus 4.6 --- packages/js-sdk/src/envd/schema.gen.ts | 53 ++++++++ packages/js-sdk/src/index.ts | 1 + .../js-sdk/src/sandbox/filesystem/index.ts | 124 +++++++++++++++++- packages/python-sdk/e2b/envd/api.py | 1 + .../sandbox_async/filesystem/filesystem.py | 124 +++++++++++++++++- .../e2b/sandbox_sync/filesystem/filesystem.py | 119 ++++++++++++++++- spec/envd/envd.yaml | 48 +++++++ 7 files changed, 463 insertions(+), 7 deletions(-) diff --git a/packages/js-sdk/src/envd/schema.gen.ts b/packages/js-sdk/src/envd/schema.gen.ts index 9c39a43d3e..cb95e50654 100644 --- a/packages/js-sdk/src/envd/schema.gen.ts +++ b/packages/js-sdk/src/envd/schema.gen.ts @@ -106,6 +106,51 @@ export interface paths { patch?: never; trace?: never; }; + "/files/compose": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Compose multiple files into a single file using zero-copy concatenation. Source files are deleted after successful composition. */ + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ComposeRequest"]; + }; + }; + responses: { + /** @description Files composed successfully */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EntryInfo"]; + }; + }; + 400: components["responses"]["InvalidPath"]; + 401: components["responses"]["InvalidUser"]; + 404: components["responses"]["FileNotFound"]; + 500: components["responses"]["InternalServerError"]; + 507: components["responses"]["NotEnoughDiskSpace"]; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/health": { parameters: { query?: never; @@ -233,6 +278,14 @@ export interface paths { export type webhooks = Record; export interface components { schemas: { + ComposeRequest: { + /** @description Destination file path for the composed file */ + destination: string; + /** @description Ordered list of source file paths to concatenate */ + source_paths: string[]; + /** @description User for setting ownership and resolving relative paths */ + username?: string; + }; EntryInfo: { /** @description Name of the file */ name: string; diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index e67092a01c..554146948b 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -27,6 +27,7 @@ export { getSignature } from './sandbox/signature' export { FileType } from './sandbox/filesystem' export type { WriteInfo, + WriteOpts, EntryInfo, Filesystem, FilesystemWriteOpts, diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 2803cc8ce3..81a6bd0eef 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -184,6 +184,20 @@ export interface FilesystemReadOpts extends FilesystemRequestOpts { gzip?: boolean } +/** + * Options for the write operation. + */ +export interface WriteOpts extends FilesystemWriteOpts { + /** + * When `true`, the file data is split into chunks and uploaded in parallel, + * then composed into the final file on the server using zero-copy concatenation. + * This is useful for uploading large files. + */ + composite?: boolean +} + +const DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 // 5 MB + export interface FilesystemListOpts extends FilesystemRequestOpts { /** * Depth of the directory to list. @@ -358,7 +372,7 @@ export class Filesystem { async write( path: string, data: string | ArrayBuffer | Blob | ReadableStream, - opts?: FilesystemWriteOpts + opts?: WriteOpts ): Promise async write( files: WriteEntry[], @@ -371,8 +385,8 @@ export class Filesystem { | ArrayBuffer | Blob | ReadableStream - | FilesystemWriteOpts, - opts?: FilesystemWriteOpts + | WriteOpts, + opts?: WriteOpts ): Promise { if (typeof pathOrFiles !== 'string' && !Array.isArray(pathOrFiles)) { throw new Error('Path or files are required') @@ -388,7 +402,7 @@ export class Filesystem { typeof pathOrFiles === 'string' ? { path: pathOrFiles, - writeOpts: opts as FilesystemWriteOpts, + writeOpts: opts as WriteOpts | undefined, writeFiles: [ { data: dataOrOpts as @@ -401,7 +415,7 @@ export class Filesystem { } : { path: undefined, - writeOpts: dataOrOpts as FilesystemWriteOpts, + writeOpts: dataOrOpts as WriteOpts | undefined, writeFiles: pathOrFiles as WriteEntry[], } @@ -418,6 +432,11 @@ export class Filesystem { const useOctetStream = compareVersions(this.envdApi.version, ENVD_OCTET_STREAM_UPLOAD) >= 0 + // Composite upload: chunk the data, upload parts in parallel, then compose + if (writeOpts?.composite && path && useOctetStream) { + return this.compositeWrite(path, writeFiles[0].data, user, writeOpts) + } + const results: WriteInfo[] = [] const useGzip = writeOpts?.gzip === true @@ -821,4 +840,99 @@ export class Filesystem { throw handleFilesystemRpcError(err) } } + + private async compositeWrite( + destination: string, + data: string | ArrayBuffer | Blob | ReadableStream, + user: string | undefined, + opts?: WriteOpts + ): Promise { + const blob = await toBlob(data) + const totalSize = blob.size + const chunkSize = DEFAULT_CHUNK_SIZE + + // If the data fits in a single chunk, no need for composite upload + if (totalSize <= chunkSize) { + const res = await this.envdApi.api.POST('/files', { + params: { + query: { + path: destination, + username: user, + }, + }, + bodySerializer: () => blob, + headers: { + 'Content-Type': 'application/octet-stream', + }, + signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + body: {}, + }) + + const err = await handleFilesystemEnvdApiError(res) + if (err) { + throw err + } + + const files = res.data as WriteInfo[] + if (!files || files.length === 0) { + throw new Error('Expected to receive information about written file') + } + + return files[0] + } + + // Split into chunks and upload in parallel + const chunkCount = Math.ceil(totalSize / chunkSize) + const uploadId = crypto.randomUUID() + const chunkPaths: string[] = [] + + for (let i = 0; i < chunkCount; i++) { + chunkPaths.push(`/tmp/.e2b-upload-${uploadId}-${i}`) + } + + await Promise.all( + chunkPaths.map(async (chunkPath, i) => { + const start = i * chunkSize + const end = Math.min(start + chunkSize, totalSize) + const chunk = blob.slice(start, end) + + const res = await this.envdApi.api.POST('/files', { + params: { + query: { + path: chunkPath, + username: user, + }, + }, + bodySerializer: () => chunk, + headers: { + 'Content-Type': 'application/octet-stream', + }, + signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + body: {}, + }) + + const err = await handleFilesystemEnvdApiError(res) + if (err) { + throw err + } + }) + ) + + // Compose chunks into the final file + const composeRes = await this.envdApi.api.POST('/files/compose', { + body: { + source_paths: chunkPaths, + destination, + username: user, + }, + signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + }) + + const composeErr = await handleFilesystemEnvdApiError(composeRes) + if (composeErr) { + throw composeErr + } + + return composeRes.data as WriteInfo + } } diff --git a/packages/python-sdk/e2b/envd/api.py b/packages/python-sdk/e2b/envd/api.py index 7e2a59c131..6c2f6fd5ec 100644 --- a/packages/python-sdk/e2b/envd/api.py +++ b/packages/python-sdk/e2b/envd/api.py @@ -14,6 +14,7 @@ ENVD_API_FILES_ROUTE = "/files" +ENVD_API_FILES_COMPOSE_ROUTE = "/files/compose" ENVD_API_HEALTH_ROUTE = "/health" _DEFAULT_API_ERROR_MAP: dict[int, Callable[[str], Exception]] = { diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 8f70f58a1c..2fbc19478a 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -1,4 +1,6 @@ import asyncio +import uuid + from io import IOBase, TextIOBase from typing import IO, AsyncIterator, List, Literal, Optional, Union, overload @@ -15,7 +17,11 @@ Username, default_username, ) -from e2b.envd.api import ENVD_API_FILES_ROUTE, ahandle_envd_api_exception +from e2b.envd.api import ( + ENVD_API_FILES_COMPOSE_ROUTE, + ENVD_API_FILES_ROUTE, + ahandle_envd_api_exception, +) from e2b.envd.filesystem import filesystem_connect, filesystem_pb2 from e2b.envd.rpc import authentication_header, handle_rpc_exception from e2b.envd.versions import ( @@ -58,6 +64,9 @@ async def _ahandle_filesystem_envd_api_exception(r): return await ahandle_envd_api_exception(r, _FILESYSTEM_HTTP_ERROR_MAP) +_DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 # 5 MB + + class Filesystem: """ Module for interacting with the filesystem in the sandbox. @@ -197,6 +206,7 @@ async def write( user: Optional[Username] = None, request_timeout: Optional[float] = None, gzip: bool = False, + composite: bool = False, ) -> WriteInfo: """ Write content to a file on the path. @@ -209,9 +219,15 @@ async def write( :param user: Run the operation as this user :param request_timeout: Timeout for the request in **seconds** :param gzip: Use gzip compression for the request + :param composite: When `True`, the file data is split into chunks and uploaded + in parallel, then composed into the final file on the server using + zero-copy concatenation. This is useful for uploading large files. :return: Information about the written file """ + if composite and self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: + return await self._composite_write(path, data, user, request_timeout) + result = await self.write_files( [WriteEntry(path=path, data=data)], user, @@ -345,6 +361,112 @@ async def _upload_file(file): return results + async def _composite_write( + self, + destination: str, + data: Union[str, bytes, IO], + user: Optional[Username] = None, + request_timeout: Optional[float] = None, + ) -> WriteInfo: + username = user + if username is None and self._envd_version < ENVD_DEFAULT_USER: + username = default_username + + if isinstance(data, str): + content = data.encode("utf-8") + elif isinstance(data, bytes): + content = data + elif isinstance(data, TextIOBase): + content = data.read().encode("utf-8") + elif isinstance(data, IOBase): + content = data.read() + else: + raise InvalidArgumentException( + f"Unsupported data type for file {destination}" + ) + + total_size = len(content) + chunk_size = _DEFAULT_CHUNK_SIZE + + # If the data fits in a single chunk, upload directly + if total_size <= chunk_size: + params = {"path": destination} + if username: + params["username"] = username + + r = await self._envd_api.post( + ENVD_API_FILES_ROUTE, + content=content, + headers={"Content-Type": "application/octet-stream"}, + params=params, + timeout=self._connection_config.get_request_timeout(request_timeout), + ) + + err = await _ahandle_filesystem_envd_api_exception(r) + if err: + raise err + + write_result = r.json() + if not isinstance(write_result, list) or len(write_result) == 0: + raise SandboxException( + "Expected to receive information about written file" + ) + return WriteInfo(**write_result[0]) + + # Split into chunks and upload in parallel + upload_id = str(uuid.uuid4()) + chunk_count = (total_size + chunk_size - 1) // chunk_size + chunk_paths: List[str] = [] + + async def _upload_chunk(i: int) -> None: + chunk_path = f"/tmp/.e2b-upload-{upload_id}-{i}" + chunk_paths.append(chunk_path) + + start = i * chunk_size + end = min(start + chunk_size, total_size) + chunk_data = content[start:end] + + params = {"path": chunk_path} + if username: + params["username"] = username + + r = await self._envd_api.post( + ENVD_API_FILES_ROUTE, + content=chunk_data, + headers={"Content-Type": "application/octet-stream"}, + params=params, + timeout=self._connection_config.get_request_timeout(request_timeout), + ) + + err = await _ahandle_filesystem_envd_api_exception(r) + if err: + raise err + + await asyncio.gather(*[_upload_chunk(i) for i in range(chunk_count)]) + + # Sort chunk_paths by index to ensure correct order + chunk_paths.sort(key=lambda p: int(p.rsplit("-", 1)[1])) + + # Compose chunks into the final file + body = { + "source_paths": chunk_paths, + "destination": destination, + } + if username: + body["username"] = username + + r = await self._envd_api.post( + ENVD_API_FILES_COMPOSE_ROUTE, + json=body, + timeout=self._connection_config.get_request_timeout(request_timeout), + ) + + err = await _ahandle_filesystem_envd_api_exception(r) + if err: + raise err + + return WriteInfo(**r.json()) + async def list( self, path: str, diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 9b26a22d0f..c50a0b1395 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -1,3 +1,5 @@ +import uuid + from io import IOBase, TextIOBase from typing import IO, Iterator, List, Literal, Optional, Union, overload @@ -15,7 +17,11 @@ ) from e2b_connect.client import Code -from e2b.envd.api import ENVD_API_FILES_ROUTE, handle_envd_api_exception +from e2b.envd.api import ( + ENVD_API_FILES_COMPOSE_ROUTE, + ENVD_API_FILES_ROUTE, + handle_envd_api_exception, +) from e2b.envd.filesystem import filesystem_connect, filesystem_pb2 from e2b.envd.rpc import authentication_header, handle_rpc_exception from e2b.envd.versions import ( @@ -56,6 +62,9 @@ def _handle_filesystem_envd_api_exception(r): return handle_envd_api_exception(r, _FILESYSTEM_HTTP_ERROR_MAP) +_DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 # 5 MB + + class Filesystem: """ Module for interacting with the filesystem in the sandbox. @@ -195,6 +204,7 @@ def write( user: Optional[Username] = None, request_timeout: Optional[float] = None, gzip: bool = False, + composite: bool = False, ) -> WriteInfo: """ Write content to a file on the path. @@ -207,9 +217,15 @@ def write( :param user: Run the operation as this user :param request_timeout: Timeout for the request in **seconds** :param gzip: Use gzip compression for the request + :param composite: When `True`, the file data is split into chunks and uploaded + in parallel, then composed into the final file on the server using + zero-copy concatenation. This is useful for uploading large files. :return: Information about the written file """ + if composite and self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: + return self._composite_write(path, data, user, request_timeout) + result = self.write_files( [WriteEntry(path=path, data=data)], user=user, @@ -222,6 +238,107 @@ def write( return result[0] + def _composite_write( + self, + destination: str, + data: Union[str, bytes, IO], + user: Optional[Username] = None, + request_timeout: Optional[float] = None, + ) -> WriteInfo: + username = user + if username is None and self._envd_version < ENVD_DEFAULT_USER: + username = default_username + + if isinstance(data, str): + content = data.encode("utf-8") + elif isinstance(data, bytes): + content = data + elif isinstance(data, TextIOBase): + content = data.read().encode("utf-8") + elif isinstance(data, IOBase): + content = data.read() + else: + raise InvalidArgumentException( + f"Unsupported data type for file {destination}" + ) + + total_size = len(content) + chunk_size = _DEFAULT_CHUNK_SIZE + + # If the data fits in a single chunk, upload directly + if total_size <= chunk_size: + params = {"path": destination} + if username: + params["username"] = username + + r = self._envd_api.post( + ENVD_API_FILES_ROUTE, + content=content, + headers={"Content-Type": "application/octet-stream"}, + params=params, + timeout=self._connection_config.get_request_timeout(request_timeout), + ) + + err = _handle_filesystem_envd_api_exception(r) + if err: + raise err + + write_result = r.json() + if not isinstance(write_result, list) or len(write_result) == 0: + raise SandboxException( + "Expected to receive information about written file" + ) + return WriteInfo(**write_result[0]) + + # Split into chunks and upload + upload_id = str(uuid.uuid4()) + chunk_count = (total_size + chunk_size - 1) // chunk_size + chunk_paths: List[str] = [] + + for i in range(chunk_count): + chunk_path = f"/tmp/.e2b-upload-{upload_id}-{i}" + chunk_paths.append(chunk_path) + + start = i * chunk_size + end = min(start + chunk_size, total_size) + chunk_data = content[start:end] + + params = {"path": chunk_path} + if username: + params["username"] = username + + r = self._envd_api.post( + ENVD_API_FILES_ROUTE, + content=chunk_data, + headers={"Content-Type": "application/octet-stream"}, + params=params, + timeout=self._connection_config.get_request_timeout(request_timeout), + ) + + err = _handle_filesystem_envd_api_exception(r) + if err: + raise err + + # Compose chunks into the final file + body = { + "source_paths": chunk_paths, + "destination": destination, + } + if username: + body["username"] = username + + r = self._envd_api.post( + ENVD_API_FILES_COMPOSE_ROUTE, + json=body, + timeout=self._connection_config.get_request_timeout(request_timeout), + ) + + err = _handle_filesystem_envd_api_exception(r) + if err: + raise err + + return WriteInfo(**r.json()) + def write_files( self, files: List[WriteEntry], diff --git a/spec/envd/envd.yaml b/spec/envd/envd.yaml index 9649c24a11..0aed1cf9d2 100644 --- a/spec/envd/envd.yaml +++ b/spec/envd/envd.yaml @@ -125,6 +125,37 @@ paths: '507': $ref: '#/components/responses/NotEnoughDiskSpace' + /files/compose: + post: + summary: Compose multiple files into a single file using zero-copy concatenation. Source files are deleted after successful composition. + tags: [files] + security: + - AccessTokenAuth: [] + - {} + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ComposeRequest' + responses: + '200': + description: Files composed successfully + content: + application/json: + schema: + $ref: '#/components/schemas/EntryInfo' + '400': + $ref: '#/components/responses/InvalidPath' + '401': + $ref: '#/components/responses/InvalidUser' + '404': + $ref: '#/components/responses/FileNotFound' + '500': + $ref: '#/components/responses/InternalServerError' + '507': + $ref: '#/components/responses/NotEnoughDiskSpace' + components: securitySchemes: AccessTokenAuth: @@ -252,6 +283,23 @@ components: description: Type of the file enum: - file + ComposeRequest: + type: object + required: + - source_paths + - destination + properties: + source_paths: + type: array + items: + type: string + description: Ordered list of source file paths to concatenate + destination: + type: string + description: Destination file path for the composed file + username: + type: string + description: User for setting ownership and resolving relative paths EnvVars: type: object description: Environment variables to set From c733a788011cb50e8c4389aac158392c7fea3d15 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 12:40:11 +0200 Subject: [PATCH 02/14] fix: address PR review comments - Build chunk_paths deterministically before asyncio.gather in async _composite_write - Use Username type instead of bare string in JS compositeWrite Co-Authored-By: Claude Opus 4.6 --- packages/js-sdk/src/sandbox/filesystem/index.ts | 2 +- .../e2b/sandbox_async/filesystem/filesystem.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 81a6bd0eef..cf1619f04f 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -844,7 +844,7 @@ export class Filesystem { private async compositeWrite( destination: string, data: string | ArrayBuffer | Blob | ReadableStream, - user: string | undefined, + user: Username | undefined, opts?: WriteOpts ): Promise { const blob = await toBlob(data) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 2fbc19478a..47d02351ff 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -416,17 +416,14 @@ async def _composite_write( # Split into chunks and upload in parallel upload_id = str(uuid.uuid4()) chunk_count = (total_size + chunk_size - 1) // chunk_size - chunk_paths: List[str] = [] + chunk_paths = [f"/tmp/.e2b-upload-{upload_id}-{i}" for i in range(chunk_count)] async def _upload_chunk(i: int) -> None: - chunk_path = f"/tmp/.e2b-upload-{upload_id}-{i}" - chunk_paths.append(chunk_path) - start = i * chunk_size end = min(start + chunk_size, total_size) chunk_data = content[start:end] - params = {"path": chunk_path} + params = {"path": chunk_paths[i]} if username: params["username"] = username @@ -444,9 +441,6 @@ async def _upload_chunk(i: int) -> None: await asyncio.gather(*[_upload_chunk(i) for i in range(chunk_count)]) - # Sort chunk_paths by index to ensure correct order - chunk_paths.sort(key=lambda p: int(p.rsplit("-", 1)[1])) - # Compose chunks into the final file body = { "source_paths": chunk_paths, From fc52b20958e437dbb2da54ad08fe0622b1fccf30 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 12:52:33 +0200 Subject: [PATCH 03/14] feat: support gzip in composite upload, increase chunk size to 64MB Co-Authored-By: Claude Opus 4.6 --- .../js-sdk/src/sandbox/filesystem/index.ts | 32 +++++++++-------- .../sandbox_async/filesystem/filesystem.py | 35 +++++++++---------- .../e2b/sandbox_sync/filesystem/filesystem.py | 35 +++++++++---------- 3 files changed, 49 insertions(+), 53 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index cf1619f04f..bc9897bd42 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -196,7 +196,7 @@ export interface WriteOpts extends FilesystemWriteOpts { composite?: boolean } -const DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 // 5 MB +const DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024 // 64 MB export interface FilesystemListOpts extends FilesystemRequestOpts { /** @@ -380,12 +380,7 @@ export class Filesystem { ): Promise async write( pathOrFiles: string | WriteEntry[], - dataOrOpts?: - | string - | ArrayBuffer - | Blob - | ReadableStream - | WriteOpts, + dataOrOpts?: string | ArrayBuffer | Blob | ReadableStream | WriteOpts, opts?: WriteOpts ): Promise { if (typeof pathOrFiles !== 'string' && !Array.isArray(pathOrFiles)) { @@ -850,9 +845,19 @@ export class Filesystem { const blob = await toBlob(data) const totalSize = blob.size const chunkSize = DEFAULT_CHUNK_SIZE + const useGzip = opts?.gzip === true + + const headers: Record = { + 'Content-Type': 'application/octet-stream', + } + if (useGzip) { + headers['Content-Encoding'] = 'gzip' + } // If the data fits in a single chunk, no need for composite upload if (totalSize <= chunkSize) { + const body = await toUploadBody(data, useGzip) + const res = await this.envdApi.api.POST('/files', { params: { query: { @@ -860,10 +865,8 @@ export class Filesystem { username: user, }, }, - bodySerializer: () => blob, - headers: { - 'Content-Type': 'application/octet-stream', - }, + bodySerializer: () => body, + headers, signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), body: {}, }) @@ -895,6 +898,7 @@ export class Filesystem { const start = i * chunkSize const end = Math.min(start + chunkSize, totalSize) const chunk = blob.slice(start, end) + const body = await toUploadBody(chunk, useGzip) const res = await this.envdApi.api.POST('/files', { params: { @@ -903,10 +907,8 @@ export class Filesystem { username: user, }, }, - bodySerializer: () => chunk, - headers: { - 'Content-Type': 'application/octet-stream', - }, + bodySerializer: () => body, + headers, signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), body: {}, }) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 47d02351ff..a95f472f26 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -64,7 +64,7 @@ async def _ahandle_filesystem_envd_api_exception(r): return await ahandle_envd_api_exception(r, _FILESYSTEM_HTTP_ERROR_MAP) -_DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 # 5 MB +_DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024 # 64 MB class Filesystem: @@ -226,7 +226,7 @@ async def write( :return: Information about the written file """ if composite and self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: - return await self._composite_write(path, data, user, request_timeout) + return await self._composite_write(path, data, user, request_timeout, gzip) result = await self.write_files( [WriteEntry(path=path, data=data)], @@ -367,37 +367,32 @@ async def _composite_write( data: Union[str, bytes, IO], user: Optional[Username] = None, request_timeout: Optional[float] = None, + use_gzip: bool = False, ) -> WriteInfo: username = user if username is None and self._envd_version < ENVD_DEFAULT_USER: username = default_username - if isinstance(data, str): - content = data.encode("utf-8") - elif isinstance(data, bytes): - content = data - elif isinstance(data, TextIOBase): - content = data.read().encode("utf-8") - elif isinstance(data, IOBase): - content = data.read() - else: - raise InvalidArgumentException( - f"Unsupported data type for file {destination}" - ) - + content = to_upload_body(data, False) total_size = len(content) chunk_size = _DEFAULT_CHUNK_SIZE + headers = {"Content-Type": "application/octet-stream"} + if use_gzip: + headers["Content-Encoding"] = "gzip" + # If the data fits in a single chunk, upload directly if total_size <= chunk_size: params = {"path": destination} if username: params["username"] = username + upload_content = to_upload_body(data, use_gzip) + r = await self._envd_api.post( ENVD_API_FILES_ROUTE, - content=content, - headers={"Content-Type": "application/octet-stream"}, + content=upload_content, + headers=headers, params=params, timeout=self._connection_config.get_request_timeout(request_timeout), ) @@ -427,10 +422,12 @@ async def _upload_chunk(i: int) -> None: if username: params["username"] = username + upload_content = to_upload_body(chunk_data, use_gzip) + r = await self._envd_api.post( ENVD_API_FILES_ROUTE, - content=chunk_data, - headers={"Content-Type": "application/octet-stream"}, + content=upload_content, + headers=headers, params=params, timeout=self._connection_config.get_request_timeout(request_timeout), ) diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index c50a0b1395..ea6a655daa 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -62,7 +62,7 @@ def _handle_filesystem_envd_api_exception(r): return handle_envd_api_exception(r, _FILESYSTEM_HTTP_ERROR_MAP) -_DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 # 5 MB +_DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024 # 64 MB class Filesystem: @@ -224,7 +224,7 @@ def write( :return: Information about the written file """ if composite and self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: - return self._composite_write(path, data, user, request_timeout) + return self._composite_write(path, data, user, request_timeout, gzip) result = self.write_files( [WriteEntry(path=path, data=data)], @@ -244,37 +244,32 @@ def _composite_write( data: Union[str, bytes, IO], user: Optional[Username] = None, request_timeout: Optional[float] = None, + use_gzip: bool = False, ) -> WriteInfo: username = user if username is None and self._envd_version < ENVD_DEFAULT_USER: username = default_username - if isinstance(data, str): - content = data.encode("utf-8") - elif isinstance(data, bytes): - content = data - elif isinstance(data, TextIOBase): - content = data.read().encode("utf-8") - elif isinstance(data, IOBase): - content = data.read() - else: - raise InvalidArgumentException( - f"Unsupported data type for file {destination}" - ) - + content = to_upload_body(data, False) total_size = len(content) chunk_size = _DEFAULT_CHUNK_SIZE + headers = {"Content-Type": "application/octet-stream"} + if use_gzip: + headers["Content-Encoding"] = "gzip" + # If the data fits in a single chunk, upload directly if total_size <= chunk_size: params = {"path": destination} if username: params["username"] = username + upload_content = to_upload_body(data, use_gzip) + r = self._envd_api.post( ENVD_API_FILES_ROUTE, - content=content, - headers={"Content-Type": "application/octet-stream"}, + content=upload_content, + headers=headers, params=params, timeout=self._connection_config.get_request_timeout(request_timeout), ) @@ -307,10 +302,12 @@ def _composite_write( if username: params["username"] = username + upload_content = to_upload_body(chunk_data, use_gzip) + r = self._envd_api.post( ENVD_API_FILES_ROUTE, - content=chunk_data, - headers={"Content-Type": "application/octet-stream"}, + content=upload_content, + headers=headers, params=params, timeout=self._connection_config.get_request_timeout(request_timeout), ) From 174b9b89cefa00eb855c3a745bc85feba1cfb0ae Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:01:59 +0200 Subject: [PATCH 04/14] fix: avoid double-consuming streams/IO in composite single-chunk path Use already-materialized blob/content instead of re-reading original data. Co-Authored-By: Claude Opus 4.6 --- packages/js-sdk/src/sandbox/filesystem/index.ts | 2 +- packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py | 2 +- packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index bc9897bd42..1e45d0f908 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -856,7 +856,7 @@ export class Filesystem { // If the data fits in a single chunk, no need for composite upload if (totalSize <= chunkSize) { - const body = await toUploadBody(data, useGzip) + const body = await toUploadBody(blob, useGzip) const res = await this.envdApi.api.POST('/files', { params: { diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index a95f472f26..d961e26e39 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -387,7 +387,7 @@ async def _composite_write( if username: params["username"] = username - upload_content = to_upload_body(data, use_gzip) + upload_content = to_upload_body(content, use_gzip) r = await self._envd_api.post( ENVD_API_FILES_ROUTE, diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index ea6a655daa..417ab3af58 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -264,7 +264,7 @@ def _composite_write( if username: params["username"] = username - upload_content = to_upload_body(data, use_gzip) + upload_content = to_upload_body(content, use_gzip) r = self._envd_api.post( ENVD_API_FILES_ROUTE, From a1447916a988ee6b292c3dd8fd1b066f6a6d7336 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:07:09 +0200 Subject: [PATCH 05/14] refactor: simplify composite upload for single-chunk files When data fits in a single chunk, fall through to the normal write path instead of duplicating the upload logic inside compositeWrite. Co-Authored-By: Claude Opus 4.6 --- .../js-sdk/src/sandbox/filesystem/index.ts | 40 ++++--------------- .../sandbox_async/filesystem/filesystem.py | 36 +++-------------- .../e2b/sandbox_sync/filesystem/filesystem.py | 34 ++-------------- 3 files changed, 17 insertions(+), 93 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 1e45d0f908..f04cf927ec 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -429,7 +429,12 @@ export class Filesystem { // Composite upload: chunk the data, upload parts in parallel, then compose if (writeOpts?.composite && path && useOctetStream) { - return this.compositeWrite(path, writeFiles[0].data, user, writeOpts) + const blob = await toBlob(writeFiles[0].data) + if (blob.size > DEFAULT_CHUNK_SIZE) { + return this.compositeWrite(path, blob, user, writeOpts) + } + // Data fits in a single chunk — fall through to normal write path + writeFiles[0] = { data: blob } } const results: WriteInfo[] = [] @@ -838,11 +843,10 @@ export class Filesystem { private async compositeWrite( destination: string, - data: string | ArrayBuffer | Blob | ReadableStream, + blob: Blob, user: Username | undefined, opts?: WriteOpts ): Promise { - const blob = await toBlob(data) const totalSize = blob.size const chunkSize = DEFAULT_CHUNK_SIZE const useGzip = opts?.gzip === true @@ -854,36 +858,6 @@ export class Filesystem { headers['Content-Encoding'] = 'gzip' } - // If the data fits in a single chunk, no need for composite upload - if (totalSize <= chunkSize) { - const body = await toUploadBody(blob, useGzip) - - const res = await this.envdApi.api.POST('/files', { - params: { - query: { - path: destination, - username: user, - }, - }, - bodySerializer: () => body, - headers, - signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), - body: {}, - }) - - const err = await handleFilesystemEnvdApiError(res) - if (err) { - throw err - } - - const files = res.data as WriteInfo[] - if (!files || files.length === 0) { - throw new Error('Expected to receive information about written file') - } - - return files[0] - } - // Split into chunks and upload in parallel const chunkCount = Math.ceil(totalSize / chunkSize) const uploadId = crypto.randomUUID() diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index d961e26e39..e5946e47fd 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -226,7 +226,11 @@ async def write( :return: Information about the written file """ if composite and self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: - return await self._composite_write(path, data, user, request_timeout, gzip) + content = to_upload_body(data, False) + if len(content) > _DEFAULT_CHUNK_SIZE: + return await self._composite_write( + path, content, user, request_timeout, gzip + ) result = await self.write_files( [WriteEntry(path=path, data=data)], @@ -364,7 +368,7 @@ async def _upload_file(file): async def _composite_write( self, destination: str, - data: Union[str, bytes, IO], + content: bytes, user: Optional[Username] = None, request_timeout: Optional[float] = None, use_gzip: bool = False, @@ -373,7 +377,6 @@ async def _composite_write( if username is None and self._envd_version < ENVD_DEFAULT_USER: username = default_username - content = to_upload_body(data, False) total_size = len(content) chunk_size = _DEFAULT_CHUNK_SIZE @@ -381,33 +384,6 @@ async def _composite_write( if use_gzip: headers["Content-Encoding"] = "gzip" - # If the data fits in a single chunk, upload directly - if total_size <= chunk_size: - params = {"path": destination} - if username: - params["username"] = username - - upload_content = to_upload_body(content, use_gzip) - - r = await self._envd_api.post( - ENVD_API_FILES_ROUTE, - content=upload_content, - headers=headers, - params=params, - timeout=self._connection_config.get_request_timeout(request_timeout), - ) - - err = await _ahandle_filesystem_envd_api_exception(r) - if err: - raise err - - write_result = r.json() - if not isinstance(write_result, list) or len(write_result) == 0: - raise SandboxException( - "Expected to receive information about written file" - ) - return WriteInfo(**write_result[0]) - # Split into chunks and upload in parallel upload_id = str(uuid.uuid4()) chunk_count = (total_size + chunk_size - 1) // chunk_size diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 417ab3af58..2dcbce29f7 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -224,7 +224,9 @@ def write( :return: Information about the written file """ if composite and self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: - return self._composite_write(path, data, user, request_timeout, gzip) + content = to_upload_body(data, False) + if len(content) > _DEFAULT_CHUNK_SIZE: + return self._composite_write(path, content, user, request_timeout, gzip) result = self.write_files( [WriteEntry(path=path, data=data)], @@ -241,7 +243,7 @@ def write( def _composite_write( self, destination: str, - data: Union[str, bytes, IO], + content: bytes, user: Optional[Username] = None, request_timeout: Optional[float] = None, use_gzip: bool = False, @@ -250,7 +252,6 @@ def _composite_write( if username is None and self._envd_version < ENVD_DEFAULT_USER: username = default_username - content = to_upload_body(data, False) total_size = len(content) chunk_size = _DEFAULT_CHUNK_SIZE @@ -258,33 +259,6 @@ def _composite_write( if use_gzip: headers["Content-Encoding"] = "gzip" - # If the data fits in a single chunk, upload directly - if total_size <= chunk_size: - params = {"path": destination} - if username: - params["username"] = username - - upload_content = to_upload_body(content, use_gzip) - - r = self._envd_api.post( - ENVD_API_FILES_ROUTE, - content=upload_content, - headers=headers, - params=params, - timeout=self._connection_config.get_request_timeout(request_timeout), - ) - - err = _handle_filesystem_envd_api_exception(r) - if err: - raise err - - write_result = r.json() - if not isinstance(write_result, list) or len(write_result) == 0: - raise SandboxException( - "Expected to receive information about written file" - ) - return WriteInfo(**write_result[0]) - # Split into chunks and upload upload_id = str(uuid.uuid4()) chunk_count = (total_size + chunk_size - 1) // chunk_size From f59ec5d8c54a7e763ef93f74eca8835ec4b71bcb Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:12:56 +0200 Subject: [PATCH 06/14] refactor: auto-detect large files for composite upload Remove the `composite` option from `write()`. Files over 64MB are now automatically chunked and uploaded via the composite path when the envd version supports it. Co-Authored-By: Claude Opus 4.6 --- packages/js-sdk/src/index.ts | 1 - .../js-sdk/src/sandbox/filesystem/index.ts | 33 ++++++++----------- .../sandbox_async/filesystem/filesystem.py | 6 +--- .../e2b/sandbox_sync/filesystem/filesystem.py | 6 +--- 4 files changed, 15 insertions(+), 31 deletions(-) diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index 554146948b..e67092a01c 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -27,7 +27,6 @@ export { getSignature } from './sandbox/signature' export { FileType } from './sandbox/filesystem' export type { WriteInfo, - WriteOpts, EntryInfo, Filesystem, FilesystemWriteOpts, diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index f04cf927ec..92958fe11c 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -184,18 +184,6 @@ export interface FilesystemReadOpts extends FilesystemRequestOpts { gzip?: boolean } -/** - * Options for the write operation. - */ -export interface WriteOpts extends FilesystemWriteOpts { - /** - * When `true`, the file data is split into chunks and uploaded in parallel, - * then composed into the final file on the server using zero-copy concatenation. - * This is useful for uploading large files. - */ - composite?: boolean -} - const DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024 // 64 MB export interface FilesystemListOpts extends FilesystemRequestOpts { @@ -372,7 +360,7 @@ export class Filesystem { async write( path: string, data: string | ArrayBuffer | Blob | ReadableStream, - opts?: WriteOpts + opts?: FilesystemWriteOpts ): Promise async write( files: WriteEntry[], @@ -380,8 +368,13 @@ export class Filesystem { ): Promise async write( pathOrFiles: string | WriteEntry[], - dataOrOpts?: string | ArrayBuffer | Blob | ReadableStream | WriteOpts, - opts?: WriteOpts + dataOrOpts?: + | string + | ArrayBuffer + | Blob + | ReadableStream + | FilesystemWriteOpts, + opts?: FilesystemWriteOpts ): Promise { if (typeof pathOrFiles !== 'string' && !Array.isArray(pathOrFiles)) { throw new Error('Path or files are required') @@ -397,7 +390,7 @@ export class Filesystem { typeof pathOrFiles === 'string' ? { path: pathOrFiles, - writeOpts: opts as WriteOpts | undefined, + writeOpts: opts as FilesystemWriteOpts | undefined, writeFiles: [ { data: dataOrOpts as @@ -410,7 +403,7 @@ export class Filesystem { } : { path: undefined, - writeOpts: dataOrOpts as WriteOpts | undefined, + writeOpts: dataOrOpts as FilesystemWriteOpts | undefined, writeFiles: pathOrFiles as WriteEntry[], } @@ -427,8 +420,8 @@ export class Filesystem { const useOctetStream = compareVersions(this.envdApi.version, ENVD_OCTET_STREAM_UPLOAD) >= 0 - // Composite upload: chunk the data, upload parts in parallel, then compose - if (writeOpts?.composite && path && useOctetStream) { + // Composite upload: automatically chunk large files, upload parts in parallel, then compose + if (path && useOctetStream) { const blob = await toBlob(writeFiles[0].data) if (blob.size > DEFAULT_CHUNK_SIZE) { return this.compositeWrite(path, blob, user, writeOpts) @@ -845,7 +838,7 @@ export class Filesystem { destination: string, blob: Blob, user: Username | undefined, - opts?: WriteOpts + opts?: FilesystemWriteOpts ): Promise { const totalSize = blob.size const chunkSize = DEFAULT_CHUNK_SIZE diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index e5946e47fd..fdb7a29ecf 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -206,7 +206,6 @@ async def write( user: Optional[Username] = None, request_timeout: Optional[float] = None, gzip: bool = False, - composite: bool = False, ) -> WriteInfo: """ Write content to a file on the path. @@ -219,13 +218,10 @@ async def write( :param user: Run the operation as this user :param request_timeout: Timeout for the request in **seconds** :param gzip: Use gzip compression for the request - :param composite: When `True`, the file data is split into chunks and uploaded - in parallel, then composed into the final file on the server using - zero-copy concatenation. This is useful for uploading large files. :return: Information about the written file """ - if composite and self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: + if self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: content = to_upload_body(data, False) if len(content) > _DEFAULT_CHUNK_SIZE: return await self._composite_write( diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 2dcbce29f7..c1d5b876c2 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -204,7 +204,6 @@ def write( user: Optional[Username] = None, request_timeout: Optional[float] = None, gzip: bool = False, - composite: bool = False, ) -> WriteInfo: """ Write content to a file on the path. @@ -217,13 +216,10 @@ def write( :param user: Run the operation as this user :param request_timeout: Timeout for the request in **seconds** :param gzip: Use gzip compression for the request - :param composite: When `True`, the file data is split into chunks and uploaded - in parallel, then composed into the final file on the server using - zero-copy concatenation. This is useful for uploading large files. :return: Information about the written file """ - if composite and self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: + if self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: content = to_upload_body(data, False) if len(content) > _DEFAULT_CHUNK_SIZE: return self._composite_write(path, content, user, request_timeout, gzip) From afd10e676745ab45d6b53056338027c7af7298cf Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:14:33 +0200 Subject: [PATCH 07/14] chore: reset index.ts to main Co-Authored-By: Claude Opus 4.6 --- packages/js-sdk/src/index.ts | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index e67092a01c..de04c3c4ab 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -5,33 +5,24 @@ export { ConnectionConfig } from './connectionConfig' export type { ConnectionOpts, Username } from './connectionConfig' export { AuthenticationError, - FileNotFoundError, GitAuthError, GitUpstreamError, InvalidArgumentError, NotEnoughSpaceError, NotFoundError, SandboxError, - SandboxNotFoundError, TemplateError, TimeoutError, RateLimitError, BuildError, FileUploadError, - VolumeError, } from './errors' export type { Logger } from './logs' export { getSignature } from './sandbox/signature' export { FileType } from './sandbox/filesystem' -export type { - WriteInfo, - EntryInfo, - Filesystem, - FilesystemWriteOpts, - FilesystemReadOpts, -} from './sandbox/filesystem' +export type { WriteInfo, EntryInfo, Filesystem } from './sandbox/filesystem' export { FilesystemEventType } from './sandbox/filesystem/watchHandle' export type { FilesystemEvent, @@ -58,8 +49,6 @@ export type { SandboxListOpts, SandboxPaginator, SandboxNetworkOpts, - SandboxLifecycle, - SandboxInfoLifecycle, SnapshotInfo, SnapshotListOpts, SnapshotPaginator, @@ -97,17 +86,6 @@ export type { GitStatus, } from './sandbox/git' -export { Volume, VolumeFileType } from './volume' -export type { - VolumeInfo, - VolumeAndToken, - VolumeEntryStat, - VolumeMetadataOptions, - VolumeWriteOptions, - VolumeApiOpts, - VolumeConnectionConfig, -} from './volume' - export { Sandbox } import { Sandbox } from './sandbox' From 12ebc8806cb84e320eaed6a7768b9d36e3e63054 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:17:43 +0200 Subject: [PATCH 08/14] fix: restore index.ts exports to match main branch Co-Authored-By: Claude Opus 4.6 --- packages/js-sdk/src/index.ts | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index de04c3c4ab..e67092a01c 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -5,24 +5,33 @@ export { ConnectionConfig } from './connectionConfig' export type { ConnectionOpts, Username } from './connectionConfig' export { AuthenticationError, + FileNotFoundError, GitAuthError, GitUpstreamError, InvalidArgumentError, NotEnoughSpaceError, NotFoundError, SandboxError, + SandboxNotFoundError, TemplateError, TimeoutError, RateLimitError, BuildError, FileUploadError, + VolumeError, } from './errors' export type { Logger } from './logs' export { getSignature } from './sandbox/signature' export { FileType } from './sandbox/filesystem' -export type { WriteInfo, EntryInfo, Filesystem } from './sandbox/filesystem' +export type { + WriteInfo, + EntryInfo, + Filesystem, + FilesystemWriteOpts, + FilesystemReadOpts, +} from './sandbox/filesystem' export { FilesystemEventType } from './sandbox/filesystem/watchHandle' export type { FilesystemEvent, @@ -49,6 +58,8 @@ export type { SandboxListOpts, SandboxPaginator, SandboxNetworkOpts, + SandboxLifecycle, + SandboxInfoLifecycle, SnapshotInfo, SnapshotListOpts, SnapshotPaginator, @@ -86,6 +97,17 @@ export type { GitStatus, } from './sandbox/git' +export { Volume, VolumeFileType } from './volume' +export type { + VolumeInfo, + VolumeAndToken, + VolumeEntryStat, + VolumeMetadataOptions, + VolumeWriteOptions, + VolumeApiOpts, + VolumeConnectionConfig, +} from './volume' + export { Sandbox } import { Sandbox } from './sandbox' From 9de5bc1fe8665bd9b77bd5a06fc985fcbd910bab Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:29:50 +0200 Subject: [PATCH 09/14] fix: avoid consuming IO streams twice when data fits in single chunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When data is an IO object and ≤64MB, to_upload_body() consumes the stream. Pass the materialized bytes to write_files() instead of the exhausted IO object. Co-Authored-By: Claude Opus 4.6 --- packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py | 2 ++ packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index fdb7a29ecf..5a9a4ccef1 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -227,6 +227,8 @@ async def write( return await self._composite_write( path, content, user, request_timeout, gzip ) + # Use materialized bytes to avoid consuming IO streams twice + data = content result = await self.write_files( [WriteEntry(path=path, data=data)], diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index c1d5b876c2..a95445e9fc 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -223,6 +223,8 @@ def write( content = to_upload_body(data, False) if len(content) > _DEFAULT_CHUNK_SIZE: return self._composite_write(path, content, user, request_timeout, gzip) + # Use materialized bytes to avoid consuming IO streams twice + data = content result = self.write_files( [WriteEntry(path=path, data=data)], From 197bc3ee8b003e9a41989299f76d20802c2744ef Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:39:59 +0200 Subject: [PATCH 10/14] chore: add changeset for composite upload Co-Authored-By: Claude Opus 4.6 --- .changeset/composite-file-upload.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/composite-file-upload.md diff --git a/.changeset/composite-file-upload.md b/.changeset/composite-file-upload.md new file mode 100644 index 0000000000..ae168277c3 --- /dev/null +++ b/.changeset/composite-file-upload.md @@ -0,0 +1,6 @@ +--- +'@e2b/python-sdk': minor +'e2b': minor +--- + +automatically split large file uploads (>64MB) into chunks and compose them server-side From f1839900981e64255871436188fe046f8588c3e0 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:59:44 +0200 Subject: [PATCH 11/14] Remove composite upload from sync Python SDK Composite upload's primary benefit is parallel chunk uploading, which the sync SDK cannot leverage (sequential HTTP requests negate the performance advantage). Only the async Python SDK and JS SDK retain composite upload support via asyncio.gather() and Promise.all(). Co-Authored-By: Claude Opus 4.6 --- .changeset/composite-file-upload.md | 2 +- .../e2b/sandbox_sync/filesystem/filesystem.py | 83 +------------------ 2 files changed, 3 insertions(+), 82 deletions(-) diff --git a/.changeset/composite-file-upload.md b/.changeset/composite-file-upload.md index ae168277c3..2e683fa13f 100644 --- a/.changeset/composite-file-upload.md +++ b/.changeset/composite-file-upload.md @@ -3,4 +3,4 @@ 'e2b': minor --- -automatically split large file uploads (>64MB) into chunks and compose them server-side +automatically split large file uploads (>64MB) into parallel chunks and compose them server-side (async Python SDK and JS SDK only) diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index a95445e9fc..8cf4d90132 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -1,5 +1,3 @@ -import uuid - from io import IOBase, TextIOBase from typing import IO, Iterator, List, Literal, Optional, Union, overload @@ -18,7 +16,6 @@ from e2b_connect.client import Code from e2b.envd.api import ( - ENVD_API_FILES_COMPOSE_ROUTE, ENVD_API_FILES_ROUTE, handle_envd_api_exception, ) @@ -62,9 +59,6 @@ def _handle_filesystem_envd_api_exception(r): return handle_envd_api_exception(r, _FILESYSTEM_HTTP_ERROR_MAP) -_DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024 # 64 MB - - class Filesystem: """ Module for interacting with the filesystem in the sandbox. @@ -220,11 +214,8 @@ def write( :return: Information about the written file """ if self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: - content = to_upload_body(data, False) - if len(content) > _DEFAULT_CHUNK_SIZE: - return self._composite_write(path, content, user, request_timeout, gzip) - # Use materialized bytes to avoid consuming IO streams twice - data = content + # Materialize IO streams to bytes to avoid consuming them twice + data = to_upload_body(data, False) result = self.write_files( [WriteEntry(path=path, data=data)], @@ -238,76 +229,6 @@ def write( return result[0] - def _composite_write( - self, - destination: str, - content: bytes, - user: Optional[Username] = None, - request_timeout: Optional[float] = None, - use_gzip: bool = False, - ) -> WriteInfo: - username = user - if username is None and self._envd_version < ENVD_DEFAULT_USER: - username = default_username - - total_size = len(content) - chunk_size = _DEFAULT_CHUNK_SIZE - - headers = {"Content-Type": "application/octet-stream"} - if use_gzip: - headers["Content-Encoding"] = "gzip" - - # Split into chunks and upload - upload_id = str(uuid.uuid4()) - chunk_count = (total_size + chunk_size - 1) // chunk_size - chunk_paths: List[str] = [] - - for i in range(chunk_count): - chunk_path = f"/tmp/.e2b-upload-{upload_id}-{i}" - chunk_paths.append(chunk_path) - - start = i * chunk_size - end = min(start + chunk_size, total_size) - chunk_data = content[start:end] - - params = {"path": chunk_path} - if username: - params["username"] = username - - upload_content = to_upload_body(chunk_data, use_gzip) - - r = self._envd_api.post( - ENVD_API_FILES_ROUTE, - content=upload_content, - headers=headers, - params=params, - timeout=self._connection_config.get_request_timeout(request_timeout), - ) - - err = _handle_filesystem_envd_api_exception(r) - if err: - raise err - - # Compose chunks into the final file - body = { - "source_paths": chunk_paths, - "destination": destination, - } - if username: - body["username"] = username - - r = self._envd_api.post( - ENVD_API_FILES_COMPOSE_ROUTE, - json=body, - timeout=self._connection_config.get_request_timeout(request_timeout), - ) - - err = _handle_filesystem_envd_api_exception(r) - if err: - raise err - - return WriteInfo(**r.json()) - def write_files( self, files: List[WriteEntry], From 051265870516537b3ec36ffcf19ac626bf1b275f Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:01:55 +0200 Subject: [PATCH 12/14] Remove redundant to_upload_body pre-materialization in sync write() write_files() already calls to_upload_body internally, so the pre-materialization in write() was unnecessary after removing the composite upload size check. Co-Authored-By: Claude Opus 4.6 --- packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 8cf4d90132..9d68059615 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -213,10 +213,6 @@ def write( :return: Information about the written file """ - if self._envd_version >= ENVD_OCTET_STREAM_UPLOAD: - # Materialize IO streams to bytes to avoid consuming them twice - data = to_upload_body(data, False) - result = self.write_files( [WriteEntry(path=path, data=data)], user=user, From 111dd1f57f4b514b29b2fa6747e2c273d1e97e15 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:03:38 +0200 Subject: [PATCH 13/14] Use TaskGroup and async gzip in async composite upload Replace asyncio.gather with asyncio.TaskGroup for structured concurrency, and offload gzip compression to a thread to avoid blocking the event loop. Co-Authored-By: Claude Opus 4.6 --- .../e2b/sandbox_async/filesystem/filesystem.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 5a9a4ccef1..9d74a66f35 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -1,4 +1,5 @@ import asyncio +import gzip import uuid from io import IOBase, TextIOBase @@ -396,7 +397,10 @@ async def _upload_chunk(i: int) -> None: if username: params["username"] = username - upload_content = to_upload_body(chunk_data, use_gzip) + if use_gzip: + upload_content = await asyncio.to_thread(gzip.compress, chunk_data) + else: + upload_content = chunk_data r = await self._envd_api.post( ENVD_API_FILES_ROUTE, @@ -410,7 +414,9 @@ async def _upload_chunk(i: int) -> None: if err: raise err - await asyncio.gather(*[_upload_chunk(i) for i in range(chunk_count)]) + async with asyncio.TaskGroup() as tg: + for i in range(chunk_count): + tg.create_task(_upload_chunk(i)) # Compose chunks into the final file body = { From 603f3179be6fde0bf8435f0bf74692ca5356e677 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Fri, 3 Apr 2026 18:10:53 +0200 Subject: [PATCH 14/14] Revert TaskGroup to asyncio.gather for Python <3.11 compat asyncio.TaskGroup requires Python 3.11+, which the SDK's type checker does not support. Revert to asyncio.gather for broader compatibility. Co-Authored-By: Claude Opus 4.6 --- .../python-sdk/e2b/sandbox_async/filesystem/filesystem.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 9d74a66f35..4fa2880d5d 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -414,9 +414,7 @@ async def _upload_chunk(i: int) -> None: if err: raise err - async with asyncio.TaskGroup() as tg: - for i in range(chunk_count): - tg.create_task(_upload_chunk(i)) + await asyncio.gather(*[_upload_chunk(i) for i in range(chunk_count)]) # Compose chunks into the final file body = {