Skip to content

Commit

Permalink
make s3 a default client (#16574)
Browse files Browse the repository at this point in the history
  • Loading branch information
cirospaciari authored Jan 22, 2025
1 parent 2cf247a commit 5d98e64
Show file tree
Hide file tree
Showing 12 changed files with 46 additions and 46 deletions.
11 changes: 6 additions & 5 deletions docs/api/s3.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ const client = new S3Client({
});

// Bun.s3 is a global singleton that is equivalent to `new Bun.S3Client()`
Bun.s3 = client;
```

### Working with S3 Files
Expand Down Expand Up @@ -375,7 +374,7 @@ If the `S3_*` environment variable is not set, Bun will also check for the `AWS_

These environment variables are read from [`.env` files](/docs/runtime/env) or from the process environment at initialization time (`process.env` is not used for this).

These defaults are overridden by the options you pass to `s3(credentials)`, `new Bun.S3Client(credentials)`, or any of the methods that accept credentials. So if, for example, you use the same credentials for different buckets, you can set the credentials once in your `.env` file and then pass `bucket: "my-bucket"` to the `s3()` helper function without having to specify all the credentials again.
These defaults are overridden by the options you pass to `s3.file(credentials)`, `new Bun.S3Client(credentials)`, or any of the methods that accept credentials. So if, for example, you use the same credentials for different buckets, you can set the credentials once in your `.env` file and then pass `bucket: "my-bucket"` to the `s3.file()` function without having to specify all the credentials again.

### `S3Client` objects

Expand Down Expand Up @@ -459,7 +458,7 @@ const exists = await client.exists("my-file.txt");

## `S3File`

`S3File` instances are created by calling the `S3` instance method or the `s3()` helper function. Like `Bun.file()`, `S3File` instances are lazy. They don't refer to something that necessarily exists at the time of creation. That's why all the methods that don't involve network requests are fully synchronous.
`S3File` instances are created by calling the `S3Client` instance method or the `s3.file()` function. Like `Bun.file()`, `S3File` instances are lazy. They don't refer to something that necessarily exists at the time of creation. That's why all the methods that don't involve network requests are fully synchronous.

```ts
interface S3File extends Blob {
Expand All @@ -482,7 +481,7 @@ interface S3File extends Blob {
| Response
| Request,
options?: BlobPropertyBag,
): Promise<void>;
): Promise<number>;

exists(options?: S3Options): Promise<boolean>;
unlink(options?: S3Options): Promise<void>;
Expand Down Expand Up @@ -600,7 +599,9 @@ const exists = await S3Client.exists("my-file.txt", credentials);
The same method also works on `S3File` instances.

```ts
const s3file = Bun.s3("my-file.txt", {
import { s3 } from "bun";

const s3file = s3.file("my-file.txt", {
...credentials,
});
const exists = await s3file.exists();
Expand Down
32 changes: 3 additions & 29 deletions packages/bun-types/bun.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1267,33 +1267,7 @@ declare module "bun" {
}

var S3Client: S3Client;

/**
* Creates a new S3File instance for working with a single file.
*
* @param path The path or key of the file
* @param options S3 configuration options
* @returns `S3File` instance for the specified path
*
* @example
* import { s3 } from "bun";
* const file = s3("my-file.txt", {
* bucket: "my-bucket",
* accessKeyId: "your-access-key",
* secretAccessKey: "your-secret-key"
* });
*
* // Read the file
* const content = await file.text();
*
* @example
* // Using s3:// protocol
* const file = s3("s3://my-bucket/my-file.txt", {
* accessKeyId: "your-access-key",
* secretAccessKey: "your-secret-key"
* });
*/
function s3(path: string | URL, options?: S3Options): S3File;
var s3: S3Client;

/**
* Configuration options for S3 operations
Expand Down Expand Up @@ -1597,15 +1571,15 @@ declare module "bun" {
*
* // Write large chunks of data efficiently
* for (const chunk of largeDataChunks) {
* await writer.write(chunk);
* writer.write(chunk);
* }
* await writer.end();
*
* @example
* // Error handling
* const writer = file.writer();
* try {
* await writer.write(data);
* writer.write(data);
* await writer.end();
* } catch (err) {
* console.error('Upload failed:', err);
Expand Down
8 changes: 6 additions & 2 deletions src/bun.js/api/BunObject.zig
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ pub const BunObject = struct {
pub const registerMacro = toJSCallback(Bun.registerMacro);
pub const resolve = toJSCallback(Bun.resolve);
pub const resolveSync = toJSCallback(Bun.resolveSync);
pub const s3 = S3File.createJSS3File;
pub const serve = toJSCallback(Bun.serve);
pub const sha = toJSCallback(JSC.wrapStaticMethod(Crypto.SHA512_256, "hash_", true));
pub const shellEscape = toJSCallback(Bun.shellEscape);
Expand Down Expand Up @@ -72,6 +71,7 @@ pub const BunObject = struct {
pub const stdout = toJSGetter(Bun.getStdout);
pub const unsafe = toJSGetter(Bun.getUnsafe);
pub const S3Client = toJSGetter(Bun.getS3ClientConstructor);
pub const s3 = toJSGetter(Bun.getS3DefaultClient);
// --- Getters ---

fn getterName(comptime baseName: anytype) [:0]const u8 {
Expand Down Expand Up @@ -133,6 +133,8 @@ pub const BunObject = struct {
@export(BunObject.semver, .{ .name = getterName("semver") });
@export(BunObject.embeddedFiles, .{ .name = getterName("embeddedFiles") });
@export(BunObject.S3Client, .{ .name = getterName("S3Client") });
@export(BunObject.s3, .{ .name = getterName("s3") });

// --- Getters --

// -- Callbacks --
Expand All @@ -157,7 +159,6 @@ pub const BunObject = struct {
@export(BunObject.resolve, .{ .name = callbackName("resolve") });
@export(BunObject.resolveSync, .{ .name = callbackName("resolveSync") });
@export(BunObject.serve, .{ .name = callbackName("serve") });
@export(BunObject.s3, .{ .name = callbackName("s3") });
@export(BunObject.sha, .{ .name = callbackName("sha") });
@export(BunObject.shellEscape, .{ .name = callbackName("shellEscape") });
@export(BunObject.shrink, .{ .name = callbackName("shrink") });
Expand Down Expand Up @@ -3451,6 +3452,9 @@ pub fn getGlobConstructor(globalThis: *JSC.JSGlobalObject, _: *JSC.JSObject) JSC
pub fn getS3ClientConstructor(globalThis: *JSC.JSGlobalObject, _: *JSC.JSObject) JSC.JSValue {
return JSC.WebCore.S3Client.getConstructor(globalThis);
}
pub fn getS3DefaultClient(globalThis: *JSC.JSGlobalObject, _: *JSC.JSObject) JSC.JSValue {
return globalThis.bunVM().rareData().s3DefaultClient(globalThis);
}
pub fn getEmbeddedFiles(globalThis: *JSC.JSGlobalObject, _: *JSC.JSObject) JSC.JSValue {
const vm = globalThis.bunVM();
const graph = vm.standalone_module_graph orelse return JSC.JSValue.createEmptyArray(globalThis, 0);
Expand Down
2 changes: 1 addition & 1 deletion src/bun.js/bindings/BunObject+exports.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
macro(semver) \
macro(embeddedFiles) \
macro(S3Client) \
macro(s3) \

// --- Callbacks ---
#define FOR_EACH_CALLBACK(macro) \
Expand All @@ -58,7 +59,6 @@
macro(registerMacro) \
macro(resolve) \
macro(resolveSync) \
macro(s3) \
macro(serve) \
macro(sha) \
macro(shrink) \
Expand Down
2 changes: 1 addition & 1 deletion src/bun.js/bindings/BunObject.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,7 @@ JSC_DEFINE_HOST_FUNCTION(functionFileURLToPath, (JSC::JSGlobalObject * globalObj
Transpiler BunObject_getter_wrap_Transpiler DontDelete|PropertyCallback
embeddedFiles BunObject_getter_wrap_embeddedFiles DontDelete|PropertyCallback
S3Client BunObject_getter_wrap_S3Client DontDelete|PropertyCallback
s3 BunObject_getter_wrap_s3 DontDelete|PropertyCallback
allocUnsafe BunObject_callback_allocUnsafe DontDelete|Function 1
argv BunObject_getter_wrap_argv DontDelete|PropertyCallback
build BunObject_callback_build DontDelete|Function 1
Expand Down Expand Up @@ -754,7 +755,6 @@ JSC_DEFINE_HOST_FUNCTION(functionFileURLToPath, (JSC::JSGlobalObject * globalObj
resolveSync BunObject_callback_resolveSync DontDelete|Function 1
revision constructBunRevision ReadOnly|DontDelete|PropertyCallback
semver BunObject_getter_wrap_semver ReadOnly|DontDelete|PropertyCallback
s3 BunObject_callback_s3 DontDelete|Function 1
sql defaultBunSQLObject DontDelete|PropertyCallback
postgres defaultBunSQLObject DontDelete|PropertyCallback
SQL constructBunSQLObject DontDelete|PropertyCallback
Expand Down
20 changes: 20 additions & 0 deletions src/bun.js/rare_data.zig
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ temp_pipe_read_buffer: ?*PipeReadBuffer = null,

aws_signature_cache: AWSSignatureCache = .{},

s3_default_client: JSC.Strong = .{},

const PipeReadBuffer = [256 * 1024]u8;
const DIGESTED_HMAC_256_LEN = 32;
pub const AWSSignatureCache = struct {
Expand Down Expand Up @@ -435,6 +437,23 @@ pub fn nodeFSStatWatcherScheduler(rare: *RareData, vm: *JSC.VirtualMachine) *Sta
};
}

pub fn s3DefaultClient(rare: *RareData, globalThis: *JSC.JSGlobalObject) JSC.JSValue {
return rare.s3_default_client.get() orelse {
const vm = globalThis.bunVM();
var aws_options = bun.S3.S3Credentials.getCredentialsWithOptions(vm.transpiler.env.getS3Credentials(), .{}, null, null, globalThis) catch bun.outOfMemory();
defer aws_options.deinit();
const client = JSC.WebCore.S3Client.new(.{
.credentials = aws_options.credentials.dupe(),
.options = aws_options.options,
.acl = aws_options.acl,
});
const js_client = client.toJS(globalThis);
js_client.ensureStillAlive();
rare.s3_default_client = JSC.Strong.create(js_client, globalThis);
return js_client;
};
}

pub fn deinit(this: *RareData) void {
if (this.temp_pipe_read_buffer) |pipe| {
this.temp_pipe_read_buffer = null;
Expand All @@ -443,6 +462,7 @@ pub fn deinit(this: *RareData) void {

this.aws_signature_cache.deinit();

this.s3_default_client.deinit();
if (this.boring_ssl_engine) |engine| {
_ = bun.BoringSSL.ENGINE_free(engine);
}
Expand Down
2 changes: 1 addition & 1 deletion src/bun.js/webcore/S3Client.zig
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ pub const S3Client = struct {
},
);
} else {
try writer.writeAll(comptime bun.Output.prettyFmt(" {{", enable_ansi_colors));
try writer.writeAll(" {");
}

try writeFormatCredentials(this.credentials, this.options, this.acl, Formatter, formatter, writer, enable_ansi_colors);
Expand Down
4 changes: 2 additions & 2 deletions test/js/bun/s3/s3-stream-leak-fixture.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions test/js/bun/s3/s3-text-leak-fixture.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion test/js/bun/s3/s3-write-leak-fixture.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion test/js/bun/s3/s3-writer-leak-fixture.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion test/js/bun/s3/s3.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { describe, expect, it, beforeAll, afterAll } from "bun:test";
import { bunExe, bunEnv, getSecret, tempDirWithFiles, isLinux } from "harness";
import { randomUUID } from "crypto";
import { S3Client, s3, file, which } from "bun";
import { S3Client, s3 as defaultS3, file, which } from "bun";
const s3 = (...args) => defaultS3.file(...args);
const S3 = (...args) => new S3Client(...args);
import child_process from "child_process";
import type { S3Options } from "bun";
Expand Down

0 comments on commit 5d98e64

Please sign in to comment.