fix: bucket URL is incorrect for local S3 emulators

.projenrc.ts

@@ -715,10 +715,13 @@ const tools = defineTools({
   ...genericCdkProps({ private: true }),
   parent: repo,
   tools: {
-    zip: {
+    'zip': {
       deps: ['archiver@^7.0.1', 'fast-glob@^3.3.3'],
       devDeps: ['@types/archiver', 'jszip'],
     },
+    's3-path-style': {
+      deps: [],
+    },
   },
 });
 configureProject(tools);
@@ -790,6 +793,7 @@ const cdkAssetsLib = configureProject(
   }),
 );
 cdkAssetsLib.with(tools.zip);
+cdkAssetsLib.with(tools['s3-path-style']);
 fixupTestTask(cdkAssetsLib);
 
 // Prevent imports of private API surface
@@ -1022,6 +1026,7 @@ const toolkitLib = configureProject(
 );
 fixupTestTask(toolkitLib);
 toolkitLib.with(tools.zip);
+toolkitLib.with(tools['s3-path-style']);
 toolkitLib.tasks.tryFind('test')?.updateStep(0, {
   // https://github.com/aws/aws-sdk-js-v3/issues/7420
   exec: 'NODE_OPTIONS="$NODE_OPTIONS --experimental-vm-modules" jest --passWithNoTests --updateSnapshot',

packages/@aws-cdk/cdk-assets-lib/lib/aws.ts

@@ -46,6 +46,7 @@ import type {
   ListObjectsV2CommandOutput,
   PutObjectCommandInput,
 } from './aws-types';
+import { forceS3PathStyle } from './private/tools';
 
 export type AssumeRoleAdditionalOptions = Partial<
   Omit<AssumeRoleCommandInput, 'ExternalId' | 'RoleArn'>
@@ -149,7 +150,11 @@ export class DefaultAwsClient implements IAws {
   }
 
   public async s3Client(options: ClientOptions): Promise<IS3Client> {
-    const client = new S3Client(await this.awsOptions(options));
+    const client = new S3Client({
+      ...(await this.awsOptions(options)),
+      // Use path-style addressing for explicit opt-in or loopback endpoints.
+      forcePathStyle: forceS3PathStyle(),
+    });
     return {
       getBucketEncryption: (
         input: GetBucketEncryptionCommandInput,

packages/@aws-cdk/cdk-assets-lib/test/aws.test.ts

@@ -1,5 +1,6 @@
 import 'aws-sdk-client-mock-jest';
 
+import * as s3 from '@aws-sdk/client-s3';
 import { GetCallerIdentityCommand } from '@aws-sdk/client-sts';
 import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
 import { mockSTS } from './mock-aws';
@@ -72,3 +73,50 @@ test('session tags are passed to fromTemporaryCredentials in awsOptions', async
     },
   });
 });
+
+describe('S3 path-style addressing', () => {
+  const ENV_VARS = ['CDK_S3_FORCE_PATH_STYLE', 'AWS_ENDPOINT_URL_S3', 'AWS_ENDPOINT_URL'];
+  const original: Record<string, string | undefined> = {};
+  let s3ClientSpy: jest.SpyInstance;
+
+  beforeEach(() => {
+    for (const v of ENV_VARS) {
+      original[v] = process.env[v];
+      delete process.env[v];
+    }
+    s3ClientSpy = jest.spyOn(s3, 'S3Client').mockImplementation(() => ({}) as any);
+  });
+
+  afterEach(() => {
+    s3ClientSpy.mockRestore();
+    for (const v of ENV_VARS) {
+      if (original[v] === undefined) {
+        delete process.env[v];
+      } else {
+        process.env[v] = original[v];
+      }
+    }
+  });
+
+  test('forces path-style addressing on the S3 client when CDK_S3_FORCE_PATH_STYLE is set', async () => {
+    process.env.CDK_S3_FORCE_PATH_STYLE = '1';
+
+    await new DefaultAwsClient().s3Client({ region: 'far-far-away' });
+
+    expect(s3ClientSpy).toHaveBeenCalledWith(expect.objectContaining({ forcePathStyle: true }));
+  });
+
+  test('auto-detects path-style addressing for a loopback endpoint', async () => {
+    process.env.AWS_ENDPOINT_URL_S3 = 'http://localhost:4566';
+
+    await new DefaultAwsClient().s3Client({ region: 'far-far-away' });
+
+    expect(s3ClientSpy).toHaveBeenCalledWith(expect.objectContaining({ forcePathStyle: true }));
+  });
+
+  test('leaves path-style addressing unset on the S3 client by default', async () => {
+    await new DefaultAwsClient().s3Client({ region: 'far-far-away' });
+
+    expect(s3ClientSpy).toHaveBeenCalledWith(expect.objectContaining({ forcePathStyle: undefined }));
+  });
+});

packages/@aws-cdk/private-tools/lib/s3-path-style/index.ts

@@ -0,0 +1,44 @@
+/**
+ * Determine whether the S3 client should use path-style addressing.
+ *
+ * The AWS SDK defaults to virtual-hosted-style addressing
+ * (`https://<bucket>.s3.amazonaws.com`). That doesn't work for S3-compatible
+ * emulators reached over loopback (e.g. LocalStack or MinIO on `localhost`),
+ * which only serve path-style URLs (`https://<endpoint>/<bucket>`).
+ *
+ * Returns `true` when path-style should be forced, or `undefined` to leave the
+ * SDK default in place.
+ *
+ * - The `CDK_S3_FORCE_PATH_STYLE` environment variable forces it explicitly.
+ * - Otherwise it is auto-detected when the configured S3 endpoint
+ *   (`AWS_ENDPOINT_URL_S3`, falling back to `AWS_ENDPOINT_URL`) points at a
+ *   loopback host.
+ */
+export function forceS3PathStyle(): boolean | undefined {
+  if (process.env.CDK_S3_FORCE_PATH_STYLE) {
+    return true;
+  }
+
+  const endpoint = process.env.AWS_ENDPOINT_URL_S3 ?? process.env.AWS_ENDPOINT_URL;
+  if (endpoint && isLoopbackEndpoint(endpoint)) {
+    return true;
+  }
+
+  return undefined;
+}
+
+function isLoopbackEndpoint(endpoint: string): boolean {
+  let host: string;
+  try {
+    host = new URL(endpoint).hostname;
+  } catch {
+    return false;
+  }
+
+  // The URL parser wraps IPv6 addresses in brackets to make use of colons unambiguous, e.g. "http://[::1]:4566"
+  // Strip them for easier comparison.
+  const normalized = host.replace(/^\[|\]$/g, '').toLowerCase();
+  return normalized === 'localhost'
+    || normalized.startsWith('127.')
+    || normalized === '::1';
+}

packages/@aws-cdk/private-tools/test/s3-path-style/s3-path-style.test.ts

@@ -0,0 +1,70 @@
+import { forceS3PathStyle } from '../../lib/s3-path-style';
+
+const ENV_VARS = ['CDK_S3_FORCE_PATH_STYLE', 'AWS_ENDPOINT_URL_S3', 'AWS_ENDPOINT_URL'];
+
+describe('forceS3PathStyle', () => {
+  const original: Record<string, string | undefined> = {};
+
+  beforeEach(() => {
+    for (const v of ENV_VARS) {
+      original[v] = process.env[v];
+      delete process.env[v];
+    }
+  });
+
+  afterEach(() => {
+    for (const v of ENV_VARS) {
+      if (original[v] === undefined) {
+        delete process.env[v];
+      } else {
+        process.env[v] = original[v];
+      }
+    }
+  });
+
+  test('returns undefined when nothing is configured', () => {
+    expect(forceS3PathStyle()).toBeUndefined();
+  });
+
+  test('CDK_S3_FORCE_PATH_STYLE forces path-style explicitly', () => {
+    process.env.CDK_S3_FORCE_PATH_STYLE = '1';
+    expect(forceS3PathStyle()).toBe(true);
+  });
+
+  test.each([
+    'http://localhost:4566',
+    'https://localhost',
+    'http://127.0.0.1:9000',
+    'http://127.1.2.3:9000',
+    'http://[::1]:4566',
+    'http://LOCALHOST:4566',
+  ])('auto-detects loopback endpoint %s from AWS_ENDPOINT_URL_S3', (endpoint) => {
+    process.env.AWS_ENDPOINT_URL_S3 = endpoint;
+    expect(forceS3PathStyle()).toBe(true);
+  });
+
+  test('falls back to AWS_ENDPOINT_URL when AWS_ENDPOINT_URL_S3 is unset', () => {
+    process.env.AWS_ENDPOINT_URL = 'http://localhost:4566';
+    expect(forceS3PathStyle()).toBe(true);
+  });
+
+  test('AWS_ENDPOINT_URL_S3 takes precedence over AWS_ENDPOINT_URL', () => {
+    process.env.AWS_ENDPOINT_URL_S3 = 'https://s3.amazonaws.com';
+    process.env.AWS_ENDPOINT_URL = 'http://localhost:4566';
+    expect(forceS3PathStyle()).toBeUndefined();
+  });
+
+  test.each([
+    'https://s3.amazonaws.com',
+    'https://s3.us-east-1.amazonaws.com',
+    'https://my-custom-endpoint.example.com',
+  ])('leaves non-loopback endpoint %s on the SDK default', (endpoint) => {
+    process.env.AWS_ENDPOINT_URL_S3 = endpoint;
+    expect(forceS3PathStyle()).toBeUndefined();
+  });
+
+  test('ignores an unparseable endpoint', () => {
+    process.env.AWS_ENDPOINT_URL_S3 = 'not a url';
+    expect(forceS3PathStyle()).toBeUndefined();
+  });
+});

packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/sdk.ts

@@ -400,6 +400,7 @@ import type { ISdkLogger } from './sdk-logger';
 import type { Account } from './sdk-provider';
 import { traceMemberMethods } from './tracing';
 import { defaultCliUserAgent } from './user-agent';
+import { forceS3PathStyle } from '../../private/tools';
 import { AuthenticationError } from '../../toolkit/toolkit-error';
 import { formatErrorMessage } from '../../util';
 import type { IoHelper } from '../io/private';
@@ -1068,7 +1069,11 @@ export class SDK {
   }
 
   public s3(): IS3Client {
-    const client = new S3Client(this.config);
+    const client = new S3Client({
+      ...this.config,
+      // Use path-style addressing for explicit opt-in or loopback endpoints.
+      forcePathStyle: forceS3PathStyle(),
+    });
     return {
       deleteObjects: (input: DeleteObjectsCommandInput): Promise<DeleteObjectsCommandOutput> =>
         client.send(new DeleteObjectsCommand({

packages/@aws-cdk/toolkit-lib/test/api/aws-auth/sdk.test.ts

@@ -0,0 +1,51 @@
+import * as s3 from '@aws-sdk/client-s3';
+import { MockSdk } from '../../_helpers/mock-sdk';
+
+describe('S3 path-style addressing', () => {
+  const ENV_VARS = ['CDK_S3_FORCE_PATH_STYLE', 'AWS_ENDPOINT_URL_S3', 'AWS_ENDPOINT_URL'];
+  const original: Record<string, string | undefined> = {};
+  let s3ClientSpy: jest.SpyInstance;
+
+  beforeEach(() => {
+    for (const v of ENV_VARS) {
+      original[v] = process.env[v];
+      delete process.env[v];
+    }
+    s3ClientSpy = jest.spyOn(s3, 'S3Client').mockImplementation(() => ({}) as any);
+  });
+
+  afterEach(() => {
+    s3ClientSpy.mockRestore();
+    for (const v of ENV_VARS) {
+      if (original[v] === undefined) {
+        delete process.env[v];
+      } else {
+        process.env[v] = original[v];
+      }
+    }
+  });
+
+  // MockSdk only supplies fake credentials and region; `.s3()` is the real SDK
+  // method under test. Returns the `forcePathStyle` value it passes when
+  // constructing the underlying S3 client.
+  function forcePathStylePassedToS3Client(): boolean | undefined {
+    new MockSdk().s3();
+    return s3ClientSpy.mock.calls[0][0].forcePathStyle;
+  }
+
+  test('is forced when CDK_S3_FORCE_PATH_STYLE is set', () => {
+    process.env.CDK_S3_FORCE_PATH_STYLE = '1';
+
+    expect(forcePathStylePassedToS3Client()).toBe(true);
+  });
+
+  test('is auto-detected for a loopback endpoint', () => {
+    process.env.AWS_ENDPOINT_URL_S3 = 'http://localhost:4566';
+
+    expect(forcePathStylePassedToS3Client()).toBe(true);
+  });
+
+  test('is left unset by default', () => {
+    expect(forcePathStylePassedToS3Client()).toBeUndefined();
+  });
+});