From 75c1da4a5a76eda4d17cdf82b5564c19ea0fd9ca Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Wed, 18 Sep 2024 18:45:40 +0200 Subject: [PATCH 1/4] use plain app id in key gen --- docs/src/app/(docs)/uploading-files/page.mdx | 42 +++----------------- packages/shared/src/crypto.ts | 5 +-- 2 files changed, 7 insertions(+), 40 deletions(-) diff --git a/docs/src/app/(docs)/uploading-files/page.mdx b/docs/src/app/(docs)/uploading-files/page.mdx index 856d6a1588..ab45162b2a 100644 --- a/docs/src/app/(docs)/uploading-files/page.mdx +++ b/docs/src/app/(docs)/uploading-files/page.mdx @@ -96,49 +96,19 @@ Once your backend adapter has received and validated the request, you will next need to generate presigned URLs. First, generate some file keys for the files to be uploaded. -To generate a file key, generate a [Sqids](https://sqids.org/) of your appId -with `{ minLength: 12 }`, then concatenate this with a file seed of your choice. +To generate a file key, concatenate your appId with a file seed of your choice. The file seed can be anything you want, but it should be unique for each file, -as well as url safe. In this example, we include a base64 encoding to ensure the -file seed is url safe, but you can do this however you want. +url safe and no less than 36 characters. In this example, we include a base64 +encoding to ensure the file seed is url safe, but you can do this however you +want. ```ts -import * as Hash from "effect/Hash"; -import SQIds, { defaultOptions } from "sqids"; - -// A simple function to shuffle the alphabet for the Sqids -function shuffle(str: string, seed: string) { - const chars = str.split(""); - const seedNum = Hash.string(seed); - - let temp: string; - let j: number; - for (let i = 0; i < chars.length; i++) { - j = ((seedNum % (i + 1)) + i) % chars.length; - temp = chars[i]; - chars[i] = chars[j]; - chars[j] = temp; - } - - return chars.join(""); -} - -function generateKey( - appId: string, - fileSeed: string, -) => - // Hash and Encode the parts and apiKey as sqids - const alphabet = shuffle(defaultOptions.alphabet, appId); - - const encodedAppId = new SQIds({ alphabet, minLength: 12 }).encode([ - Math.abs(Hash.string(appId)), - ]); - +function generateKey(appId: string, fileSeed: string) { // We use a base64 encoding here to ensure the file seed is url safe, but // you can do this however you want const encodedFileSeed = encodeBase64(fileSeed); - return `${encodedAppId}${encodedFileSeed}`; + return `${appId}${encodedFileSeed}`; } ``` diff --git a/packages/shared/src/crypto.ts b/packages/shared/src/crypto.ts index 2d4634132f..50c6bf6f72 100644 --- a/packages/shared/src/crypto.ts +++ b/packages/shared/src/crypto.ts @@ -107,12 +107,9 @@ export const generateKey = ( const encodedFileSeed = new SQIds({ alphabet, minLength: 36 }).encode([ Math.abs(Hash.string(hashParts)), ]); - const encodedAppId = new SQIds({ alphabet, minLength: 12 }).encode([ - Math.abs(Hash.string(appId)), - ]); // Concatenate them - return encodedAppId + encodedFileSeed; + return appId + encodedFileSeed; }).pipe(Micro.withTrace("generateKey")); // Verify that the key was generated with the same appId From b281a5c7d9c7d2c09c6b57658556e012ddd46535 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Wed, 18 Sep 2024 20:13:34 +0200 Subject: [PATCH 2/4] rm verifykey helper --- packages/shared/src/crypto.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/shared/src/crypto.ts b/packages/shared/src/crypto.ts index 50c6bf6f72..0cad8086e0 100644 --- a/packages/shared/src/crypto.ts +++ b/packages/shared/src/crypto.ts @@ -112,20 +112,6 @@ export const generateKey = ( return appId + encodedFileSeed; }).pipe(Micro.withTrace("generateKey")); -// Verify that the key was generated with the same appId -export const verifyKey = (key: string, appId: string) => - Micro.sync(() => { - const alphabet = shuffle(defaultOptions.alphabet, appId); - const expectedPrefix = new SQIds({ alphabet, minLength: 12 }).encode([ - Math.abs(Hash.string(appId)), - ]); - - return key.startsWith(expectedPrefix); - }).pipe( - Micro.withTrace("verifyKey"), - Micro.orElseSucceed(() => false), - ); - export const generateSignedURL = ( url: string | URL, secretKey: string, From 6376d33dd731688cb2693f08fb5293fd61bc51b6 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Wed, 6 Nov 2024 10:24:33 +0100 Subject: [PATCH 3/4] rm --- docs/src/app/(docs)/uploading-files/page.mdx | 173 ------------------- 1 file changed, 173 deletions(-) diff --git a/docs/src/app/(docs)/uploading-files/page.mdx b/docs/src/app/(docs)/uploading-files/page.mdx index 2d7b8b322b..77116639ae 100644 --- a/docs/src/app/(docs)/uploading-files/page.mdx +++ b/docs/src/app/(docs)/uploading-files/page.mdx @@ -102,179 +102,6 @@ url safe and no less than 36 characters. In this example, we include a base64 encoding to ensure the file seed is url safe, but you can do this however you want. -Although we currently only offer a JavaScript SDK, here are some reference -implementations you can use to generate valid file keys. We also plan on making -this process easier in the future. - - - If you struggle to implement it for your language, you can also request one - from the [`/v7/prepareUpload` REST - endpoint](/api-reference/openapi-spec#tag/default/POST/v7/prepareUpload). Keep - in mind that this adds extra latency to your uploads. - - - - - -```ts -function generateKey(appId: string, fileSeed: string) { - // We use a base64 encoding here to ensure the file seed is url safe, but - // you can do this however you want - const encodedFileSeed = encodeBase64(fileSeed); - - return `${appId}${encodedFileSeed}`; -} -``` - - - - -```py -import math -import base64 -from sqids import Sqids -from sqids.constants import DEFAULT_ALPHABET - -def hash_string(s: str) -> int: - h = 5381 - for char in reversed(s): - h = (h * 33) ^ ord(char) - # 32-bit integer overflow - h &= 0xFFFFFFFF - h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000) - - # Convert to signed 32-bit integer - if h >= 0x80000000: - h -= 0x100000000 - - return h - - -def shuffle(string: str, seed: str) -> str: - chars = list(string) - seed_num = hash_string(seed) - - for i in range(len(chars)): - j = int(math.fmod(math.fmod(seed_num, i + 1) + i, len(chars))) - chars[i], chars[j] = chars[j], chars[i] - - return "".join(chars) - - -def generate_key(file_seed: str, app_id: str) -> str: - alphabet = shuffle(DEFAULT_ALPHABET, app_id) - - encoded_app_id = Sqids(alphabet, min_length=12).encode( - [abs(hash_string(app_id))] - ) - - return encoded_app_id + file_seed -``` - - - - -```php -use Sqids\Sqids; - -function hash_string(string $string): int { - $h = 5381; - for ($i = strlen($string) - 1; $i >= 0; $i--) { - $char = $string[$i]; - $h = ($h * 33) ^ ord($char); - // 32-bit integer overflow - $h &= 0xFFFFFFFF; - } - $h = ($h & 0xBFFFFFFF) | (($h >> 1) & 0x40000000); - - // Convert to signed 32-bit integer - if ($h >= 0x80000000) { - $h -= 0x100000000; - } - - return $h; - } - -function shuffle_string(string $string, string $seed): string { - $chars = str_split($string); - $seed_num = hash_string($seed); - - for ($i = 0; $i < count($chars); $i++) { - $j = (($seed_num % ($i + 1)) + $i) % count($chars); - [$chars[$i], $chars[$j]] = [$chars[$j], $chars[$i]]; - } - - return implode('', $chars); -} - -function generate_key(string $file_seed, string $appId): string { - $alphabet = shuffle_string(Sqids::DEFAULT_ALPHABET, $appId); - $sqids = new Sqids($alphabet, 12); - - $encodedAppId = $sqids->encode( - [abs(hash_string($appId))] - ); - - return $encodedAppId . base64_encode($file_seed); -} -``` - - - - -```go -import ( - "math" - "github.com/sqids/sqids-go" -) - -func hashString(s string) int32 { - h := int64(5381) - for i := len(s) - 1; i >= 0; i-- { - h = (h * 33) ^ int64(s[i]) - // 32-bit integer overflow - h &= 0xFFFFFFFF - } - h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000) - - // Convert to signed 32-bit integer - if h >= 0x80000000 { - h -= 0x100000000 - } - - return int32(h) -} - -func shuffle(input string, seed string) string { - chars := []rune(input) - seedNum := hashString(seed) - - for i := 0; i < len(chars); i++ { - j := (int(seedNum)%(i+1) + i) % len(chars) - chars[i], chars[j] = chars[j], chars[i] - } - - return string(chars) -} - -func generateKey(fileSeed string, appId string) string { - alphabet := shuffle("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", appId) - s, _ := sqids.New(sqids.Options{ - MinLength: 12, - Alphabet: alphabet, - }) - - encodedAppId, _ := s.Encode( - []uint64{uint64(math.Abs(float64(hashString(appId))))}, - ) - - return encodedAppId + fileSeed -} -``` - - - - The URL, to which you will upload the file, will depend on your app's region. You can find the list of regions in the [regions documentation](/concepts/regions-acl#regions). The upload URL can then From badab68432210a1dc13b15fd08e91a3b65e16ca7 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Wed, 6 Nov 2024 10:29:22 +0100 Subject: [PATCH 4/4] rm unused test --- packages/shared/test/crypto.test.ts | 93 ----------------------------- 1 file changed, 93 deletions(-) diff --git a/packages/shared/test/crypto.test.ts b/packages/shared/test/crypto.test.ts index 02ff9a6945..4be760e0b8 100644 --- a/packages/shared/test/crypto.test.ts +++ b/packages/shared/test/crypto.test.ts @@ -7,7 +7,6 @@ import { generateKey, generateSignedURL, signPayload, - verifyKey, verifySignature, } from "../src/crypto"; @@ -106,96 +105,4 @@ describe("key gen", () => { expect(key).toBeTruthy(); }), ); - - it.effect("verifies a key", () => - Effect.gen(function* () { - const appI = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appI, - ); - - const verified = yield* verifyKey(key, appI); - expect(verified).toBe(true); - }), - ); - - it.effect("doesn't verify a key with a bad appI", () => - Effect.gen(function* () { - const appI = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appI, - ); - - const verified = yield* verifyKey(key, "bad"); - expect(verified).toBe(false); - }), - ); - - it.effect("doesn't verify a key with a bad key", () => - Effect.gen(function* () { - const appId = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appId, - ); - - const verified = yield* verifyKey("badseed" + key.substring(7), appId); - expect(verified).toBe(false); - }), - ); - - it.effect("verifies with a custom hash function", () => - Effect.gen(function* () { - const appI = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appI, - (file) => [file.name], - ); - - const verified = yield* verifyKey(key, appI); - expect(verified).toBe(true); - }), - ); - - it.effect("works even when there's nothing to seed", () => - Effect.gen(function* () { - const appI = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appI, - () => [], - ); - - const verified = yield* verifyKey(key, appI); - expect(verified).toBe(true); - }), - ); });