diff --git a/docs/src/app/(docs)/uploading-files/page.mdx b/docs/src/app/(docs)/uploading-files/page.mdx index 0b0c13c0b3..d2a83d896f 100644 --- a/docs/src/app/(docs)/uploading-files/page.mdx +++ b/docs/src/app/(docs)/uploading-files/page.mdx @@ -96,275 +96,11 @@ Once your backend adapter has received and validated the request, you will next need to generate presigned URLs. First, generate some file keys for the files to be uploaded. -To generate a file key, generate a [Sqids](https://sqids.org/) of your appId -with `{ minLength: 12 }`, then concatenate this with a file seed of your choice. +To generate a file key, concatenate your appId with a file seed of your choice. The file seed can be anything you want, but it should be unique for each file, -as well as url safe. In this example, we include a base64 encoding to ensure the -file seed is url safe, but you can do this however you want. - -Although we currently only offer a JavaScript SDK, here are some reference -implementations you can use to generate valid file keys. We also plan on making -this process easier in the future. - - - If you struggle to implement it for your language, you can also request one - from the [`/v7/prepareUpload` REST - endpoint](/api-reference/openapi-spec#tag/default/POST/v7/prepareUpload). Keep - in mind that this adds extra latency to your uploads. - - - - - -```ts -import SQIds, { defaultOptions } from "sqids"; - -function djb2(s: string) { - const h = 5381; - let i = s.length; - while (i) { - h = (h * 33) ^ s.charCodeAt(--i); - } - return (h & 0xbfffffff) | ((h >>> 1) & 0x40000000); -} - -// A simple function to shuffle the alphabet for the Sqids -function shuffle(str: string, seed: string) { - const chars = str.split(""); - const seedNum = djb2(seed); - - let temp: string; - let j: number; - for (let i = 0; i < chars.length; i++) { - j = ((seedNum % (i + 1)) + i) % chars.length; - temp = chars[i]; - chars[i] = chars[j]; - chars[j] = temp; - } - - return chars.join(""); -} - -function generateKey(appId: string, fileSeed: string) { - // Hash and Encode the parts and apiKey as sqids - const alphabet = shuffle(defaultOptions.alphabet, appId); - - const encodedAppId = new SQIds({ alphabet, minLength: 12 }).encode([ - Math.abs(djb2(appId)), - ]); - - // We use a base64 encoding here to ensure the file seed is url safe, but - // you can do this however you want - const encodedFileSeed = encodeBase64(fileSeed); - - return `${encodedAppId}${encodedFileSeed}`; -} -``` - - - - -```py -import math -import base64 -from sqids import Sqids -from sqids.constants import DEFAULT_ALPHABET - -def djb2(s: str) -> int: - h = 5381 - for char in reversed(s): - h = (h * 33) ^ ord(char) - # 32-bit integer overflow - h &= 0xFFFFFFFF - h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000) - - # Convert to signed 32-bit integer - if h >= 0x80000000: - h -= 0x100000000 - - return h - - -def shuffle(string: str, seed: str) -> str: - chars = list(string) - seed_num = djb2(seed) - - for i in range(len(chars)): - j = int(math.fmod(math.fmod(seed_num, i + 1) + i, len(chars))) - chars[i], chars[j] = chars[j], chars[i] - - return "".join(chars) - - -def generate_key(file_seed: str, app_id: str) -> str: - alphabet = shuffle(DEFAULT_ALPHABET, app_id) - - encoded_app_id = Sqids(alphabet, min_length=12).encode( - [abs(djb2(app_id))] - ) - - return encoded_app_id + file_seed -``` - - - - -```php -use Sqids\Sqids; - -function djb2(string $string): int { - $h = 5381; - for ($i = strlen($string) - 1; $i >= 0; $i--) { - $char = $string[$i]; - $h = ($h * 33) ^ ord($char); - // 32-bit integer overflow - $h &= 0xFFFFFFFF; - } - $h = ($h & 0xBFFFFFFF) | (($h >> 1) & 0x40000000); - - // Convert to signed 32-bit integer - if ($h >= 0x80000000) { - $h -= 0x100000000; - } - - return $h; - } - -function shuffle_string(string $string, string $seed): string { - $chars = str_split($string); - $seed_num = djb2($seed); - - for ($i = 0; $i < count($chars); $i++) { - $j = (($seed_num % ($i + 1)) + $i) % count($chars); - [$chars[$i], $chars[$j]] = [$chars[$j], $chars[$i]]; - } - - return implode('', $chars); -} - -function generate_key(string $file_seed, string $appId): string { - $alphabet = shuffle_string(Sqids::DEFAULT_ALPHABET, $appId); - $sqids = new Sqids($alphabet, 12); - - $encodedAppId = $sqids->encode( - [abs(djb2($appId))] - ); - - return $encodedAppId . base64_encode($file_seed); -} -``` - - - - -```go -import ( - "math" - "github.com/sqids/sqids-go" -) - -func djb2(s string) int32 { - h := int64(5381) - for i := len(s) - 1; i >= 0; i-- { - h = (h * 33) ^ int64(s[i]) - // 32-bit integer overflow - h &= 0xFFFFFFFF - } - h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000) - - // Convert to signed 32-bit integer - if h >= 0x80000000 { - h -= 0x100000000 - } - - return int32(h) -} - -func shuffle(input string, seed string) string { - chars := []rune(input) - seedNum := djb2(seed) - - for i := 0; i < len(chars); i++ { - j := (int(seedNum)%(i+1) + i) % len(chars) - chars[i], chars[j] = chars[j], chars[i] - } - - return string(chars) -} - -func generateKey(fileSeed string, appId string) string { - alphabet := shuffle("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", appId) - s, _ := sqids.New(sqids.Options{ - MinLength: 12, - Alphabet: alphabet, - }) - - encodedAppId, _ := s.Encode( - []uint64{uint64(math.Abs(float64(djb2(appId))))}, - ) - - return encodedAppId + fileSeed -} -``` - - - - -```rust -fn djb2_hash(s: &str) -> i32 { - let mut h: i64 = 5381; - for &byte in s.as_bytes().iter().rev() { - h = (h * 33) ^ (byte as i64); - // Simulate 32-bit integer overflow - h &= 0xFFFFFFFF; - } - // Convert to signed 32-bit integer with the same bit manipulation - h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000); - if h >= 0x80000000 { - h -= 0x100000000; - } - h as i32 -} - -fn shuffle(input: &str, seed: &str) -> String { - let mut chars: Vec = input.chars().collect(); - let seed_num = djb2_hash(seed); - for i in 0..chars.len() { - let j = ((seed_num % (i as i32 + 1)) + i as i32) as usize % chars.len(); - let temp = chars[i]; - chars[i] = chars[j]; - chars[j] = temp; - } - chars.iter().collect() -} -fn generate_file_key(app_id: String) -> String { - let app_hash = djb2_hash(&app_id); - let alphabet: Vec = shuffle(sqids::DEFAULT_ALPHABET, &app_id).chars().collect(); - - // https://sqids.org/rust - let sqids = sqids::Sqids::builder() - .alphabet(alphabet) - .min_length(12) - .build() - .expect("Could not create sqid builder"); - let encoded_app_id = sqids.encode(&vec![app_hash.abs() as u64]).expect("Could not encode sqid"); - - // https://github.com/uuid-rs/uuid - let file_seed = uuid::Uuid::new_v4().to_string(); - - // We use a base64 encoding here to ensure the file seed is url safe, but - // you can do this however you want - // https://github.com/marshallpierce/rust-base64 - use base64::prelude::*; - let encoded_file_seed = BASE64_URL_SAFE.encode(file_seed.as_bytes()); - - format!("{}{}", encoded_app_id, encoded_file_seed) -} - -``` - - - +url safe and no less than 36 characters. In this example, we include a base64 +encoding to ensure the file seed is url safe, but you can do this however you +want. The URL, to which you will upload the file, will depend on your app's region. You can find the list of regions in the diff --git a/packages/shared/src/crypto.ts b/packages/shared/src/crypto.ts index af9d28530e..7491497d38 100644 --- a/packages/shared/src/crypto.ts +++ b/packages/shared/src/crypto.ts @@ -111,28 +111,11 @@ export const generateKey = ( const encodedFileSeed = new SQIds({ alphabet, minLength: 36 }).encode([ Math.abs(Hash.string(hashParts)), ]); - const encodedAppId = new SQIds({ alphabet, minLength: 12 }).encode([ - Math.abs(Hash.string(appId)), - ]); // Concatenate them - return encodedAppId + encodedFileSeed; + return appId + encodedFileSeed; }).pipe(Micro.withTrace("generateKey")); -// Verify that the key was generated with the same appId -export const verifyKey = (key: string, appId: string) => - Micro.sync(() => { - const alphabet = shuffle(defaultOptions.alphabet, appId); - const expectedPrefix = new SQIds({ alphabet, minLength: 12 }).encode([ - Math.abs(Hash.string(appId)), - ]); - - return key.startsWith(expectedPrefix); - }).pipe( - Micro.withTrace("verifyKey"), - Micro.orElseSucceed(() => false), - ); - export const generateSignedURL = ( url: string | URL, secretKey: Redacted.Redacted, diff --git a/packages/shared/test/crypto.test.ts b/packages/shared/test/crypto.test.ts index 02ff9a6945..4be760e0b8 100644 --- a/packages/shared/test/crypto.test.ts +++ b/packages/shared/test/crypto.test.ts @@ -7,7 +7,6 @@ import { generateKey, generateSignedURL, signPayload, - verifyKey, verifySignature, } from "../src/crypto"; @@ -106,96 +105,4 @@ describe("key gen", () => { expect(key).toBeTruthy(); }), ); - - it.effect("verifies a key", () => - Effect.gen(function* () { - const appI = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appI, - ); - - const verified = yield* verifyKey(key, appI); - expect(verified).toBe(true); - }), - ); - - it.effect("doesn't verify a key with a bad appI", () => - Effect.gen(function* () { - const appI = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appI, - ); - - const verified = yield* verifyKey(key, "bad"); - expect(verified).toBe(false); - }), - ); - - it.effect("doesn't verify a key with a bad key", () => - Effect.gen(function* () { - const appId = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appId, - ); - - const verified = yield* verifyKey("badseed" + key.substring(7), appId); - expect(verified).toBe(false); - }), - ); - - it.effect("verifies with a custom hash function", () => - Effect.gen(function* () { - const appI = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appI, - (file) => [file.name], - ); - - const verified = yield* verifyKey(key, appI); - expect(verified).toBe(true); - }), - ); - - it.effect("works even when there's nothing to seed", () => - Effect.gen(function* () { - const appI = "foo-123"; - const key = yield* generateKey( - { - name: "foo.txt", - size: 123, - type: "text/plain", - lastModified: Date.now(), - }, - appI, - () => [], - ); - - const verified = yield* verifyKey(key, appI); - expect(verified).toBe(true); - }), - ); });