From a65dee0f538c9fe48da11d9070536fef07856e60 Mon Sep 17 00:00:00 2001 From: CultPodcastsBot <142722442+cultpodcasts@users.noreply.github.com> Date: Thu, 9 May 2024 22:06:39 +0100 Subject: [PATCH] Use submit endpoint when authorised (#22) * Use submit endpoint when authorised * Added prisma * Deploy from branch * Null check on member before accessing * Improved handling for no token * Fallback to non-secure endpoint on failure * Remove unused variable * Corrected log message * Fixed syntax * Read body once * Carry over auth header * log some of auth header * Change header name * Fix body on secure submit * Add accept header * add user-agent * Added Host header * Fix typo in header --------- Co-authored-by: Jon Breen --- .env | 7 ++ .github/workflows/buildAndDeploy.yml | 8 +- migrations/0001_init.sql | 13 +++ package-lock.json | 107 +++++++++++++++++- package.json | 7 +- prisma/schema.prisma | 22 ++++ src/Auth0JwtPayload.ts | 7 ++ src/index.ts | 156 ++++++++++++++++----------- wrangler.toml | 8 +- 9 files changed, 260 insertions(+), 75 deletions(-) create mode 100644 .env create mode 100644 migrations/0001_init.sql create mode 100644 prisma/schema.prisma create mode 100644 src/Auth0JwtPayload.ts diff --git a/.env b/.env new file mode 100644 index 0000000..c4a55b5 --- /dev/null +++ b/.env @@ -0,0 +1,7 @@ +# Environment variables declared in this file are automatically made available to Prisma. +# See the documentation for more detail: https://pris.ly/d/prisma-schema#accessing-environment-variables-from-the-schema + +# Prisma supports the native connection string format for PostgreSQL, MySQL, SQLite, SQL Server, MongoDB and CockroachDB. +# See the documentation for all the connection string options: https://pris.ly/d/connection-strings + +DATABASE_URL=./.wrangler/state/d1/DB.sqlite3 \ No newline at end of file diff --git a/.github/workflows/buildAndDeploy.yml b/.github/workflows/buildAndDeploy.yml index 9765771..fe9ed43 100644 --- a/.github/workflows/buildAndDeploy.yml +++ b/.github/workflows/buildAndDeploy.yml @@ -3,14 +3,16 @@ on: push: branches: - main - - feature/authorization-header - - feature/new-router + - feature/direct-submit jobs: deploy: runs-on: ubuntu-latest name: Deploy steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 + - name: Build + run: npx prisma generate - name: Publish uses: cloudflare/wrangler-action@v3 with: diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..9da5248 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,13 @@ +-- Migration number: 0001 2024-05-09T09:28:45.679Z + +-- CreateTable +CREATE TABLE "Submissions" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "url" TEXT NOT NULL, + "state" INTEGER NOT NULL DEFAULT 0, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "ip_address" TEXT, + "country" TEXT, + "user_agent" TEXT +); + diff --git a/package-lock.json b/package-lock.json index ee92aa1..5ff9e58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,12 +9,15 @@ "version": "0.0.0", "dependencies": { "@cfworker/jwt": "^4.0.6", - "hono": "^4.2.9" + "@prisma/adapter-d1": "^5.13.0", + "@prisma/client": "^5.13.0", + "hono": "^4.2.9", + "prisma": "^5.13.0" }, "devDependencies": { "@cloudflare/workers-types": "^4.20240423.0", "typescript": "^5.0.4", - "wrangler": "^3.47.0" + "wrangler": "^3.53.1" } }, "node_modules/@cfworker/jwt": { @@ -555,6 +558,85 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@prisma/adapter-d1": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@prisma/adapter-d1/-/adapter-d1-5.13.0.tgz", + "integrity": "sha512-1VuQZyA0bEIEHv3joEsGi/7+XBsZIQSubTDhagOkmeTHKME/eXP8S5r7BRPyGcNk4Yp7WRuLuIMFM1LgGXytLQ==", + "dependencies": { + "@cloudflare/workers-types": "4.20240405.0", + "@prisma/driver-adapter-utils": "5.13.0" + } + }, + "node_modules/@prisma/adapter-d1/node_modules/@cloudflare/workers-types": { + "version": "4.20240405.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20240405.0.tgz", + "integrity": "sha512-sEVOhyOgXUwfLkgHqbLZa/sfkSYrh7/zLmI6EZNibPaVPvAnAcItbNNl3SAlLyLKuwf8m4wAIAgu9meKWCvXjg==" + }, + "node_modules/@prisma/client": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.13.0.tgz", + "integrity": "sha512-uYdfpPncbZ/syJyiYBwGZS8Gt1PTNoErNYMuqHDa2r30rNSFtgTA/LXsSk55R7pdRTMi5pHkeP9B14K6nHmwkg==", + "hasInstallScript": true, + "engines": { + "node": ">=16.13" + }, + "peerDependencies": { + "prisma": "*" + }, + "peerDependenciesMeta": { + "prisma": { + "optional": true + } + } + }, + "node_modules/@prisma/debug": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.13.0.tgz", + "integrity": "sha512-699iqlEvzyCj9ETrXhs8o8wQc/eVW+FigSsHpiskSFydhjVuwTJEfj/nIYqTaWFYuxiWQRfm3r01meuW97SZaQ==" + }, + "node_modules/@prisma/driver-adapter-utils": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@prisma/driver-adapter-utils/-/driver-adapter-utils-5.13.0.tgz", + "integrity": "sha512-SaimwvGuvXJTsWH+FOfl7PlkZZlPiRGeMEchC0kvB08Xw9B3CLYo/Q7zaY5Fp+u5D8asrpcFQmCjXEgDvOmPkg==", + "dependencies": { + "@prisma/debug": "5.13.0" + } + }, + "node_modules/@prisma/engines": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.13.0.tgz", + "integrity": "sha512-hIFLm4H1boj6CBZx55P4xKby9jgDTeDG0Jj3iXtwaaHmlD5JmiDkZhh8+DYWkTGchu+rRF36AVROLnk0oaqhHw==", + "hasInstallScript": true, + "dependencies": { + "@prisma/debug": "5.13.0", + "@prisma/engines-version": "5.13.0-23.b9a39a7ee606c28e3455d0fd60e78c3ba82b1a2b", + "@prisma/fetch-engine": "5.13.0", + "@prisma/get-platform": "5.13.0" + } + }, + "node_modules/@prisma/engines-version": { + "version": "5.13.0-23.b9a39a7ee606c28e3455d0fd60e78c3ba82b1a2b", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.13.0-23.b9a39a7ee606c28e3455d0fd60e78c3ba82b1a2b.tgz", + "integrity": "sha512-AyUuhahTINGn8auyqYdmxsN+qn0mw3eg+uhkp8zwknXYIqoT3bChG4RqNY/nfDkPvzWAPBa9mrDyBeOnWSgO6A==" + }, + "node_modules/@prisma/fetch-engine": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.13.0.tgz", + "integrity": "sha512-Yh4W+t6YKyqgcSEB3odBXt7QyVSm0OQlBSldQF2SNXtmOgMX8D7PF/fvH6E6qBCpjB/yeJLy/FfwfFijoHI6sA==", + "dependencies": { + "@prisma/debug": "5.13.0", + "@prisma/engines-version": "5.13.0-23.b9a39a7ee606c28e3455d0fd60e78c3ba82b1a2b", + "@prisma/get-platform": "5.13.0" + } + }, + "node_modules/@prisma/get-platform": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.13.0.tgz", + "integrity": "sha512-B/WrQwYTzwr7qCLifQzYOmQhZcFmIFhR81xC45gweInSUn2hTEbfKUPd2keAog+y5WI5xLAFNJ3wkXplvSVkSw==", + "dependencies": { + "@prisma/debug": "5.13.0" + } + }, "node_modules/acorn": { "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", @@ -1026,6 +1108,21 @@ "integrity": "sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==", "dev": true }, + "node_modules/prisma": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.13.0.tgz", + "integrity": "sha512-kGtcJaElNRAdAGsCNykFSZ7dBKpL14Cbs+VaQ8cECxQlRPDjBlMHNFYeYt0SKovAVy2Y65JXQwB3A5+zIQwnTg==", + "hasInstallScript": true, + "dependencies": { + "@prisma/engines": "5.13.0" + }, + "bin": { + "prisma": "build/index.js" + }, + "engines": { + "node": ">=16.13" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -1223,9 +1320,9 @@ } }, "node_modules/wrangler": { - "version": "3.53.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.53.0.tgz", - "integrity": "sha512-JxkvCQekL9j8Mu4CEKM/HEVyDnymWzKQuMUuJH0yum1AilutD5HAP9kVVYmvu7BvwlRyRUAj8TI5OUxXnLCEpQ==", + "version": "3.53.1", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.53.1.tgz", + "integrity": "sha512-bdMRQdHYdvowIwOhEMFkARIZUh56aDw7HLUZ/2JreBjj760osXE4Fc4L1TCkfRRBWgB6/LKF5LA4OcvORMYmHg==", "dev": true, "dependencies": { "@cloudflare/kv-asset-handler": "0.3.2", diff --git a/package.json b/package.json index 3142081..09522bb 100644 --- a/package.json +++ b/package.json @@ -9,10 +9,13 @@ "devDependencies": { "@cloudflare/workers-types": "^4.20240423.0", "typescript": "^5.0.4", - "wrangler": "^3.47.0" + "wrangler": "^3.53.1" }, "dependencies": { "@cfworker/jwt": "^4.0.6", - "hono": "^4.2.9" + "@prisma/adapter-d1": "^5.13.0", + "@prisma/client": "^5.13.0", + "hono": "^4.2.9", + "prisma": "^5.13.0" } } diff --git a/prisma/schema.prisma b/prisma/schema.prisma new file mode 100644 index 0000000..28626f9 --- /dev/null +++ b/prisma/schema.prisma @@ -0,0 +1,22 @@ +// This is your Prisma schema file, +// learn more about it in the docs: https://pris.ly/d/prisma-schema + +generator client { + provider = "prisma-client-js" + previewFeatures = ["driverAdapters"] +} + +datasource db { + provider = "sqlite" + url = env("DATABASE_URL") +} + +model Submissions { + id Int @id @default(autoincrement()) + url String + state Int @default(0) + createdAt DateTime @default(now()) + ip_address String? + country String? + user_agent String? +} diff --git a/src/Auth0JwtPayload.ts b/src/Auth0JwtPayload.ts new file mode 100644 index 0000000..680dabb --- /dev/null +++ b/src/Auth0JwtPayload.ts @@ -0,0 +1,7 @@ +import { JwtPayload } from '@cfworker/jwt'; + +export interface Auth0JwtPayload extends JwtPayload { + azp: string; + scope: string; + permissions: string[]; +} diff --git a/src/index.ts b/src/index.ts index 0940aa7..c75f903 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,18 +3,22 @@ import { Hono } from 'hono'; import { cors } from 'hono/cors' import { stream } from 'hono/streaming' import { createMiddleware } from 'hono/factory' -import { Bindings } from 'hono/types'; +import { Auth0JwtPayload } from './Auth0JwtPayload'; +import { Prisma, PrismaClient } from '@prisma/client' +import { PrismaD1 } from "@prisma/adapter-d1"; type Env = { Content: R2Bucket; Data: R2Bucket; Analytics: AnalyticsEngineDataset; DB: D1Database; + apiDB: D1Database; apikey: string; apihost: string; gatewayKey: string; auth0Issuer: string; auth0Audience: string; + secureSubmitEndpoint: string; } const allowedOrigins: Array = [ @@ -42,12 +46,12 @@ const auth0Middleware = createMiddleware<{ }>(async (c, next) => { const authorization = c.req.header('Authorization'); const bearer = "Bearer "; - c.set('auth0', (payload) => {}) + c.set('auth0', (payload) => { }) if (authorization && authorization.startsWith(bearer)) { const token = authorization.slice(bearer.length); const result = await parseJwt(token, c.env.auth0Issuer, c.env.auth0Audience); if (result.valid) { - c.set('auth0', (payload) => result.payload) + c.set('auth0', (payload) => result.payload as Auth0JwtPayload) } else { console.log(result.reason); } @@ -234,79 +238,109 @@ app.post("/search", async (c) => { } }); -app.get("/submit", async (c) => { - if (c.req.header("key") != c.env.gatewayKey) { - return c.json({ message: "Unauthorised" }, 401); - } - let submissionIds = c.env.DB - .prepare("SELECT id FROM urls WHERE state=0"); - let result = await submissionIds.all(); - if (result.success) { - const inClause = result.results - .map((urlId) => { - if (!Number.isInteger(urlId.id)) { throw Error("invalid id, expected an integer"); } - return urlId.id; +app.get("/submit", auth0Middleware, async (c) => { + const adapter = new PrismaD1(c.env.apiDB); + const prisma = new PrismaClient({ adapter }); + const auth0Payload: Auth0JwtPayload = c.var.auth0('payload'); + + if (auth0Payload?.permissions && auth0Payload.permissions.includes('submit')) { + try { + const submissionIds = await prisma.submissions.findMany({ + where: { + state: 0 + }, + select: { + id: true + } }) - .join(','); - let urls = "SELECT id, url, timestamp_date, ip_address, country, user_agent FROM urls WHERE id IN ($urlIds)"; - urls = urls.replace('$urlIds', inClause); - let urlResults = await c.env.DB - .prepare(urls) - .run(); - if (urlResults.success) { - let update = "UPDATE urls SET state=1 WHERE id IN ($urlIds)"; - update = update.replace('$urlIds', inClause); - let raiseState = await c.env.DB - .prepare(update) - .all(); - if (raiseState.success) { - return c.json(urlResults); - } else { - return c.text("Failure to raise state of new submissons in ids " + result.results.join(", "), 400); + const urlResults = await prisma.submissions.findMany({ + where: { + id: { in: submissionIds.map((record) => record.id) }, + } + }) + const updates = await prisma.submissions.updateMany({ + where: { + id: { in: submissionIds.map((record) => record.id) } + }, + data: { + state: 1, + }, + }); + return c.json(urlResults); + } catch (e) { + if (e instanceof Prisma.PrismaClientKnownRequestError) { + console.log(`PrismaClientKnownRequestError code: '${e.code}'`, e); } - } else { - return c.text("Unable to retrieve new submissions", 500); + return c.json({ error: "Unable to accept" }, 400); } } else { - return c.text(result.error!, 500); + return c.json({ message: "Unauthorised" }, 401); } }); app.post("/submit", auth0Middleware, async (c) => { - const auth0Payload = c.var.auth0('payload'); + const auth0Payload: Auth0JwtPayload = c.var.auth0('payload'); c.header("Cache-Control", "max-age=600"); c.header("Content-Type", "application/json"); c.header("Access-Control-Allow-Origin", getOrigin(c.req.header("Origin"))); c.header("Access-Control-Allow-Methods", "POST,GET,OPTIONS"); - return c.req - .json() - .then(async (data: any) => { - let url: URL | undefined; - let urlParam = data.url; - if (urlParam == null) { - return c.json({ error: "Missing url param." }, 400); - } - try { - url = new URL(urlParam); - } catch { - return c.json({ error: `Invalid url '${data.url}'.` }, 400); - } - let insert = c.env.DB - .prepare("INSERT INTO urls (url, timestamp, timestamp_date, ip_address, country, user_agent) VALUES (?, ?, ?, ?, ?, ?)") - .bind(url.toString(), Date.now(), new Date().toLocaleString(), c.req.header("CF-Connecting-IP"), c.req.header("CF-IPCountry"), c.req.header("User-Agent")); - let result = await insert.run(); - - if (result.success) { - return c.json({ success: "Submitted" }); - } else { - return c.json({ error: "Unable to accept" }, 400); - } + const data = await c.req.json(); + if (auth0Payload?.permissions && auth0Payload.permissions.includes('submit')) { + const authorisation: string = c.req.header("Authorization")!; + console.log(`Using auth header '${authorisation.slice(0, 20)}..'`); + const resp = await fetch(c.env.secureSubmitEndpoint, { + headers: { + 'Accept': "*/*", + 'Authorization': authorisation, + "Content-type": "application/json", + "Cache-Control": "no-cache", + "User-Agent": "cultvault-podcasts-api", + "Host": new URL(c.env.secureSubmitEndpoint).host + }, + body: JSON.stringify(data), + method: "POST" }); -}); - -export default app; + if (resp.status == 200) { + console.log(`Successfully used secure enpoint.`); + return c.json({ success: "Submitted" }); + } else { + console.log(`Failed to use secure submit endpoint. Response code: '${resp.status}'.`); + } + } + console.log(`Storing submission in d1.`); + const adapter = new PrismaD1(c.env.apiDB); + const prisma = new PrismaClient({ adapter }); + let url: URL | undefined; + let urlParam = data.url; + if (urlParam == null) { + return c.json({ error: "Missing url param." }, 400); + } + try { + url = new URL(urlParam); + } catch { + return c.json({ error: `Invalid url '${data.url}'.` }, 400); + } + try { + const record = { + url: url.toString(), + ip_address: c.req.header("CF-Connecting-IP") ?? null, + user_agent: c.req.header("User-Agent") ?? null, + country: c.req.header("CF-IPCountry") ?? null + }; + const submission = await prisma.submissions.create({ + data: record + }); + } catch (e) { + if (e instanceof Prisma.PrismaClientKnownRequestError) { + console.log(`PrismaClientKnownRequestError code: '${e.code}'`, e); + } + return c.json({ error: "Unable to accept" }, 400); + } + return c.json({ success: "Submitted" }); +}); +export default app; diff --git a/wrangler.toml b/wrangler.toml index e55680d..ec5c4b6 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -7,10 +7,10 @@ r2_buckets = [ { binding = "Data", bucket_name = "data", preview_bucket_name = "data" } ] -[[d1_databases]] -binding = "DB" -database_name = "submissions" -database_id = "6d00eb71-b420-47f7-9e74-96d53bcb943a" +d1_databases= [ + {binding="DB", database_name = "submissions", database_id = "6d00eb71-b420-47f7-9e74-96d53bcb943a"}, + {binding = "apiDB", database_name = "cultpodcasts-api", database_id = "17d94e5b-3bd9-4cec-95b7-cf418fe8c870"} +] [[analytics_engine_datasets]] binding = "Analytics"