From fe1b63424d363193039cfead42c925a425a08931 Mon Sep 17 00:00:00 2001 From: Adam Janis Date: Wed, 18 Nov 2020 21:01:56 +0100 Subject: [PATCH] feat: move gc monitors from cron schedule to deploy postCommands --- .github/workflows/deploy.yml | 6 ++- README.md | 19 +++++++-- package.json | 4 +- src/cli/gcMonitors.js | 74 ++++++++++++++++++++++++++++++++++++ src/functions/cronTrigger.js | 4 -- src/functions/helpers.js | 20 ---------- yarn.lock | 13 +++++-- 7 files changed, 105 insertions(+), 35 deletions(-) create mode 100644 src/cli/gcMonitors.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d57bfc887..15bb8f87f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,10 +24,12 @@ jobs: preCommands: | wrangler kv:namespace create KV_STATUS_PAGE apt-get update && apt-get install -y jq - kv_namespace_id=$(wrangler kv:namespace list | jq -c 'map(select(.title | contains("KV_STATUS_PAGE")))' | jq ".[0].id") + export KV_NAMESPACE_ID=$(wrangler kv:namespace list | jq -c 'map(select(.title | contains("KV_STATUS_PAGE")))' | jq -r ".[0].id") echo "[env.production]" >> wrangler.toml - echo "kv_namespaces = [{binding=\"KV_STATUS_PAGE\", id=${kv_namespace_id}}]" >> wrangler.toml + echo "kv_namespaces = [{binding=\"KV_STATUS_PAGE\", id=\"${KV_NAMESPACE_ID}\"}]" >> wrangler.toml [ -z "$SECRET_SLACK_WEBHOOK_URL" ] && echo "Secret SECRET_SLACK_WEBHOOK_URL not set, creating dummy one..." && SECRET_SLACK_WEBHOOK_URL="default-gh-action-secret" || true + postCommands: | + yarn kv-gc secrets: | SECRET_SLACK_WEBHOOK_URL environment: production diff --git a/README.md b/README.md index 0e1e8ef9d..f5cbfc579 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ You'll need a [Cloudflare Workers account](https://dash.cloudflare.com/sign-up/w * A workers domain set up * The Workers Bundled subscription \($5/mo\) - * [Try it now with the free tier!](https://blog.cloudflare.com/workers-kv-free-tier/) Stay tuned while we make some changes so it will completely fit in the Free Tier with a 5min check interval. + * [Try it now with the free tier!](https://blog.cloudflare.com/workers-kv-free-tier/) Check [more info](#workers-kv-free-tier) on how to run on Workers Free. * Some websites/APIs to watch 🙂 Also, prepare the following secrets @@ -40,7 +40,7 @@ You can either deploy with **Cloudflare Deploy Button** using GitHub Actions or - Value: your-slack-webhook-url ``` 3. Navigate to the **Actions** settings in your repository and enable them -4. Edit [config.yaml](https://github.com/eidam/cf-workers-status-page/blob/main/config.yaml) to adjust configuration and list all of your websites/APIs you want to monitor +4. Edit [config.yaml](./config.yaml) to adjust configuration and list all of your websites/APIs you want to monitor ```yaml settings: @@ -74,16 +74,20 @@ You can either deploy with **Cloudflare Deploy Button** using GitHub Actions or 6. 🎉 7. _\(optional\)_ Go to [Cloudflare Workers settings](https://dash.cloudflare.com/?to=/workers) and assign custom domain/route * e.g. `status-page.eidam.dev/*` _\(make sure you include `/*` as the Worker also serve static files\)_ -8. _\(optional\)_ Edit [wrangler.toml](https://github.com/eidam/cf-workers-github-releases/blob/main/wrangler.toml) to adjust Worker settings or CRON Trigger schedule +8. _\(optional\)_ Edit [wrangler.toml](./wrangler.toml) to adjust Worker settings or CRON Trigger schedule, especially if you are on [Workers Free plan](#workers-kv-free-tier) ### Deploy on your own You can clone the repository yourself and use Wrangler CLI to develop/deploy, extra list of things you need to take care of: -* create KV namespace and add the `KV_STATUS_PAGE` binding to [wrangler.toml](https://github.com/eidam/cf-workers-github-releases/blob/main/wrangler.toml) +* create KV namespace and add the `KV_STATUS_PAGE` binding to [wrangler.toml](./wrangler.toml) * create Worker secrets _\(optional\)_ * `SECRET_SLACK_WEBHOOK_URL` +## Workers KV free tier +The Workers Free plan includes limited KV usage, in order to not deplete the quota and still have enough room for monitor status changes we recommend the following changes: +* Change the CRON trigger to 5 minutes interval (`crons = ["*/5 * * * *"]`) in [wrangler.toml](./wrangler.toml) + ## Known issues * **Max 25 monitors to watch in case you are using Slack notifications**, due to the limit of subrequests Cloudflare Worker can make \(50\). @@ -93,3 +97,10 @@ You can clone the repository yourself and use Wrangler CLI to develop/deploy, ex * **KV replication lag** - You might get Slack notification instantly, however it may take couple of more seconds to see the change on your status page as [Cron Triggers are usually running on underutilized quiet hours machines](https://blog.cloudflare.com/introducing-cron-triggers-for-cloudflare-workers/#how-are-you-able-to-offer-this-feature-at-no-additional-cost). * **Initial delay (no data)** - It takes couple of minutes to schedule and run CRON Triggers for the first time + +* **Slack message for monitor just removed from the config** - It takes a couple of minutes to schedule new version of CRON Triggers, so older version of the configuration might be scheduled after deployment/gc - just ignore it or re-run the latest Github Action after a couple of minutes again. + +## Future plans +Stay tuned for more features coming in, like leveraging the fact that CRON instances are scheduled around the world during the day + so we can monitor the response times. However, we will most probably wait for the [Durable Objects](https://blog.cloudflare.com/introducing-workers-durable-objects/) to be in open beta + as they are better fit to reliably store such info. diff --git a/package.json b/package.json index 612caf2ed..675c23803 100644 --- a/package.json +++ b/package.json @@ -9,16 +9,18 @@ "dev": "flareact dev", "build": "flareact build", "deploy": "flareact publish", + "kv-gc": "node ./src/cli/gcMonitors.js", "format": "prettier --write '**/*.{js,css,json,md}'" }, "dependencies": { - "flareact": "^0.8.0", + "flareact": "^0.9.0", "laco": "^1.2.1", "laco-react": "^1.1.0", "react": "^16.13.1", "react-dom": "^16.13.1" }, "devDependencies": { + "node-fetch": "^2.6.1", "prettier": "^1.18.2", "yaml-loader": "^0.6.0" } diff --git a/src/cli/gcMonitors.js b/src/cli/gcMonitors.js new file mode 100644 index 000000000..c60d8570e --- /dev/null +++ b/src/cli/gcMonitors.js @@ -0,0 +1,74 @@ +const yaml = require('yaml-loader') +const fetch = require('node-fetch') +const fs = require('fs') + +const accountId = process.env.CF_ACCOUNT_ID +const namespaceId = process.env.KV_NAMESPACE_ID +const apiToken = process.env.CF_API_TOKEN + +const kvPrefix = 's_' + +if (!accountId || !namespaceId || !apiToken) { + console.error("Missing required environment variables: CF_ACCOUNT_ID, KV_NAMESPACE_ID, CF_API_TOKEN") + process.exit(0) +} + +async function getKvMonitors(kvPrefix) { + const init = { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiToken}`, + }, + } + + const res = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/keys?limit=100&prefix=${kvPrefix}`, + init, + ) + const json = await res.json() + return json.result +} + +async function deleteKvBulk(keys) { + const init = { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiToken}`, + }, + method: 'DELETE', + body: JSON.stringify(keys), + } + + return await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/bulk`, + init, + ) +} + +function loadConfig() { + const configFile = fs.readFileSync('./config.yaml', 'utf8') + const config = yaml(configFile) + return JSON.parse(config) +} + +getKvMonitors(kvPrefix).then(async kvMonitors => { + const config = loadConfig() + const monitors = config.monitors.map(key => { + return key.id + }) + const kvState = kvMonitors.map(key => { + return key.name + }) + const keysForRemoval = kvState.filter( + x => !monitors.includes(x.replace(kvPrefix, '')), + ) + + if (keysForRemoval.length > 0) { + console.log( + `Removing following keys from KV storage as they are no longer in the config: ${keysForRemoval.join( + ', ', + )}`, + ) + await deleteKvBulk(keysForRemoval) + } +}).catch(e => console.log(e)) diff --git a/src/functions/cronTrigger.js b/src/functions/cronTrigger.js index da7082cd9..0aadc3607 100644 --- a/src/functions/cronTrigger.js +++ b/src/functions/cronTrigger.js @@ -3,7 +3,6 @@ import config from '../../config.yaml' import { setKV, getKVWithMetadata, - gcMonitors, getKV, notifySlack, } from './helpers' @@ -70,8 +69,5 @@ export async function processCronTrigger(event) { const loc = res.headers.get('cf-ray').split('-')[1] await setKV('lastUpdate', Date.now(), { loc }) - // gc monitor statuses - event.waitUntil(gcMonitors(config)) - return new Response('OK') } diff --git a/src/functions/helpers.js b/src/functions/helpers.js index 3b5cab4df..7df2a20b4 100644 --- a/src/functions/helpers.js +++ b/src/functions/helpers.js @@ -56,26 +56,6 @@ export async function deleteKV(key) { return KV_STATUS_PAGE.delete(key) } -export async function gcMonitors(config) { - const checkKvPrefix = 's_' - - const monitors = config.monitors.map(key => { - return key.id - }) - - const kvMonitors = await listKV(checkKvPrefix) - const kvState = kvMonitors.keys.map(key => { - return key.metadata.id - }) - - const keysForRemoval = kvState.filter(x => !monitors.includes(x)) - - for (const key of keysForRemoval) { - console.log('gc: deleting ' + checkKvPrefix + key) - await deleteKV(checkKvPrefix + key) - } -} - export async function notifySlack(monitor, newMetadata) { const payload = { attachments: [ diff --git a/yarn.lock b/yarn.lock index 332a6ee39..aa1149d76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2947,10 +2947,10 @@ findup-sync@^3.0.0: micromatch "^3.0.4" resolve-dir "^1.0.1" -flareact@^0.8.0: - version "0.8.0" - resolved "https://registry.yarnpkg.com/flareact/-/flareact-0.8.0.tgz#c7653f7278abee04353c5bf087af4105b7895c0a" - integrity sha512-ewjFqrxSXPBppyZtVBTD3W4iIcMB2wORpYX1QofOa9QRuy7dJ2nK4oMvCNmPJPelqdN5MF+fUBTIfMGqO3A1OA== +flareact@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/flareact/-/flareact-0.9.0.tgz#c16ded48f217010452a509e02b754f84eb26878c" + integrity sha512-YT1nGqusHTJDreU5gQezKQNU2Pszez+M3v5IrKIEtOD3ABQal+cVoWzRQGQTWMKryrUpWB0Z0nRhLYDutD8xdQ== dependencies: "@babel/core" "^7.11.0" "@babel/plugin-transform-runtime" "^7.11.0" @@ -4354,6 +4354,11 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +node-fetch@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" + integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== + node-forge@^0.10.0: version "0.10.0" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3"