Merge branch 'main' into richard/add-outline-generation

.github/ISSUE_TEMPLATE/ossgg_sidequest.yml

@@ -1,7 +1,11 @@
 name: oss.gg hack submission 🕹️
 description: "Submit your contribution for the for the oss.gg hackathon"
 title: "[oss.gg hackathon]"
-labels: 🕹️ oss.gg, player submission
+labels: 
+  - "🕹️ oss.gg"
+  - "Needs Approval"
+  - "player submission"
+
 assignees: []
 body:
   - type: textarea

.github/workflows/changesets.yaml

@@ -26,7 +26,7 @@ jobs:
 
       - name: Create Version PR or Publish to NPM
         id: changesets
-        uses: changesets/[email protected].8
+        uses: changesets/[email protected].9
         with:
           commit: "chore(release): version packages"
           title: "chore(release): version packages"

.github/workflows/pr-alerts-campsite.yml

@@ -0,0 +1,53 @@
+name: Campsite PR Alerts
+
+on:
+  pull_request:
+    types: [opened, closed, merged, ready_for_review, reopened]
+
+jobs:
+  post_to_campsite:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Post to Campsite
+        id: post_to_campsite
+        run: |
+          if [[ "${{ github.event.pull_request.draft }}" == "true" ]]; then
+            STATUS_EMOJI="⚪"
+          elif [[ "${{ github.event.action }}" == "opened" || "${{ github.event.action }}" == "reopened" || "${{ github.event.action }}" == "ready_for_review" ]]; then
+            STATUS_EMOJI="🟢"
+          elif [[ "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then
+            STATUS_EMOJI="🟣"
+          elif [[ "${{ github.event.action }}" == "closed" ]]; then
+            STATUS_EMOJI="🔴"
+          fi
+
+          ACTION=${{ github.event.action }}
+          if [[ "${ACTION}" == "ready_for_review" ]]; then
+            ACTION="ready for review"
+          elif [[ "${ACTION}" == "reopened" ]]; then
+            ACTION="reopened"
+          elif [[ "${ACTION}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then
+            ACTION="merged"
+          fi
+
+          ESCAPED_TITLE=$(echo "${{ github.event.pull_request.title }}" | jq -Rr @json)
+
+          CONTENT="${STATUS_EMOJI} Pull request ${ACTION} by ${{ github.event.pull_request.user.login }}: [#${{ github.event.pull_request.number }} ${ESCAPED_TITLE}](${{ github.event.pull_request.html_url }})"
+
+          echo "content=${CONTENT}" >> $GITHUB_OUTPUT
+      - name: Create Campsite message
+        uses: campsite/campsite-github-action@v1
+        with:
+          api_key: ${{ secrets.CAMPSITE_API_KEY }}
+          action_type: create_message
+          thread_id: ${{ secrets.CAMPSITE_PR_ALERTS_THREAD_ID }}
+          content: ${{ steps.post_to_campsite.outputs.content }}

.gitignore

@@ -19,4 +19,7 @@ dist
 .dev.vars
 .wrangler
 .vitest
-.react-email
+.react-email
+
+.secrets.json
+secrets.ts

Taskfile.yml

@@ -23,7 +23,7 @@ tasks:
     env:
       GOOSE_DRIVER: clickhouse
       GOOSE_DBSTRING: "tcp://default:[email protected]:9000"
-      GOOSE_MIGRATION_DIR: ./apps/agent/pkg/clickhouse/schema
+      GOOSE_MIGRATION_DIR: ./internal/clickhouse/schema
     cmds:
       - goose up
 

apps/agent/integration/identities/ratelimits_with_cost_load_test.go

@@ -0,0 +1,146 @@
+package identities_test
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+	unkey "github.com/unkeyed/unkey-go"
+	"github.com/unkeyed/unkey-go/models/components"
+	"github.com/unkeyed/unkey-go/models/operations"
+	attack "github.com/unkeyed/unkey/apps/agent/pkg/testutil"
+	"github.com/unkeyed/unkey/apps/agent/pkg/uid"
+	"github.com/unkeyed/unkey/apps/agent/pkg/util"
+)
+
+func TestIdentityRatelimitsWithCost0Accuracy(t *testing.T) {
+	// Step 1 --------------------------------------------------------------------
+	// Setup the sdk, create an API
+	// ---------------------------------------------------------------------------
+
+	ctx := context.Background()
+	rootKey := os.Getenv("INTEGRATION_TEST_ROOT_KEY")
+	require.NotEmpty(t, rootKey, "INTEGRATION_TEST_ROOT_KEY must be set")
+	baseURL := os.Getenv("UNKEY_BASE_URL")
+	require.NotEmpty(t, baseURL, "UNKEY_BASE_URL must be set")
+
+	options := []unkey.SDKOption{
+		unkey.WithSecurity(rootKey),
+	}
+
+	if baseURL != "" {
+		options = append(options, unkey.WithServerURL(baseURL))
+	}
+	sdk := unkey.New(options...)
+
+	for _, tc := range []struct {
+		rate         attack.Rate
+		testDuration time.Duration
+	}{
+		{
+			rate:         attack.Rate{Freq: 100, Per: time.Second},
+			testDuration: 1 * time.Minute,
+		},
+		{
+			rate:         attack.Rate{Freq: 100, Per: time.Second},
+			testDuration: 5 * time.Minute,
+		},
+		{
+			rate:         attack.Rate{Freq: 100, Per: time.Second},
+			testDuration: 30 * time.Minute,
+		},
+	} {
+		t.Run(fmt.Sprintf("[%s] over %s", tc.rate.String(), tc.testDuration), func(t *testing.T) {
+			api, err := sdk.Apis.CreateAPI(ctx, operations.CreateAPIRequestBody{
+				Name: uid.New("testapi"),
+			})
+			require.NoError(t, err)
+
+			// Step 2 --------------------------------------------------------------------
+			// Create the identity with ratelimits
+			// ---------------------------------------------------------------------------
+
+			ratelimit := operations.Ratelimits{
+				Name:     "ratelimit-a",
+				Limit:    600,
+				Duration: time.Minute.Milliseconds(),
+			}
+
+			externalID := uuid.NewString()
+			_, err = sdk.Identities.CreateIdentity(ctx, operations.CreateIdentityRequestBody{
+				ExternalID: externalID,
+				Ratelimits: []operations.Ratelimits{
+					ratelimit,
+				},
+			})
+			require.NoError(t, err)
+
+			// Step 3 --------------------------------------------------------------------
+			// Create key for this identity
+			// ---------------------------------------------------------------------------
+
+			key, err := sdk.Keys.CreateKey(ctx, operations.CreateKeyRequestBody{
+				APIID:      api.Object.APIID,
+				ExternalID: util.Pointer(externalID),
+			})
+			require.NoError(t, err)
+
+			// Step 5 --------------------------------------------------------------------
+			// Test ratelimits
+			// ---------------------------------------------------------------------------
+
+			total := 0
+			passed := 0
+			withCost := 0
+			errors := 0
+
+			results := attack.Attack(t, tc.rate, tc.testDuration, func() bool {
+
+				cost := int64(0)
+				if rand.Intn(100) == 0 {
+					withCost++
+					cost = 1
+				}
+
+				res, err := sdk.Keys.VerifyKey(context.Background(), components.V1KeysVerifyKeyRequest{
+					APIID: unkey.String(api.Object.APIID),
+					Key:   key.Object.Key,
+					Ratelimits: []components.Ratelimits{
+						{Name: ratelimit.Name,
+							Cost: util.Pointer(cost),
+						},
+					},
+				})
+				if err != nil {
+					errors++
+					return false
+				}
+
+				return res.V1KeysVerifyKeyResponse.Valid
+
+			})
+
+			for valid := range results {
+				total++
+				if valid {
+					passed++
+				}
+
+			}
+
+			// Step 6 --------------------------------------------------------------------
+			// Assert ratelimits worked
+			// ---------------------------------------------------------------------------
+
+			t.Logf("Total: %d, Passed: %d, withCost=1: %d", total, passed, withCost)
+
+			// check requests::api is not exceeded
+		})
+
+	}
+}

apps/api/CHANGELOG.md

@@ -1,5 +1,11 @@
 # api
 
+## 0.1.0
+
+### Minor Changes
+
+- 09d36ad: add /v1/keys.whoami route
+
 ## 0.0.14
 
 ### Patch Changes

apps/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "api",
-  "version": "0.0.14",
+  "version": "0.1.0",
   "private": true,
   "scripts": {
     "build": "tsc",
@@ -18,7 +18,7 @@
     "@vitest/ui": "^1.6.0",
     "typescript": "^5.5.3",
     "vitest": "^1.6.0",
-    "wrangler": "^3.62.0"
+    "wrangler": "^3.80.5"
   },
   "dependencies": {
     "@axiomhq/js": "1.0.0-rc.2",
@@ -27,6 +27,7 @@
     "@hono/zod-validator": "^0.2.1",
     "@planetscale/database": "^1.16.0",
     "@unkey/cache": "workspace:^",
+    "@unkey/clickhouse": "workspace:^",
     "@unkey/db": "workspace:^",
     "@unkey/encryption": "workspace:^",
     "@unkey/error": "workspace:^",
@@ -36,7 +37,6 @@
     "@unkey/logs": "workspace:^",
     "@unkey/metrics": "workspace:^",
     "@unkey/rbac": "workspace:^",
-    "@unkey/clickhouse-zod": "workspace:^",
     "@unkey/schema": "workspace:^",
     "@unkey/worker-logging": "workspace:^",
     "hono": "^4.6.3",

apps/api/src/pkg/analytics.ts

@@ -1,5 +1,5 @@
 import { NoopTinybird, Tinybird } from "@chronark/zod-bird";
-import * as ch from "@unkey/clickhouse-zod";
+import { ClickHouse } from "@unkey/clickhouse";
 import { newId } from "@unkey/id";
 import { auditLogSchemaV1, unkeyAuditLogEvents } from "@unkey/schema/src/auditlog";
 import { ratelimitSchemaV1 } from "@unkey/schema/src/ratelimit-tinybird";
@@ -18,7 +18,7 @@ const dateToUnixMilli = z.string().transform((t) => new Date(t.split(" ").at(0)
 export class Analytics {
   public readonly readClient: Tinybird | NoopTinybird;
   public readonly writeClient: Tinybird | NoopTinybird;
-  private clickhouse: ch.Clickhouse;
+  private clickhouse: ClickHouse;
 
   constructor(opts: {
     tinybirdToken?: string;
@@ -38,12 +38,12 @@ export class Analytics {
       ? new Tinybird({ token: opts.tinybirdProxy.token, baseUrl: opts.tinybirdProxy.url })
       : this.readClient;
 
-    this.clickhouse = opts.clickhouse ? new ch.Client({ url: opts.clickhouse.url }) : new ch.Noop();
+    this.clickhouse = new ClickHouse({ url: opts.clickhouse?.url });
   }
 
   public get insertSdkTelemetry() {
-    return this.clickhouse.insert({
-      table: "default.raw_telemetry_sdks_v1",
+    return this.clickhouse.client.insert({
+      table: "telemetry.raw_sdks_v1",
       schema: z.object({
         request_id: z.string(),
         time: z.number().int(),
@@ -107,6 +107,20 @@ export class Analytics {
       })),
     });
   }
+  public get insertRatelimit() {
+    return this.clickhouse.client.insert({
+      table: "ratelimits.raw_ratelimits_v1",
+      schema: z.object({
+        request_id: z.string(),
+        time: z.number().int(),
+        workspace_id: z.string(),
+        namespace_id: z.string(),
+        identifier: z.string(),
+        passed: z.boolean(),
+      }),
+    });
+  }
+
   //tinybird
   public get ingestRatelimit() {
     return this.writeClient.buildIngestEndpoint({
@@ -116,8 +130,8 @@ export class Analytics {
   }
 
   public get insertKeyVerification() {
-    return this.clickhouse.insert({
-      table: "default.raw_key_verifications_v1",
+    return this.clickhouse.client.insert({
+      table: "verifications.raw_key_verifications_v1",
       schema: z.object({
         request_id: z.string(),
         time: z.number().int(),
@@ -140,8 +154,8 @@ export class Analytics {
   }
 
   public get insertApiRequest() {
-    return this.clickhouse.insert({
-      table: "default.raw_api_requests_v1",
+    return this.clickhouse.client.insert({
+      table: "metrics.raw_api_requests_v1",
       schema: z.object({
         request_id: z.string(),
         time: z.number().int(),
@@ -158,6 +172,10 @@ export class Analytics {
         service_latency: z.number().int(),
         user_agent: z.string(),
         ip_address: z.string(),
+        continent: z.string().nullable().default(""),
+        city: z.string().nullable().default(""),
+        country: z.string().nullable().default(""),
+        colo: z.string().nullable().default(""),
       }),
     });
   }