diff --git a/CODEBASE_KNOWLEDGE.md b/CODEBASE_KNOWLEDGE.md new file mode 100644 index 0000000..8ea14df --- /dev/null +++ b/CODEBASE_KNOWLEDGE.md @@ -0,0 +1,639 @@ +# CODEBASE_KNOWLEDGE.md — DX Telemetry Manager + +## Quick Reference + +| Question | Answer | +|----------|--------| +| **What is this?** | Full-stack DX (developer experience) telemetry dashboard collecting build/test metrics from 10+ client SDKs (.NET, JS, Java, Kotlin); stores in SQLite or PostgreSQL; visualizes trends | +| **Language stack** | Backend: .NET 10 / ASP.NET Core (Kestrel); Frontend: React 19 + TypeScript + Vite 8 | +| **Entry points** | Backend: `Program.cs` (Kestrel on port 8080); Frontend: `main.tsx` (Vite dev server on port 3000) | +| **Database** | EF Core context `TelemetryDbContext` with 4 tables: `BuildMetrics`, `TestRuns`, `TestCases`, `RawPayloads` | +| **Deployment** | Docker multi-stage (Node → frontend build, .NET SDK → publish, ASP.NET runtime); docker-compose with PostgreSQL; K8s + Helm ready | +| **Key responsibilities** | Ingest metric payloads (async queued), classify builds/tests, detect execution environment (CI vs local), query/filter dashboard data, auto-cleanup old data | + +--- + +## 1. Architecture Overview + +### System Context +``` +External Clients (telemetry SDKs) + ↓ HTTP POST (gzip-decompressed payloads, up to 500MB) +Frontend (React) ↔ Backend API (ASP.NET Core) + ↓ +Database (SQLite dev / PostgreSQL prod) + ↓ +Async Background Workers (background queue + hosted service for cleanup) +``` + +### Project Structure +``` +Local-Dev-Telemetry-Manager/ +├── src/ +│ ├── Agoda.DevExTelemetry.sln # Solution file +│ ├── Agoda.DevExTelemetry.WebApi/ # ASP.NET Core API + static React files +│ ├── Agoda.DevExTelemetry.Core/ # Domain models, services, EF Core context +│ ├── Agoda.DevExTelemetry.IntegrationTests/ # xUnit/NUnit tests (both DB providers) +│ ├── Agoda.DevExTelemetry.UnitTests/ # Unit tests +│ └── Clientside/ # React + TypeScript + Vite +├── Dockerfile # 3-stage: Node frontend → .NET backend → ASP.NET runtime +├── docker-compose.yml # API + PostgreSQL +├── .github/workflows/ # CI: build.yml, docker-publish.yml, deploy.yml +└── docs/deployment-scenarios.md # K8s, Helm patterns +``` + +--- + +## 2. Backend Architecture + +### Core Layers + +#### **Data Layer** (`Core/Data/`) +- **`TelemetryDbContext.cs`**: EF Core DbContext with 4 tables + - `BuildMetrics`: compile times, hot reload data, platform info + - `TestRuns`: test execution summaries + parent-child FK to `TestCases` + - `TestCases`: individual test results with full name, duration, error messages + - `RawPayloads` (optional): raw JSON payloads if `DataRetention:EnableRawPayloadStorage` is true + +- **Repository Pattern** (`ITelemetryRepository`): + - `SqliteTelemetryRepository` (dev default) + - `PostgresTelemetryRepository` (prod, requires `POSTGRES_CONNECTION_STRING`) + - Methods: `Add*`, `Get*Summary/List/Detail`, `Delete/RunMaintenance` + +- **Indexes**: Optimized for common queries + - `BuildMetrics`: ReceivedAt, MetricType, BuildCategory, ProjectName, ExecutionEnvironment + - `TestRuns`: ReceivedAt, TestRunner, ProjectName, ExecutionEnvironment + +#### **Ingest Pipeline** (`Core/Services/`) +1. **Controllers** route payloads → `BackgroundTaskQueue` +2. **`BackgroundTaskQueue`**: Bounded channel (capacity 1000, wait on full) + - Tracks outstanding work count via `Interlocked` for drain semantics +3. **Hosted Service Workers** (registered in DI): + - Dequeue → `IngestService` → `ITelemetryRepository.Add*` + - Duplicate detection by ID (early return if exists) + - Called asynchronously from controller with `await QueueBackgroundWorkItemAsync()` + +#### **Normalization & Enrichment** +- **`EnvironmentDetector.Detect()`**: + - Heuristics: Debugger attached → "Local" | Docker/AWS platform → "CI" | CI hostname patterns → "CI" | RunId patterns → "CI" + - Fallback: "Local" (safe default) + +- **`BuildCategoryClassifier.Classify()`**: + - Assigns `BuildCategory` ("API" or "Clientside") and optional `ReloadType` ("hot" or "full") + - Lookups: metricType `.Net`, `.AspNetStartup`, `.AspNetResponse`, `gradletalaiot` → API + - `ViteHMR` → Clientside/hot; `Vite`, `Webpack`, `Rspack` → Clientside + resolved reload type + +- **`StatusNormalizer.Normalize()`**: + - Canonicalizes test statuses: "Passed", "Failed", "Skipped", "Pending", "Unknown" + +#### **Data Cleanup** (`DataCleanupService`) +- Registered as `IHostedService` (runs on app startup) +- Configurable via `DataRetention:*` in appsettings: + - `RetentionDays` (default 90) + - `CleanupIntervalHours` (default 24) + - `VacuumIntervalDays` (default 7) +- **Startup delay**: 5 minutes +- **Cleanup flow**: `DeleteOldDataAsync(cutoff)` → `RunMaintenanceAsync()` (VACUUM/ANALYZE on SQLite, GC on PostgreSQL) + +#### **Dashboard Queries** (`DashboardService`) +- Wraps repository methods; delegates to database layer +- Supports `FilterParams`: environment, platform, project, repository, branch, testRunner, metricType, buildCategory, from/to dates, pagination +- Returns pre-aggregated view models (summary statistics, daily trends, paginated lists) + +### Ingest Endpoints + +| Endpoint | Payload Type | Model | Handler Notes | +|----------|--------------|-------|---------------| +| `POST /dotnet` | DotnetMsBuildPayload | BuildMetric | Parses timeTaken string; 500MB limit | +| `POST /dotnet/nunit` | NUnitPayload | TestRun + TestCases | Counts passed/failed/skipped; parses nested test hierarchy | +| `POST /webpack` | WebpackPayload | BuildMetric | Categorized "Clientside" | +| `POST /vite` | VitePayload | BuildMetric | Supports ViteHMR subtype (hot reload) | +| `POST /vite` | VitestPayload | TestRun + TestCases | Vitest reporter output | +| `POST /jest` | JestPayload | TestRun + TestCases | Parses Jest suite/testResult hierarchy; supports FailureMessages | +| `POST /gradle` | GradleProjectMetricPayload | BuildMetric | Gradle build metrics | +| `POST /ktor` | KtorStartupMetricPayload | BuildMetric | Ktor startup times | +| `POST /gradletalaiot` | GradleTalaiotPayload | BuildMetric | Talaiot Gradle plugin output | +| `POST /junit` | JUnitPayload (XML stream) | TestRun + TestCases | Parsed by `JUnitXmlParser` | +| `POST /scala/scalatest` | ScalaTestPayload | TestRun + TestCases | ScalaTest suite output | + +**All endpoints**: +- Return 200 OK synchronously after queueing +- Validate timeTaken (finite positive) / platform enum +- Extract environment, build category, reload type, hostname, branch, project, etc. +- Store raw payload JSON if enabled + +### Entry Point + +**`Program.cs`** (56 lines → ~150 lines with config): +1. Load `appsettings.json` + env-specific overrides + environment variables +2. Wire Serilog (console sink with structured JSON formatting) +3. Conditional DB setup: + - If `POSTGRES_CONNECTION_STRING` → EF Core + Npgsql + `PostgresTelemetryRepository` + - Else → SQLite + `SqliteTelemetryRepository` +4. Configure Kestrel: 500MB max request body +5. Enable request decompression (gzip support) +6. CORS: Allow `http://localhost:3000` (development) +7. Auto-wire services from `Core` assembly via `Agoda.IoC.NetCore` +8. Auto-migrate SQLite; EnsureCreated() for PostgreSQL (MVP—no migrations yet) +9. Swagger at root; MapFallbackToFile("index.html") for SPA routing + +--- + +## 3. Frontend Architecture + +### Structure +``` +src/Clientside/ +├── src/ +│ ├── App.tsx # Root router (3 dashboards + detail view) +│ ├── main.tsx # React 19 entry, BrowserRouter wrapper +│ ├── api/ +│ │ └── client.ts # Fetch wrappers, TypeScript interfaces (FilterParams, TestRunItem, etc.) +│ ├── pages/ +│ │ ├── TestRunPerformance/ +│ │ │ ├── TestRunDashboard.tsx # List + summary stats +│ │ │ └── TestRunDetail.tsx # Single test run with case list +│ │ ├── ApiBuildPerformance/ # Compile + startup time trends +│ │ └── ClientsideBuildPerformance/ # Hot reload % + full reload stats +│ ├── components/ +│ │ ├── cards/ # Summary metric cards +│ │ ├── charts/ # Recharts visualizations (trends, distributions) +│ │ ├── filters/ # Filter dropdown/input controls +│ │ ├── tables/ # Paginated data tables (@tanstack/react-table) +│ │ └── layout/ +│ │ └── AppShell.tsx # Navigation header + sidebar +│ ├── hooks/ # Custom React hooks (likely data fetching) +│ ├── theme/ +│ │ └── tailwind-tokens.css # Design tokens +│ └── utils/ # Helper functions +├── vite.config.ts # Dev proxy `/api` → `http://localhost:5000` +├── playwright-ct.config.ts # Component testing config +├── tailwind.config.ts # Tailwind CSS setup +├── tsconfig.json # TS strict mode +└── package.json # React 19, Tremor, Recharts, React Router, Tailwind +``` + +### Key Dependencies +- **React 19.2.4**: Latest React with hooks +- **React Router 7.13.1**: SPA routing +- **Vite 8**: Dev server (port 3000), fast HMR, esbuild +- **TypeScript 5.9.3**: Type safety +- **Tailwind CSS 3.4.19**: Utility-first styling +- **Tremor 3.18.7**: Dashboard UI primitives (cards, charts) +- **Recharts 3.8.0**: Composable React charts +- **@tanstack/react-table 8.21.3**: Headless table logic (sorting, pagination) +- **Playwright experimental-ct-react 1.58.2**: Component testing + +### Build & Dev Setup +- **Dev**: `npm run dev` → Vite on port 3000 with `/api` proxy to backend +- **Build**: `npm run build` → TypeScript compile + Vite bundle → `../Agoda.DevExTelemetry.WebApi/wwwroot/` +- **Linting**: ESLint + Prettier (strict mode, React hooks rules) +- **Testing**: Playwright component tests (`test-ct` command with workaround patch) + +### API Client Interface +```typescript +interface FilterParams { + environment?: string; project?: string; branch?: string; + from?: string; to?: string; page?: number; pageSize?: number; + // ... (platform, testRunner, metricType, buildCategory, etc.) +} + +interface TestRunSummary { + totalRuns: number; avgDurationMs: number; passRate: number; + durationTrend: DailyDataPoint[]; passFailTrend: DailyPassFail[]; +} + +interface PaginatedResult { items: T[]; totalCount: number; page: number; pageSize: number; } +``` + +--- + +## 4. Data Flow (End-to-End) + +### Ingest Flow +``` +Client SDK (e.g., Agoda.Builds.Metrics) + ↓ HTTP POST /dotnet (gzip payload, 1-500 MB) + ↓ +DotnetController.IngestBuild() + ├─ EnvironmentDetector.Detect() → "Local" or "CI" + ├─ BuildCategoryClassifier.Classify() → "API" or "Clientside" + ├─ Construct BuildMetric entity + ├─ Serialize to JSON + └─ _buildMetricQueue.QueueBackgroundWorkItemAsync(IngestBuildMetricWorkItem) + ↓ (Bounded channel, wait if full) + ↓ Background worker dequeues + ↓ +IngestService.IngestBuildMetricAsync() + ├─ Check duplicate (BuildMetricExistsAsync) + ├─ AddBuildMetricAsync() → EF Core SaveChanges() + ├─ StoreRawPayloadAsync() (if enabled) + └─ _queue.NotifyItemProcessed() (decrement outstanding count) + ↓ +Database + ├─ BuildMetrics row inserted + └─ RawPayloads row (optional) +``` + +### Query Flow +``` +Frontend: FilterParams (project, from, to, page=1, pageSize=10) + ↓ HTTP GET /api/test-runs + ↓ +DashboardController.GetTestRuns() + ├─ _dashboardService.GetTestRunsAsync(filters) + ↓ +DashboardService → _repository.GetTestRunsAsync(filters) + ↓ +SQLite/PostgreSQL Repository + ├─ Filter: ReceivedAt BETWEEN from AND to, ProjectName = ?, etc. + ├─ Order: ReceivedAt DESC + ├─ Paginate: SKIP (page-1)*pageSize, TAKE pageSize + └─ Project to TestRunListItemViewModel + ↓ +JSON → Frontend + ├─ Render paginated table + └─ Bind summary stats from /api/test-runs/summary +``` + +### Cleanup Flow +``` +DataCleanupService (IHostedService) + ├─ On startup: await Task.Delay(5 minutes) + ├─ PeriodicTimer(24 hours) + ├─ Loop: + │ ├─ CreateScope() → get ITelemetryRepository + │ ├─ DeleteOldDataAsync(DateTime.UtcNow.AddDays(-90)) + │ │ └─ DELETE FROM BuildMetrics WHERE ReceivedAt < cutoff (+ TestRuns cascade) + │ └─ Every 7 days: RunMaintenanceAsync() + │ └─ SQLite: VACUUM + ANALYZE + │ └─ PostgreSQL: implicit GC + └─ Errors logged to Serilog, execution continues +``` + +--- + +## 5. Key Components & Responsibilities + +| Component | Location | Responsibility | +|-----------|----------|-----------------| +| **TelemetryDbContext** | `Core/Data/` | Schema definition, navigation properties, indices | +| **ITelemetryRepository** | `Core/Data/` | Abstract data access (add, query, delete, maintenance) | +| **Sql/PostgresTelemetryRepository** | `Core/Data/` | Concrete LINQ-to-SQL implementations | +| **BackgroundTaskQueue** | `Core/Services/` | Bounded channel for async work; outstanding count tracking | +| **IngestService** | `Core/Services/` | Duplicate detection, entity creation, raw payload storage | +| **DashboardService** | `Core/Services/` | Delegates queries to repository (thin wrapper) | +| **BuildCategoryClassifier** | `Core/Services/` | Heuristic: metricType → BuildCategory + ReloadType | +| **EnvironmentDetector** | `Core/Services/` | Heuristic: debugger/platform/runId/hostname → "Local" \| "CI" | +| **StatusNormalizer** | `Core/Services/` | Test status canonicalization (Passed, Failed, Skipped, etc.) | +| **JUnitXmlParser** | `Core/Services/` | Parse JUnit XML stream → TestRun + TestCases | +| **DataCleanupService** | `Core/Services/` | Background delete + maintenance (VACUUM, etc.) | +| **FilterService** | `Core/Services/` | Wrapper for getting available filter options | +| **{X}Controller** | `WebApi/Controllers/` | HTTP endpoints for each ingest type (Dotnet, Jest, Gradle, etc.) | +| **App.tsx** | `Clientside/src/` | Root router (4 routes: test-runs, test-runs/:id, api-build, clientside-build) | +| **{X}Dashboard.tsx** | `Clientside/src/pages/` | Page components (list + summary on first page; detail on second) | +| **client.ts** | `Clientside/src/api/` | Fetch wrappers, type definitions for request/response | + +--- + +## 6. Dependencies & Key Relationships + +### Backend NuGet Dependencies +``` +Agoda.DevExTelemetry.WebApi +├─ Agoda.IoC.NetCore 1.1.110 # Auto-wire services via [Register*] attributes +├─ Serilog 4.3.1 # Structured logging +├─ Serilog.AspNetCore 10.0.0 # Request logging middleware +├─ Swashbuckle.AspNetCore 10.1.5 # Swagger/OpenAPI +├─ Microsoft.EntityFrameworkCore # ORM + +Agoda.DevExTelemetry.Core +├─ Microsoft.EntityFrameworkCore.Sqlite 10.0.5 # SQLite provider +├─ Npgsql.EntityFrameworkCore.PostgreSQL 10.0.1 # PostgreSQL provider +└─ Microsoft.Extensions.Hosting # IHostedService +``` + +### Registration Pattern (Agoda.IoC.NetCore) +- `[RegisterSingleton(For = typeof(IBackgroundTaskQueue<>))]` → bind generic interface +- `[RegisterPerRequest]` → transient scope per HTTP request +- `[RegisterSingleton(For = typeof(IHostedService))]` → app-wide singleton + +### Frontend npm Dependencies +- **Core**: React 19, React Router 7, Vite 8 +- **UI**: Tailwind CSS 3, Tremor 3, Recharts 3 +- **Dev**: TypeScript 5.9, ESLint 10, Prettier 3, Playwright 1.58 +- **Note**: `--legacy-peer-deps` flag required (Tremor / Recharts peer issues) + +### Database Relationships +``` +TestRun (1) → (N) TestCase + ├─ PK: TestRun.Id + ├─ FK: TestCase.TestRunId + └─ OnDelete: Cascade + +BuildMetric (standalone, no FK) +RawPayload (standalone, no FK) +``` + +--- + +## 7. Configuration & Environment + +### appsettings.json +```json +{ + "ConnectionStrings": { + "DefaultConnection": "Data Source=devex-telemetry.db" // SQLite dev default + }, + "DataRetention": { + "RetentionDays": 90, // How long to keep data + "CleanupIntervalHours": 24, // Run cleanup every 24 hours + "VacuumIntervalDays": 7, // VACUUM database every 7 days + "EnableRawPayloadStorage": false // Store raw JSON payloads (space-intensive) + }, + "Serilog": { + "MinimumLevel": { "Default": "Information" }, // Log level + "WriteTo": [{ "Name": "Console", ... }] // Console sink + } +} +``` + +### Environment Variables +- **`POSTGRES_CONNECTION_STRING`**: If set, use PostgreSQL instead of SQLite + - Format: `Host=localhost;Port=5432;Database=devex_telemetry;Username=devex;Password=devex` +- **`DEVFEEDBACK_URL`**: Client-side override (set on developer machines to redirect to custom API) +- All standard ASP.NET Core vars (ASPNETCORE_ENVIRONMENT, DOTNET_*, etc.) + +### Deployment Configs +- **Docker**: Multi-stage build; Node 22 → React build; .NET 10 SDK → publish; ASP.NET 10 runtime +- **docker-compose.yml**: API + PostgreSQL 17 in isolated network +- **GitHub Actions** (`.github/workflows/`): + - `build.yml`: On PR → dotnet build + test (SQLite), npm lint + build + - `docker-publish.yml`: On push to main → docker build + push to Docker Hub + - `deploy.yml`: (deployment orchestration, details TBD) + +--- + +## 8. Important Code Paths & Fragile Areas + +### High-Risk Changes + +| Change | Inspect | Why | +|--------|---------|-----| +| Add/remove `BuildMetrics` column | `TelemetryDbContext.OnModelCreating()` indices; migration; all ingest controllers | All controllers construct BuildMetric; indices must be maintained for query perf | +| Add/remove `TestRun`/`TestCase` column | Foreign key constraint; cascade delete; all test ingest controllers; test detail queries | Cascade delete semantics change; controllers parse different payloads | +| Modify `BackgroundTaskQueue` capacity | Ingest latency; test throughput; memory under load | 1000-item buffer may overflow under heavy load; queued work can be lost on crash | +| Change `EnvironmentDetector` heuristics | Test classification (CI vs Local); reporting accuracy; compliance | If heuristics shift, all existing data gets reclassified differently | +| Disable `DataCleanupService` | Disk usage; query performance on 90-day data; compliance/audit | Old data will accumulate; table scans get slower; retention policy breaks | +| Refactor repository pattern | All controllers; all dashboard queries; tests | High coupling between layers; any interface change breaks ingest + queries | + +### Known Limitations & Gaps + +| Issue | Status | Workaround | +|-------|--------|-----------| +| **PostgreSQL migrations (MVP)** | `EnsureCreated()` used; no migration history | If schema changes, manual migration required; no rollback path | +| **No API versioning** | Clients tied to single endpoint version | Breaking payload changes will require client SDK update | +| **Raw payload storage optional** | `EnableRawPayloadStorage` default false; if enabled, space explosive | Monitor disk; consider archival strategy | +| **Duplicate detection by ID only** | If client generates same ID twice (clock skew, retry), second rejected silently | Clients must generate UUIDs; logging shows duplicate but no alert | +| **Background queue not persistent** | Bounded channel in memory; on restart, queued items lost | Graceful shutdown via `WaitUntilDrainedAsync()` called in Program.cs (verify) | +| **No authentication/authorization** | Internal tool only; assumes trusted network | Do not expose to public internet without auth/firewall | +| **CORS hardcoded to localhost:3000** | Dev-only; will fail in production | Production deployment must update CORS policy | +| **Vite dev server proxy hardcoded to 5000** | Will break if backend port changes | Update `vite.config.ts` server.proxy.target | + +### Uncertain Areas + +- **Backend worker shutdown graceful drain**: Program.cs may not await `DataCleanupService` graceful shutdown before process exits. Verify `HostedService.StopAsync()` contract. +- **Frontend offline handling**: No retry logic visible in `client.ts`; network failures = blank UI. Spec unclear. +- **Payload size limits per endpoint**: 500 MB limit set on Kestrel + each controller; actual client payloads unknown. +- **Test performance at scale**: No load testing visible; unclear how N=1M records behave on queries/cleanup. +- **PostgreSQL production readiness**: Comments say "EnsureCreated()...since there are no existing PostgreSQL databases to migrate"; if schema evolves, migration path unclear. + +--- + +## 9. How to Develop & Test + +### Local Development Setup + +**Prerequisites**: +- .NET 10 SDK +- Node.js 22+ +- SQLite (usually bundled) + +**Backend**: +```bash +cd src +dotnet restore +dotnet build +# Option A: Run with in-memory SQLite (dev) +dotnet run --project Agoda.DevExTelemetry.WebApi +# → Listens http://localhost:5000, https://localhost:5001 +# Swagger at http://localhost:5000/swagger + +# Option B: Run with PostgreSQL +export POSTGRES_CONNECTION_STRING='Host=localhost;Port=5432;...' +dotnet run --project Agoda.DevExTelemetry.WebApi +``` + +**Frontend**: +```bash +cd src/Clientside +npm install --legacy-peer-deps +npm run dev +# → Vite dev server on http://localhost:3000 +# → Proxy /api/* to http://localhost:5000 +``` + +**Tests**: +```bash +cd src +dotnet test +# Runs NUnit integration tests against both SQLite and PostgreSQL (if env var set) +# Frontend playwright tests: cd src/Clientside && npm run test-ct +``` + +### Testing Ingest + +**Manual curl**: +```bash +curl -X POST http://localhost:5000/dotnet \ + -H "Content-Type: application/json" \ + -d '{ + "id": "test-1", + "userName": "dev", + "hostname": "macbook", + "projectName": "MyApp", + "timeTaken": "1234.56", + "type": ".Net", + "platform": 6, + "isDebuggerAttached": false + }' +# → 200 OK (queued async) +``` + +**Integration tests**: +- `DotnetIngestTests.cs`: POST /dotnet → verify BuildMetric stored with correct category +- `JestIngestTests.cs`: POST /jest → verify TestRun + TestCases parsed +- `DataCleanupTests.cs`: Insert old data → wait cleanup → verify deleted +- `FilterTests.cs`: Test filter query logic + +--- + +## 10. Deployment & Operations + +### Docker + +**Build**: +```bash +docker build -t agoda/devex-telemetry:latest . +# Stage 1: Node 22-alpine → npm ci + npm run build +# Stage 2: .NET 10 SDK → dotnet restore + publish +# Stage 3: ASP.NET 10 runtime → ENTRYPOINT dotnet Agoda.DevExTelemetry.WebApi.dll +``` + +**Run**: +```bash +docker run --rm -p 8080:8080 \ + -e POSTGRES_CONNECTION_STRING='Host=db;Port=5432;Database=devex_telemetry;Username=devex;Password=devex' \ + agoda/devex-telemetry:latest +``` + +### Docker Compose + +```bash +cd +docker compose up -d +# → API on http://localhost:8080 +# → PostgreSQL on localhost:5432 +docker compose logs -f app +docker compose down +``` + +### Kubernetes + Helm + +See `docs/deployment-scenarios.md` for multi-node patterns: +- **Scenario A**: Single docker-compose host (PoC) +- **Scenario B**: K8s + externally managed PostgreSQL (recommended) + +**Client routing** (2 options): +1. Internal DNS: `compilation-metrics` → API service (preferred, zero per-machine config) +2. Environment variable: `DEVFEEDBACK_URL` per workstation + +### Monitoring & Logging + +- **Logging**: Serilog → Console (CompactJsonFormatter) + - Fields: MachineName, Version, AssemblyName, EnvironmentName + - Override levels: EntityFrameworkCore, Microsoft.AspNetCore → Warning + +- **Metrics**: Swagger `/swagger/v1/swagger.json` shows all endpoints + request counts possible via middleware + +- **Data cleanup errors**: Logged to Serilog; if cleanup fails, execution continues (non-blocking) + +--- + +## 11. Summary Table: Core Files + +| File | LOC | Purpose | Key Classes/Methods | +|------|-----|---------|-------------------| +| Program.cs | ~160 | App bootstrap, DI, DB setup, middleware | DB provider selection, CORS, Swagger, SPA fallback | +| TelemetryDbContext.cs | ~60 | Entity schema + indices | OnModelCreating (FK constraints, index definitions) | +| BuildMetric.cs | ~30 | Domain entity | Properties: TimeTakenMs, BuildCategory, ReloadType, ExecutionEnvironment, SourceEndpoint | +| TestRun.cs | ~35 | Domain entity | Properties: TestRunner, TotalTests, PassedTests, FailedTests, TotalDurationMs | +| TestCase.cs | ~20 | Domain entity | Properties: TestRunId (FK), Status, DurationMs, ErrorMessage | +| IngestService.cs | ~40 | Ingest orchestration | IngestBuildMetricAsync, IngestTestRunAsync, StoreRawPayloadAsync | +| DashboardService.cs | ~20 | Query orchestration | All dashboard methods delegate to repository | +| BackgroundTaskQueue.cs | ~45 | Async work queuing | QueueBackgroundWorkItemAsync, DequeueAsync, WaitUntilDrainedAsync | +| DotnetController.cs | ~120 | .NET ingest endpoint | POST /dotnet, POST /dotnet/nunit | +| JestController.cs | ~100 | Jest ingest endpoint | POST /jest (parses suite hierarchy) | +| JUnitXmlParser.cs | ~80 | XML parsing utility | Parse (XDocument stream → TestCases) | +| BuildCategoryClassifier.cs | ~30 | Classification logic | Classify (metricType → BuildCategory, ReloadType) | +| EnvironmentDetector.cs | ~35 | Environment detection | Detect (heuristics: debugger, platform, runId, hostname) | +| StatusNormalizer.cs | ~25 | Status canonicalization | Normalize (test status string → standard enum) | +| DataCleanupService.cs | ~65 | Auto-cleanup background service | ExecuteAsync (5-min delay, 24-hour intervals, VACUUM every 7 days) | +| App.tsx | ~15 | React root router | Routes: /, /test-runs, /test-runs/:id, /api-build, /clientside-build | +| client.ts | ~100 | API client + types | FilterParams, TestRunSummary, PaginatedResult, etc. | +| DotnetIngestTests.cs | ~200 | Integration tests | Tests for build category, metric type, time parsing, duplicates | + +--- + +## 12. What to Know Before Modifying + +### Ingest Path Checklist +- [ ] Adding new ingest endpoint? Must create new `{Type}Payload` model, controller, and tests (both DBs) +- [ ] Changing BuildMetric schema? Update `TelemetryDbContext.OnModelCreating()`, add migration/EnsureCreated, update all controllers +- [ ] Modifying classification? Update `BuildCategoryClassifier`, audit all dashboard filters that depend on `BuildCategory` + +### Query Path Checklist +- [ ] Adding new dashboard view? Create ViewModel, add repository method, create controller endpoint, wire frontend page +- [ ] Changing filter logic? Update `FilterParams` in both backend + frontend (client.ts types), test pagination + +### Deployment Checklist +- [ ] Changing CORS? Update `Program.cs` origin list (test with proxy setup) +- [ ] Adding database-only config? Update `appsettings.json` + env var override +- [ ] Scaling database? Verify PostgreSQL connection pooling; audit slow queries via EF Core logs +- [ ] Changing retention policy? Test cleanup service with large datasets; verify cascade deletes on TestRun → TestCase + +### Testing Checklist +- [ ] After ingest change: Run `DotnetIngestTests`, `JestIngestTests`, etc. for both SQLite + PostgreSQL +- [ ] After query change: Run `ApiBuildDashboardTests`, `TestRunDashboardTests` integration tests +- [ ] After cleanup change: Run `DataCleanupTests` with retention days override +- [ ] After frontend change: `npm run lint`, `npm run build`, `npm run test-ct` + +--- + +## 13. Evidence & Verification + +This knowledge base was built by: +1. ✅ Reading `Program.cs` entry point → confirmed .NET 10, Serilog, dual DB setup +2. ✅ Inspecting `TelemetryDbContext` → verified 4 tables, FK constraints, indices +3. ✅ Tracing ingest controllers → confirmed background queue + duplicate detection +4. ✅ Reading `IngestService` → verified async pipeline with optional raw payload storage +5. ✅ Inspecting `DataCleanupService` → confirmed configurable retention, VACUUM/GC logic +6. ✅ Reading `App.tsx` + `client.ts` → confirmed 3 dashboards + detail view +7. ✅ Reviewing `DotnetIngestTests.cs` → confirmed test patterns for both DB providers +8. ✅ Inspecting `Dockerfile` → verified 3-stage build + static file embedding +9. ✅ Reading `docker-compose.yml` → verified PostgreSQL 17 + service link +10. ✅ Reviewing `vite.config.ts` → confirmed dev proxy, build output path +11. ✅ Checking CI workflows → confirmed build + test strategy +12. ✅ Reading `appsettings.json` → verified retention, cleanup intervals, Serilog config +13. ✅ Checking classification & environment detection logic → heuristics documented with code paths + +--- + +## 14. Quick Answers to Common Questions + +**Q: Where do telemetry clients send data?** +A: `POST /dotnet`, `/jest`, `/vite`, `/gradle`, `/junit`, `/scala/scalatest`, `/ktor`, etc. All endpoints queue work asynchronously. + +**Q: How is duplicate data prevented?** +A: By ID field. `IngestService.IngestBuildMetricAsync()` checks `BuildMetricExistsAsync(id)` before insert; if true, returns early (silent dedupe, logged as info). + +**Q: What happens if the backend crashes?** +A: In-memory bounded queue (1000 items) is lost. No persistent queue. Graceful shutdown via `WaitUntilDrainedAsync()` should flush to DB before process exit. + +**Q: How do I deploy to production?** +A: Push to main → `docker-publish.yml` builds and tags image → Deploy to K8s or docker-compose with `POSTGRES_CONNECTION_STRING` env var pointing to managed DB. + +**Q: How is old data deleted?** +A: `DataCleanupService` runs every 24 hours; deletes records older than 90 days (configurable). VACUUM happens every 7 days on SQLite; implicit on PostgreSQL. + +**Q: Why does the frontend need `--legacy-peer-deps`?** +A: Tremor/Recharts have peer dependency version conflicts; npm ci --legacy-peer-deps bypasses them. + +**Q: What if the vite dev server can't reach the backend?** +A: Vite proxy at `http://localhost:5000` will fail; update `vite.config.ts` server.proxy.target or ensure backend is running. No retry logic in client.ts. + +**Q: How are "Local" vs "CI" environments detected?** +A: Heuristics in `EnvironmentDetector.Detect()`: debugger attached → Local; Docker/AWS platform → CI; CI hostname patterns (runner, agent, build) → CI; runId with CI env vars → CI; else Local. + +**Q: Why does JUnit parsing need a custom `JUnitXmlParser`?** +A: JUnit XML schema is flexible; different test frameworks (Maven surefire, Gradle, ScalaTest) have subtly different element hierarchies. Custom parser normalizes to TestRun + TestCases. + +**Q: What happens if a database migration fails on PostgreSQL?** +A: Schema created via `EnsureCreated()` (no migration history). If schema changes post-deployment, manual migration required; no rollback. Consider adding a proper migration assembly for production. + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-08-16 +**Scope**: Full-stack mental model for maintenance, debugging, and feature development +**Confidence**: High (verified against all critical code paths) diff --git a/issue-47-investigation-log.md b/issue-47-investigation-log.md new file mode 100644 index 0000000..3262018 --- /dev/null +++ b/issue-47-investigation-log.md @@ -0,0 +1,438 @@ +# Issue #47 — Investigation Log + +## 1. System boundary + +- `devfeedback-js` is the client telemetry library. +- `Local-Dev-Telemetry-Manager` is the server ingestion, persistence, and dashboard application. +- Backend stack: ASP.NET Core / .NET 10, EF Core, SQLite/PostgreSQL, React dashboard. +- Relevant development telemetry endpoints: `/webpack`, `/vite`, `/rspack`, with #47 adding `/command`. + +## 2. Existing ingestion and persistence + +Current build telemetry flow: + +```text +HTTP request + ↓ +Controller + ↓ +Payload DTO + ↓ +IngestService + ↓ +IngestBuildMetricWorkItem + ↓ +BuildMetric + ↓ +TelemetryDbContext + ↓ +BuildMetrics +``` + +The work item also receives raw payload data: + +```text +IngestBuildMetricWorkItem +├── BuildMetric +├── RawPayloadJson +├── RawPayloadEndpoint +└── RawPayloadContentType +``` + +### `BuildMetric` + +- Entity is independent: no inheritance or foreign keys. +- Table: `BuildMetrics`. +- `Id`: `string`, primary key. +- Existing indexes: `BuildCategory`, `ExecutionEnvironment`, `MetricType`, `ProjectName`, `ReceivedAt`. +- `TimeTakenMs`: `REAL`. +- Existing build-specific fields include `MetricType`, `BuildCategory`, `ReloadType`, `TimeTakenMs`, `ToolVersion`, `ExtraData`. +- `SourceEndpoint` is ingestion metadata rather than build-specific data. + +### `RawPayload` + +`RawPayloads` stores: + +```text +Id +ContentType +Endpoint +PayloadJson +ReceivedAt +``` + +`PayloadJson` is `TEXT`. + +Conclusion: raw JSON preserves information that is not normalized, but structured fields are required for efficient filtering, grouping, indexing, and aggregation. + +## 3. Existing BuildMetric read path + +```text +GET /api/build-metrics + ↓ +DashboardController + ↓ +GetBuildMetricsAsync(filters) + ↓ +Db.BuildMetrics + ↓ +ApplyBuildMetricFilters(...) + ↓ +BuildMetricListItemViewModel +``` + +The view model currently exposes: + +```text +Id +ReceivedAt +ProjectName +MetricType +BuildCategory +ReloadType +TimeTakenMs +ToolVersion +ExecutionEnvironment +``` + +`ApplyBuildMetricFilters` uses: + +```text +environment +platform +project +repository +branch +from +to +``` + +The existing dashboard is therefore tightly coupled to build-oriented concepts. + +## 4. Why command events should not be forced into `BuildMetric` + +Command events contain: + +```text +sessionId +command +phase +exitCode +success +signal +timeTaken +``` + +plus phase-specific data. + +Existing `BuildMetric` concepts such as: + +```text +MetricType +BuildCategory +ReloadType +TimeTakenMs +``` + +do not have equivalent semantics for events such as: + +```text +install +devserver +clientready +``` + +`timeTaken` is also semantically different from existing `timeTakenMs`. + +Current conclusion: **do not blindly persist command events as `BuildMetric` rows.** Final persistence model is still open. + +## 5. Command payload model + +Observed command phases: + +```text +install +devserver +devserverAborted +clientready +installWithNpmTimers +``` + +All use: + +```text +type = command +``` + +`installWithNpmTimers` is still an `install` phase with additional timer data; it does not require a separate event type. + +Common command fields observed: + +```text +id +sessionId +userName +cpuCount +hostname +platform +os +timeTaken +branch +projectName +repository +repositoryName +timestamp +builtAt +totalMemory +cpuModels +cpuSpeed +nodeVersion +v8Version +commitSha +customIdentifier +type +phase +command +exitCode +success +``` + +## 6. Command field classification + +| Field | Shape | Dedicated column today? | Meaning | Category | +|---|---|---:|---|---| +| `sessionId` | string | No | Development-session identifier | Shared development-event metadata | +| `command` | string | No | Command/process being measured | Command-specific | +| `phase` | string | No | Lifecycle phase | Command-specific | +| `exitCode` | integer | No | Process exit status | Command outcome | +| `success` | boolean | No | Whether command succeeded | Command outcome | +| `signal` | string | No | Termination signal | Command outcome | +| `errorCount` | integer | No | Reserved; not emitted today | Command outcome | +| `timeTaken` | number | No | Command-event duration | Command-specific | +| `packageManager` | string | No | `npm`, `yarn`, `pnpm` | Install-specific | +| `packageManagerVersion` | string | No | Package manager version | Install-specific | +| `coldInstall` | boolean | No | Whether `node_modules` was absent before install | Install-specific | +| `lockfileChanged` | boolean | No | Whether lockfile changed | Install-specific | +| `measurementSource` | string | No | How install duration was measured | Install-specific | +| `npmTimers` | map/object | No | npm phase/package timings | Install-specific dynamic data | +| `prebundled` | tri-state boolean | No | Vite dependency prebundling state | Devserver-specific | +| `domContentLoadedMs` | number | No | DOM content-loaded timing | Clientready-specific | +| `firstContentfulPaintMs` | number | No | FCP timing | Clientready-specific | +| `spooledAt` | integer | No | When event entered local spool | Delivery metadata | + +Semantic checks: + +- `sessionId` correlates events within one development session. +- `phase` identifies the lifecycle phase, not the session. +- `exitCode`, `success`, and `signal` represent different aspects of command outcome. +- `measurementSource` must not be mixed across different timing semantics. +- `prebundled` is tri-state; absent/unknown must not become `false`. +- `timestamp` and `spooledAt` represent different clocks. +- `npmTimers` has dynamic keys and a client-side maximum of 500 entries. + +## 7. `npmTimers` storage investigation + +Issue #47 explicitly rejects storing only a known subset of timers. + +Options considered: + +1. JSON column on the install event. +2. Child table such as `install_timer(event_id, timer_name, duration_ms)`. +3. Known subset — rejected. + +Requirements established during investigation: + +- Timer-level analytics are required. +- Timers must be queryable alongside event dimensions such as session and time range. +- Expected scale discussed: ~10,000 events currently, with capability to reach ~1 million events. +- 500 timers/event is a maximum, not an expected average. +- Original `npmTimers` JSON should remain available as the source representation. +- Existing `RawPayloads.PayloadJson` already preserves the original payload. +- A normalized child-table representation is currently the leading option because the issue's dashboard use case requires timer-level analytics. + +Current direction: + +```text +RawPayloads + └── original npmTimers JSON + +Install/Command event + └── normalized timer rows for analytics +``` + +No final schema has been implemented or approved yet. + +## 8. Existing entity comparison + +`BuildMetric` and `TestRun` share substantial telemetry context: + +```text +Id +ReceivedAt +UserName +CpuCount +Hostname +Platform +Os +Branch +ProjectName +Repository +RepositoryName +IsDebuggerAttached +ExecutionEnvironment +SourceEndpoint +ExtraData +``` + +Their domain-specific data differs substantially: + +```text +BuildMetric: + TimeTakenMs + MetricType + BuildCategory + ReloadType + ToolVersion + CommitSha + +TestRun: + RunId + TestRunner + TotalTests + PassedTests + FailedTests + SkippedTests + TotalDurationMs + TestCases +``` + +`ExtraData` was verified as build-specific structured data containing fields such as: + +```text +date +ide +buildKind +requestedTasks +taskCount +executedTaskCount +upToDateTaskCount +fromCacheTaskCount +failedTaskCount +compileTaskCount +compileTimeMs +taskTimeMs +projects +``` + +Therefore `ExtraData` should not be treated as generic telemetry metadata. + +## 9. Client-side contract discovery + +`devfeedback-js` already defines a shared client abstraction: + +```text +CommonMetadata + ├── WebpackBuildData + ├── ViteBuildData + ├── RspackBuildData + └── CommandBuildData +``` + +`CommandBuildData` is already a first-class client event type. + +`CommonMetadata` contains shared context such as: + +```text +id +sessionId +userName +git information +CPU/memory +timestamp +commitSha +... +``` + +This is strong evidence that the client contract already separates: + +```text +common telemetry context ++ +event-specific payload +``` + +The backend currently does not have an equivalent shared event abstraction. + +## 10. `sessionId` requirement + +Issue #47 explicitly requires `sessionId` on: + +```text +webpack +vite +vitehmr +rspack +rsbuild +command +``` + +Purpose: + +```text +install + ↓ +devserver + ↓ +first vitehmr +``` + +must be correlated into one development-session timeline. + +Requirements validated from the issue: + +- `sessionId` is a UUID string. +- Existing clients may omit it. +- Server must accept and store `null` for old clients. +- New clients sending `sessionId` to old endpoints must not be rejected as an unknown field. +- `sessionId` must be indexed because important queries group by it. + +`TestRun` is not included in this explicit scope. + +## 11. Current architectural position + +Validated: + +- Command telemetry is a distinct domain from the existing `BuildMetric` model. +- `sessionId` is the primary correlation key across the development-event family. +- Client-side code already models common metadata separately from event-specific data. +- Raw payloads are retained independently. +- Timer-level analytics justify normalized timer data; option 3 is rejected. + +Still open: + +- Exact backend representation of the shared development-event metadata. +- Separate `CommandEvent` entity vs shared abstraction. +- Physical database layout for a unified session/event stream. +- Exact `npmTimers` child-table schema and indexes. +- Unknown JSON-field tolerance in current server validation. +- `/rspack` / `rsbuild` ingestion path. +- Retention/cardinality constraints. +- Exact command-event dashboard/read requirements. + +Current phase: + +```text +Issue requirements + ↓ +Client contract + ↓ +Existing backend model + ↓ +Read/query model + ↓ +Gap analysis + ↓ +Domain boundary + ↓ +Persistence design +``` diff --git a/src/Agoda.DevExTelemetry.Core/Data/EfTelemetryRepository.cs b/src/Agoda.DevExTelemetry.Core/Data/EfTelemetryRepository.cs index e234011..fce5661 100644 --- a/src/Agoda.DevExTelemetry.Core/Data/EfTelemetryRepository.cs +++ b/src/Agoda.DevExTelemetry.Core/Data/EfTelemetryRepository.cs @@ -22,6 +22,16 @@ public async Task AddBuildMetricAsync(BuildMetric metric) await Db.SaveChangesAsync(); } + public async Task CommandEventExistsAsync(string id) => + await Db.CommandEvents.AnyAsync(e => e.Id == id); + + public async Task AddCommandEventAsync(CommandEvent commandEvent, IEnumerable npmTimers) + { + Db.CommandEvents.Add(commandEvent); + await Db.CommandEventNpmTimers.AddRangeAsync(npmTimers); + await Db.SaveChangesAsync(); + } + public async Task TestRunExistsAsync(string id) => await Db.TestRuns.AnyAsync(tr => tr.Id == id); @@ -348,6 +358,17 @@ await Db.BuildMetrics .Where(bm => bm.ReceivedAt < cutoff) .ExecuteDeleteAsync(); + await Db.CommandEventNpmTimers + .Where(timer => Db.CommandEvents + .Where(evt => evt.ReceivedAt < cutoff) + .Select(evt => evt.Id) + .Contains(timer.CommandEventId)) + .ExecuteDeleteAsync(); + + await Db.CommandEvents + .Where(evt => evt.ReceivedAt < cutoff) + .ExecuteDeleteAsync(); + await Db.RawPayloads .Where(rp => rp.ReceivedAt < cutoff) .ExecuteDeleteAsync(); diff --git a/src/Agoda.DevExTelemetry.Core/Data/ITelemetryRepository.cs b/src/Agoda.DevExTelemetry.Core/Data/ITelemetryRepository.cs index e803db2..c8dc447 100644 --- a/src/Agoda.DevExTelemetry.Core/Data/ITelemetryRepository.cs +++ b/src/Agoda.DevExTelemetry.Core/Data/ITelemetryRepository.cs @@ -7,6 +7,8 @@ public interface ITelemetryRepository { Task BuildMetricExistsAsync(string id); Task AddBuildMetricAsync(BuildMetric metric); + Task CommandEventExistsAsync(string id); + Task AddCommandEventAsync(CommandEvent commandEvent, IEnumerable npmTimers); Task TestRunExistsAsync(string id); Task AddTestRunAsync(TestRun run, IEnumerable testCases); Task AddRawPayloadAsync(RawPayload payload); diff --git a/src/Agoda.DevExTelemetry.Core/Data/TelemetryDbContext.cs b/src/Agoda.DevExTelemetry.Core/Data/TelemetryDbContext.cs index 27a47b5..7afef75 100644 --- a/src/Agoda.DevExTelemetry.Core/Data/TelemetryDbContext.cs +++ b/src/Agoda.DevExTelemetry.Core/Data/TelemetryDbContext.cs @@ -8,6 +8,8 @@ public class TelemetryDbContext : DbContext public TelemetryDbContext(DbContextOptions options) : base(options) { } public DbSet BuildMetrics => Set(); + public DbSet CommandEvents => Set(); + public DbSet CommandEventNpmTimers => Set(); public DbSet TestRuns => Set(); public DbSet TestCases => Set(); public DbSet RawPayloads => Set(); @@ -22,6 +24,31 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.HasIndex(e => e.BuildCategory); entity.HasIndex(e => e.ProjectName); entity.HasIndex(e => e.ExecutionEnvironment); + entity.HasIndex(e => e.SessionId); + }); + + modelBuilder.Entity(entity => + { + entity.HasKey(e => e.Id); + entity.HasIndex(e => e.ReceivedAt); + entity.HasIndex(e => e.SessionId); + entity.HasIndex(e => e.Phase); + entity.HasIndex(e => e.ProjectName); + entity.HasIndex(e => e.MeasurementSource); + entity.HasIndex(e => e.SourceEndpoint); + }); + + modelBuilder.Entity(entity => + { + entity.HasKey(e => e.Id); + entity.Property(e => e.Id).ValueGeneratedOnAdd(); + entity.HasIndex(e => e.CommandEventId); + entity.HasIndex(e => e.TimerName); + + entity.HasOne(e => e.CommandEvent) + .WithMany(e => e.NpmTimers) + .HasForeignKey(e => e.CommandEventId) + .OnDelete(DeleteBehavior.Cascade); }); modelBuilder.Entity(entity => diff --git a/src/Agoda.DevExTelemetry.Core/Migrations/20260824073108_AddCommandEventsAndSessionId.Designer.cs b/src/Agoda.DevExTelemetry.Core/Migrations/20260824073108_AddCommandEventsAndSessionId.Designer.cs new file mode 100644 index 0000000..7405f7a --- /dev/null +++ b/src/Agoda.DevExTelemetry.Core/Migrations/20260824073108_AddCommandEventsAndSessionId.Designer.cs @@ -0,0 +1,479 @@ +// +using System; +using Agoda.DevExTelemetry.Core.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace Agoda.DevExTelemetry.Core.Migrations +{ + [DbContext(typeof(TelemetryDbContext))] + [Migration("20260824073108_AddCommandEventsAndSessionId")] + partial class AddCommandEventsAndSessionId + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.5"); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.BuildMetric", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Branch") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("BuildCategory") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CommitSha") + .HasColumnType("TEXT"); + + b.Property("CpuCount") + .HasColumnType("INTEGER"); + + b.Property("ExecutionEnvironment") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ExtraData") + .HasColumnType("TEXT"); + + b.Property("Hostname") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IsDebuggerAttached") + .HasColumnType("INTEGER"); + + b.Property("MetricType") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Os") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Platform") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ProjectName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReceivedAt") + .HasColumnType("TEXT"); + + b.Property("ReloadType") + .HasColumnType("TEXT"); + + b.Property("Repository") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RepositoryName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SessionId") + .HasColumnType("TEXT"); + + b.Property("SourceEndpoint") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TimeTakenMs") + .HasColumnType("REAL"); + + b.Property("ToolVersion") + .HasColumnType("TEXT"); + + b.Property("UserName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("BuildCategory"); + + b.HasIndex("ExecutionEnvironment"); + + b.HasIndex("MetricType"); + + b.HasIndex("ProjectName"); + + b.HasIndex("ReceivedAt"); + + b.HasIndex("SessionId"); + + b.ToTable("BuildMetrics"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.CommandEvent", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Branch") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ColdInstall") + .HasColumnType("INTEGER"); + + b.Property("Command") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CommitSha") + .HasColumnType("TEXT"); + + b.Property("CpuCount") + .HasColumnType("INTEGER"); + + b.Property("DomContentLoadedMs") + .HasColumnType("REAL"); + + b.Property("ErrorCount") + .HasColumnType("INTEGER"); + + b.Property("ExitCode") + .HasColumnType("INTEGER"); + + b.Property("ExtraData") + .HasColumnType("TEXT"); + + b.Property("FirstContentfulPaintMs") + .HasColumnType("REAL"); + + b.Property("Hostname") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("LockfileChanged") + .HasColumnType("INTEGER"); + + b.Property("MeasurementSource") + .HasColumnType("TEXT"); + + b.Property("Os") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("PackageManager") + .HasColumnType("TEXT"); + + b.Property("PackageManagerVersion") + .HasColumnType("TEXT"); + + b.Property("Phase") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Platform") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Prebundled") + .HasColumnType("INTEGER"); + + b.Property("ProjectName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReceivedAt") + .HasColumnType("TEXT"); + + b.Property("Repository") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RepositoryName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SessionId") + .HasColumnType("TEXT"); + + b.Property("Signal") + .HasColumnType("TEXT"); + + b.Property("SourceEndpoint") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SpooledAt") + .HasColumnType("INTEGER"); + + b.Property("Success") + .HasColumnType("INTEGER"); + + b.Property("TimeTakenMs") + .HasColumnType("REAL"); + + b.Property("Type") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("UserName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("MeasurementSource"); + + b.HasIndex("Phase"); + + b.HasIndex("ProjectName"); + + b.HasIndex("ReceivedAt"); + + b.HasIndex("SessionId"); + + b.HasIndex("SourceEndpoint"); + + b.ToTable("CommandEvents"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.CommandEventNpmTimer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CommandEventId") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("REAL"); + + b.Property("TimerName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CommandEventId"); + + b.HasIndex("TimerName"); + + b.ToTable("CommandEventNpmTimers"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.RawPayload", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ContentType") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("PayloadJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReceivedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("RawPayloads"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.TestCase", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ClassName") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("REAL"); + + b.Property("EndTime") + .HasColumnType("TEXT"); + + b.Property("ErrorMessage") + .HasColumnType("TEXT"); + + b.Property("FullName") + .HasColumnType("TEXT"); + + b.Property("MethodName") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("OriginalId") + .HasColumnType("TEXT"); + + b.Property("StartTime") + .HasColumnType("TEXT"); + + b.Property("Status") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TestRunId") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ClassName"); + + b.HasIndex("Status"); + + b.HasIndex("TestRunId"); + + b.ToTable("TestCases"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.TestRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Branch") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CpuCount") + .HasColumnType("INTEGER"); + + b.Property("ExecutionEnvironment") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ExtraData") + .HasColumnType("TEXT"); + + b.Property("FailedTests") + .HasColumnType("INTEGER"); + + b.Property("Hostname") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IsDebuggerAttached") + .HasColumnType("INTEGER"); + + b.Property("Os") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("PassedTests") + .HasColumnType("INTEGER"); + + b.Property("Platform") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ProjectName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReceivedAt") + .HasColumnType("TEXT"); + + b.Property("Repository") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RepositoryName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RunId") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SkippedTests") + .HasColumnType("INTEGER"); + + b.Property("SourceEndpoint") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TestRunner") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TotalDurationMs") + .HasColumnType("REAL"); + + b.Property("TotalTests") + .HasColumnType("INTEGER"); + + b.Property("UserName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ExecutionEnvironment"); + + b.HasIndex("ProjectName"); + + b.HasIndex("ReceivedAt"); + + b.HasIndex("TestRunner"); + + b.ToTable("TestRuns"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.CommandEventNpmTimer", b => + { + b.HasOne("Agoda.DevExTelemetry.Core.Models.Entities.CommandEvent", "CommandEvent") + .WithMany("NpmTimers") + .HasForeignKey("CommandEventId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CommandEvent"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.TestCase", b => + { + b.HasOne("Agoda.DevExTelemetry.Core.Models.Entities.TestRun", "TestRun") + .WithMany("TestCases") + .HasForeignKey("TestRunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("TestRun"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.CommandEvent", b => + { + b.Navigation("NpmTimers"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.TestRun", b => + { + b.Navigation("TestCases"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Agoda.DevExTelemetry.Core/Migrations/20260824073108_AddCommandEventsAndSessionId.cs b/src/Agoda.DevExTelemetry.Core/Migrations/20260824073108_AddCommandEventsAndSessionId.cs new file mode 100644 index 0000000..502fd4b --- /dev/null +++ b/src/Agoda.DevExTelemetry.Core/Migrations/20260824073108_AddCommandEventsAndSessionId.cs @@ -0,0 +1,147 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Agoda.DevExTelemetry.Core.Migrations +{ + /// + public partial class AddCommandEventsAndSessionId : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "SessionId", + table: "BuildMetrics", + type: "TEXT", + nullable: true); + + migrationBuilder.CreateTable( + name: "CommandEvents", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + SessionId = table.Column(type: "TEXT", nullable: true), + ReceivedAt = table.Column(type: "TEXT", nullable: false), + UserName = table.Column(type: "TEXT", nullable: false), + CpuCount = table.Column(type: "INTEGER", nullable: false), + Hostname = table.Column(type: "TEXT", nullable: false), + Platform = table.Column(type: "TEXT", nullable: false), + Os = table.Column(type: "TEXT", nullable: false), + Branch = table.Column(type: "TEXT", nullable: false), + ProjectName = table.Column(type: "TEXT", nullable: false), + Repository = table.Column(type: "TEXT", nullable: false), + RepositoryName = table.Column(type: "TEXT", nullable: false), + Type = table.Column(type: "TEXT", nullable: false), + Phase = table.Column(type: "TEXT", nullable: false), + Command = table.Column(type: "TEXT", nullable: false), + ExitCode = table.Column(type: "INTEGER", nullable: false), + Success = table.Column(type: "INTEGER", nullable: false), + Signal = table.Column(type: "TEXT", nullable: true), + ErrorCount = table.Column(type: "INTEGER", nullable: true), + TimeTakenMs = table.Column(type: "REAL", nullable: false), + PackageManager = table.Column(type: "TEXT", nullable: true), + PackageManagerVersion = table.Column(type: "TEXT", nullable: true), + ColdInstall = table.Column(type: "INTEGER", nullable: true), + LockfileChanged = table.Column(type: "INTEGER", nullable: true), + MeasurementSource = table.Column(type: "TEXT", nullable: true), + Prebundled = table.Column(type: "INTEGER", nullable: true), + DomContentLoadedMs = table.Column(type: "REAL", nullable: true), + FirstContentfulPaintMs = table.Column(type: "REAL", nullable: true), + SpooledAt = table.Column(type: "INTEGER", nullable: true), + CommitSha = table.Column(type: "TEXT", nullable: true), + SourceEndpoint = table.Column(type: "TEXT", nullable: false), + ExtraData = table.Column(type: "TEXT", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_CommandEvents", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "CommandEventNpmTimers", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + CommandEventId = table.Column(type: "TEXT", nullable: false), + TimerName = table.Column(type: "TEXT", nullable: false), + DurationMs = table.Column(type: "REAL", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_CommandEventNpmTimers", x => x.Id); + table.ForeignKey( + name: "FK_CommandEventNpmTimers_CommandEvents_CommandEventId", + column: x => x.CommandEventId, + principalTable: "CommandEvents", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_BuildMetrics_SessionId", + table: "BuildMetrics", + column: "SessionId"); + + migrationBuilder.CreateIndex( + name: "IX_CommandEventNpmTimers_CommandEventId", + table: "CommandEventNpmTimers", + column: "CommandEventId"); + + migrationBuilder.CreateIndex( + name: "IX_CommandEventNpmTimers_TimerName", + table: "CommandEventNpmTimers", + column: "TimerName"); + + migrationBuilder.CreateIndex( + name: "IX_CommandEvents_MeasurementSource", + table: "CommandEvents", + column: "MeasurementSource"); + + migrationBuilder.CreateIndex( + name: "IX_CommandEvents_Phase", + table: "CommandEvents", + column: "Phase"); + + migrationBuilder.CreateIndex( + name: "IX_CommandEvents_ProjectName", + table: "CommandEvents", + column: "ProjectName"); + + migrationBuilder.CreateIndex( + name: "IX_CommandEvents_ReceivedAt", + table: "CommandEvents", + column: "ReceivedAt"); + + migrationBuilder.CreateIndex( + name: "IX_CommandEvents_SessionId", + table: "CommandEvents", + column: "SessionId"); + + migrationBuilder.CreateIndex( + name: "IX_CommandEvents_SourceEndpoint", + table: "CommandEvents", + column: "SourceEndpoint"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "CommandEventNpmTimers"); + + migrationBuilder.DropTable( + name: "CommandEvents"); + + migrationBuilder.DropIndex( + name: "IX_BuildMetrics_SessionId", + table: "BuildMetrics"); + + migrationBuilder.DropColumn( + name: "SessionId", + table: "BuildMetrics"); + } + } +} diff --git a/src/Agoda.DevExTelemetry.Core/Migrations/TelemetryDbContextModelSnapshot.cs b/src/Agoda.DevExTelemetry.Core/Migrations/TelemetryDbContextModelSnapshot.cs index 77f743d..7398e82 100644 --- a/src/Agoda.DevExTelemetry.Core/Migrations/TelemetryDbContextModelSnapshot.cs +++ b/src/Agoda.DevExTelemetry.Core/Migrations/TelemetryDbContextModelSnapshot.cs @@ -80,6 +80,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("TEXT"); + b.Property("SessionId") + .HasColumnType("TEXT"); + b.Property("SourceEndpoint") .IsRequired() .HasColumnType("TEXT"); @@ -106,9 +109,164 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("ReceivedAt"); + b.HasIndex("SessionId"); + b.ToTable("BuildMetrics"); }); + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.CommandEvent", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Branch") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ColdInstall") + .HasColumnType("INTEGER"); + + b.Property("Command") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CommitSha") + .HasColumnType("TEXT"); + + b.Property("CpuCount") + .HasColumnType("INTEGER"); + + b.Property("DomContentLoadedMs") + .HasColumnType("REAL"); + + b.Property("ErrorCount") + .HasColumnType("INTEGER"); + + b.Property("ExitCode") + .HasColumnType("INTEGER"); + + b.Property("ExtraData") + .HasColumnType("TEXT"); + + b.Property("FirstContentfulPaintMs") + .HasColumnType("REAL"); + + b.Property("Hostname") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("LockfileChanged") + .HasColumnType("INTEGER"); + + b.Property("MeasurementSource") + .HasColumnType("TEXT"); + + b.Property("Os") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("PackageManager") + .HasColumnType("TEXT"); + + b.Property("PackageManagerVersion") + .HasColumnType("TEXT"); + + b.Property("Phase") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Platform") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Prebundled") + .HasColumnType("INTEGER"); + + b.Property("ProjectName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReceivedAt") + .HasColumnType("TEXT"); + + b.Property("Repository") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RepositoryName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SessionId") + .HasColumnType("TEXT"); + + b.Property("Signal") + .HasColumnType("TEXT"); + + b.Property("SourceEndpoint") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SpooledAt") + .HasColumnType("INTEGER"); + + b.Property("Success") + .HasColumnType("INTEGER"); + + b.Property("TimeTakenMs") + .HasColumnType("REAL"); + + b.Property("Type") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("UserName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("MeasurementSource"); + + b.HasIndex("Phase"); + + b.HasIndex("ProjectName"); + + b.HasIndex("ReceivedAt"); + + b.HasIndex("SessionId"); + + b.HasIndex("SourceEndpoint"); + + b.ToTable("CommandEvents"); + }); + + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.CommandEventNpmTimer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CommandEventId") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("REAL"); + + b.Property("TimerName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CommandEventId"); + + b.HasIndex("TimerName"); + + b.ToTable("CommandEventNpmTimers"); + }); + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.RawPayload", b => { b.Property("Id") @@ -281,6 +439,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("TestRuns"); }); + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.CommandEventNpmTimer", b => + { + b.HasOne("Agoda.DevExTelemetry.Core.Models.Entities.CommandEvent", "CommandEvent") + .WithMany("NpmTimers") + .HasForeignKey("CommandEventId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CommandEvent"); + }); + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.TestCase", b => { b.HasOne("Agoda.DevExTelemetry.Core.Models.Entities.TestRun", "TestRun") @@ -292,6 +461,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("TestRun"); }); + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.CommandEvent", b => + { + b.Navigation("NpmTimers"); + }); + modelBuilder.Entity("Agoda.DevExTelemetry.Core.Models.Entities.TestRun", b => { b.Navigation("TestCases"); diff --git a/src/Agoda.DevExTelemetry.Core/Models/Entities/BuildMetric.cs b/src/Agoda.DevExTelemetry.Core/Models/Entities/BuildMetric.cs index 60361f5..7bdf9dd 100644 --- a/src/Agoda.DevExTelemetry.Core/Models/Entities/BuildMetric.cs +++ b/src/Agoda.DevExTelemetry.Core/Models/Entities/BuildMetric.cs @@ -3,6 +3,7 @@ namespace Agoda.DevExTelemetry.Core.Models.Entities; public class BuildMetric { public string Id { get; set; } = string.Empty; + public string? SessionId { get; set; } public DateTime ReceivedAt { get; set; } public string UserName { get; set; } = string.Empty; public int CpuCount { get; set; } diff --git a/src/Agoda.DevExTelemetry.Core/Models/Entities/CommandEvent.cs b/src/Agoda.DevExTelemetry.Core/Models/Entities/CommandEvent.cs new file mode 100644 index 0000000..513cbe2 --- /dev/null +++ b/src/Agoda.DevExTelemetry.Core/Models/Entities/CommandEvent.cs @@ -0,0 +1,39 @@ +namespace Agoda.DevExTelemetry.Core.Models.Entities; + +public class CommandEvent +{ + public string Id { get; set; } = string.Empty; + public string? SessionId { get; set; } + public DateTime ReceivedAt { get; set; } + public string UserName { get; set; } = string.Empty; + public int CpuCount { get; set; } + public string Hostname { get; set; } = string.Empty; + public string Platform { get; set; } = string.Empty; + public string Os { get; set; } = string.Empty; + public string Branch { get; set; } = string.Empty; + public string ProjectName { get; set; } = string.Empty; + public string Repository { get; set; } = string.Empty; + public string RepositoryName { get; set; } = string.Empty; + public string Type { get; set; } = string.Empty; + public string Phase { get; set; } = string.Empty; + public string Command { get; set; } = string.Empty; + public int ExitCode { get; set; } + public bool Success { get; set; } + public string? Signal { get; set; } + public int? ErrorCount { get; set; } + public double TimeTakenMs { get; set; } + public string? PackageManager { get; set; } + public string? PackageManagerVersion { get; set; } + public bool? ColdInstall { get; set; } + public bool? LockfileChanged { get; set; } + public string? MeasurementSource { get; set; } + public bool? Prebundled { get; set; } + public double? DomContentLoadedMs { get; set; } + public double? FirstContentfulPaintMs { get; set; } + public long? SpooledAt { get; set; } + public string? CommitSha { get; set; } + public string SourceEndpoint { get; set; } = string.Empty; + public string? ExtraData { get; set; } + + public ICollection NpmTimers { get; set; } = new List(); +} diff --git a/src/Agoda.DevExTelemetry.Core/Models/Entities/CommandEventNpmTimer.cs b/src/Agoda.DevExTelemetry.Core/Models/Entities/CommandEventNpmTimer.cs new file mode 100644 index 0000000..4098066 --- /dev/null +++ b/src/Agoda.DevExTelemetry.Core/Models/Entities/CommandEventNpmTimer.cs @@ -0,0 +1,11 @@ +namespace Agoda.DevExTelemetry.Core.Models.Entities; + +public class CommandEventNpmTimer +{ + public int Id { get; set; } + public string CommandEventId { get; set; } = string.Empty; + public string TimerName { get; set; } = string.Empty; + public double DurationMs { get; set; } + + public CommandEvent CommandEvent { get; set; } = null!; +} diff --git a/src/Agoda.DevExTelemetry.Core/Models/Ingest/CommandPayload.cs b/src/Agoda.DevExTelemetry.Core/Models/Ingest/CommandPayload.cs new file mode 100644 index 0000000..38cebff --- /dev/null +++ b/src/Agoda.DevExTelemetry.Core/Models/Ingest/CommandPayload.cs @@ -0,0 +1,124 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Agoda.DevExTelemetry.Core.Models.Ingest; + +public class CommandPayload +{ + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("sessionId")] + public string? SessionId { get; set; } + + [JsonPropertyName("userName")] + public string? UserName { get; set; } + + [JsonPropertyName("cpuCount")] + public int CpuCount { get; set; } + + [JsonPropertyName("hostname")] + public string? Hostname { get; set; } + + [JsonPropertyName("platform")] + public JsonElement Platform { get; set; } + + [JsonPropertyName("os")] + public string? Os { get; set; } + + [JsonPropertyName("branch")] + public string? Branch { get; set; } + + [JsonPropertyName("projectName")] + public string? ProjectName { get; set; } + + [JsonPropertyName("repository")] + public string? Repository { get; set; } + + [JsonPropertyName("repositoryName")] + public string? RepositoryName { get; set; } + + [JsonPropertyName("timestamp")] + public long? Timestamp { get; set; } + + [JsonPropertyName("builtAt")] + public JsonElement BuiltAt { get; set; } + + [JsonPropertyName("totalMemory")] + public long? TotalMemory { get; set; } + + [JsonPropertyName("cpuModels")] + public JsonElement CpuModels { get; set; } + + [JsonPropertyName("cpuSpeed")] + public JsonElement CpuSpeed { get; set; } + + [JsonPropertyName("nodeVersion")] + public string? NodeVersion { get; set; } + + [JsonPropertyName("v8Version")] + public string? V8Version { get; set; } + + [JsonPropertyName("commitSha")] + public string? CommitSha { get; set; } + + [JsonPropertyName("customIdentifier")] + public string? CustomIdentifier { get; set; } + + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("phase")] + public string? Phase { get; set; } + + [JsonPropertyName("command")] + public string? Command { get; set; } + + [JsonPropertyName("exitCode")] + public int ExitCode { get; set; } + + [JsonPropertyName("success")] + public bool Success { get; set; } + + [JsonPropertyName("signal")] + public string? Signal { get; set; } + + [JsonPropertyName("errorCount")] + public int? ErrorCount { get; set; } + + [JsonPropertyName("timeTaken")] + public JsonElement TimeTaken { get; set; } + + [JsonPropertyName("packageManager")] + public string? PackageManager { get; set; } + + [JsonPropertyName("packageManagerVersion")] + public string? PackageManagerVersion { get; set; } + + [JsonPropertyName("coldInstall")] + public bool? ColdInstall { get; set; } + + [JsonPropertyName("lockfileChanged")] + public bool? LockfileChanged { get; set; } + + [JsonPropertyName("measurementSource")] + public string? MeasurementSource { get; set; } + + [JsonPropertyName("npmTimers")] + public Dictionary? NpmTimers { get; set; } + + [JsonPropertyName("prebundled")] + public bool? Prebundled { get; set; } + + [JsonPropertyName("domContentLoadedMs")] + public double? DomContentLoadedMs { get; set; } + + [JsonPropertyName("firstContentfulPaintMs")] + public double? FirstContentfulPaintMs { get; set; } + + [JsonPropertyName("spooledAt")] + public long? SpooledAt { get; set; } + + [JsonExtensionData] + public Dictionary? AdditionalData { get; set; } +} diff --git a/src/Agoda.DevExTelemetry.Core/Models/Ingest/IngestCommandEventWorkItem.cs b/src/Agoda.DevExTelemetry.Core/Models/Ingest/IngestCommandEventWorkItem.cs new file mode 100644 index 0000000..b68a5ba --- /dev/null +++ b/src/Agoda.DevExTelemetry.Core/Models/Ingest/IngestCommandEventWorkItem.cs @@ -0,0 +1,12 @@ +using Agoda.DevExTelemetry.Core.Models.Entities; + +namespace Agoda.DevExTelemetry.Core.Models.Ingest; + +public class IngestCommandEventWorkItem +{ + public required CommandEvent CommandEvent { get; init; } + public required IReadOnlyList NpmTimers { get; init; } + public string? RawPayloadJson { get; init; } + public string? RawPayloadEndpoint { get; init; } + public string? RawPayloadContentType { get; init; } +} diff --git a/src/Agoda.DevExTelemetry.Core/Models/Ingest/VitePayload.cs b/src/Agoda.DevExTelemetry.Core/Models/Ingest/VitePayload.cs index f7380c6..78ae4a9 100644 --- a/src/Agoda.DevExTelemetry.Core/Models/Ingest/VitePayload.cs +++ b/src/Agoda.DevExTelemetry.Core/Models/Ingest/VitePayload.cs @@ -7,6 +7,9 @@ public class VitePayload [JsonPropertyName("id")] public string? Id { get; set; } + [JsonPropertyName("sessionId")] + public string? SessionId { get; set; } + [JsonPropertyName("userName")] public string? UserName { get; set; } diff --git a/src/Agoda.DevExTelemetry.Core/Models/Ingest/WebpackPayload.cs b/src/Agoda.DevExTelemetry.Core/Models/Ingest/WebpackPayload.cs index 3d9b82d..9bb40fe 100644 --- a/src/Agoda.DevExTelemetry.Core/Models/Ingest/WebpackPayload.cs +++ b/src/Agoda.DevExTelemetry.Core/Models/Ingest/WebpackPayload.cs @@ -7,6 +7,12 @@ public class WebpackPayload [JsonPropertyName("id")] public string? Id { get; set; } + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("sessionId")] + public string? SessionId { get; set; } + [JsonPropertyName("userName")] public string? UserName { get; set; } diff --git a/src/Agoda.DevExTelemetry.Core/Services/IIngestService.cs b/src/Agoda.DevExTelemetry.Core/Services/IIngestService.cs index a02a7d5..8036319 100644 --- a/src/Agoda.DevExTelemetry.Core/Services/IIngestService.cs +++ b/src/Agoda.DevExTelemetry.Core/Services/IIngestService.cs @@ -5,6 +5,7 @@ namespace Agoda.DevExTelemetry.Core.Services; public interface IIngestService { Task IngestBuildMetricAsync(BuildMetric metric); + Task IngestCommandEventAsync(CommandEvent commandEvent, IEnumerable npmTimers); Task IngestTestRunAsync(TestRun run, IEnumerable testCases); Task StoreRawPayloadAsync(string endpoint, string contentType, string json); } diff --git a/src/Agoda.DevExTelemetry.Core/Services/IngestService.cs b/src/Agoda.DevExTelemetry.Core/Services/IngestService.cs index e926161..210461a 100644 --- a/src/Agoda.DevExTelemetry.Core/Services/IngestService.cs +++ b/src/Agoda.DevExTelemetry.Core/Services/IngestService.cs @@ -33,6 +33,19 @@ public async Task IngestBuildMetricAsync(BuildMetric metric) await _repository.AddBuildMetricAsync(metric); } + public async Task IngestCommandEventAsync(CommandEvent commandEvent, IEnumerable npmTimers) + { + commandEvent.ReceivedAt = DateTime.UtcNow; + + if (await _repository.CommandEventExistsAsync(commandEvent.Id)) + { + _logger.LogInformation("Ignoring duplicate CommandEvent with Id={Id}", commandEvent.Id); + return; + } + + await _repository.AddCommandEventAsync(commandEvent, npmTimers); + } + public async Task IngestTestRunAsync(TestRun run, IEnumerable testCases) { run.ReceivedAt = DateTime.UtcNow; diff --git a/src/Agoda.DevExTelemetry.IntegrationTests/CommandIngestTests.cs b/src/Agoda.DevExTelemetry.IntegrationTests/CommandIngestTests.cs new file mode 100644 index 0000000..ba2d1ef --- /dev/null +++ b/src/Agoda.DevExTelemetry.IntegrationTests/CommandIngestTests.cs @@ -0,0 +1,112 @@ +using System.Net; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Shouldly; + +namespace Agoda.DevExTelemetry.IntegrationTests; + +[TestFixture(DatabaseProvider.Sqlite)] +[TestFixture(DatabaseProvider.PostgreSql)] +public class CommandIngestTests +{ + private readonly DatabaseProvider _provider; + private CustomWebApplicationFactory _factory = null!; + private HttpClient _client = null!; + + public CommandIngestTests(DatabaseProvider provider) => _provider = provider; + + [SetUp] + public void SetUp() + { + _factory = new CustomWebApplicationFactory(_provider); + _client = _factory.CreateClient(); + } + + [TearDown] + public void TearDown() + { + _client.Dispose(); + _factory.Dispose(); + } + + [Test] + public async Task POST_Command_InstallWithTimers_PersistsEventAndTimerRows() + { + var payload = TestFixtures.CreateCommandPayload( + phase: "install", + measurementSource: "npm-timing", + npmTimers: new Dictionary + { + ["idealTree"] = 8120, + ["reify"] = 39880 + }); + + var response = await TestFixtures.PostJsonAsync(_client, "/command", payload); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + await _factory.DrainBackgroundQueuesAsync(); + + using var db = _factory.CreateDbContext(); + var evt = await db.CommandEvents.FirstOrDefaultAsync(); + evt.ShouldNotBeNull(); + evt.MeasurementSource.ShouldBe("npm-timing"); + + var timers = await db.CommandEventNpmTimers + .OrderBy(t => t.TimerName) + .ToListAsync(); + timers.Count.ShouldBe(2); + timers[0].TimerName.ShouldBe("idealTree"); + timers[0].DurationMs.ShouldBe(8120); + timers[1].TimerName.ShouldBe("reify"); + timers[1].DurationMs.ShouldBe(39880); + } + + [Test] + public async Task POST_Command_Devserver_WithPrebundledFalse_PersistsFalse() + { + var payload = TestFixtures.CreateCommandPayload( + phase: "devserver", + prebundled: false); + + await TestFixtures.PostJsonAsync(_client, "/command", payload); + await _factory.DrainBackgroundQueuesAsync(); + + using var db = _factory.CreateDbContext(); + var evt = await db.CommandEvents.FirstOrDefaultAsync(); + evt.ShouldNotBeNull(); + evt.Prebundled.ShouldBe(false); + } + + [Test] + public async Task POST_Command_Devserver_WithoutPrebundled_PersistsNull() + { + var payload = TestFixtures.CreateCommandPayload( + phase: "devserver", + includePrebundled: false); + + await TestFixtures.PostJsonAsync(_client, "/command", payload); + await _factory.DrainBackgroundQueuesAsync(); + + using var db = _factory.CreateDbContext(); + var evt = await db.CommandEvents.FirstOrDefaultAsync(); + evt.ShouldNotBeNull(); + evt.Prebundled.ShouldBeNull(); + } + + [Test] + public async Task POST_Command_WithSpooledAt_PersistsValue() + { + const long spooledAt = 1785900160500; + var payload = TestFixtures.CreateCommandPayload( + phase: "devserver", + spooledAt: spooledAt); + + await TestFixtures.PostJsonAsync(_client, "/command", payload); + await _factory.DrainBackgroundQueuesAsync(); + + using var db = _factory.CreateDbContext(); + var evt = await db.CommandEvents.FirstOrDefaultAsync(); + evt.ShouldNotBeNull(); + evt.SpooledAt.ShouldBe(spooledAt); + } +} diff --git a/src/Agoda.DevExTelemetry.IntegrationTests/CustomWebApplicationFactory.cs b/src/Agoda.DevExTelemetry.IntegrationTests/CustomWebApplicationFactory.cs index 9508d2e..588c26b 100644 --- a/src/Agoda.DevExTelemetry.IntegrationTests/CustomWebApplicationFactory.cs +++ b/src/Agoda.DevExTelemetry.IntegrationTests/CustomWebApplicationFactory.cs @@ -84,13 +84,15 @@ public async Task DrainBackgroundQueuesAsync(CancellationToken cancellationToken { var testRunQueue = Services.GetRequiredService>(); var buildMetricQueue = Services.GetRequiredService>(); + var commandEventQueue = Services.GetRequiredService>(); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); await Task.WhenAll( testRunQueue.WaitUntilDrainedAsync(cts.Token), - buildMetricQueue.WaitUntilDrainedAsync(cts.Token)); + buildMetricQueue.WaitUntilDrainedAsync(cts.Token), + commandEventQueue.WaitUntilDrainedAsync(cts.Token)); } protected override void Dispose(bool disposing) diff --git a/src/Agoda.DevExTelemetry.IntegrationTests/RspackIngestTests.cs b/src/Agoda.DevExTelemetry.IntegrationTests/RspackIngestTests.cs new file mode 100644 index 0000000..51e6c6e --- /dev/null +++ b/src/Agoda.DevExTelemetry.IntegrationTests/RspackIngestTests.cs @@ -0,0 +1,80 @@ +using System.Net; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Shouldly; + +namespace Agoda.DevExTelemetry.IntegrationTests; + +[TestFixture(DatabaseProvider.Sqlite)] +[TestFixture(DatabaseProvider.PostgreSql)] +public class RspackIngestTests +{ + private readonly DatabaseProvider _provider; + private CustomWebApplicationFactory _factory = null!; + private HttpClient _client = null!; + + public RspackIngestTests(DatabaseProvider provider) => _provider = provider; + + [SetUp] + public void SetUp() + { + _factory = new CustomWebApplicationFactory(_provider); + _client = _factory.CreateClient(); + } + + [TearDown] + public void TearDown() + { + _client.Dispose(); + _factory.Dispose(); + } + + [Test] + public async Task POST_Rspack_TypeRspack_PersistsRspackMetric() + { + var payload = TestFixtures.CreateWebpackPayload(type: "rspack", sessionId: "session-rspack"); + var response = await TestFixtures.PostJsonAsync(_client, "/rspack", payload); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + await _factory.DrainBackgroundQueuesAsync(); + + using var db = _factory.CreateDbContext(); + var metric = await db.BuildMetrics.FirstOrDefaultAsync(); + metric.ShouldNotBeNull(); + metric.MetricType.ShouldBe("rspack"); + metric.SourceEndpoint.ShouldBe("/rspack"); + metric.SessionId.ShouldBe("session-rspack"); + } + + [Test] + public async Task POST_Rspack_TypeRsbuild_PersistsRsbuildMetric() + { + var payload = TestFixtures.CreateWebpackPayload(type: "rsbuild"); + var response = await TestFixtures.PostJsonAsync(_client, "/rspack", payload); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + await _factory.DrainBackgroundQueuesAsync(); + + using var db = _factory.CreateDbContext(); + var metric = await db.BuildMetrics.FirstOrDefaultAsync(); + metric.ShouldNotBeNull(); + metric.MetricType.ShouldBe("rsbuild"); + metric.SourceEndpoint.ShouldBe("/rspack"); + } + + [Test] + public async Task POST_Webpack_TypeRsbuild_StillAccepted_ForBackCompat() + { + var payload = TestFixtures.CreateWebpackPayload(type: "rsbuild"); + var response = await TestFixtures.PostJsonAsync(_client, "/webpack", payload); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + await _factory.DrainBackgroundQueuesAsync(); + + using var db = _factory.CreateDbContext(); + var metric = await db.BuildMetrics.FirstOrDefaultAsync(); + metric.ShouldNotBeNull(); + metric.MetricType.ShouldBe("rsbuild"); + metric.SourceEndpoint.ShouldBe("/webpack"); + } +} diff --git a/src/Agoda.DevExTelemetry.IntegrationTests/TestFixtures.cs b/src/Agoda.DevExTelemetry.IntegrationTests/TestFixtures.cs index 34c7473..004e4e0 100644 --- a/src/Agoda.DevExTelemetry.IntegrationTests/TestFixtures.cs +++ b/src/Agoda.DevExTelemetry.IntegrationTests/TestFixtures.cs @@ -195,10 +195,16 @@ public static object CreateScalaTestPayload(int testCaseCount = 2, string? id = }).ToList() }; - public static object CreateWebpackPayload(bool withHmrFeedback = false, string? id = null) => + public static object CreateWebpackPayload( + bool withHmrFeedback = false, + string? id = null, + string? type = null, + string? sessionId = null) => new { id = id ?? Guid.NewGuid().ToString(), + type, + sessionId, userName = "testuser", cpuCount = 8, hostname = "dev-workstation", @@ -223,6 +229,65 @@ public static object CreateWebpackPayload(bool withHmrFeedback = false, string? : Array.Empty() }; + public static object CreateCommandPayload( + string phase, + string? id = null, + string? sessionId = null, + string command = "vite dev", + int exitCode = 0, + bool success = true, + double timeTaken = 4120, + string? signal = null, + bool includePrebundled = true, + bool? prebundled = true, + string? measurementSource = null, + Dictionary? npmTimers = null, + long? spooledAt = null, + double? domContentLoadedMs = null, + double? firstContentfulPaintMs = null) + { + var payload = new Dictionary + { + ["id"] = id ?? Guid.NewGuid().ToString(), + ["sessionId"] = sessionId ?? Guid.NewGuid().ToString(), + ["userName"] = "testuser", + ["cpuCount"] = 8, + ["hostname"] = "dev-workstation", + ["platform"] = "Darwin", + ["os"] = "23.5.0", + ["branch"] = "main", + ["projectName"] = "TestProject", + ["repository"] = "https://github.com/test/repo", + ["repositoryName"] = "test-repo", + ["timestamp"] = 1711094400000L, + ["builtAt"] = "2026-08-05T03:21:00.000Z", + ["totalMemory"] = 16_000_000_000L, + ["cpuModels"] = new[] { "Apple M3 Pro" }, + ["cpuSpeed"] = new[] { 0 }, + ["nodeVersion"] = "v22.14.0", + ["v8Version"] = "12.4.254.21-node.35", + ["commitSha"] = "abc123", + ["customIdentifier"] = "dev", + ["type"] = "command", + ["phase"] = phase, + ["command"] = command, + ["exitCode"] = exitCode, + ["success"] = success, + ["signal"] = signal, + ["timeTaken"] = timeTaken, + ["measurementSource"] = measurementSource, + ["npmTimers"] = npmTimers, + ["spooledAt"] = spooledAt, + ["domContentLoadedMs"] = domContentLoadedMs, + ["firstContentfulPaintMs"] = firstContentfulPaintMs + }; + + if (includePrebundled) + payload["prebundled"] = prebundled; + + return payload; + } + public static object CreateVitePayload(string type = "vite", string? id = null) => new { diff --git a/src/Agoda.DevExTelemetry.WebApi/Controllers/CommandController.cs b/src/Agoda.DevExTelemetry.WebApi/Controllers/CommandController.cs new file mode 100644 index 0000000..6607b9d --- /dev/null +++ b/src/Agoda.DevExTelemetry.WebApi/Controllers/CommandController.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Threading.Tasks; +using Agoda.DevExTelemetry.Core.Models.Entities; +using Agoda.DevExTelemetry.Core.Models.Ingest; +using Agoda.DevExTelemetry.Core.Services; +using Microsoft.AspNetCore.Mvc; + +namespace Agoda.DevExTelemetry.WebApi.Controllers; + +[ApiController] +public class CommandController : ControllerBase +{ + private static readonly string[] AllowedPhases = + { + "install", + "codegen", + "typecheck", + "lint", + "test", + "build", + "devserver", + "clientready" + }; + + private readonly IBackgroundTaskQueue _queue; + + public CommandController(IBackgroundTaskQueue queue) + { + _queue = queue; + } + + [HttpPost("command")] + [RequestSizeLimit(500 * 1024 * 1024)] + public async Task Ingest([FromBody] CommandPayload payload) + { + if (!string.Equals(payload.Type, "command", StringComparison.OrdinalIgnoreCase)) + return BadRequest(new { error = "Invalid type value" }); + + if (string.IsNullOrWhiteSpace(payload.Phase) || + Array.IndexOf(AllowedPhases, payload.Phase.ToLowerInvariant()) < 0) + { + return BadRequest(new { error = "Invalid phase value" }); + } + + if (string.IsNullOrWhiteSpace(payload.Command)) + return BadRequest(new { error = "command is required" }); + + if (!TryGetDouble(payload.TimeTaken, out var timeTakenMs) || !double.IsFinite(timeTakenMs) || timeTakenMs < 0) + return BadRequest(new { error = "Invalid timeTaken value" }); + + var platform = ResolvePlatform(payload.Platform); + + object? builtAt = payload.BuiltAt.ValueKind == JsonValueKind.Undefined ? null : payload.BuiltAt; + object? cpuModels = payload.CpuModels.ValueKind == JsonValueKind.Undefined ? null : payload.CpuModels; + object? cpuSpeed = payload.CpuSpeed.ValueKind == JsonValueKind.Undefined ? null : payload.CpuSpeed; + + var extraData = new + { + payload.Timestamp, + BuiltAt = builtAt, + payload.TotalMemory, + CpuModels = cpuModels, + CpuSpeed = cpuSpeed, + payload.NodeVersion, + payload.V8Version, + payload.CustomIdentifier, + payload.AdditionalData + }; + + var commandEventId = payload.Id ?? Guid.NewGuid().ToString(); + var commandEvent = new CommandEvent + { + Id = commandEventId, + SessionId = payload.SessionId, + UserName = payload.UserName ?? string.Empty, + CpuCount = payload.CpuCount, + Hostname = payload.Hostname ?? string.Empty, + Platform = platform, + Os = payload.Os ?? string.Empty, + Branch = payload.Branch ?? string.Empty, + ProjectName = payload.ProjectName ?? string.Empty, + Repository = payload.Repository ?? string.Empty, + RepositoryName = payload.RepositoryName ?? string.Empty, + Type = "command", + Phase = payload.Phase!.ToLowerInvariant(), + Command = payload.Command, + ExitCode = payload.ExitCode, + Success = payload.Success, + Signal = payload.Signal, + ErrorCount = payload.ErrorCount, + TimeTakenMs = timeTakenMs, + PackageManager = payload.PackageManager, + PackageManagerVersion = payload.PackageManagerVersion, + ColdInstall = payload.ColdInstall, + LockfileChanged = payload.LockfileChanged, + MeasurementSource = payload.MeasurementSource, + Prebundled = payload.Prebundled, + DomContentLoadedMs = payload.DomContentLoadedMs, + FirstContentfulPaintMs = payload.FirstContentfulPaintMs, + SpooledAt = payload.SpooledAt, + CommitSha = payload.CommitSha, + SourceEndpoint = "/command", + ExtraData = JsonSerializer.Serialize(extraData) + }; + + var npmTimers = new List(); + if (payload.NpmTimers != null) + { + foreach (var timer in payload.NpmTimers) + { + npmTimers.Add(new CommandEventNpmTimer + { + CommandEventId = commandEventId, + TimerName = timer.Key, + DurationMs = timer.Value + }); + } + } + + await _queue.QueueBackgroundWorkItemAsync(new IngestCommandEventWorkItem + { + CommandEvent = commandEvent, + NpmTimers = npmTimers, + RawPayloadJson = JsonSerializer.Serialize(payload), + RawPayloadEndpoint = "/command", + RawPayloadContentType = "application/json" + }); + + return Ok(); + } + + private static bool TryGetDouble(JsonElement value, out double result) + { + if (value.ValueKind == JsonValueKind.Number) + return value.TryGetDouble(out result); + + if (value.ValueKind == JsonValueKind.String) + { + var text = value.GetString(); + return double.TryParse(text, NumberStyles.Float | NumberStyles.AllowThousands, + CultureInfo.InvariantCulture, out result); + } + + result = default; + return false; + } + + private static string ResolvePlatform(JsonElement value) + { + if (value.ValueKind == JsonValueKind.String) + return value.GetString() ?? string.Empty; + + if (value.ValueKind == JsonValueKind.Number && value.TryGetInt32(out var platformId)) + return ((PlatformID)platformId).ToString(); + + return string.Empty; + } +} diff --git a/src/Agoda.DevExTelemetry.WebApi/Controllers/ViteController.cs b/src/Agoda.DevExTelemetry.WebApi/Controllers/ViteController.cs index 8a750ad..59c2e74 100644 --- a/src/Agoda.DevExTelemetry.WebApi/Controllers/ViteController.cs +++ b/src/Agoda.DevExTelemetry.WebApi/Controllers/ViteController.cs @@ -68,6 +68,7 @@ public async Task Ingest([FromBody] VitePayload payload) var metric = new BuildMetric { Id = payload.Id ?? Guid.NewGuid().ToString(), + SessionId = payload.SessionId, UserName = payload.UserName ?? string.Empty, CpuCount = payload.CpuCount, Hostname = payload.Hostname ?? string.Empty, diff --git a/src/Agoda.DevExTelemetry.WebApi/Controllers/WebpackController.cs b/src/Agoda.DevExTelemetry.WebApi/Controllers/WebpackController.cs index e28b047..756bd22 100644 --- a/src/Agoda.DevExTelemetry.WebApi/Controllers/WebpackController.cs +++ b/src/Agoda.DevExTelemetry.WebApi/Controllers/WebpackController.cs @@ -30,11 +30,26 @@ public WebpackController( [HttpPost("webpack")] [RequestSizeLimit(500 * 1024 * 1024)] public async Task Ingest([FromBody] WebpackPayload payload) + => await IngestInternal(payload, "/webpack"); + + [HttpPost("rspack")] + [RequestSizeLimit(500 * 1024 * 1024)] + public async Task IngestRspack([FromBody] WebpackPayload payload) + => await IngestInternal(payload, "/rspack"); + + private async Task IngestInternal(WebpackPayload payload, string sourceEndpoint) { var platformStr = ((PlatformID)payload.Platform).ToString(); var environment = _environmentDetector.Detect( payload.Hostname, payload.IsDebuggerAttached, platformStr, null); + var metricType = (payload.Type ?? "webpack").ToLowerInvariant() switch + { + "rspack" => "rspack", + "rsbuild" => "rsbuild", + _ => "webpack" + }; + string? devFeedbackType = null; if (payload.DevFeedback?.Any(df => string.Equals(df.Type, "hmr", StringComparison.OrdinalIgnoreCase)) == true) @@ -42,7 +57,7 @@ public async Task Ingest([FromBody] WebpackPayload payload) devFeedbackType = "hmr"; } - var (buildCategory, reloadType) = _classifier.Classify("webpack", devFeedbackType); + var (buildCategory, reloadType) = _classifier.Classify(metricType, devFeedbackType); if (!double.TryParse(payload.TimeTaken, NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out var timeTakenMs) && !string.IsNullOrWhiteSpace(payload.TimeTaken)) @@ -65,6 +80,7 @@ public async Task Ingest([FromBody] WebpackPayload payload) var metric = new BuildMetric { Id = payload.Id ?? Guid.NewGuid().ToString(), + SessionId = payload.SessionId, UserName = payload.UserName ?? string.Empty, CpuCount = payload.CpuCount, Hostname = payload.Hostname ?? string.Empty, @@ -75,14 +91,14 @@ public async Task Ingest([FromBody] WebpackPayload payload) Repository = payload.Repository ?? string.Empty, RepositoryName = payload.RepositoryName ?? string.Empty, TimeTakenMs = timeTakenMs, - MetricType = "webpack", + MetricType = metricType, BuildCategory = buildCategory, ReloadType = reloadType, ToolVersion = payload.NodeVersion, CommitSha = payload.CommitSha, IsDebuggerAttached = payload.IsDebuggerAttached, ExecutionEnvironment = environment, - SourceEndpoint = "/webpack", + SourceEndpoint = sourceEndpoint, ExtraData = JsonSerializer.Serialize(extraData) }; @@ -90,7 +106,7 @@ await _queue.QueueBackgroundWorkItemAsync(new IngestBuildMetricWorkItem { BuildMetric = metric, RawPayloadJson = JsonSerializer.Serialize(payload), - RawPayloadEndpoint = "/webpack", + RawPayloadEndpoint = sourceEndpoint, RawPayloadContentType = "application/json" }); diff --git a/src/Agoda.DevExTelemetry.WebApi/Services/CommandEventIngestQueue.cs b/src/Agoda.DevExTelemetry.WebApi/Services/CommandEventIngestQueue.cs new file mode 100644 index 0000000..74bdf3e --- /dev/null +++ b/src/Agoda.DevExTelemetry.WebApi/Services/CommandEventIngestQueue.cs @@ -0,0 +1,43 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Agoda.DevExTelemetry.Core.Models.Ingest; +using Agoda.DevExTelemetry.Core.Services; +using Agoda.IoC.Core; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Agoda.DevExTelemetry.WebApi.Services; + +[RegisterSingleton(For = typeof(IHostedService))] +public class CommandEventIngestQueue : QueuedHostedService +{ + private readonly IServiceProvider _serviceProvider; + + public CommandEventIngestQueue( + IBackgroundTaskQueue taskQueue, + IServiceProvider serviceProvider, + ILogger logger) + : base(taskQueue, logger) + { + _serviceProvider = serviceProvider; + } + + protected override async Task ProcessWorkItem( + IngestCommandEventWorkItem workItem, CancellationToken stoppingToken) + { + using var scope = _serviceProvider.CreateScope(); + var ingestService = scope.ServiceProvider.GetRequiredService(); + + await ingestService.IngestCommandEventAsync(workItem.CommandEvent, workItem.NpmTimers); + + if (workItem.RawPayloadJson != null && workItem.RawPayloadEndpoint != null) + { + await ingestService.StoreRawPayloadAsync( + workItem.RawPayloadEndpoint, + workItem.RawPayloadContentType ?? "application/json", + workItem.RawPayloadJson); + } + } +} diff --git a/table-comparison.md b/table-comparison.md new file mode 100644 index 0000000..fc5bd95 --- /dev/null +++ b/table-comparison.md @@ -0,0 +1,8 @@ +| Concern | JSON column | Child table | +| --- | --- | --- | +| Query one timer | Yes cuz JSON is easy to scan through and storage-efficient but as the json getting larger, it becomes slower | Relation database but storage heavy | +| Aggregate timer durations | uses a little slower searches | uses B-tree for more efficient searches | +| Dynamic timer names | Not dedicated timer names | It can have child databases with specific names | +| Indexing | can index in postgresql | yes the same as json column | +| Storage/row count | More row count less storage | less row count but more storage | +| Query complexity | GIN / Expression | B-tree |