From bf41a1e5ea79fe065ebb3e79b019e293216dfe88 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 8 Jun 2025 21:57:49 +0200
Subject: [PATCH 1/8] validate params
---
src/__tests__/routes.test.js | 6 ++++--
src/controllers/categoriesController.js | 11 +++++++++++
src/controllers/reportController.js | 11 +++++++++++
src/controllers/technologiesController.js | 11 +++++++++++
src/controllers/versionsController.js | 11 +++++++++++
src/utils/controllerHelpers.js | 9 +++++++--
test-api.sh | 1 +
7 files changed, 56 insertions(+), 4 deletions(-)
diff --git a/src/__tests__/routes.test.js b/src/__tests__/routes.test.js
index 90ac268..b26fe47 100644
--- a/src/__tests__/routes.test.js
+++ b/src/__tests__/routes.test.js
@@ -335,8 +335,10 @@ describe('API Routes', () => {
it('should handle invalid query parameters gracefully', async () => {
const res = await request(app).get('/v1/technologies?invalid=parameter');
- expect(res.statusCode).toEqual(200);
- expect(Array.isArray(res.body)).toBe(true);
+ expect(res.statusCode).toEqual(400);
+ expect(res.body).toHaveProperty('errors');
+ expect(res.body.errors[0]).toHaveProperty('error');
+ expect(res.body.errors[0].error).toContain('Unsupported parameters: ');
});
});
diff --git a/src/controllers/categoriesController.js b/src/controllers/categoriesController.js
index 6d46972..f7d1148 100644
--- a/src/controllers/categoriesController.js
+++ b/src/controllers/categoriesController.js
@@ -6,6 +6,17 @@ import { executeQuery, validateArrayParameter } from '../utils/controllerHelpers
*/
const listCategories = async (req, res) => {
const queryBuilder = async (params) => {
+ // Validate parameters
+ const supportedParams = ['category', 'onlyname', 'fields'];
+ const providedParams = Object.keys(params);
+ const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param));
+
+ if (unsupportedParams.length > 0) {
+ const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`);
+ error.statusCode = 400;
+ throw error;
+ }
+
const isOnlyNames = params.onlyname || typeof params.onlyname === 'string';
const hasCustomFields = params.fields && !isOnlyNames;
diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js
index fc42a58..7cf9f67 100644
--- a/src/controllers/reportController.js
+++ b/src/controllers/reportController.js
@@ -49,6 +49,17 @@ const createReportController = (reportType) => {
try {
const params = req.query;
+ // Validate supported parameters
+ const supportedParams = ['technology', 'geo', 'rank', 'start', 'end'];
+ const providedParams = Object.keys(params);
+ const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param));
+
+ if (unsupportedParams.length > 0) {
+ const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`);
+ error.statusCode = 400;
+ throw error;
+ }
+
// Validate required parameters using shared utility
const errors = validateRequiredParams(params, [
REQUIRED_PARAMS.GEO,
diff --git a/src/controllers/technologiesController.js b/src/controllers/technologiesController.js
index 23b4029..e5171b3 100644
--- a/src/controllers/technologiesController.js
+++ b/src/controllers/technologiesController.js
@@ -6,6 +6,17 @@ import { executeQuery, validateTechnologyArray, validateArrayParameter, FIRESTOR
*/
const listTechnologies = async (req, res) => {
const queryBuilder = async (params) => {
+ // Validate parameters
+ const supportedParams = ['technology', 'category', 'onlyname', 'fields'];
+ const providedParams = Object.keys(params);
+ const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param));
+
+ if (unsupportedParams.length > 0) {
+ const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`);
+ error.statusCode = 400;
+ throw error;
+ }
+
const isOnlyNames = params.onlyname || typeof params.onlyname === 'string';
const hasCustomFields = params.fields && !isOnlyNames;
diff --git a/src/controllers/versionsController.js b/src/controllers/versionsController.js
index 17cd8f7..aa19d55 100644
--- a/src/controllers/versionsController.js
+++ b/src/controllers/versionsController.js
@@ -6,6 +6,17 @@ import { executeQuery, validateTechnologyArray, FIRESTORE_IN_LIMIT } from '../ut
*/
const listVersions = async (req, res) => {
const queryBuilder = async (params) => {
+ // Validate parameters
+ const supportedParams = ['version', 'technology', 'category', 'onlyname', 'fields'];
+ const providedParams = Object.keys(params);
+ const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param));
+
+ if (unsupportedParams.length > 0) {
+ const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`);
+ error.statusCode = 400;
+ throw error;
+ }
+
let query = firestore.collection('versions');
// Apply technology filter with validation
diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js
index 047f7bf..56aa46e 100644
--- a/src/utils/controllerHelpers.js
+++ b/src/utils/controllerHelpers.js
@@ -230,9 +230,14 @@ const getCacheStats = () => {
*/
const handleControllerError = (res, error, operation) => {
console.error(`Error ${operation}:`, error);
- res.statusCode = 500;
+ const statusCode = error.statusCode || 500;
+ res.statusCode = statusCode;
+
+ // Use custom error message for client errors (4xx), generic message for server errors (5xx)
+ const errorMessage = statusCode >= 400 && statusCode < 500 ? error.message : `Failed to ${operation}`;
+
res.end(JSON.stringify({
- errors: [{ error: `Failed to ${operation}` }]
+ errors: [{ error: errorMessage }]
}));
};
diff --git a/test-api.sh b/test-api.sh
index ea98db3..c16bfe4 100755
--- a/test-api.sh
+++ b/test-api.sh
@@ -85,6 +85,7 @@ test_endpoint "/" ""
# Test technologies endpoint
test_cors_preflight "/v1/technologies"
+test_endpoint "/v1/technologies" "?onlyname=true"
test_endpoint "/v1/technologies" "?technology=WordPress&onlyname=true"
test_endpoint "/v1/technologies" "?technology=WordPress&onlyname=true&fields=technology,icon"
test_endpoint "/v1/technologies" "?technology=WordPress&category=CMS&fields=technology,icon"
From 3a005b8ecba2eaa559ceab33e2e98480b1d3598d Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 8 Jun 2025 23:04:54 +0200
Subject: [PATCH 2/8] skip xss
---
src/__tests__/routes.test.js | 10 ++++++----
src/controllers/categoriesController.js | 2 ++
src/controllers/reportController.js | 2 ++
src/controllers/technologiesController.js | 2 ++
src/controllers/versionsController.js | 2 ++
src/index.js | 8 ++++++++
6 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/src/__tests__/routes.test.js b/src/__tests__/routes.test.js
index b26fe47..b88c66d 100644
--- a/src/__tests__/routes.test.js
+++ b/src/__tests__/routes.test.js
@@ -335,10 +335,12 @@ describe('API Routes', () => {
it('should handle invalid query parameters gracefully', async () => {
const res = await request(app).get('/v1/technologies?invalid=parameter');
- expect(res.statusCode).toEqual(400);
- expect(res.body).toHaveProperty('errors');
- expect(res.body.errors[0]).toHaveProperty('error');
- expect(res.body.errors[0].error).toContain('Unsupported parameters: ');
+ expect(res.statusCode).toEqual(200);
+ expect(Array.isArray(res.body)).toBe(true);
+
+ //expect(res.body).toHaveProperty('errors');
+ //expect(res.body.errors[0]).toHaveProperty('error');
+ //expect(res.body.errors[0].error).toContain('Unsupported parameters: ');
});
});
diff --git a/src/controllers/categoriesController.js b/src/controllers/categoriesController.js
index f7d1148..6a76918 100644
--- a/src/controllers/categoriesController.js
+++ b/src/controllers/categoriesController.js
@@ -6,6 +6,7 @@ import { executeQuery, validateArrayParameter } from '../utils/controllerHelpers
*/
const listCategories = async (req, res) => {
const queryBuilder = async (params) => {
+ /*
// Validate parameters
const supportedParams = ['category', 'onlyname', 'fields'];
const providedParams = Object.keys(params);
@@ -16,6 +17,7 @@ const listCategories = async (req, res) => {
error.statusCode = 400;
throw error;
}
+ */
const isOnlyNames = params.onlyname || typeof params.onlyname === 'string';
const hasCustomFields = params.fields && !isOnlyNames;
diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js
index 7cf9f67..1e297f0 100644
--- a/src/controllers/reportController.js
+++ b/src/controllers/reportController.js
@@ -49,6 +49,7 @@ const createReportController = (reportType) => {
try {
const params = req.query;
+ /*
// Validate supported parameters
const supportedParams = ['technology', 'geo', 'rank', 'start', 'end'];
const providedParams = Object.keys(params);
@@ -59,6 +60,7 @@ const createReportController = (reportType) => {
error.statusCode = 400;
throw error;
}
+ */
// Validate required parameters using shared utility
const errors = validateRequiredParams(params, [
diff --git a/src/controllers/technologiesController.js b/src/controllers/technologiesController.js
index e5171b3..5d75aba 100644
--- a/src/controllers/technologiesController.js
+++ b/src/controllers/technologiesController.js
@@ -6,6 +6,7 @@ import { executeQuery, validateTechnologyArray, validateArrayParameter, FIRESTOR
*/
const listTechnologies = async (req, res) => {
const queryBuilder = async (params) => {
+ /*
// Validate parameters
const supportedParams = ['technology', 'category', 'onlyname', 'fields'];
const providedParams = Object.keys(params);
@@ -16,6 +17,7 @@ const listTechnologies = async (req, res) => {
error.statusCode = 400;
throw error;
}
+ */
const isOnlyNames = params.onlyname || typeof params.onlyname === 'string';
const hasCustomFields = params.fields && !isOnlyNames;
diff --git a/src/controllers/versionsController.js b/src/controllers/versionsController.js
index aa19d55..8af580a 100644
--- a/src/controllers/versionsController.js
+++ b/src/controllers/versionsController.js
@@ -6,6 +6,7 @@ import { executeQuery, validateTechnologyArray, FIRESTORE_IN_LIMIT } from '../ut
*/
const listVersions = async (req, res) => {
const queryBuilder = async (params) => {
+ /*
// Validate parameters
const supportedParams = ['version', 'technology', 'category', 'onlyname', 'fields'];
const providedParams = Object.keys(params);
@@ -16,6 +17,7 @@ const listVersions = async (req, res) => {
error.statusCode = 400;
throw error;
}
+ */
let query = firestore.collection('versions');
diff --git a/src/index.js b/src/index.js
index 46a01d4..1c553b6 100644
--- a/src/index.js
+++ b/src/index.js
@@ -105,6 +105,14 @@ const handleRequest = async (req, res) => {
return;
}
+ // Validate URL to skip XSS attacks
+ const unsafe = /onerror|onload|javascript:/i;
+ if (unsafe.test(req.url)) {
+ res.statusCode = 400
+ res.end(JSON.stringify({ error: 'Invalid input' }));
+ return;
+ }
+
// Parse URL
const parsedUrl = url.parse(req.url, true);
const pathname = parsedUrl.pathname;
From 256f07f58f1ba8ef89723f2f6adda444da552cdc Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 9 Jun 2025 00:37:45 +0200
Subject: [PATCH 3/8] bq init
---
src/controllers/geosBQController.js | 17 +
src/controllers/ranksBQController.js | 17 +
src/index.js | 4 +-
src/package-lock.json | 467 +++++++++++++++++++++++++++
src/package.json | 1 +
src/utils/bigquery.js | 89 +++++
src/utils/controllerHelpers.js | 45 +++
7 files changed, 638 insertions(+), 2 deletions(-)
create mode 100644 src/controllers/geosBQController.js
create mode 100644 src/controllers/ranksBQController.js
create mode 100644 src/utils/bigquery.js
diff --git a/src/controllers/geosBQController.js b/src/controllers/geosBQController.js
new file mode 100644
index 0000000..9a6b417
--- /dev/null
+++ b/src/controllers/geosBQController.js
@@ -0,0 +1,17 @@
+import { getGeosFromBQ } from '../utils/bigquery.js';
+import { executeBigQuery } from '../utils/controllerHelpers.js';
+
+/**
+ * List all geographic locations from BigQuery
+ */
+const listGeos = async (req, res) => {
+ const queryExecutor = async () => {
+ return await getGeosFromBQ();
+ };
+
+ await executeBigQuery(req, res, 'geos', queryExecutor);
+};
+
+export {
+ listGeos
+};
diff --git a/src/controllers/ranksBQController.js b/src/controllers/ranksBQController.js
new file mode 100644
index 0000000..182fa73
--- /dev/null
+++ b/src/controllers/ranksBQController.js
@@ -0,0 +1,17 @@
+import { getRanksFromBQ } from '../utils/bigquery.js';
+import { executeBigQuery } from '../utils/controllerHelpers.js';
+
+/**
+ * List all rank options from BigQuery
+ */
+const listRanks = async (req, res) => {
+ const queryExecutor = async () => {
+ return await getRanksFromBQ();
+ };
+
+ await executeBigQuery(req, res, 'ranks', queryExecutor);
+};
+
+export {
+ listRanks
+};
diff --git a/src/index.js b/src/index.js
index 1c553b6..c886f73 100644
--- a/src/index.js
+++ b/src/index.js
@@ -33,10 +33,10 @@ const getController = async (name) => {
controllers[name] = await import('./controllers/reportController.js');
break;
case 'ranks':
- controllers[name] = await import('./controllers/ranksController.js');
+ controllers[name] = await import('./controllers/ranksBQController.js');
break;
case 'geos':
- controllers[name] = await import('./controllers/geosController.js');
+ controllers[name] = await import('./controllers/geosBQController.js');
break;
case 'versions':
controllers[name] = await import('./controllers/versionsController.js');
diff --git a/src/package-lock.json b/src/package-lock.json
index d4e94bb..62391c7 100644
--- a/src/package-lock.json
+++ b/src/package-lock.json
@@ -8,6 +8,7 @@
"name": "tech-report-api",
"version": "1.0.0",
"dependencies": {
+ "@google-cloud/bigquery": "^8.1.0",
"@google-cloud/firestore": "7.11.1",
"@google-cloud/functions-framework": "4.0.0"
},
@@ -568,6 +569,310 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/@google-cloud/bigquery": {
+ "version": "8.1.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-8.1.0.tgz",
+ "integrity": "sha512-eDleD/IHKQIRm4GmMnwJvPkx4PgSaK8m8DCmDmVOf0gIhqPLSdvOAEeM4QjyyZGUGjV4yHyJfEJxzULTzl22Aw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@google-cloud/common": "^6.0.0",
+ "@google-cloud/paginator": "^6.0.0",
+ "@google-cloud/precise-date": "^5.0.0",
+ "@google-cloud/promisify": "^5.0.0",
+ "arrify": "^3.0.0",
+ "big.js": "^6.2.2",
+ "duplexify": "^4.1.3",
+ "extend": "^3.0.2",
+ "is": "^3.3.0",
+ "stream-events": "^1.0.5",
+ "teeny-request": "^10.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/bigquery/node_modules/agent-base": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+ "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+ "license": "MIT",
+ "dependencies": {
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6.0.0"
+ }
+ },
+ "node_modules/@google-cloud/bigquery/node_modules/debug": {
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
+ "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
+ "license": "MIT",
+ "dependencies": {
+ "ms": "^2.1.3"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "peerDependenciesMeta": {
+ "supports-color": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/@google-cloud/bigquery/node_modules/https-proxy-agent": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+ "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+ "license": "MIT",
+ "dependencies": {
+ "agent-base": "6",
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
+ "node_modules/@google-cloud/bigquery/node_modules/ms": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+ "license": "MIT"
+ },
+ "node_modules/@google-cloud/bigquery/node_modules/node-fetch": {
+ "version": "3.3.2",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
+ "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
+ "license": "MIT",
+ "dependencies": {
+ "data-uri-to-buffer": "^4.0.0",
+ "fetch-blob": "^3.1.4",
+ "formdata-polyfill": "^4.0.10"
+ },
+ "engines": {
+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/node-fetch"
+ }
+ },
+ "node_modules/@google-cloud/bigquery/node_modules/teeny-request": {
+ "version": "10.1.0",
+ "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz",
+ "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "http-proxy-agent": "^5.0.0",
+ "https-proxy-agent": "^5.0.0",
+ "node-fetch": "^3.3.2",
+ "stream-events": "^1.0.5"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-6.0.0.tgz",
+ "integrity": "sha512-IXh04DlkLMxWgYLIUYuHHKXKOUwPDzDgke1ykkkJPe48cGIS9kkL2U/o0pm4ankHLlvzLF/ma1eO86n/bkumIA==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@google-cloud/projectify": "^4.0.0",
+ "@google-cloud/promisify": "^4.0.0",
+ "arrify": "^2.0.0",
+ "duplexify": "^4.1.3",
+ "extend": "^3.0.2",
+ "google-auth-library": "^10.0.0-rc.1",
+ "html-entities": "^2.5.2",
+ "retry-request": "^8.0.0",
+ "teeny-request": "^10.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/@google-cloud/promisify": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.1.0.tgz",
+ "integrity": "sha512-G/FQx5cE/+DqBbOpA5jKsegGwdPniU6PuIEMt+qxWgFxvxuFOzVmp6zYchtYuwAWV5/8Dgs0yAmjvNZv3uXLQg==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/agent-base": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+ "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+ "license": "MIT",
+ "dependencies": {
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6.0.0"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/arrify": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz",
+ "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/debug": {
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
+ "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
+ "license": "MIT",
+ "dependencies": {
+ "ms": "^2.1.3"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "peerDependenciesMeta": {
+ "supports-color": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/gaxios": {
+ "version": "7.0.0-rc.6",
+ "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.0.0-rc.6.tgz",
+ "integrity": "sha512-osVFpgeBiwTM2AVI9MXvb8iWzM6oSMbTVWc65Gm5BgBlE+nUA6PBHFMaYpqjZx1AhUH7aPOZq78WcRAM6hhAwA==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "extend": "^3.0.2",
+ "https-proxy-agent": "^7.0.1",
+ "node-fetch": "^3.3.2"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/gcp-metadata": {
+ "version": "7.0.0-rc.1",
+ "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.0-rc.1.tgz",
+ "integrity": "sha512-E6c+AdIaK1LNA839OyotiTca+B2IG1nDlMjnlcck8JjXn3fVgx57Ib9i6iL1/iqN7bA3EUQdcRRu+HqOCOABIg==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "gaxios": "^7.0.0-rc.1",
+ "google-logging-utils": "^1.0.0",
+ "json-bigint": "^1.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/google-auth-library": {
+ "version": "10.0.0-rc.3",
+ "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.0.0-rc.3.tgz",
+ "integrity": "sha512-WC9wfEKK0bk3seWKsDn2loduLth6JWKTsrbWftzrhPuzpwnVXb5oi2+aa0JDBxLBDdkGesLvTQ67F2nZ7leq1Q==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "base64-js": "^1.3.0",
+ "ecdsa-sig-formatter": "^1.0.11",
+ "gaxios": "^7.0.0-rc.4",
+ "gcp-metadata": "^7.0.0-rc.1",
+ "google-logging-utils": "^1.0.0",
+ "gtoken": "^8.0.0-rc.1",
+ "jws": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/google-logging-utils": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz",
+ "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=14"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/gtoken": {
+ "version": "8.0.0-rc.1",
+ "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0-rc.1.tgz",
+ "integrity": "sha512-UjE/egX6ixArdcCKOkheuFQ4XN4/0gX92nd2JPVEYuRU2sWHAWuOVGnowm1fQUdQtaxqn1n8H0hOb2LCaUhJ3A==",
+ "license": "MIT",
+ "dependencies": {
+ "gaxios": "^7.0.0-rc.1",
+ "jws": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/ms": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+ "license": "MIT"
+ },
+ "node_modules/@google-cloud/common/node_modules/node-fetch": {
+ "version": "3.3.2",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
+ "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
+ "license": "MIT",
+ "dependencies": {
+ "data-uri-to-buffer": "^4.0.0",
+ "fetch-blob": "^3.1.4",
+ "formdata-polyfill": "^4.0.10"
+ },
+ "engines": {
+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/node-fetch"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/retry-request": {
+ "version": "8.0.0",
+ "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.0.tgz",
+ "integrity": "sha512-dJkZNmyV9C8WKUmbdj1xcvVlXBSvsUQCkg89TCK8rD72RdSn9A2jlXlS2VuYSTHoPJjJEfUHhjNYrlvuksF9cg==",
+ "license": "MIT",
+ "dependencies": {
+ "@types/request": "^2.48.12",
+ "extend": "^3.0.2",
+ "teeny-request": "^10.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/teeny-request": {
+ "version": "10.1.0",
+ "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz",
+ "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "http-proxy-agent": "^5.0.0",
+ "https-proxy-agent": "^5.0.0",
+ "node-fetch": "^3.3.2",
+ "stream-events": "^1.0.5"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/common/node_modules/teeny-request/node_modules/https-proxy-agent": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+ "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+ "license": "MIT",
+ "dependencies": {
+ "agent-base": "6",
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/@google-cloud/firestore": {
"version": "7.11.1",
"resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-7.11.1.tgz",
@@ -619,6 +924,45 @@
"node": ">=10"
}
},
+ "node_modules/@google-cloud/paginator": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-6.0.0.tgz",
+ "integrity": "sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "extend": "^3.0.2"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/precise-date": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-5.0.0.tgz",
+ "integrity": "sha512-9h0Gvw92EvPdE8AK8AgZPbMnH5ftDyPtKm7/KUfcJVaPEPjwGDsJd1QV0H8esBDV4II41R/2lDWH1epBqIoKUw==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@google-cloud/projectify": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz",
+ "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=14.0.0"
+ }
+ },
+ "node_modules/@google-cloud/promisify": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-5.0.0.tgz",
+ "integrity": "sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=18"
+ }
+ },
"node_modules/@grpc/grpc-js": {
"version": "1.13.3",
"resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.3.tgz",
@@ -1824,6 +2168,18 @@
"integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==",
"license": "MIT"
},
+ "node_modules/arrify": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz",
+ "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=12"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/asap": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
@@ -2036,6 +2392,19 @@
],
"license": "MIT"
},
+ "node_modules/big.js": {
+ "version": "6.2.2",
+ "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz",
+ "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==",
+ "license": "MIT",
+ "engines": {
+ "node": "*"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/bigjs"
+ }
+ },
"node_modules/bignumber.js": {
"version": "9.3.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.0.tgz",
@@ -2488,6 +2857,15 @@
"node": ">= 8"
}
},
+ "node_modules/data-uri-to-buffer": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
+ "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 12"
+ }
+ },
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -2943,6 +3321,29 @@
"bser": "2.1.1"
}
},
+ "node_modules/fetch-blob": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
+ "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/jimmywarting"
+ },
+ {
+ "type": "paypal",
+ "url": "https://paypal.me/jimmywarting"
+ }
+ ],
+ "license": "MIT",
+ "dependencies": {
+ "node-domexception": "^1.0.0",
+ "web-streams-polyfill": "^3.0.3"
+ },
+ "engines": {
+ "node": "^12.20 || >= 14.13"
+ }
+ },
"node_modules/fill-range": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
@@ -3031,6 +3432,18 @@
"node": ">= 0.12"
}
},
+ "node_modules/formdata-polyfill": {
+ "version": "4.0.10",
+ "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
+ "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
+ "license": "MIT",
+ "dependencies": {
+ "fetch-blob": "^3.1.2"
+ },
+ "engines": {
+ "node": ">=12.20.0"
+ }
+ },
"node_modules/formidable": {
"version": "3.5.4",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz",
@@ -3405,6 +3818,22 @@
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"license": "ISC"
},
+ "node_modules/html-entities": {
+ "version": "2.6.0",
+ "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz",
+ "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/mdevils"
+ },
+ {
+ "type": "patreon",
+ "url": "https://patreon.com/mdevils"
+ }
+ ],
+ "license": "MIT"
+ },
"node_modules/html-escaper": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
@@ -3604,6 +4033,15 @@
"node": ">= 0.10"
}
},
+ "node_modules/is": {
+ "version": "3.3.0",
+ "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz",
+ "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==",
+ "license": "MIT",
+ "engines": {
+ "node": "*"
+ }
+ },
"node_modules/is-arguments": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz",
@@ -4916,6 +5354,26 @@
"node": ">= 0.6"
}
},
+ "node_modules/node-domexception": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+ "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+ "deprecated": "Use your platform's native DOMException instead",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/jimmywarting"
+ },
+ {
+ "type": "github",
+ "url": "https://paypal.me/jimmywarting"
+ }
+ ],
+ "license": "MIT",
+ "engines": {
+ "node": ">=10.5.0"
+ }
+ },
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
@@ -6416,6 +6874,15 @@
"makeerror": "1.0.12"
}
},
+ "node_modules/web-streams-polyfill": {
+ "version": "3.3.3",
+ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
+ "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 8"
+ }
+ },
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
diff --git a/src/package.json b/src/package.json
index b129f94..1286c68 100644
--- a/src/package.json
+++ b/src/package.json
@@ -13,6 +13,7 @@
"test:live": "bash ../test-api.sh"
},
"dependencies": {
+ "@google-cloud/bigquery": "^8.1.0",
"@google-cloud/firestore": "7.11.1",
"@google-cloud/functions-framework": "4.0.0"
},
diff --git a/src/utils/bigquery.js b/src/utils/bigquery.js
new file mode 100644
index 0000000..753e00a
--- /dev/null
+++ b/src/utils/bigquery.js
@@ -0,0 +1,89 @@
+import { BigQuery } from '@google-cloud/bigquery';
+
+// Initialize BigQuery client
+const bigquery = new BigQuery({
+ projectId: process.env.PROJECT || 'httparchive'
+});
+
+// BigQuery configuration optimizations
+const BQ_CONFIG = {
+ // Optimize for BI Engine
+ location: 'US', // Use the same location as your BI Engine
+ // Use maximum parallelism for BI Engine
+ maximumBytesBilled: '100000000', // 100MB limit for safety
+ // Labels for monitoring
+ labels: {
+ 'app': 'tech-report-api',
+ 'source': 'bigquery-direct',
+ }
+};
+
+/**
+ * Execute a BigQuery query with caching support
+ * @param {string} query - SQL query string
+ * @param {Object} options - Query options
+ * @returns {Array} - Query results
+ */
+const executeBigQueryQuery = async (query, options = {}) => {
+ try {
+ const queryOptions = {
+ query,
+
+ jobCreationMode: 'JOB_CREATION_OPTIONAL', // Returning immediate results is prioritized.
+ timeoutMs: 10000, // 10 seconds
+ // Use query cache when possible
+ useQueryCache: true,
+ // Apply BI Engine optimizations
+ ...BQ_CONFIG,
+ ...options
+ };
+
+ console.log('Executing BigQuery:', query);
+ const [rows, , metadata] = await bigquery.query(queryOptions);
+ if(metadata.jobReference) {
+ console.log(`BigQuery job ${metadata.jobReference.jobId} completed. Rows: ${rows.length}`);
+ }
+
+ return rows;
+ } catch (error) {
+ console.error('BigQuery execution error:', error);
+ throw error;
+ }
+};
+
+/**
+ * Get ranks from BigQuery
+ * @returns {Array} - Array of rank objects
+ */
+const getRanksFromBQ = async () => {
+ const query = `
+ SELECT rank
+ FROM \`httparchive.reports.tech_report_ranks\`
+ ORDER BY mobile_origins DESC
+ `;
+
+ const rows = await executeBigQueryQuery(query);
+ return rows.map(row => ({ rank: row.rank }));
+};
+
+/**
+ * Get geos from BigQuery
+ * @returns {Array} - Array of geo objects
+ */
+const getGeosFromBQ = async () => {
+ const query = `
+ SELECT geo
+ FROM \`httparchive.reports.tech_report_geos\`
+ ORDER BY mobile_origins DESC
+ `;
+
+ const rows = await executeBigQueryQuery(query);
+ return rows.map(row => ({ geo: row.geo }));
+};
+
+export {
+ bigquery,
+ executeBigQueryQuery,
+ getRanksFromBQ,
+ getGeosFromBQ
+};
diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js
index 56aa46e..8f0a132 100644
--- a/src/utils/controllerHelpers.js
+++ b/src/utils/controllerHelpers.js
@@ -318,6 +318,50 @@ const validateTechnologyArray = (technologyParam) => {
}
};
+/**
+ * Generic BigQuery-enabled query executor
+ * Handles caching, query execution, and response for BigQuery queries
+ * @param {Object} req - Request object
+ * @param {Object} res - Response object
+ * @param {string} queryName - Query name for caching and error handling
+ * @param {Function} queryExecutor - Function that executes BigQuery and returns results
+ * @param {Function} dataProcessor - Optional function to process results
+ */
+const executeBigQuery = async (req, res, queryName, queryExecutor, dataProcessor = null) => {
+ try {
+ const params = req.query;
+
+ // Generate cache key
+ const cacheKey = generateQueryCacheKey(`bq_${queryName}`, params);
+
+ // Check cache first
+ const cachedResult = getCachedQueryResult(cacheKey);
+ if (cachedResult) {
+ res.statusCode = 200;
+ res.end(JSON.stringify(cachedResult));
+ return;
+ }
+
+ // Execute BigQuery
+ let data = await queryExecutor(params);
+
+ // Process data if processor provided
+ if (dataProcessor) {
+ data = dataProcessor(data, params);
+ }
+
+ // Cache the result
+ setCachedQueryResult(cacheKey, data);
+
+ // Send response
+ res.statusCode = 200;
+ res.end(JSON.stringify(data));
+
+ } catch (error) {
+ handleControllerError(res, error, `executing BigQuery ${queryName}`);
+ }
+};
+
export {
REQUIRED_PARAMS,
FIRESTORE_IN_LIMIT,
@@ -331,5 +375,6 @@ export {
setCachedQueryResult,
getCacheStats,
executeQuery,
+ executeBigQuery,
validateTechnologyArray
};
From d9f1a0c880984f6f4af3480be997b4a019e798c2 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Wed, 18 Jun 2025 19:24:18 +0200
Subject: [PATCH 4/8] audits endpoint
---
README.md | 74 +++++++++++++++++++++++++--
src/controllers/reportController.js | 11 +++-
src/index.js | 5 ++
terraform/modules/run-service/main.tf | 2 +-
test-api.sh | 3 ++
5 files changed, 87 insertions(+), 8 deletions(-)
diff --git a/README.md b/README.md
index 500b339..f66526e 100644
--- a/README.md
+++ b/README.md
@@ -174,9 +174,9 @@ Lists available versions.
#### Versions Parameters
-- `version` (optional): Filter by version name(s) - comma-separated list
- `technology` (optional): Filter by technology name(s) - comma-separated list
- `category` (optional): Filter by category - comma-separated list
+- `version` (optional): Filter by version name(s) - comma-separated list
- `onlyname` (optional): If present, returns only version names
- `fields` (optional): Comma-separated list of fields to include in the response (see [Field Selection API Documentation](#field-selection-api-documentation) for details)
@@ -209,10 +209,10 @@ Provides technology adoption data.
#### Adoption Parameters
- `technology` (required): Filter by technology name(s) - comma-separated list
+- `geo` (required): Filter by geographic location
+- `rank` (required): Filter by rank
- `start` (optional): Filter by date range start (YYYY-MM-DD or 'latest')
- `end` (optional): Filter by date range end (YYYY-MM-DD)
-- `geo` (optional): Filter by geographic location
-- `rank` (optional): Filter by rank
#### Adoption Response
@@ -334,8 +334,8 @@ Provides Page Weight metrics for technologies.
#### Page Weight Parameters
- `technology` (required): Filter by technology name(s) - comma-separated list
-- `geo` (optional): Filter by geographic location
-- `rank` (optional): Filter by rank
+- `geo` (required): Filter by geographic location
+- `rank` (required): Filter by rank
- `start` (optional): Filter by date range start (YYYY-MM-DD or 'latest')
- `end` (optional): Filter by date range end (YYYY-MM-DD)
@@ -387,6 +387,70 @@ Returns a JSON object with the following schema:
]
```
+
+### `GET /audits`
+
+Provides Lighthouse audits for technologies.
+
+#### Audits Parameters
+
+- `technology` (required): Filter by technology name(s) - comma-separated list
+- `geo` (required): Filter by geographic location
+- `rank` (required): Filter by rank
+- `start` (optional): Filter by date range start (YYYY-MM-DD or 'latest')
+- `end` (optional): Filter by date range end (YYYY-MM-DD)
+
+#### Audits Response
+
+```bash
+curl --request GET \
+ --url 'https://{{HOST}}/v1/audits?start=latest&geo=ALL&technology=WordPress&rank=ALL'
+```
+
+Returns a JSON object with the following schema:
+
+```json
+[
+ {
+ "date": "2025-06-01",
+ "audits": [
+ {
+ "desktop": {
+ "pass_origins": 2428028
+ },
+ "mobile": {
+ "pass_origins": 2430912
+ },
+ "id": "first-contentful-paint",
+ "category": "performance"
+ },
+ {
+ "desktop": {
+ "pass_origins": 490451
+ },
+ "mobile": {
+ "pass_origins": 477218
+ },
+ "id": "largest-contentful-paint",
+ "category": "performance"
+ },
+ {
+ "desktop": {
+ "pass_origins": 1221876
+ },
+ "mobile": {
+ "pass_origins": 1296673
+ },
+ "id": "cumulative-layout-shift",
+ "category": "performance"
+ }
+ ],
+ "technology": "WordPress"
+ },
+ ...
+]
+```
+
### `GET /ranks`
Lists all available ranks.
diff --git a/src/controllers/reportController.js b/src/controllers/reportController.js
index 1e297f0..9f24123 100644
--- a/src/controllers/reportController.js
+++ b/src/controllers/reportController.js
@@ -32,6 +32,10 @@ const REPORT_CONFIGS = {
cwv: {
table: 'core_web_vitals',
dataField: 'vitals'
+ },
+ audits: {
+ table: 'audits',
+ dataField: 'audits'
}
};
@@ -146,7 +150,10 @@ const createReportController = (reportType) => {
};
// Export individual controller functions
+export const listAuditsData = createReportController('audits');
export const listAdoptionData = createReportController('adoption');
-export const listPageWeightData = createReportController('pageWeight');
-export const listLighthouseData = createReportController('lighthouse');
export const listCWVTechData = createReportController('cwv');
+export const listLighthouseData = createReportController('lighthouse');
+export const listPageWeightData = createReportController('pageWeight');
+
+
diff --git a/src/index.js b/src/index.js
index c886f73..a775263 100644
--- a/src/index.js
+++ b/src/index.js
@@ -11,6 +11,7 @@ const controllers = {
cwvtech: null,
lighthouse: null,
pageWeight: null,
+ audits: null,
ranks: null,
geos: null,
versions: null
@@ -30,6 +31,7 @@ const getController = async (name) => {
case 'cwvtech':
case 'lighthouse':
case 'pageWeight':
+ case 'audits':
controllers[name] = await import('./controllers/reportController.js');
break;
case 'ranks':
@@ -144,6 +146,9 @@ const handleRequest = async (req, res) => {
} else if (pathname === '/v1/page-weight' && req.method === 'GET') {
const { listPageWeightData } = await getController('pageWeight');
await listPageWeightData(req, res);
+ } else if (pathname === '/v1/audits' && req.method === 'GET') {
+ const { listAuditsData } = await getController('audits');
+ await listAuditsData(req, res);
} else if (pathname === '/v1/ranks' && req.method === 'GET') {
const { listRanks } = await getController('ranks');
await listRanks(req, res);
diff --git a/terraform/modules/run-service/main.tf b/terraform/modules/run-service/main.tf
index e2c6760..3969282 100644
--- a/terraform/modules/run-service/main.tf
+++ b/terraform/modules/run-service/main.tf
@@ -1,5 +1,5 @@
locals {
- bucketName = "tf-cloudfunctions-backingapi-20230314"
+ bucketName = "gcf-v2-uploads-226352634162-us-central1"
}
data "archive_file" "source" {
type = "zip"
diff --git a/test-api.sh b/test-api.sh
index c16bfe4..d97d9ff 100755
--- a/test-api.sh
+++ b/test-api.sh
@@ -113,4 +113,7 @@ test_endpoint "/v1/lighthouse" "?technology=WordPress&geo=ALL&rank=ALL&start=lat
# Test page-weight endpoint
test_endpoint "/v1/page-weight" "?technology=WordPress&geo=ALL&rank=ALL&start=latest"
+# Test audits endpoint
+test_endpoint "/v1/audits" "?technology=WordPress&geo=ALL&rank=ALL&start=latest"
+
echo "API tests complete! All endpoints returned 200 status code and CORS is properly configured."
From 5425aef2daf7f67e7155429621a1c08c77ed6074 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Wed, 18 Jun 2025 20:16:17 +0200
Subject: [PATCH 5/8] cleanup
---
src/controllers/geosBQController.js | 17 -
src/controllers/ranksBQController.js | 17 -
src/index.js | 4 +-
src/package-lock.json | 467 ---------------------------
src/package.json | 1 -
src/utils/bigquery.js | 89 -----
src/utils/controllerHelpers.js | 45 ---
7 files changed, 2 insertions(+), 638 deletions(-)
delete mode 100644 src/controllers/geosBQController.js
delete mode 100644 src/controllers/ranksBQController.js
delete mode 100644 src/utils/bigquery.js
diff --git a/src/controllers/geosBQController.js b/src/controllers/geosBQController.js
deleted file mode 100644
index 9a6b417..0000000
--- a/src/controllers/geosBQController.js
+++ /dev/null
@@ -1,17 +0,0 @@
-import { getGeosFromBQ } from '../utils/bigquery.js';
-import { executeBigQuery } from '../utils/controllerHelpers.js';
-
-/**
- * List all geographic locations from BigQuery
- */
-const listGeos = async (req, res) => {
- const queryExecutor = async () => {
- return await getGeosFromBQ();
- };
-
- await executeBigQuery(req, res, 'geos', queryExecutor);
-};
-
-export {
- listGeos
-};
diff --git a/src/controllers/ranksBQController.js b/src/controllers/ranksBQController.js
deleted file mode 100644
index 182fa73..0000000
--- a/src/controllers/ranksBQController.js
+++ /dev/null
@@ -1,17 +0,0 @@
-import { getRanksFromBQ } from '../utils/bigquery.js';
-import { executeBigQuery } from '../utils/controllerHelpers.js';
-
-/**
- * List all rank options from BigQuery
- */
-const listRanks = async (req, res) => {
- const queryExecutor = async () => {
- return await getRanksFromBQ();
- };
-
- await executeBigQuery(req, res, 'ranks', queryExecutor);
-};
-
-export {
- listRanks
-};
diff --git a/src/index.js b/src/index.js
index a775263..4269c99 100644
--- a/src/index.js
+++ b/src/index.js
@@ -35,10 +35,10 @@ const getController = async (name) => {
controllers[name] = await import('./controllers/reportController.js');
break;
case 'ranks':
- controllers[name] = await import('./controllers/ranksBQController.js');
+ controllers[name] = await import('./controllers/ranksController.js');
break;
case 'geos':
- controllers[name] = await import('./controllers/geosBQController.js');
+ controllers[name] = await import('./controllers/geosController.js');
break;
case 'versions':
controllers[name] = await import('./controllers/versionsController.js');
diff --git a/src/package-lock.json b/src/package-lock.json
index 62391c7..d4e94bb 100644
--- a/src/package-lock.json
+++ b/src/package-lock.json
@@ -8,7 +8,6 @@
"name": "tech-report-api",
"version": "1.0.0",
"dependencies": {
- "@google-cloud/bigquery": "^8.1.0",
"@google-cloud/firestore": "7.11.1",
"@google-cloud/functions-framework": "4.0.0"
},
@@ -569,310 +568,6 @@
"dev": true,
"license": "MIT"
},
- "node_modules/@google-cloud/bigquery": {
- "version": "8.1.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-8.1.0.tgz",
- "integrity": "sha512-eDleD/IHKQIRm4GmMnwJvPkx4PgSaK8m8DCmDmVOf0gIhqPLSdvOAEeM4QjyyZGUGjV4yHyJfEJxzULTzl22Aw==",
- "license": "Apache-2.0",
- "dependencies": {
- "@google-cloud/common": "^6.0.0",
- "@google-cloud/paginator": "^6.0.0",
- "@google-cloud/precise-date": "^5.0.0",
- "@google-cloud/promisify": "^5.0.0",
- "arrify": "^3.0.0",
- "big.js": "^6.2.2",
- "duplexify": "^4.1.3",
- "extend": "^3.0.2",
- "is": "^3.3.0",
- "stream-events": "^1.0.5",
- "teeny-request": "^10.0.0"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/bigquery/node_modules/agent-base": {
- "version": "6.0.2",
- "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
- "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
- "license": "MIT",
- "dependencies": {
- "debug": "4"
- },
- "engines": {
- "node": ">= 6.0.0"
- }
- },
- "node_modules/@google-cloud/bigquery/node_modules/debug": {
- "version": "4.4.1",
- "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
- "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
- "license": "MIT",
- "dependencies": {
- "ms": "^2.1.3"
- },
- "engines": {
- "node": ">=6.0"
- },
- "peerDependenciesMeta": {
- "supports-color": {
- "optional": true
- }
- }
- },
- "node_modules/@google-cloud/bigquery/node_modules/https-proxy-agent": {
- "version": "5.0.1",
- "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
- "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
- "license": "MIT",
- "dependencies": {
- "agent-base": "6",
- "debug": "4"
- },
- "engines": {
- "node": ">= 6"
- }
- },
- "node_modules/@google-cloud/bigquery/node_modules/ms": {
- "version": "2.1.3",
- "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
- "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
- "license": "MIT"
- },
- "node_modules/@google-cloud/bigquery/node_modules/node-fetch": {
- "version": "3.3.2",
- "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
- "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
- "license": "MIT",
- "dependencies": {
- "data-uri-to-buffer": "^4.0.0",
- "fetch-blob": "^3.1.4",
- "formdata-polyfill": "^4.0.10"
- },
- "engines": {
- "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
- },
- "funding": {
- "type": "opencollective",
- "url": "https://opencollective.com/node-fetch"
- }
- },
- "node_modules/@google-cloud/bigquery/node_modules/teeny-request": {
- "version": "10.1.0",
- "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz",
- "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==",
- "license": "Apache-2.0",
- "dependencies": {
- "http-proxy-agent": "^5.0.0",
- "https-proxy-agent": "^5.0.0",
- "node-fetch": "^3.3.2",
- "stream-events": "^1.0.5"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common": {
- "version": "6.0.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-6.0.0.tgz",
- "integrity": "sha512-IXh04DlkLMxWgYLIUYuHHKXKOUwPDzDgke1ykkkJPe48cGIS9kkL2U/o0pm4ankHLlvzLF/ma1eO86n/bkumIA==",
- "license": "Apache-2.0",
- "dependencies": {
- "@google-cloud/projectify": "^4.0.0",
- "@google-cloud/promisify": "^4.0.0",
- "arrify": "^2.0.0",
- "duplexify": "^4.1.3",
- "extend": "^3.0.2",
- "google-auth-library": "^10.0.0-rc.1",
- "html-entities": "^2.5.2",
- "retry-request": "^8.0.0",
- "teeny-request": "^10.0.0"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common/node_modules/@google-cloud/promisify": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.1.0.tgz",
- "integrity": "sha512-G/FQx5cE/+DqBbOpA5jKsegGwdPniU6PuIEMt+qxWgFxvxuFOzVmp6zYchtYuwAWV5/8Dgs0yAmjvNZv3uXLQg==",
- "license": "Apache-2.0",
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common/node_modules/agent-base": {
- "version": "6.0.2",
- "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
- "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
- "license": "MIT",
- "dependencies": {
- "debug": "4"
- },
- "engines": {
- "node": ">= 6.0.0"
- }
- },
- "node_modules/@google-cloud/common/node_modules/arrify": {
- "version": "2.0.1",
- "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz",
- "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==",
- "license": "MIT",
- "engines": {
- "node": ">=8"
- }
- },
- "node_modules/@google-cloud/common/node_modules/debug": {
- "version": "4.4.1",
- "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
- "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
- "license": "MIT",
- "dependencies": {
- "ms": "^2.1.3"
- },
- "engines": {
- "node": ">=6.0"
- },
- "peerDependenciesMeta": {
- "supports-color": {
- "optional": true
- }
- }
- },
- "node_modules/@google-cloud/common/node_modules/gaxios": {
- "version": "7.0.0-rc.6",
- "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.0.0-rc.6.tgz",
- "integrity": "sha512-osVFpgeBiwTM2AVI9MXvb8iWzM6oSMbTVWc65Gm5BgBlE+nUA6PBHFMaYpqjZx1AhUH7aPOZq78WcRAM6hhAwA==",
- "license": "Apache-2.0",
- "dependencies": {
- "extend": "^3.0.2",
- "https-proxy-agent": "^7.0.1",
- "node-fetch": "^3.3.2"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common/node_modules/gcp-metadata": {
- "version": "7.0.0-rc.1",
- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.0-rc.1.tgz",
- "integrity": "sha512-E6c+AdIaK1LNA839OyotiTca+B2IG1nDlMjnlcck8JjXn3fVgx57Ib9i6iL1/iqN7bA3EUQdcRRu+HqOCOABIg==",
- "license": "Apache-2.0",
- "dependencies": {
- "gaxios": "^7.0.0-rc.1",
- "google-logging-utils": "^1.0.0",
- "json-bigint": "^1.0.0"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common/node_modules/google-auth-library": {
- "version": "10.0.0-rc.3",
- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.0.0-rc.3.tgz",
- "integrity": "sha512-WC9wfEKK0bk3seWKsDn2loduLth6JWKTsrbWftzrhPuzpwnVXb5oi2+aa0JDBxLBDdkGesLvTQ67F2nZ7leq1Q==",
- "license": "Apache-2.0",
- "dependencies": {
- "base64-js": "^1.3.0",
- "ecdsa-sig-formatter": "^1.0.11",
- "gaxios": "^7.0.0-rc.4",
- "gcp-metadata": "^7.0.0-rc.1",
- "google-logging-utils": "^1.0.0",
- "gtoken": "^8.0.0-rc.1",
- "jws": "^4.0.0"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common/node_modules/google-logging-utils": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz",
- "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==",
- "license": "Apache-2.0",
- "engines": {
- "node": ">=14"
- }
- },
- "node_modules/@google-cloud/common/node_modules/gtoken": {
- "version": "8.0.0-rc.1",
- "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0-rc.1.tgz",
- "integrity": "sha512-UjE/egX6ixArdcCKOkheuFQ4XN4/0gX92nd2JPVEYuRU2sWHAWuOVGnowm1fQUdQtaxqn1n8H0hOb2LCaUhJ3A==",
- "license": "MIT",
- "dependencies": {
- "gaxios": "^7.0.0-rc.1",
- "jws": "^4.0.0"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common/node_modules/ms": {
- "version": "2.1.3",
- "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
- "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
- "license": "MIT"
- },
- "node_modules/@google-cloud/common/node_modules/node-fetch": {
- "version": "3.3.2",
- "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
- "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
- "license": "MIT",
- "dependencies": {
- "data-uri-to-buffer": "^4.0.0",
- "fetch-blob": "^3.1.4",
- "formdata-polyfill": "^4.0.10"
- },
- "engines": {
- "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
- },
- "funding": {
- "type": "opencollective",
- "url": "https://opencollective.com/node-fetch"
- }
- },
- "node_modules/@google-cloud/common/node_modules/retry-request": {
- "version": "8.0.0",
- "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.0.tgz",
- "integrity": "sha512-dJkZNmyV9C8WKUmbdj1xcvVlXBSvsUQCkg89TCK8rD72RdSn9A2jlXlS2VuYSTHoPJjJEfUHhjNYrlvuksF9cg==",
- "license": "MIT",
- "dependencies": {
- "@types/request": "^2.48.12",
- "extend": "^3.0.2",
- "teeny-request": "^10.0.0"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common/node_modules/teeny-request": {
- "version": "10.1.0",
- "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz",
- "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==",
- "license": "Apache-2.0",
- "dependencies": {
- "http-proxy-agent": "^5.0.0",
- "https-proxy-agent": "^5.0.0",
- "node-fetch": "^3.3.2",
- "stream-events": "^1.0.5"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/common/node_modules/teeny-request/node_modules/https-proxy-agent": {
- "version": "5.0.1",
- "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
- "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
- "license": "MIT",
- "dependencies": {
- "agent-base": "6",
- "debug": "4"
- },
- "engines": {
- "node": ">= 6"
- }
- },
"node_modules/@google-cloud/firestore": {
"version": "7.11.1",
"resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-7.11.1.tgz",
@@ -924,45 +619,6 @@
"node": ">=10"
}
},
- "node_modules/@google-cloud/paginator": {
- "version": "6.0.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-6.0.0.tgz",
- "integrity": "sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==",
- "license": "Apache-2.0",
- "dependencies": {
- "extend": "^3.0.2"
- },
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/precise-date": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-5.0.0.tgz",
- "integrity": "sha512-9h0Gvw92EvPdE8AK8AgZPbMnH5ftDyPtKm7/KUfcJVaPEPjwGDsJd1QV0H8esBDV4II41R/2lDWH1epBqIoKUw==",
- "license": "Apache-2.0",
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@google-cloud/projectify": {
- "version": "4.0.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz",
- "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==",
- "license": "Apache-2.0",
- "engines": {
- "node": ">=14.0.0"
- }
- },
- "node_modules/@google-cloud/promisify": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-5.0.0.tgz",
- "integrity": "sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==",
- "license": "Apache-2.0",
- "engines": {
- "node": ">=18"
- }
- },
"node_modules/@grpc/grpc-js": {
"version": "1.13.3",
"resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.3.tgz",
@@ -2168,18 +1824,6 @@
"integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==",
"license": "MIT"
},
- "node_modules/arrify": {
- "version": "3.0.0",
- "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz",
- "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==",
- "license": "MIT",
- "engines": {
- "node": ">=12"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/asap": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
@@ -2392,19 +2036,6 @@
],
"license": "MIT"
},
- "node_modules/big.js": {
- "version": "6.2.2",
- "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz",
- "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==",
- "license": "MIT",
- "engines": {
- "node": "*"
- },
- "funding": {
- "type": "opencollective",
- "url": "https://opencollective.com/bigjs"
- }
- },
"node_modules/bignumber.js": {
"version": "9.3.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.0.tgz",
@@ -2857,15 +2488,6 @@
"node": ">= 8"
}
},
- "node_modules/data-uri-to-buffer": {
- "version": "4.0.1",
- "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
- "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
- "license": "MIT",
- "engines": {
- "node": ">= 12"
- }
- },
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -3321,29 +2943,6 @@
"bser": "2.1.1"
}
},
- "node_modules/fetch-blob": {
- "version": "3.2.0",
- "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
- "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/jimmywarting"
- },
- {
- "type": "paypal",
- "url": "https://paypal.me/jimmywarting"
- }
- ],
- "license": "MIT",
- "dependencies": {
- "node-domexception": "^1.0.0",
- "web-streams-polyfill": "^3.0.3"
- },
- "engines": {
- "node": "^12.20 || >= 14.13"
- }
- },
"node_modules/fill-range": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
@@ -3432,18 +3031,6 @@
"node": ">= 0.12"
}
},
- "node_modules/formdata-polyfill": {
- "version": "4.0.10",
- "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
- "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
- "license": "MIT",
- "dependencies": {
- "fetch-blob": "^3.1.2"
- },
- "engines": {
- "node": ">=12.20.0"
- }
- },
"node_modules/formidable": {
"version": "3.5.4",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz",
@@ -3818,22 +3405,6 @@
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"license": "ISC"
},
- "node_modules/html-entities": {
- "version": "2.6.0",
- "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz",
- "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==",
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/mdevils"
- },
- {
- "type": "patreon",
- "url": "https://patreon.com/mdevils"
- }
- ],
- "license": "MIT"
- },
"node_modules/html-escaper": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
@@ -4033,15 +3604,6 @@
"node": ">= 0.10"
}
},
- "node_modules/is": {
- "version": "3.3.0",
- "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz",
- "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==",
- "license": "MIT",
- "engines": {
- "node": "*"
- }
- },
"node_modules/is-arguments": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz",
@@ -5354,26 +4916,6 @@
"node": ">= 0.6"
}
},
- "node_modules/node-domexception": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
- "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
- "deprecated": "Use your platform's native DOMException instead",
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/jimmywarting"
- },
- {
- "type": "github",
- "url": "https://paypal.me/jimmywarting"
- }
- ],
- "license": "MIT",
- "engines": {
- "node": ">=10.5.0"
- }
- },
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
@@ -6874,15 +6416,6 @@
"makeerror": "1.0.12"
}
},
- "node_modules/web-streams-polyfill": {
- "version": "3.3.3",
- "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
- "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
- "license": "MIT",
- "engines": {
- "node": ">= 8"
- }
- },
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
diff --git a/src/package.json b/src/package.json
index 1286c68..b129f94 100644
--- a/src/package.json
+++ b/src/package.json
@@ -13,7 +13,6 @@
"test:live": "bash ../test-api.sh"
},
"dependencies": {
- "@google-cloud/bigquery": "^8.1.0",
"@google-cloud/firestore": "7.11.1",
"@google-cloud/functions-framework": "4.0.0"
},
diff --git a/src/utils/bigquery.js b/src/utils/bigquery.js
deleted file mode 100644
index 753e00a..0000000
--- a/src/utils/bigquery.js
+++ /dev/null
@@ -1,89 +0,0 @@
-import { BigQuery } from '@google-cloud/bigquery';
-
-// Initialize BigQuery client
-const bigquery = new BigQuery({
- projectId: process.env.PROJECT || 'httparchive'
-});
-
-// BigQuery configuration optimizations
-const BQ_CONFIG = {
- // Optimize for BI Engine
- location: 'US', // Use the same location as your BI Engine
- // Use maximum parallelism for BI Engine
- maximumBytesBilled: '100000000', // 100MB limit for safety
- // Labels for monitoring
- labels: {
- 'app': 'tech-report-api',
- 'source': 'bigquery-direct',
- }
-};
-
-/**
- * Execute a BigQuery query with caching support
- * @param {string} query - SQL query string
- * @param {Object} options - Query options
- * @returns {Array} - Query results
- */
-const executeBigQueryQuery = async (query, options = {}) => {
- try {
- const queryOptions = {
- query,
-
- jobCreationMode: 'JOB_CREATION_OPTIONAL', // Returning immediate results is prioritized.
- timeoutMs: 10000, // 10 seconds
- // Use query cache when possible
- useQueryCache: true,
- // Apply BI Engine optimizations
- ...BQ_CONFIG,
- ...options
- };
-
- console.log('Executing BigQuery:', query);
- const [rows, , metadata] = await bigquery.query(queryOptions);
- if(metadata.jobReference) {
- console.log(`BigQuery job ${metadata.jobReference.jobId} completed. Rows: ${rows.length}`);
- }
-
- return rows;
- } catch (error) {
- console.error('BigQuery execution error:', error);
- throw error;
- }
-};
-
-/**
- * Get ranks from BigQuery
- * @returns {Array} - Array of rank objects
- */
-const getRanksFromBQ = async () => {
- const query = `
- SELECT rank
- FROM \`httparchive.reports.tech_report_ranks\`
- ORDER BY mobile_origins DESC
- `;
-
- const rows = await executeBigQueryQuery(query);
- return rows.map(row => ({ rank: row.rank }));
-};
-
-/**
- * Get geos from BigQuery
- * @returns {Array} - Array of geo objects
- */
-const getGeosFromBQ = async () => {
- const query = `
- SELECT geo
- FROM \`httparchive.reports.tech_report_geos\`
- ORDER BY mobile_origins DESC
- `;
-
- const rows = await executeBigQueryQuery(query);
- return rows.map(row => ({ geo: row.geo }));
-};
-
-export {
- bigquery,
- executeBigQueryQuery,
- getRanksFromBQ,
- getGeosFromBQ
-};
diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js
index 8f0a132..56aa46e 100644
--- a/src/utils/controllerHelpers.js
+++ b/src/utils/controllerHelpers.js
@@ -318,50 +318,6 @@ const validateTechnologyArray = (technologyParam) => {
}
};
-/**
- * Generic BigQuery-enabled query executor
- * Handles caching, query execution, and response for BigQuery queries
- * @param {Object} req - Request object
- * @param {Object} res - Response object
- * @param {string} queryName - Query name for caching and error handling
- * @param {Function} queryExecutor - Function that executes BigQuery and returns results
- * @param {Function} dataProcessor - Optional function to process results
- */
-const executeBigQuery = async (req, res, queryName, queryExecutor, dataProcessor = null) => {
- try {
- const params = req.query;
-
- // Generate cache key
- const cacheKey = generateQueryCacheKey(`bq_${queryName}`, params);
-
- // Check cache first
- const cachedResult = getCachedQueryResult(cacheKey);
- if (cachedResult) {
- res.statusCode = 200;
- res.end(JSON.stringify(cachedResult));
- return;
- }
-
- // Execute BigQuery
- let data = await queryExecutor(params);
-
- // Process data if processor provided
- if (dataProcessor) {
- data = dataProcessor(data, params);
- }
-
- // Cache the result
- setCachedQueryResult(cacheKey, data);
-
- // Send response
- res.statusCode = 200;
- res.end(JSON.stringify(data));
-
- } catch (error) {
- handleControllerError(res, error, `executing BigQuery ${queryName}`);
- }
-};
-
export {
REQUIRED_PARAMS,
FIRESTORE_IN_LIMIT,
@@ -375,6 +331,5 @@ export {
setCachedQueryResult,
getCacheStats,
executeQuery,
- executeBigQuery,
validateTechnologyArray
};
From d8244f955aab9bd91e6cab5f2b6058de27052e25 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Wed, 18 Jun 2025 21:40:51 +0200
Subject: [PATCH 6/8] tf backend migrated
---
terraform/dev/main.tf | 4 ++--
terraform/prod/main.tf | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf
index 037840f..dc9c003 100644
--- a/terraform/dev/main.tf
+++ b/terraform/dev/main.tf
@@ -1,7 +1,7 @@
terraform {
backend "gcs" {
- bucket = "tf-state-backingapi-20230314"
- prefix = "dev"
+ bucket = "tfstate-httparchive"
+ prefix = "tech-report-apis/dev"
}
}
diff --git a/terraform/prod/main.tf b/terraform/prod/main.tf
index 50c36a0..9439dac 100644
--- a/terraform/prod/main.tf
+++ b/terraform/prod/main.tf
@@ -1,7 +1,7 @@
terraform {
backend "gcs" {
- bucket = "tf-state-backingapi-20230314"
- prefix = "prod"
+ bucket = "tfstate-httparchive"
+ prefix = "tech-report-apis/prod"
}
}
From 5a0212b1db7f5d6fde317413c2cac4e895f806cc Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 19 Jun 2025 01:38:11 +0200
Subject: [PATCH 7/8] api config update
---
terraform/dev/main.tf | 13 ++++++++++---
terraform/prod/main.tf | 13 ++++++++++---
2 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf
index dc9c003..db15672 100644
--- a/terraform/dev/main.tf
+++ b/terraform/dev/main.tf
@@ -52,14 +52,14 @@ paths:
/v1/adoption:
get:
summary: adoption
- operationId: getadoptionReports
+ operationId: getAdoptionReports
responses:
200:
description: String
/v1/page-weight:
get:
summary: pageWeight
- operationId: getpageWeight
+ operationId: getPageWeightReports
responses:
200:
description: String
@@ -73,7 +73,7 @@ paths:
/v1/cwv:
get:
summary: cwv
- operationId: getCwv
+ operationId: getCWVReports
responses:
200:
description: String
@@ -98,6 +98,13 @@ paths:
responses:
200:
description: String
+ /v1/audits:
+ get:
+ summary: audits
+ operationId: getAuditReports
+ responses:
+ 200:
+ description: String
EOF
)
}
diff --git a/terraform/prod/main.tf b/terraform/prod/main.tf
index 9439dac..721b575 100644
--- a/terraform/prod/main.tf
+++ b/terraform/prod/main.tf
@@ -52,14 +52,14 @@ paths:
/v1/adoption:
get:
summary: adoption
- operationId: getadoptionReports
+ operationId: getAdoptionReports
responses:
200:
description: String
/v1/page-weight:
get:
summary: pageWeight
- operationId: getpageWeight
+ operationId: getPageWeightReports
responses:
200:
description: String
@@ -73,7 +73,7 @@ paths:
/v1/cwv:
get:
summary: cwv
- operationId: getCwv
+ operationId: getCWVReports
responses:
200:
description: String
@@ -98,6 +98,13 @@ paths:
responses:
200:
description: String
+ /v1/audits:
+ get:
+ summary: audits
+ operationId: getAuditReports
+ responses:
+ 200:
+ description: String
EOF
)
}
From 2383e75ecf697cd2c7b370dbeb6e2718f689bae4 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 19 Jun 2025 01:55:41 +0200
Subject: [PATCH 8/8] cache reset
---
README.md | 26 ++++++++++++++++++++++++++
src/__tests__/routes.test.js | 31 +++++++++++++++++++++++++++++++
src/index.js | 7 ++++++-
src/utils/controllerHelpers.js | 28 +++++++++++++++++++++++++++-
test-api.sh | 23 +++++++++++++++++++++++
5 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index f66526e..2724719 100644
--- a/README.md
+++ b/README.md
@@ -520,6 +520,32 @@ Returns a JSON object with the following schema:
}
```
+### `POST /v1/cache-reset`
+
+Resets all caches in the API. This endpoint requires a POST request.
+
+```bash
+curl --request POST \
+ --url 'https://{{HOST}}/v1/cache-reset'
+```
+
+Returns a JSON object with the following schema:
+
+```json
+{
+ "success": true,
+ "message": "All caches have been reset",
+ "before": {
+ "queryCache": 150,
+ "dateCache": 12
+ },
+ "after": {
+ "queryCache": 0,
+ "dateCache": 0
+ }
+}
+```
+
## Testing
```bash
diff --git a/src/__tests__/routes.test.js b/src/__tests__/routes.test.js
index b88c66d..f84cd9b 100644
--- a/src/__tests__/routes.test.js
+++ b/src/__tests__/routes.test.js
@@ -362,4 +362,35 @@ describe('API Routes', () => {
expect(res.headers['timing-allow-origin']).toEqual('*');
});
});
+
+ describe('Cache Management', () => {
+ it('should provide cache stats', async () => {
+ const res = await request(app)
+ .get('/v1/cache-stats')
+ .expect(200);
+
+ expect(res.body).toHaveProperty('queryCache');
+ expect(res.body).toHaveProperty('dateCache');
+ expect(res.body).toHaveProperty('config');
+ });
+
+ it('should reset cache on POST request', async () => {
+ const res = await request(app)
+ .post('/v1/cache-reset')
+ .expect(200);
+
+ expect(res.body).toHaveProperty('success', true);
+ expect(res.body).toHaveProperty('message');
+ expect(res.body).toHaveProperty('before');
+ expect(res.body).toHaveProperty('after');
+ });
+
+ it('should handle cache reset OPTIONS request', async () => {
+ const res = await request(app)
+ .options('/v1/cache-reset')
+ .expect(204);
+
+ expect(res.headers['access-control-allow-methods']).toContain('POST');
+ });
+ });
});
diff --git a/src/index.js b/src/index.js
index 4269c99..cf2baeb 100644
--- a/src/index.js
+++ b/src/index.js
@@ -51,7 +51,7 @@ const getController = async (name) => {
// Helper function to set CORS headers
const setCORSHeaders = (res) => {
res.setHeader('Access-Control-Allow-Origin', '*');
- res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
+ res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Timing-Allow-Origin');
res.setHeader('Access-Control-Max-Age', '86400');
};
@@ -163,6 +163,11 @@ const handleRequest = async (req, res) => {
const { getCacheStats } = await import('./utils/controllerHelpers.js');
const stats = getCacheStats();
sendJSONResponse(res, stats);
+ } else if (pathname === '/v1/cache-reset' && req.method === 'POST') {
+ // Cache reset endpoint
+ const { resetCache } = await import('./utils/controllerHelpers.js');
+ const result = resetCache();
+ sendJSONResponse(res, result);
} else {
// 404 Not Found
res.statusCode = 404;
diff --git a/src/utils/controllerHelpers.js b/src/utils/controllerHelpers.js
index 56aa46e..b22313f 100644
--- a/src/utils/controllerHelpers.js
+++ b/src/utils/controllerHelpers.js
@@ -318,6 +318,31 @@ const validateTechnologyArray = (technologyParam) => {
}
};
+/**
+ * Reset all caches
+ * @returns {Object} Reset operation result
+ */
+const resetCache = () => {
+ const beforeStats = {
+ queryCache: queryResultCache.size,
+ dateCache: latestDateCache.size
+ };
+
+ // Clear both caches
+ queryResultCache.clear();
+ latestDateCache.clear();
+
+ return {
+ success: true,
+ message: 'All caches have been reset',
+ before: beforeStats,
+ after: {
+ queryCache: queryResultCache.size,
+ dateCache: latestDateCache.size
+ }
+ };
+};
+
export {
REQUIRED_PARAMS,
FIRESTORE_IN_LIMIT,
@@ -331,5 +356,6 @@ export {
setCachedQueryResult,
getCacheStats,
executeQuery,
- validateTechnologyArray
+ validateTechnologyArray,
+ resetCache
};
diff --git a/test-api.sh b/test-api.sh
index d97d9ff..e73e836 100755
--- a/test-api.sh
+++ b/test-api.sh
@@ -116,4 +116,27 @@ test_endpoint "/v1/page-weight" "?technology=WordPress&geo=ALL&rank=ALL&start=la
# Test audits endpoint
test_endpoint "/v1/audits" "?technology=WordPress&geo=ALL&rank=ALL&start=latest"
+# Test cache stats endpoint
+echo "Testing cache stats endpoint..."
+test_endpoint "/v1/cache-stats" ""
+
+# Test cache reset endpoint
+echo "Testing cache reset endpoint..."
+echo "Checking cache reset: http://localhost:3000/v1/cache-reset"
+response=$(curl -s -w "\n%{http_code}" -X POST "http://localhost:3000/v1/cache-reset")
+http_code=$(echo "$response" | tail -n1)
+body=$(echo "$response" | sed '$d')
+
+echo "$body" | jq .
+echo "Status code: $http_code"
+
+if [[ $http_code -ne 200 ]]; then
+ echo "Error: Cache reset endpoint returned non-200 status code"
+ exit 1
+fi
+
+echo ""
+echo "----------------------"
+echo ""
+
echo "API tests complete! All endpoints returned 200 status code and CORS is properly configured."