Merge pull request #86 from duckduckgo/configurable-region

Make region configurable
duckduckgo · Aug 30, 2024 · 4770f4d · 4770f4d
2 parents 90b90f7 + d2906a1
commit 4770f4d
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 17 deletions.
diff --git a/config.js b/config.js
@@ -8,4 +8,5 @@ module.exports = {
     categoryPagesPath: './docs/categories/',
     hostPath: '/tracker-radar-wiki',
     appendTrackerRadarRevisions: [], // in addition to all Git tags, checkout these tracker-radar revisions
+    region: 'US',
 };
diff --git a/scripts/generateHistoricalData.js b/scripts/generateHistoricalData.js
@@ -9,7 +9,6 @@ const getListOfJSONPathsFromFolder = require('./helpers/getListOfJSONPathsFromFo
 
 const TRACKER_RADAR_DOMAINS_PATH = path.join(config.trackerRadarRepoPath, '/domains/');
 const TRACKER_RADAR_ENTITIES_PATH = path.join(config.trackerRadarRepoPath, '/entities/');
-const REGION = 'US';
 
 const domainMap = new Map();
 const entityMap = new Map();
@@ -135,8 +134,8 @@ async function main() {
         await git.raw('checkout', tag, '--force');
 
         let domainPath = TRACKER_RADAR_DOMAINS_PATH;
-        if (fs.existsSync(path.join(domainPath, REGION))) {
-            domainPath = path.join(domainPath, REGION);
+        if (fs.existsSync(path.join(domainPath, config.region))) {
+            domainPath = path.join(domainPath, config.region);
         }
 
         const domainFiles = getListOfJSONPathsFromFolder(domainPath);

diff --git a/scripts/generateIndexPage.js b/scripts/generateIndexPage.js
@@ -15,11 +15,10 @@ const lastTagInfo = {
 
 let TRACKER_RADAR_DOMAINS_PATH = path.join(config.trackerRadarRepoPath, '/domains/');
 const TRACKER_RADAR_ENTITIES_PATH = path.join(config.trackerRadarRepoPath, '/entities/');
-const REGION = 'US';
 
 // Backwards compatibility for regions updates
-if (fs.existsSync(path.join(TRACKER_RADAR_DOMAINS_PATH, REGION))) {
-    TRACKER_RADAR_DOMAINS_PATH = path.join(TRACKER_RADAR_DOMAINS_PATH, REGION);
+if (fs.existsSync(path.join(TRACKER_RADAR_DOMAINS_PATH, config.region))) {
+    TRACKER_RADAR_DOMAINS_PATH = path.join(TRACKER_RADAR_DOMAINS_PATH, config.region);
 }
 
 const domainFiles = getListOfJSONPathsFromFolder(TRACKER_RADAR_DOMAINS_PATH);
@@ -89,7 +88,8 @@ const renderData = {
     categories: Array.from(categories.values()),
     lastTagInfo,
     historicDataString,
-    hostPath: config.hostPath
+    hostPath: config.hostPath,
+    region: config.region,
 };
 
 try {
@@ -106,7 +106,8 @@ fs.writeFileSync(path.join(config.basePagesPath, 'index.html'), output);
 const top100RenderData = {
     domains: domains.slice(0, 100),
     entities: entities.slice(0, 100),
-    hostPath: config.hostPath
+    hostPath: config.hostPath,
+    region: config.region,
 };
 
 const top100Output = mustache.render(getTemplate('top100'), top100RenderData, getTemplate);
@@ -122,7 +123,8 @@ const apiHistoryRenderData = {
         .map(item => ({api: item.api, entries: item.entries, lastValue: item.entries[item.entries.length - 1].value}))
         .sort((a, b) => b.lastValue - a.lastValue),
     apiHistoryString,
-    hostPath: config.hostPath
+    hostPath: config.hostPath,
+    region: config.region,
 };
 
 const apiHistoryOutput = mustache.render(getTemplate('apiHistory'), apiHistoryRenderData, getTemplate);

diff --git a/scripts/generateSearchIndex.js b/scripts/generateSearchIndex.js
@@ -8,11 +8,10 @@ const getListOfJSONPathsFromFolder = require('./helpers/getListOfJSONPathsFromFo
 
 let TRACKER_RADAR_DOMAINS_PATH = path.join(config.trackerRadarRepoPath, '/domains/');
 const TRACKER_RADAR_ENTITIES_PATH = path.join(config.trackerRadarRepoPath, '/entities/');
-const REGION = 'US';
 
 // Backwards compatibility for regions updates
-if (fs.existsSync(path.join(TRACKER_RADAR_DOMAINS_PATH, REGION))) {
-    TRACKER_RADAR_DOMAINS_PATH = path.join(TRACKER_RADAR_DOMAINS_PATH, REGION);
+if (fs.existsSync(path.join(TRACKER_RADAR_DOMAINS_PATH, config.region))) {
+    TRACKER_RADAR_DOMAINS_PATH = path.join(TRACKER_RADAR_DOMAINS_PATH, config.region);
 }
 
 const domainFiles = getListOfJSONPathsFromFolder(TRACKER_RADAR_DOMAINS_PATH);

diff --git a/scripts/generateSubpages.js b/scripts/generateSubpages.js
@@ -10,11 +10,10 @@ const escapeEntityName = require('./helpers/escapeEntityName');
 
 let TRACKER_RADAR_DOMAINS_PATH = path.join(config.trackerRadarRepoPath, '/domains/');
 const TRACKER_RADAR_ENTITIES_PATH = path.join(config.trackerRadarRepoPath, '/entities/');
-const REGION = 'US';
 
 // Backwards compatibility for regions updates
-if (fs.existsSync(path.join(TRACKER_RADAR_DOMAINS_PATH, REGION))) {
-    TRACKER_RADAR_DOMAINS_PATH = path.join(TRACKER_RADAR_DOMAINS_PATH, REGION);
+if (fs.existsSync(path.join(TRACKER_RADAR_DOMAINS_PATH, config.region))) {
+    TRACKER_RADAR_DOMAINS_PATH = path.join(TRACKER_RADAR_DOMAINS_PATH, config.region);
 }
 
 const fingerprintTexts = [
@@ -140,6 +139,8 @@ domainFiles.forEach(({file, resolvedPath}) => {
             sites: res.sites
         };
     }).sort((a, b) => b.sites - a.sites);
+
+    data.region = config.region;
 
     const output = mustache.render(getTemplate('domain'), data, getTemplate);
 
@@ -203,6 +204,7 @@ entityFiles.forEach(({file, resolvedPath}) => {
     }
 
     data.hostPath = config.hostPath;
+    data.region = config.region;
 
     const output = mustache.render(getTemplate('entity'), data, getTemplate);
 
@@ -238,6 +240,7 @@ Array.from(categories.values()).forEach(data => {
     }
 
     data.hostPath = config.hostPath;
+    data.region = config.region;
 
     const output = mustache.render(getTemplate('category'), data, getTemplate);
 

diff --git a/scripts/helpers/getListOfJSONPathsFromFolder.js b/scripts/helpers/getListOfJSONPathsFromFolder.js
@@ -12,8 +12,7 @@ module.exports = function getListOfJSONPathsFromFolder(folderPath) {
                 return true;
             }
 
-            console.trace('Skipping path', resolvedPath, stat);
-            process.exit(1);
+            console.log('Skipping path', resolvedPath, stat);
             return false;
         })
         // map file names to full paths