fix: failing english builds (#1082)

.github/workflows/deploy-eng.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [18.x]
+        node-version: [20.x]
         languages: [english]
         site_tlds: [org]
 

src/_data/datasource.js

@@ -6,22 +6,25 @@ const fetchFromGhost = require('../../utils/ghost/fetch-from-ghost');
 const fetchFromHashnode = require('../../utils/hashnode/fetch-from-hashnode');
 const { postsPerPage, siteURL } = require('../../config');
 const pingEditorialTeam = require('../../utils/ping-editorial-team');
+const idleTimeoutMS = 3600000; // 1 hour
 
 const getUniqueList = (arr, key) => [
   ...new Map(arr.map(item => [item[key], item])).values()
 ];
 
 const piscinaGhost = new Piscina({
-  filename: resolve(__dirname, '../../utils/ghost/process-batch')
+  filename: resolve(__dirname, '../../utils/ghost/process-batch'),
+  idleTimeout: idleTimeoutMS
 });
 const piscinaHashnode = new Piscina({
-  filename: resolve(__dirname, '../../utils/hashnode/process-batch')
+  filename: resolve(__dirname, '../../utils/hashnode/process-batch'),
+  idleTimeout: idleTimeoutMS
 });
 
 module.exports = async () => {
   // Chunk raw Ghost posts and pages and process them in batches
   // with a pool of workers to create posts and pages global data
-  const batchSize = 200;
+  const batchSize = 300;
   const allGhostPosts = await fetchFromGhost('posts');
   const allHashnodePosts = await fetchFromHashnode('posts');
   const allGhostPages = await fetchFromGhost('pages');

utils/ghost/process-batch.js

@@ -192,6 +192,10 @@ const processBatch = async ({
     })
   );
 
+  console.log(
+    `Finished processing Ghost ${contentType} batch ${currBatchNo} of ${totalBatches}...and using ${process.memoryUsage.rss() / 1024 / 1024} MB of memory`
+  );
+
   return batch;
 };
 

utils/hashnode/process-batch.js

@@ -148,6 +148,10 @@ const processBatch = async ({
     })
   );
 
+  console.log(
+    `Finished processing Hashnode ${contentType} batch ${currBatchNo} of ${totalBatches}...and using ${process.memoryUsage.rss() / 1024 / 1024} MB of memory`
+  );
+
   return batch;
 };