[api refactor] Minor refactoring

Author: indexzero

src/cache.js

@@ -81,11 +81,10 @@ function entryToTuple({ id, key, value }) {
 
 /**
  * Checks if a partition file is cached and contains valid JSON.
- * @param {Partition} partition
+ * @param {String} filename
  * @returns {Promise<boolean>}
  */
-async function isPartitionCached(partition) {
-  const { filename } = partition;
+async function isJsonCached(filename) {
   try {
     await access(filename, fs.constants.F_OK);
     debug(`${filename} cache | hit`);
@@ -110,7 +109,7 @@ async function isPartitionCached(partition) {
  * @returns {Promise<{ partition: Partition, cached: boolean }>}
  */
 async function refreshPartition({ partition }) {
-  const cached = await isPartitionCached(partition);
+  const cached = await isJsonCached(partition.filename);
   if (!cached && !dryrun) {
     await writePartition({ partition });
   }
@@ -150,7 +149,7 @@ async function getPartition(partition, dryrun) {
 }
 
 module.exports = {
-  isPartitionCached,
+  isJsonCached,
   refreshPartition,
   getPartition,
   listPartitions,

src/http.js

@@ -29,7 +29,7 @@ const agent = new RetryAgent(dispatch, {
 });
 
 module.exports = {
-  default: {
+  defaults: {
     agent,
     dispatch,
     config: agentDefaults

src/map-reduce.js

@@ -78,7 +78,7 @@ async function reduceAllDocsIndex({ partitions, cacheDir, reduceFn, concurrency
     if (!_all_docs || !_all_docs.rows || !_all_docs.rows.length) {
       return acc;
     }
-    
+
     const addToIndex = reduceFn
       ? _all_docs.rows.map(reduceFn)
       : _all_docs.rows;

src/packument.js

@@ -1,6 +1,8 @@
+const { join } = require('path');
 const pMap = require('p-map').default;
 const pMapSeries = require('p-map-series').default;
-const { default } = require('./http');
+const { isJsonCached } = require('./cache');
+const { defaults } = require('./http');
 
 const debug = require('debug')('_all_docs/packument');
 
@@ -10,7 +12,7 @@ const debug = require('debug')('_all_docs/packument');
  * @returns {Promise<any>} A promise resolving to the parsed packument.
  */
 async function getPackument(name) {
-  const agent = default.agent;
+  const agent = defaults.agent;
   const options = {
     origin: 'https://replicate.npmjs.com',
     path: `/${name}`,
@@ -75,11 +77,13 @@ async function cachePackumentsSeries(names, writeFn) {
  * @param {(packument: any) => Promise<any>} writeFn
  * @returns {Promise<any[]>} A promise resolving to an array of cached packuments.
  */
-async function cachePackumentsLimit(names, writeFn, limit) {
-  return await pMap(names, async function (name) {
+async function cachePackumentsLimit(names, writeFn, { limit, cacheDir }) {
+  await pMap(names, async function (name) {
+    const filename = join(cacheDir, `${name}.json`);
+    if (await isJsonCached(filename)) return;
+
     const packument = await getPackument(name);
     await writeFn(packument);
-    return packument;
   }, { concurrency: limit });
 }
 

src/undtils.js

@@ -0,0 +1,4 @@
+
+async function batchRequestsBySize() {
+
+}