-
Notifications
You must be signed in to change notification settings - Fork 192
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1884 from mashehu/reduce-build-time
reduce built time through content collections
- Loading branch information
Showing
22 changed files
with
309 additions
and
364 deletions.
There are no files selected for viewing
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,58 +1,117 @@ | ||
#! /usr/bin/env node | ||
import { getGitHubFile, getCurrentRateLimitRemaining } from '../src/components/octokit.js'; | ||
import Cache from 'file-system-cache'; | ||
import { readFileSync } from 'fs'; | ||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs'; | ||
import path from 'path'; | ||
import ProgressBar from 'progress'; | ||
|
||
|
||
const cache = Cache.default({ | ||
basePath: './.cache', | ||
ns: 'nf-core', | ||
}); | ||
import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3'; | ||
import cache from './cache.js'; | ||
|
||
// get current path | ||
const __dirname = path.resolve(); | ||
|
||
// check for `--force` flag | ||
const force = process.argv.includes('--force'); | ||
|
||
console.log(await getCurrentRateLimitRemaining()); | ||
export const buildCache = async () => { | ||
const pipelinesJson = readFileSync(path.join(__dirname, '/public/pipelines.json')); | ||
const pipelines = JSON.parse(pipelinesJson); | ||
|
||
let bar = new ProgressBar(' caching markdown [:bar] :percent :etas', { total: pipelines.remote_workflows.length }); | ||
|
||
// go through the releases of each pipeline and get the files which are needed for the pipeline pages | ||
for (const pipeline of pipelines.remote_workflows) { | ||
// console.log(`Caching ${pipeline.name}`); | ||
const { name } = pipeline; | ||
let releases = pipeline.releases; | ||
for (const release of releases) { | ||
// console.log(`Caching ${name} ${release.tag_name}`); | ||
release.doc_files.push('README.md'); // add the README to the cache | ||
if (release.has_schema) { | ||
release.doc_files.push('nextflow_schema.json'); // add the nextflow_schema.json to the cache | ||
async function getKeysWithPrefixes(prefixes) { | ||
let client = new S3Client({ | ||
region: 'eu-west-1', | ||
signer: { sign: async (request) => request }, | ||
}); | ||
const keys = []; | ||
const commonPrefixes = []; | ||
|
||
for (const prefix of prefixes) { | ||
let continuationToken = undefined; | ||
do { | ||
const command = new ListObjectsV2Command({ | ||
Bucket: 'nf-core-awsmegatests', | ||
Prefix: prefix, | ||
ContinuationToken: continuationToken, | ||
Delimiter: '/', | ||
}); | ||
try { | ||
const response = await client.send(command); | ||
if (response.Contents) { | ||
for (const object of response.Contents) { | ||
keys.push(object); | ||
} | ||
} | ||
if (response.CommonPrefixes) { | ||
for (const object of response.CommonPrefixes) { | ||
commonPrefixes.push(object); | ||
} | ||
} | ||
|
||
continuationToken = response.NextContinuationToken; | ||
} catch (error) { | ||
console.error('Error retrieving keys:', error); | ||
return []; | ||
} | ||
const version = release.tag_name; | ||
for (const f of release.doc_files) { | ||
const cache_key = `${name}/${version}/${f}`; | ||
// console.log(`Checking ${cache_key}`); | ||
const is_cached = cache.getSync(cache_key, false) && cache.getSync(cache_key, false).length > 0; | ||
if (!is_cached || force || version === 'dev') { | ||
const content = await getGitHubFile(name, f, version); | ||
// console.log(`Caching ${cache_key}`); | ||
cache.set(cache_key, content); | ||
// console.log(`Cached ${cache_key}`); | ||
} else { | ||
// console.log(`Already cached ${cache_key}`); | ||
} while (continuationToken); | ||
} | ||
|
||
return { keys: keys, commonPrefixes: commonPrefixes }; | ||
} | ||
|
||
(async () => { | ||
console.log(await getCurrentRateLimitRemaining()); | ||
const buildCache = async () => { | ||
// build the pipeline cache | ||
const pipelinesJson = readFileSync(path.join(__dirname, '/public/pipelines.json')); | ||
const pipelines = JSON.parse(pipelinesJson); | ||
let bar = new ProgressBar(' caching markdown [:bar] :percent :etas', { total: pipelines.remote_workflows.length }); | ||
// go through the releases of each pipeline and get the files which are needed for the pipeline pages | ||
for (const pipeline of pipelines.remote_workflows) { | ||
// console.log(`Caching ${pipeline.name}`); | ||
const { name } = pipeline; | ||
let releases = pipeline.releases; | ||
for (const release of releases) { | ||
// console.log(`Caching ${name} ${release.tag_name}`); | ||
release.doc_files.push('README.md'); // add the README to the cache | ||
if (release.has_schema) { | ||
release.doc_files.push('nextflow_schema.json'); // add the nextflow_schema.json to the cache | ||
} | ||
const version = release.tag_name; | ||
for (const f of release.doc_files) { | ||
const cache_key = `${name}/${version}/${f}`; | ||
// console.log(`Checking ${cache_key}`); | ||
const is_cached = existsSync(path.join(__dirname, '.cache', cache_key)); | ||
if (!is_cached || force || version === 'dev') { | ||
const content = await getGitHubFile(name, f, version); | ||
// console.log(`Caching ${cache_key}`); | ||
// cache.set(cache_key, content); | ||
// console.log(`Cached ${cache_key}`); | ||
//generate folder structure | ||
const parent = cache_key.split('/').slice(0, -1).join('/'); | ||
mkdirSync(path.join(__dirname, '.cache', parent), { recursive: true }); | ||
writeFileSync(path.join(__dirname, '.cache', cache_key), content); | ||
} else { | ||
// console.log(`Already cached ${cache_key}`); | ||
} | ||
} | ||
} | ||
|
||
bar.tick(); | ||
} | ||
// cache aws results | ||
const cache_key = `aws.json`; | ||
const is_cached = existsSync(path.join(__dirname, '.cache', cache_key)); | ||
if (!is_cached || force) { | ||
const prefixes = pipelines.remote_workflows.flatMap((pipeline) => { | ||
return pipeline.releases | ||
.filter((rel) => rel.tag_name !== 'dev') | ||
.map((release) => { | ||
return `${pipeline.name}/results-${release.tag_sha}/`; | ||
}); | ||
}); | ||
|
||
bar.tick(); | ||
} | ||
return true; | ||
}; | ||
buildCache(); | ||
const awsResponse = await getKeysWithPrefixes(prefixes); | ||
console.log(`Caching ${cache_key}`); | ||
cache.set(cache_key, { commonPrefixes: awsResponse.commonPrefixes, bucketContents: awsResponse.keys }); | ||
} | ||
|
||
console.log('Done'); | ||
return true; | ||
}; | ||
await buildCache(); | ||
})(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
#! /usr/bin/env node | ||
// move markdown files from cache to src/content | ||
import { promises, readdirSync, statSync, existsSync, mkdirSync } from 'fs'; | ||
import path from 'path'; | ||
|
||
|
||
(async () => { | ||
async function buildContentCollection() { | ||
// go through all files in .cache, move them to src/content | ||
|
||
const getAllMDFiles = (dir) => | ||
readdirSync(dir).reduce((files, file) => { | ||
const name = path.join(dir, file); | ||
const isDirectory = statSync(name).isDirectory(); | ||
if (isDirectory) { | ||
return [...files, ...getAllMDFiles(name)]; | ||
} else { | ||
if (/\.mdx?$/.test(name)) { | ||
return [...files, name]; | ||
} | ||
return files; | ||
} | ||
}, []); | ||
|
||
const files = getAllMDFiles('.cache'); | ||
if (!existsSync('src/content/pipelines')) { | ||
mkdirSync('src/content/pipelines', { recursive: true }); | ||
} | ||
Promise.all( | ||
// create src/content/pipelines folder if it doesn't exist | ||
|
||
files.map(async (f) => { | ||
let content = await promises.readFile(f, 'utf8'); | ||
const pathParts = f.split('/'); | ||
const pipeline = pathParts[1]; | ||
const version = pathParts[2]; | ||
// make relative links to png and svg files absolute in markdown to current Astro.url.pathname | ||
content = content.replaceAll( | ||
/(\]\()(docs\/images\/.*?\.png|svg)/gmu, | ||
`$1${`https://raw.githubusercontent.com/nf-core/${pipeline}/${version}/$2`}` | ||
); | ||
const newPath = f.replace('.cache', 'src/content/pipelines'); | ||
const parent = newPath.split('/').slice(0, -1).join('/'); | ||
await promises.mkdir(parent, { recursive: true }); | ||
await promises.writeFile(newPath, content); | ||
}) | ||
); | ||
return true; | ||
}; | ||
|
||
await buildContentCollection(); | ||
})(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,5 @@ | ||
[functions] | ||
external_node_modules = ["vscode-oniguruma","shiki"] | ||
node_bundler = "esbuild" | ||
[build] | ||
environment = { NODE_VERSION = "20.4.0" } |
Oops, something went wrong.