From dfc55da4c3fcb79944541bb24e29a182c0585ef3 Mon Sep 17 00:00:00 2001 From: Jeroen Ooms Date: Mon, 13 Jan 2025 16:06:04 +0100 Subject: [PATCH] Start with packages api --- routes/api.js | 17 +++++++++++++++-- routes/repos.js | 19 +------------------ src/db.js | 23 ++++++++++++++++------- src/tools.js | 17 +++++++++++++++++ 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/routes/api.js b/routes/api.js index 42d78c8..7068249 100644 --- a/routes/api.js +++ b/routes/api.js @@ -1,6 +1,6 @@ import express from 'express'; import url from 'node:url'; -import {ls_packages} from '../src/db.js'; +import {ls_packages, get_package_info, get_universe_packages} from '../src/db.js'; const router = express.Router(); @@ -9,8 +9,21 @@ router.get('/api', function(req, res, next) { }); router.get('/api/ls', function(req, res, next) { - ls_packages(res.locals.universe).then(x => res.send(x)); + return ls_packages(res.locals.universe).then(x => res.send(x)); }); +router.get('/api/packages', function(req, res, next) { + var fields = req.query.fields && req.query.fields.split(","); + return get_universe_packages(res.locals.universe, fields).then(function(x){ + res.send(x); + }); +}); + +router.get('/api/packages/:package', function(req, res, next) { + var fields = req.query.fields && req.query.fields.split(","); + return get_package_info(req.params.package, res.locals.universe, fields).then(function(x){ + res.send(x) + }); +}); export default router; diff --git a/routes/repos.js b/routes/repos.js index 4885554..18305ce 100644 --- a/routes/repos.js +++ b/routes/repos.js @@ -1,27 +1,10 @@ import express from 'express'; -import zlib from 'node:zlib'; import createError from 'http-errors'; -import {doc_to_dcf, match_macos_arch} from '../src/tools.js'; +import {doc_to_dcf, match_macos_arch, doc_to_ndjson, cursor_stream} from '../src/tools.js'; import {get_universe_binaries, get_packages_index, get_package_hash} from '../src/db.js'; const router = express.Router(); -function doc_to_ndjson(x){ - return JSON.stringify(x) + '\n'; -} - -// Somehow node:stream/promises does not catch input on-error callbacks properly -// so we promisify ourselves. See https://github.com/r-universe-org/help/issues/540 -function cursor_stream(cursor, output, transform, gzip){ - return new Promise(function(resolve, reject) { - var input = cursor.stream({transform: transform}).on('error', reject); - if(gzip){ - input = input.pipe(zlib.createGzip()).on('error', reject); - } - input.pipe(output).on('finish', resolve).on('error', reject); - }); -} - function packages_index(query, req, res, mixed = false){ query._user = res.locals.universe; var format = req.params.format || "PACKAGES.json"; diff --git a/src/db.js b/src/db.js index 39df984..389e732 100644 --- a/src/db.js +++ b/src/db.js @@ -127,9 +127,18 @@ function array_first(key){ } function build_projection(fields){ + if(!fields || !fields.length) return {_id:0}; var projection = {Package:1, _type:1, _user:1, _indexed: 1, _id:0}; fields.forEach(function (f) { - projection[f] = 1; + if(f == '_binaries'){ + projection['Built'] = 1; + projection['_status'] = 1; + projection['_check'] = 1; + if(!fields.includes("_commit")) + projection['_commit.id'] = 1; + } else { + projection[f] = 1; + } }); return projection; } @@ -138,8 +147,9 @@ function mongo_package_files(pkg, universe){ return mongo_find({_user: universe, Package: pkg, _registered: true}).toArray(); } -function mongo_package_info(pkg, universe){ - return mongo_find({_user: universe, Package: pkg, _registered: true}).toArray().then(function(docs){ +function mongo_package_info(pkg, universe, fields){ + var proj = build_projection(fields); + return mongo_find({_user: universe, Package: pkg, _registered: true}).project(proj).toArray().then(function(docs){ if(!docs.length) //should never happen because we checked earlier throw createError(404, `Package ${pkg} not found in ${universe}`); var pkgdata = group_package_data(docs); @@ -416,9 +426,9 @@ export function get_bucket_stream(hash){ } } -export function get_package_info(pkg, universe){ +export function get_package_info(pkg, universe, fields){ if(production){ - return mongo_package_info(pkg, universe); + return mongo_package_info(pkg, universe, fields); } else { console.warn(`Fetching ${pkg} info from API...`) if(universe){ @@ -463,7 +473,7 @@ export function get_universe_vignettes(universe){ export function get_universe_packages(universe, fields, limit = 5000){ if(production){ - return mongo_universe_packages(universe, fields, limit) + return mongo_universe_packages(universe, fields, limit); } else { console.warn(`Fetching ${universe} packages from API...`) var apiurl = `https://${universe}.r-universe.dev/api/packages?stream=1&all=true&limit=${limit}&fields=${fields.join()}`; @@ -554,7 +564,6 @@ export function get_packages_index(query, fields = [], mixed = false){ if(production){ let projection = {...pkgfields}; fields.forEach(function (f) { - console.log("adding", f) projection[f] = 1; }); var cursor = mixed ? mongo_aggregate([ diff --git a/src/tools.js b/src/tools.js index f43deca..0e74ce4 100644 --- a/src/tools.js +++ b/src/tools.js @@ -1,5 +1,6 @@ import {Buffer} from "node:buffer"; import tar from 'tar-stream'; +import zlib from 'node:zlib'; import gunzip from 'gunzip-maybe'; import {load as cheerio_load} from 'cheerio'; import hljs from 'highlight.js'; @@ -165,6 +166,22 @@ export function doc_to_dcf(doc){ }).join("\n") + "\n\n"; } +export function doc_to_ndjson(x){ + return JSON.stringify(x) + '\n'; +} + +// Somehow node:stream/promises does not catch input on-error callbacks properly +// so we promisify ourselves. See https://github.com/r-universe-org/help/issues/540 +export function cursor_stream(cursor, output, transform, gzip){ + return new Promise(function(resolve, reject) { + var input = cursor.stream({transform: transform}).on('error', reject); + if(gzip){ + input = input.pipe(zlib.createGzip()).on('error', reject); + } + input.pipe(output).on('finish', resolve).on('error', reject); + }); +} + export function match_macos_arch(platform){ if(platform.match("arm64|aarch64")){ return {$not : /x86_64/};