build.binaries.mjs

import {default as fetch} from 'node-fetch';
import {default as he} from 'he';
import {fixPerms, allFilesExist, zipAll, rmDir, rmFile,
  checkMakeFolder, curlDownloadFile, untargzFile, unzipFile} from './src/lib/download.js';
import {getPlatTag, runToolBuffered} from './src/lib/tools.js';
import * as path from 'path';
import {fileURLToPath} from 'url';
import * as fsSync from 'fs';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const buildFolder = path.resolve(__dirname, './build');
const buildInfoFile = path.resolve(__dirname, './build.txt');

/**
 * @typedef {{[P in import('./src/lib/tools.js').PlatformTag]: RegExp}} PlatformReleaseMatcher
 * @typedef {{name: string, url: string}} MatchedAsset
 * @typedef {{[P in import('./src/lib/tools.js').PlatformTag]: MatchedAsset}} PlatformMatchedAssets
 * @typedef {{[P in import('./src/lib/tools.js').PlatformTag]: string[]}} PlatformFileList
 * @typedef {{[P in import('./src/lib/tools.js').PlatformTag]: string}} PlatformURLs
 */

/**
 * @typedef {object} SourceConfigBase
 * @property {string} name
 * @property {string[]} verargs
 * @property {RegExp} vermatch
 * @property {PlatformReleaseMatcher} matchers
 * @property {PlatformFileList} filelist
 * @typedef {SourceConfigBase & {type:'githubrelease', repo:string}} SourceConfigGithubRelease
 * @typedef {SourceConfigBase & {type:'spirvtoolsci', urls:PlatformURLs}} SourceConfigSpirvToolsCI
 */
/**
 * @typedef {SourceConfigGithubRelease|SourceConfigSpirvToolsCI} SourceConfig
 */

/**
 * @typedef {'unzip'|'untargz'} BuildFetchStepAction
 * @typedef {object} BuildFetchStep
 * @property {string} name
 * @property {string} url
 * @property {BuildFetchStepAction} [action]
 * @property {string[]} fileList
 * @typedef {Object<string,BuildFetchStep[]>} BuildTargets
 * @typedef {object} BuildConfig
 * @property {string[]} targets
 * @property {SourceConfig[]} sources
 * @property {BuildFetchStep[]} include
 */

/**
 * @param {BuildFetchStep} buildStep
 * @param {string} targetDir
 * @return {string[]} resultant files
*/
const runBuildFetchStep = async (buildStep, targetDir) => {
  let targetFile = '';
  if (buildStep.action) {
    targetFile = buildStep.action === 'untargz' ? 'temp.tar.gz' : 'temp.zip';
  } else {
    if (buildStep.fileList.length !== 1) throw new Error('buildStep.fileList.length !== 1');
    targetFile = buildStep.fileList[0];
  }
  const targetFileAbs = path.join(targetDir, targetFile);
  console.log(`Fetch ${buildStep.name}`);
  await curlDownloadFile(buildStep.url, targetFile, targetDir);

  if (!fsSync.existsSync(targetFileAbs)) {
    throw new Error(`No file downloaded`);
  }

  if (buildStep.action) {
    if (buildStep.action === 'untargz') {
      await untargzFile(targetFile, buildStep.fileList, targetDir);
    } else if (buildStep.action === 'unzip') {
      await unzipFile(targetFile, buildStep.fileList, targetDir);
    }

    fsSync.unlinkSync(targetFileAbs); // clean temp archive

    const fileList = buildStep.fileList.map((file) => path.basename(file));
    const absFileList = fileList.map((file) => path.join(targetDir, file));
    if (!allFilesExist(absFileList)) {
      throw new Error(`Archive did not extract expected files`);
    }
    fixPerms(absFileList);
    return fileList;
  } else {
    return [targetFile];
  }

};

/**
 * @param {string} url
 * @return {Promise<any>}
 */
async function jsonRequest(url) {
  try {
    const response = await fetch(url, {
      method: 'GET',
    });
    if (!response.ok) throw new Error(`Bad response code: ${response.status}`);
    return response.json();
  } catch (err) {
    throw new Error(`JSON request ${url} failed\n${err.message}`);
  }
}

/**
 * @param {string} repo
 * @param {PlatformReleaseMatcher} releaseMatchers platform -> regex matching
 */
async function fetchMatchingGithubRelease(repo, releaseMatchers) {
  /** @type {Array<object>} */
  const releaseInfos = await jsonRequest(`https://api.github.com/repos/${repo}/releases?per_page=100`);
  if (!releaseInfos || !releaseInfos.length) {
    throw new Error(`Invalid release info for ${repo}`);
  }
  // Sort by created_at (required since glslang's CI replaces assets on the same release ID)
  releaseInfos.sort(({created_at: a}, {created_at: b}) => a ? b ? (new Date(b) - new Date(a)) : -1 : 1);
  let foundReleaseInfo;
  for (const releaseInfo of releaseInfos) {
    if (releaseInfo.draft || // Skip draft releases
      !releaseInfo.html_url || !releaseInfo.created_at || !releaseInfo.target_commitish || // Missing info
      !releaseInfo.assets || !releaseInfo.assets.length // No assets
    ) continue;

    const matchAssets = new Set(releaseInfo.assets);
    const matchPreds = new Map(Object.entries(releaseMatchers));
    /** @type {PlatformMatchedAssets} */
    const matchedPlats = {};
    nextAssset:
    for (const testAsset of matchAssets) {
      if (!testAsset.name || !testAsset.browser_download_url) continue;
      for (const [platName, testPred] of matchPreds) { // Try each matcher on this asset
        if (testAsset.name.match(testPred)) {
          matchedPlats[platName] = /** @type {MatchedAsset} */(
            {name: testAsset.name, url: testAsset.browser_download_url}
          );
          matchPreds.delete(platName);
          continue nextAssset;
        }
      }
    }
    if (matchPreds.size === 0) {
      foundReleaseInfo = {
        url: releaseInfo.html_url,
        hash: releaseInfo.target_commitish,
        date: releaseInfo.created_at,
        tag: new Date(releaseInfo.created_at).toISOString().split('T')[0] +
            '-' + releaseInfo.target_commitish.slice(0, 7),
        platforms: matchedPlats,
      };
      break;
    } else {
      // We could continue searching down releases, but better to throw an error in case
      // naming scheme changes in future
      throw new Error(`Release ${releaseInfo.html_url} failed to match ${[...matchPreds.keys()].join(', ')}`);
    }
  }
  if (!foundReleaseInfo) {
    throw new Error(`Could not find candidate release for ${repo}`);
  }
  return foundReleaseInfo;
}

/**
 * Fetch spirv tools CI badge page and extract meta redirect url
 * @param {string} url
 */
async function fetchSpirvToolsCIPage(url) {
  let resultHTML;
  try {
    const response = await fetch(url, {
      method: 'GET',
    });
    if (!response.ok) throw new Error(`Bad response code: ${response.status}`);
    resultHTML = await response.text();
  } catch (err) {
    throw new Error(`Download CI page ${url} failed\n${err.message}`);
  }
  const redirMatch = resultHTML.match(/<meta\s+http-equiv\s*=\s*"refresh"\s+content\s*=\s*"([^"]*)"[^>]*>/i);
  if (redirMatch && redirMatch.length === 2) {
    const redirContent = he.decode(redirMatch[1]);
    const redirURLMatch = redirContent.match(/url\s*=\s*(\S+)\s*$/i);
    if (redirURLMatch && redirURLMatch.length === 2) {
      return redirURLMatch[1];
    }
  }
  throw new Error(`Failed to parse CI Page ${url}`);
}

/**
 * @typedef {{[P in import('./src/lib/tools.js').PlatformTag]: string}} SpirvToolsCIUrls
 */
/**
 * @param {SpirvToolsCIUrls} urls platform -> CI badge download urls
 * @param {PlatformReleaseMatcher} releaseMatchers platform -> regex matching [version, filename]
 */
async function fetchMatchingSpirvToolsCI(urls, releaseMatchers) {
  /** @type {PlatformMatchedAssets} */
  const platforms = {};
  let version;
  for (const [platform, url] of Object.entries(urls)) {
    const downloadURL = await fetchSpirvToolsCIPage(url);
    if (!releaseMatchers[platform]) throw new Error(`Release matcher missing ${platform}`);
    const matchResult = downloadURL.match(releaseMatchers[platform]);
    if (!matchResult || matchResult.length !== 3) {
      throw new Error(`CI platform ${platform} URL ${downloadURL} failed to match`);
    }
    const assetVersion = matchResult[1], name = matchResult[2];
    platforms[platform] = /** @type {MatchedAsset} */({name, url: downloadURL});
    if (version !== undefined && assetVersion !== version) {
      throw new Error(`Detected version mismatch platform ${platform} : '${assetVersion}' !== '${version}'`);
    }
    version = assetVersion;
  }
  return {version, tag: version, url: '', platforms};
}

(async function main() {
  const thisPlat = getPlatTag();
  if (!thisPlat) throw new Error(`Couldn't identify this platform's tag`);

  const buildConfig = /** @type {BuildConfig} */((await import('./build.binaries.config.mjs')).default);
  if (!buildConfig || !buildConfig.targets || !buildConfig.sources || !buildConfig.include) {
    throw new Error(`Bad build config`);
  }

  rmFile(buildInfoFile);
  rmDir(buildFolder);
  checkMakeFolder(buildFolder);

  const sources = [];

  for (const source of buildConfig.sources) {
    let sourceResult;
    switch (source.type) {
      case 'githubrelease':
        sourceResult = await fetchMatchingGithubRelease(source.repo, source.matchers);
        break;
      case 'spirvtoolsci':
        sourceResult = await fetchMatchingSpirvToolsCI(source.urls, source.matchers);
        break;
      default: throw new Error(`build config error: Unknown type '${source.type}' for source '${source.name}'`);
    }
    for (const target of buildConfig.targets) {
      if (!sourceResult.platforms[target]) throw new Error(`source '${source.name}' missing target '${target}'`);
    }
    console.log(`source '${source.name}' - chose '${sourceResult.tag}' (${sourceResult.url})`);
    sources.push({...sourceResult, name: source.name, filelist: source.filelist,
      verargs: source.verargs, vermatch: source.vermatch});
  }

  const includeFiles = [];
  // Fetch includes (License files)
  for (const includeStep of buildConfig.include) {
    includeFiles.push(...await runBuildFetchStep(includeStep, buildFolder));
  }
  const buildArtefacts = [...includeFiles];

  for (const target of buildConfig.targets) {
    const targetPath = path.join(buildFolder, target);
    console.log(`\n-----\nTarget: ${target} -> ${targetPath}\n-----\n`);
    checkMakeFolder(targetPath);

    for (const includeFile of includeFiles) {
      fsSync.copyFileSync(path.join(buildFolder, includeFile), path.join(targetPath, includeFile),
          fsSync.constants.COPYFILE_EXCL); // Don't overwrite
    }

    for (const source of sources) {
      const url = source.platforms[target].url;
      const extMatch = url.match(/\.(tar\.gz|tgz|zip)$/i) || ['', ''];
      let action;
      switch (extMatch[1].toLowerCase()) {
        case 'tar.gz': case 'tgz': action = 'untargz'; break;
        case 'zip': action = 'unzip'; break;
        default: throw new Error(`Unknown file extension for ${url}`);
      }
      await runBuildFetchStep({
        name: source.name,
        fileList: source.filelist[target],
        url,
        action,
      }, targetPath);
      if (target === thisPlat) { // Run the tool and get version
        const toolBinPath = path.join(targetPath, path.basename(source.filelist[target][0]));
        const toolRunResult = await runToolBuffered(toolBinPath, targetPath, source.name, source.verargs);
        const toolOutput = toolRunResult.out || toolRunResult.err;
        const toolOutputMatch = toolOutput.match(source.vermatch);
        if (!toolOutputMatch || toolOutputMatch.length !== 2) {
          console.log(toolOutput);
          throw new Error(`Couldn't extract version string from ${source.name}`);
        }
        const toolVersion = toolOutputMatch[1];
        source.veroutput = toolVersion;
      }
    }


    console.log('Creating archive');
    const targetArchive = `${target}.zip`;
    const targetArchiveAbs = path.join(buildFolder, targetArchive);
    await zipAll(targetArchiveAbs, targetPath);

    if (!fsSync.existsSync(targetArchiveAbs)) {
      throw new Error(`No archive created`);
    }
    buildArtefacts.push(targetArchive);

    rmDir(targetPath);
    console.log(`\nTarget completed\n`);
  }

  console.log('Build completed');

  console.log(`\nListing:\n${buildArtefacts.join('\n')}`);

  const versionLines = [];
  for (const source of sources) {
    versionLines.push(`${source.name} ${source.veroutput} (${source.tag})`);
  }
  const versionInfo = versionLines.join('\n');
  fsSync.writeFileSync(buildInfoFile, versionInfo);

  console.log(`\nDescription:\n${versionInfo}`);

  process.exit(0);
})();