Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improvements to the build system #1245

Merged
merged 10 commits into from
Nov 27, 2023
Merged

Improvements to the build system #1245

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d448556
Improvements to the build system
ibc Nov 27, 2023
894d785
npm-scripts.mjs: use spawnSync instead of execSync
ibc Nov 27, 2023
ba9840f
fix typo
ibc Nov 27, 2023
ecf9961
Try to make Windows happy again
ibc Nov 27, 2023
384dd54
npm-scripts: Use fs.rmSync() to delete node/lib folder
ibc Nov 27, 2023
ce82667
worker/Cargo.toml: add "tasks.py" file
ibc Nov 27, 2023
d40e732
I hate Windows
ibc Nov 27, 2023
8d1c9b1
hell
ibc Nov 27, 2023
e39731d
Revert changes in npm-scripts and use execSync() again
ibc Nov 27, 2023
e89375a
lint
ibc Nov 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 55 additions & 52 deletions npm-scripts.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import process from 'node:process';
import os from 'node:os';
import fs from 'node:fs';
import path from 'node:path';
import { execSync } from 'node:child_process';
import { execSync, spawnSync } from 'node:child_process';
import fetch from 'node-fetch';
import tar from 'tar';

Expand All @@ -11,7 +11,6 @@ const IS_WINDOWS = os.platform() === 'win32';
const MAYOR_VERSION = PKG.version.split('.')[0];
const PYTHON = getPython();
const PIP_INVOKE_DIR = path.resolve('worker/pip_invoke');
const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.2.0';
const FLATBUFFERS_VERSION = '23.3.3';
const WORKER_RELEASE_DIR = 'worker/out/Release';
const WORKER_RELEASE_BIN = IS_WINDOWS ? 'mediasoup-worker.exe' : 'mediasoup-worker';
Expand Down Expand Up @@ -133,7 +132,7 @@ async function run()
case 'typescript:watch':
{
deleteNodeLib();
executeCmd('tsc --project node --watch');
spawnCmd('tsc', [ '--project', 'node', '--watch' ]);

break;
}
Expand Down Expand Up @@ -168,7 +167,7 @@ async function run()

case 'format:worker':
{
executeCmd(`"${PYTHON}" -m invoke -r worker format`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'format' ]);

break;
}
Expand Down Expand Up @@ -205,8 +204,8 @@ async function run()
case 'coverage:node':
{
buildTypescript({ force: false });
executeCmd('jest --coverage');
executeCmd('open-cli coverage/lcov-report/index.html');
spawnCmd('jest', [ '--coverage' ]);
spawnCmd('open-cli', [ 'coverage/lcov-report/index.html' ]);

break;
}
Expand Down Expand Up @@ -236,10 +235,10 @@ async function run()
}

checkRelease();
executeCmd(`git commit -am '${PKG.version}'`);
executeCmd(`git tag -a ${PKG.version} -m '${PKG.version}'`);
executeCmd(`git push origin v${MAYOR_VERSION}`);
executeCmd(`git push origin '${PKG.version}'`);
spawnCmd('git', [ 'commit', '-am', PKG.version ]);
spawnCmd('git', [ 'tag', '-a', PKG.version, '-m', PKG.version ]);
spawnCmd('git', [ 'push', 'origin', `v${MAYOR_VERSION}` ]);
spawnCmd('git', [ 'push', 'origin', PKG.version ]);

logInfo('creating release in GitHub');

Expand All @@ -263,7 +262,7 @@ async function run()
await uploadMacArmPrebuiltWorker();
}

executeCmd('npm publish');
spawnCmd('npm', [ 'publish' ]);

break;
}
Expand Down Expand Up @@ -317,8 +316,9 @@ function installInvoke()

// Install pip invoke into custom location, so we don't depend on system-wide
// installation.
executeCmd(
`"${PYTHON}" -m pip install --upgrade --target="${PIP_INVOKE_DIR}" invoke==${INVOKE_VERSION}`, /* exitOnError */ true
spawnCmd(
PYTHON,
[ '-m', 'pip', 'install', '--upgrade', '--target', PIP_INVOKE_DIR, 'invoke' ]
);
}

Expand All @@ -331,15 +331,7 @@ function deleteNodeLib()

logInfo('deleteNodeLib()');

if (!IS_WINDOWS)
{
executeCmd('rm -rf node/lib');
}
else
{
// NOTE: This command fails in Windows if the dir doesn't exist.
executeCmd('rmdir /s /q "node/lib"', /* exitOnError */ false);
}
fs.rmSync('node/lib', { recursive: true, force: true });
}

function buildTypescript({ force = false } = { force: false })
Expand All @@ -352,48 +344,51 @@ function buildTypescript({ force = false } = { force: false })
logInfo('buildTypescript()');

deleteNodeLib();
executeCmd('tsc --project node');
spawnCmd('tsc', [ '--project', 'node' ]);
}

function buildWorker()
{
logInfo('buildWorker()');

executeCmd(`"${PYTHON}" -m invoke -r worker mediasoup-worker`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'mediasoup-worker' ]);
}

function cleanWorkerArtifacts()
{
logInfo('cleanWorkerArtifacts()');

// Clean build artifacts except `mediasoup-worker`.
executeCmd(`"${PYTHON}" -m invoke -r worker clean-build`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'clean-build' ]);
// Clean downloaded dependencies.
executeCmd(`"${PYTHON}" -m invoke -r worker clean-subprojects`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'clean-subprojects' ]);
// Clean PIP/Meson/Ninja.
executeCmd(`"${PYTHON}" -m invoke -r worker clean-pip`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'clean-pip' ]);
}

function lintNode()
{
logInfo('lintNode()');

executeCmd('eslint -c node/.eslintrc.js --ignore-path node/.eslintignore --max-warnings 0 node/src node/.eslintrc.js npm-scripts.mjs worker/scripts/clang-format.mjs');
spawnCmd(
'eslint',
[ '-c', 'node/.eslintrc.js', '--ignore-path', 'node/.eslintignore', '--max-warnings', '0', 'node/src', 'node/.eslintrc.js', 'npm-scripts.mjs', 'worker/scripts/clang-format.mjs' ]
);
}

function lintWorker()
{
logInfo('lintWorker()');

executeCmd(`"${PYTHON}" -m invoke -r worker lint`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'lint' ]);
}

function flatcNode()
{
logInfo('flatcNode()');

// Build flatc if needed.
executeCmd(`"${PYTHON}" -m invoke -r worker flatc`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'flatc' ]);

const buildType = process.env.MEDIASOUP_BUILDTYPE || 'Release';
const extension = IS_WINDOWS ? '.exe' : '';
Expand All @@ -410,15 +405,18 @@ function flatcNode()

const filePath = path.resolve(path.join('worker', 'fbs', dirent.name));

executeCmd(`"${flatc}" --ts --ts-no-import-ext --gen-object-api -o "${out}" "${filePath}"`);
spawnCmd(
flatc,
[ '--ts', '--ts-no-import-ext', '--gen-object-api', '-o', out, filePath ]
);
}
}

function flatcWorker()
{
logInfo('flatcWorker()');

executeCmd(`"${PYTHON}" -m invoke -r worker flatc`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'flatc' ]);
}

function testNode()
Expand All @@ -427,29 +425,29 @@ function testNode()

if (!process.env.TEST_FILE)
{
executeCmd('jest');
spawnCmd('jest');
}
else
{
executeCmd(`jest --testPathPattern "${process.env.TEST_FILE}"`);
spawnCmd('jest', [ '--testPathPattern', process.env.TEST_FILE ]);
}
}

function testWorker()
{
logInfo('testWorker()');

executeCmd(`"${PYTHON}" -m invoke -r worker test`);
spawnCmd(PYTHON, [ '-m', 'invoke', '-r', 'worker', 'test' ]);
}

function installNodeDeps()
{
logInfo('installNodeDeps()');

// Install/update Node deps.
executeCmd('npm ci --ignore-scripts');
spawnCmd('npm', [ 'ci', '--ignore-scripts' ]);
// Update package-lock.json.
executeCmd('npm install --package-lock-only --ignore-scripts');
spawnCmd('npm', [ 'install', '--package-lock-only', '--ignore-scripts' ]);
}

function checkRelease()
Expand Down Expand Up @@ -584,7 +582,7 @@ async function downloadPrebuiltWorker()
const resolvedBinPath = path.resolve(WORKER_RELEASE_BIN_PATH);

execSync(
resolvedBinPath,
`"${resolvedBinPath}"`,
{
stdio : [ 'ignore', 'ignore', 'ignore' ],
// Ensure no env is passed to avoid accidents.
Expand Down Expand Up @@ -707,26 +705,31 @@ async function getVersionChanges()
throw new Error(`no entry found in CHANGELOG.md for version '${PKG.version}'`);
}

function executeCmd(command, exitOnError = true)
function spawnCmd(command, args = [], exitOnError = true)
{
logInfo(`executeCmd(): ${command}`);
logInfo(`spawnCmd(): ${command} ${args.join(' ')}`);

try
const result = spawnSync(
command, args, { stdio: [ 'ignore', process.stdout, process.stderr ] }
);

if (result.status === 0)
{
execSync(command, { stdio: [ 'ignore', process.stdout, process.stderr ] });
return;
}
catch (error)
else if (exitOnError)
{
if (exitOnError)
{
logError(`executeCmd() failed, exiting: ${error}`);
logError(
`spawnCmd() failed, exiting [status:${result.status}, signal:${result.signal}, error:${result.error}]`
);

exitWithError();
}
else
{
logInfo(`executeCmd() failed, ignoring: ${error}`);
}
exitWithError();
}
else
{
logInfo(
`spawnCmd() failed, ignoring [status:${result.status}, signal:${result.signal}, error:${result.error}]`
);
}
}

Expand Down
17 changes: 8 additions & 9 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,22 @@
"types": "node/lib/index.d.ts",
"files": [
"node/lib",
"worker/src",
"worker/include",
"worker/deps/libwebrtc",
"worker/fbs",
"worker/test/src",
"worker/test/include",
"worker/fuzzer/src",
"worker/fuzzer/include",
"worker/scripts/*.mjs",
"worker/fuzzer/src",
"worker/include",
"worker/src",
"worker/scripts/*.json",
"worker/scripts/*.mjs",
"worker/scripts/*.py",
"worker/scripts/*.sh",
"worker/subprojects/*.wrap",
"worker/deps/libwebrtc",
"worker/tasks.py",
"worker/Makefile",
"worker/test/include",
"worker/test/src",
"worker/meson.build",
"worker/meson_options.txt",
"worker/tasks.py",
"npm-scripts.mjs"
],
"engines": {
Expand Down
1 change: 0 additions & 1 deletion worker/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ include = [
"/test/src",
"/build.rs",
"/Cargo.toml",
"/Makefile",
ibc marked this conversation as resolved.
Show resolved Hide resolved
"/meson.build",
"/meson_options.txt",
]
Expand Down
3 changes: 1 addition & 2 deletions worker/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

PYTHON ?= $(shell command -v python3 2> /dev/null || echo python)
PIP_INVOKE_DIR = $(shell pwd)/pip_invoke
INVOKE_VERSION ?= 2.2.0

# Instruct Python where to look for invoke module.
ifeq ($(OS),Windows_NT)
Expand Down Expand Up @@ -48,7 +47,7 @@ invoke:
ifeq ($(wildcard $(PIP_INVOKE_DIR)),)
# Install pip invoke into custom location, so we don't depend on system-wide
# installation.
$(PYTHON) -m pip install --upgrade --target="$(PIP_INVOKE_DIR)" invoke==$(INVOKE_VERSION)
$(PYTHON) -m pip install --upgrade --target="$(PIP_INVOKE_DIR)" invoke
endif

meson-ninja: invoke
Expand Down
9 changes: 4 additions & 5 deletions worker/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,8 @@ fn main() {

// Install Python invoke package in custom folder
let pip_invoke_dir = format!("{out_dir}/pip_invoke");
let invoke_version = "2.2.0";
let python = env::var("PYTHON").unwrap_or("python3".to_string());
let mut pythonpath = if env::var("PYTHONPATH").is_ok() {
let original_pythonpath = env::var("PYTHONPATH").unwrap();
let mut pythonpath = if let Ok(original_pythonpath) = env::var("PYTHONPATH") {
format!("{pip_invoke_dir}:{original_pythonpath}")
} else {
pip_invoke_dir.clone()
Expand All @@ -126,8 +124,9 @@ fn main() {
.arg("pip")
.arg("install")
.arg("--upgrade")
.arg(format!("--target={pip_invoke_dir}"))
.arg(format!("invoke=={invoke_version}"))
.arg("--target")
.arg(pip_invoke_dir)
.arg("invoke")
.spawn()
.expect("Failed to start")
.wait()
Expand Down
Loading