Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci(github): add eth and stellar connect plugin to release process #15940

Merged
merged 1 commit into from
Dec 12, 2024
Merged

ci(github): add eth and stellar connect plugin to release process #15940

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 53 additions & 36 deletions scripts/ci/check-npm-and-local.js → scripts/ci/check-npm-and-local.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
const { execSync } = require('child_process');
import { execSync } from 'child_process';

const fs = require('fs');
const util = require('util');
const fetch = require('cross-fetch');
const tar = require('tar');
const path = require('path');
const crypto = require('crypto');
const semver = require('semver');
import fs from 'fs';
import util from 'util';
import fetch from 'cross-fetch';
import path from 'path';
import * as tar from 'tar';
import crypto from 'crypto';
import semver from 'semver';

const mkdir = util.promisify(fs.mkdir);
const existsDirectory = util.promisify(fs.exists);

const makeSureDirExists = async dirPath => {
const makeSureDirExists = async (dirPath: string) => {
if (!(await existsDirectory(dirPath))) {
// Make sure there is dirPath directory.
return mkdir(dirPath, { recursive: true });
}
};

async function extractTarball(tarballPath, extractPath) {
async function extractTarball(tarballPath: string, extractPath: string) {
try {
await makeSureDirExists(extractPath);
await tar.x({ file: tarballPath, C: extractPath });
Expand All @@ -28,7 +28,7 @@ async function extractTarball(tarballPath, extractPath) {
}
}

const downloadFile = (url, filePath) =>
const downloadFile = (url: string, filePath: string) =>
new Promise((resolve, reject) => {
fetch(url)
.then(res => {
Expand All @@ -46,9 +46,10 @@ const downloadFile = (url, filePath) =>
// Create a file stream
const file = fs.createWriteStream(filePath);

// Pipe the response stream to the file stream
stream.pipe(file);

if (stream) {
// Pipe the response stream to the file stream
(stream as any).pipe(file);
}
file.on('error', err => {
file.close();
reject(err);
Expand All @@ -65,7 +66,7 @@ const downloadFile = (url, filePath) =>
});
});

const packModule = (moduleName, modulePath, outputDirectory) => {
const packModule = (moduleName: string, modulePath: string, outputDirectory: string) => {
try {
const currentPwd = __dirname;
// Change the current working directory
Expand All @@ -85,14 +86,14 @@ const packModule = (moduleName, modulePath, outputDirectory) => {
}
};

const calculateChecksumForFile = filePath => {
const calculateChecksumForFile = (filePath: string) => {
const fileBuffer = fs.readFileSync(filePath);
const hashSum = crypto.createHash('sha256');
hashSum.update(fileBuffer);
return hashSum.digest('hex');
};

const calculateChecksum = directoryPath => {
const calculateChecksum = (directoryPath: string) => {
const combinedHash = crypto.createHash('sha256');

fs.readdirSync(directoryPath).forEach(file => {
Expand All @@ -107,7 +108,22 @@ const calculateChecksum = directoryPath => {
return combinedHash.digest('hex');
};

const getLocalAndRemoteChecksums = async moduleName => {
export const getLocalAndRemoteChecksums = async (
moduleName: string,
): Promise<
| {
success: true;
data: {
localChecksum: string;
remoteChecksum: string;
distributionTags: {
beta: string;
latest: string;
};
};
}
| { success: false; error: string }
> => {
const ROOT = path.join(__dirname, '..', '..');

const [_prefix, name] = moduleName.split('/');
Expand All @@ -116,51 +132,56 @@ const getLocalAndRemoteChecksums = async moduleName => {
const npmRegistryUrl = `https://registry.npmjs.org/${moduleName}`;

try {
console.log(`fetching npm registry info from: ${npmRegistryUrl}`);
console.info(`fetching npm registry info from: ${npmRegistryUrl}`);
const response = await fetch(npmRegistryUrl);
const data = await response.json();
if (data.error) {
return { success: false };
return { success: false, error: 'Error fetching from npm registry' };
}

const betaVersion = data['dist-tags'].beta;
console.log(`beta remote version in npm registry: ${betaVersion}`);
console.info(`beta remote version in npm registry: ${betaVersion}`);
const latestVersion = data['dist-tags'].latest;
console.log(`latest remote version in npm registry: ${latestVersion}`);
console.info(`latest remote version in npm registry: ${latestVersion}`);

// When beta version has greatest semver in NPM then we need to check with
// that one since that was released latest in time, so it includes oldest changes.
const greatestSemver =
betaVersion && semver.gt(betaVersion, latestVersion) ? betaVersion : latestVersion;

console.log(`greatest remove version in npm registry: ${greatestSemver}`);
console.info(`greatest remove version in npm registry: ${greatestSemver}`);

const tarballUrl = data.versions[greatestSemver].dist.tarball;

const tarballDestination = path.join(__dirname, 'tmp', name);
console.log(`downloading tarball from ${tarballUrl} to `);
console.info(`downloading tarball from ${tarballUrl} to `);
const fileName = await downloadFile(tarballUrl, tarballDestination);
console.log(`File downloaded!: ${fileName}`);
console.info(`File downloaded!: ${fileName}`);

const extractRemotePath = path.join(__dirname, 'tmp', 'remote', name);
console.log(`extracting remote tarball from ${tarballDestination} to ${extractRemotePath}`);
console.info(
`extracting remote tarball from ${tarballDestination} to ${extractRemotePath}`,
);
await extractTarball(tarballDestination, extractRemotePath);

console.log(`packaging local npm module from ${PACKAGE_PATH} to ${tmpDir}`);
console.info(`packaging local npm module from ${PACKAGE_PATH} to ${tmpDir}`);
const tarballPath = packModule(name, PACKAGE_PATH, tmpDir);
if (!tarballPath) {
return { success: false, error: 'Error packing module tarball' };
}

const extractLocalPath = path.join(__dirname, 'tmp', 'local', name);

console.log(`extracting local tarball from ${tarballPath} to ${extractLocalPath}`);
console.info(`extracting local tarball from ${tarballPath} to ${extractLocalPath}`);
await extractTarball(tarballPath, extractLocalPath);

console.log('calculating remote package checksum');
console.info('calculating remote package checksum');
const remoteChecksum = calculateChecksum(`${extractRemotePath}/package`);
console.log('remoteChecksum', remoteChecksum);
console.info('remoteChecksum', remoteChecksum);

console.log('calculating local package checksum');
console.info('calculating local package checksum');
const localChecksum = calculateChecksum(`${extractLocalPath}/package`);
console.log('localChecksum', localChecksum);
console.info('localChecksum', localChecksum);

return {
success: true,
Expand All @@ -171,7 +192,3 @@ const getLocalAndRemoteChecksums = async moduleName => {
return { success: false, error };
}
};

module.exports = {
getLocalAndRemoteChecksums,
};
54 changes: 42 additions & 12 deletions scripts/ci/connect-bump-versions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import path from 'path';
import fs from 'fs';

import { promisify } from 'util';
import { gettingNpmDistributionTags } from './helpers';
import { getPackagesAndDependenciesRequireUpdate, gettingNpmDistributionTags } from './helpers';

const readFile = promisify(fs.readFile);
const existsDirectory = promisify(fs.exists);
Expand Down Expand Up @@ -116,18 +116,48 @@ const updateConnectChangelog = async (

const bumpConnect = async () => {
try {
const checkResult: { update: string[]; errors: string[] } = await checkPackageDependencies(
'connect',
deploymentType,
const connectDependenciesToUpdate: { update: string[]; errors: string[] } =
await checkPackageDependencies('connect', deploymentType);

console.info('connectDependenciesToUpdate', connectDependenciesToUpdate);

const update = connectDependenciesToUpdate.update.map((pkg: string) =>
pkg.replace('@trezor/', ''),
);
const errors = connectDependenciesToUpdate.errors.map((pkg: string) =>
pkg.replace('@trezor/', ''),
);

console.log('checkResult', checkResult);
// We also have some packages that we want to update with connect but they are not
// dependencies of connect, and they have their own version. We also do not want
// to release them every time we release connect but when there are changes applied in
// those packages.
const independentPackagesNames = [
'@trezor/connect-plugin-ethereum',
'@trezor/connect-plugin-stellar',
];

const independentPackagesAndDependenciesToUpdate =
await getPackagesAndDependenciesRequireUpdate(independentPackagesNames);

console.info(
'independentPackagesAndDependenciesToUpdate',
independentPackagesAndDependenciesToUpdate,
);

const allPackagesToUpdate = [
...independentPackagesAndDependenciesToUpdate.map((pkg: string) =>
pkg.replace('@trezor/', ''),
),
...update,
];
console.info('allPackagesToUpdate', allPackagesToUpdate);

const update = checkResult.update.map((pkg: string) => pkg.replace('@trezor/', ''));
const errors = checkResult.errors.map((pkg: string) => pkg.replace('@trezor/', ''));
const allUniquePackagesToUpdate = [...new Set(allPackagesToUpdate)];
console.info('allUniquePackagesToUpdate', allUniquePackagesToUpdate);

if (update) {
for (const packageName of update) {
if (allUniquePackagesToUpdate) {
for (const packageName of allUniquePackagesToUpdate) {
const PACKAGE_PATH = path.join(ROOT, 'packages', packageName);
const PACKAGE_JSON_PATH = path.join(PACKAGE_PATH, 'package.json');

Expand Down Expand Up @@ -246,7 +276,7 @@ const bumpConnect = async () => {
});
}

const depsChecklist = update.reduce(
const depsChecklist = allUniquePackagesToUpdate.reduce(
(acc, packageName) =>
`${acc}\n- [ ] [![NPM](https://img.shields.io/npm/v/@trezor/${packageName}.svg)](https://www.npmjs.org/package/@trezor/${packageName}) @trezor/${packageName}`,
'',
Expand Down Expand Up @@ -281,7 +311,7 @@ const bumpConnect = async () => {
);

if (!connectGitLogText) {
console.log('no changelog for @trezor/connect');
console.info('no changelog for @trezor/connect');
return;
}

Expand All @@ -291,7 +321,7 @@ const bumpConnect = async () => {
});
}
} catch (error) {
console.log('error:', error);
console.info('error:', error);
}
};

Expand Down
56 changes: 46 additions & 10 deletions scripts/ci/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@ import fs from 'fs';
import path from 'path';
import semver from 'semver';
import fetch from 'cross-fetch';

import { promisify } from 'util';
import { ChildProcessWithoutNullStreams, spawn } from 'child_process';

import { promisify } from 'util';
import { getLocalAndRemoteChecksums } from './check-npm-and-local';

const readFile = promisify(fs.readFile);

const { getLocalAndRemoteChecksums } = require('./check-npm-and-local');

const ROOT = path.join(__dirname, '..', '..');

const updateNeeded: string[] = [];
Expand Down Expand Up @@ -43,12 +41,49 @@ export const getNpmRemoteGreatestVersion = async (moduleName: string) => {
}
};

export const getPackagesAndDependenciesRequireUpdate = async (packages: string[]) => {
let packagesRequireUpdate = [];
let dependenciesRequireUpdate = [];
for (const packageName of packages) {
const response = await getLocalAndRemoteChecksums(packageName);

if (!response.success) {
console.error('Error when getting local and remote checksums');
} else {
const { localChecksum, remoteChecksum, distributionTags } = response.data;
console.info('localChecksum', localChecksum);
console.info('remoteChecksum', remoteChecksum);
console.info('distributionTags', distributionTags);
if (localChecksum !== remoteChecksum) {
packagesRequireUpdate.push(packageName);
}
}
}

for (const packageName of packagesRequireUpdate) {
const checkResult: { update: string[]; errors: string[] } = await checkPackageDependencies(
packageName.replace('@trezor/', ''),
'stable',
);
console.info('checkResult', checkResult);
if (checkResult.update) {
console.info('checkResult.update', checkResult.update);
dependenciesRequireUpdate.push(...checkResult.update);
}
}

console.info('packagesRequireUpdate', packagesRequireUpdate);
console.info('dependenciesRequireUpdate', dependenciesRequireUpdate);

return [...packagesRequireUpdate, ...dependenciesRequireUpdate];
};

export const checkPackageDependencies = async (
packageName: string,
deploymentType: 'stable' | 'canary',
): Promise<{ update: string[]; errors: string[] }> => {
console.log('######################################################');
console.log(`Checking package ${packageName}`);
console.info('######################################################');
console.info(`Checking package ${packageName}`);
const rawPackageJSON = await readFile(
path.join(ROOT, 'packages', packageName, 'package.json'),
'utf-8',
Expand Down Expand Up @@ -76,15 +111,15 @@ export const checkPackageDependencies = async (
if (!response.success) {
// If the package was not found it might be it has not been release yet or other issue, so we include it in errors.
const index = errors.findIndex(lib => lib === dependency);
console.log('index', index);
console.info('index', index);
if (index > -1) {
errors.splice(index, 1);
}

errors.push(dependency);
} else {
const { localChecksum, remoteChecksum, distributionTags } = response.data;
console.log('distributionTags', distributionTags);
console.info('distributionTags', distributionTags);

if (localChecksum !== remoteChecksum) {
// if the checked dependency is already in the array, remove it and push it to the end of array
Expand Down Expand Up @@ -124,7 +159,7 @@ export const exec = async (
cmd: string,
params: any[],
): Promise<{ stdout: string; stderr: string }> => {
console.log(cmd, ...params);
console.info(cmd, ...params);

const res: ChildProcessWithoutNullStreams = spawn(cmd, params, {
cwd: ROOT,
Expand Down Expand Up @@ -166,7 +201,8 @@ export const exec = async (

export const commit = async ({ path, message }: { path: string; message: string }) => {
await exec('git', ['add', path]);
await exec('git', ['commit', '-m', `${message}`]);
// We need to add `-n` so we do not have problems with git hooks when committing in CI.
await exec('git', ['commit', '-m', `${message}`, '-n']);
};

export const comment = async ({ prNumber, body }: { prNumber: string; body: string }) => {
Expand Down
Loading