Skip to content

[rush] Add bridge-cache plugin #5190

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ These GitHub repositories provide supplementary resources for Rush Stack:
| [/rigs/heft-web-rig](./rigs/heft-web-rig/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig.svg)](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig) | [changelog](./rigs/heft-web-rig/CHANGELOG.md) | [@rushstack/heft-web-rig](https://www.npmjs.com/package/@rushstack/heft-web-rig) |
| [/rush-plugins/rush-amazon-s3-build-cache-plugin](./rush-plugins/rush-amazon-s3-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin) | | [@rushstack/rush-amazon-s3-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-amazon-s3-build-cache-plugin) |
| [/rush-plugins/rush-azure-storage-build-cache-plugin](./rush-plugins/rush-azure-storage-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin) | | [@rushstack/rush-azure-storage-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-azure-storage-build-cache-plugin) |
| [/rush-plugins/rush-bridge-cache-plugin](./rush-plugins/rush-bridge-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-bridge-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-bridge-cache-plugin) | | [@rushstack/rush-bridge-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-bridge-cache-plugin) |
| [/rush-plugins/rush-buildxl-graph-plugin](./rush-plugins/rush-buildxl-graph-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-buildxl-graph-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-buildxl-graph-plugin) | | [@rushstack/rush-buildxl-graph-plugin](https://www.npmjs.com/package/@rushstack/rush-buildxl-graph-plugin) |
| [/rush-plugins/rush-http-build-cache-plugin](./rush-plugins/rush-http-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin) | | [@rushstack/rush-http-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-http-build-cache-plugin) |
| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) |
Expand Down
2 changes: 1 addition & 1 deletion apps/rush/src/start-dev-docs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ import { Colorize, ConsoleTerminalProvider, Terminal } from '@rushstack/terminal
const terminal: Terminal = new Terminal(new ConsoleTerminalProvider());

terminal.writeLine('For instructions on debugging Rush, please see this documentation:');
terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/debugging/'));
terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/#debugging-rush'));
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "Introduce a `@rushstack/rush-bridge-cache-plugin` package that adds a `--set-cache-only` flag to phased commands, which sets the cache entry without performing the operation.",
"type": "none"
}
],
"packageName": "@microsoft/rush"
}
4 changes: 4 additions & 0 deletions common/config/rush/nonbrowser-approved-packages.json
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,10 @@
"name": "@rushstack/rush-azure-storage-build-cache-plugin",
"allowedCategories": [ "libraries" ]
},
{
"name": "@rushstack/rush-bridge-cache-plugin",
"allowedCategories": [ "libraries" ]
},
{
"name": "@rushstack/rush-http-build-cache-plugin",
"allowedCategories": [ "libraries" ]
Expand Down
21 changes: 21 additions & 0 deletions common/config/subspaces/default/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 28 additions & 0 deletions common/reviews/api/rush-lib.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -583,6 +583,12 @@ export class IndividualVersionPolicy extends VersionPolicy {
export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase {
}

// @internal (undocumented)
export interface _IOperationBuildCacheOptions {
buildCacheConfiguration: BuildCacheConfiguration;
terminal: ITerminal;
}

// @alpha
export interface IOperationExecutionResult {
readonly error: Error | undefined;
Expand Down Expand Up @@ -779,6 +785,14 @@ export interface IPnpmPeerDependencyRules {

export { IPrefixMatch }

// @internal (undocumented)
export type _IProjectBuildCacheOptions = _IOperationBuildCacheOptions & {
projectOutputFolderNames: ReadonlyArray<string>;
project: RushConfigurationProject;
operationStateHash: string;
phaseName: string;
};

// @beta
export interface IRushCommand {
readonly actionName: string;
Expand Down Expand Up @@ -947,6 +961,20 @@ export class Operation {
weight: number;
}

// @internal (undocumented)
export class _OperationBuildCache {
// (undocumented)
get cacheId(): string | undefined;
// (undocumented)
static forOperation(executionResult: IOperationExecutionResult, options: _IOperationBuildCacheOptions): _OperationBuildCache;
// (undocumented)
static getOperationBuildCache(options: _IProjectBuildCacheOptions): _OperationBuildCache;
// (undocumented)
tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean>;
// (undocumented)
trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean>;
}

// @internal
export class _OperationMetadataManager {
constructor(options: _IOperationMetadataManagerOptions);
Expand Down
6 changes: 6 additions & 0 deletions libraries/rush-lib/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,3 +197,9 @@ export {
type IRushCommandLineParameter,
type IRushCommandLineAction
} from './api/RushCommandLine';

export { OperationBuildCache as _OperationBuildCache } from './logic/buildCache/OperationBuildCache';
export type {
IOperationBuildCacheOptions as _IOperationBuildCacheOptions,
IProjectBuildCacheOptions as _IProjectBuildCacheOptions
} from './logic/buildCache/OperationBuildCache';
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,11 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider';
import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider';
import { TarExecutable } from '../../utilities/TarExecutable';
import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration';
import type { OperationExecutionRecord } from '../operations/OperationExecutionRecord';
import type { IOperationExecutionResult } from '../operations/IOperationExecutionResult';

/**
* @internal
*/
export interface IOperationBuildCacheOptions {
/**
* The repo-wide configuration for the build cache.
Expand All @@ -26,6 +29,9 @@ export interface IOperationBuildCacheOptions {
terminal: ITerminal;
}

/**
* @internal
*/
export type IProjectBuildCacheOptions = IOperationBuildCacheOptions & {
/**
* Value from rush-project.json
Expand All @@ -50,7 +56,10 @@ interface IPathsToCache {
outputFilePaths: string[];
}

export class ProjectBuildCache {
/**
* @internal
*/
export class OperationBuildCache {
private static _tarUtilityPromise: Promise<TarExecutable | undefined> | undefined;

private readonly _project: RushConfigurationProject;
Expand Down Expand Up @@ -82,40 +91,40 @@ export class ProjectBuildCache {
}

private static _tryGetTarUtility(terminal: ITerminal): Promise<TarExecutable | undefined> {
if (!ProjectBuildCache._tarUtilityPromise) {
ProjectBuildCache._tarUtilityPromise = TarExecutable.tryInitializeAsync(terminal);
if (!OperationBuildCache._tarUtilityPromise) {
OperationBuildCache._tarUtilityPromise = TarExecutable.tryInitializeAsync(terminal);
}

return ProjectBuildCache._tarUtilityPromise;
return OperationBuildCache._tarUtilityPromise;
}

public get cacheId(): string | undefined {
return this._cacheId;
}

public static getProjectBuildCache(options: IProjectBuildCacheOptions): ProjectBuildCache {
const cacheId: string | undefined = ProjectBuildCache._getCacheId(options);
return new ProjectBuildCache(cacheId, options);
public static getOperationBuildCache(options: IProjectBuildCacheOptions): OperationBuildCache {
const cacheId: string | undefined = OperationBuildCache._getCacheId(options);
return new OperationBuildCache(cacheId, options);
}

public static forOperation(
operation: OperationExecutionRecord,
executionResult: IOperationExecutionResult,
options: IOperationBuildCacheOptions
): ProjectBuildCache {
const outputFolders: string[] = [...(operation.operation.settings?.outputFolderNames ?? [])];
if (operation.metadataFolderPath) {
outputFolders.push(operation.metadataFolderPath);
): OperationBuildCache {
const outputFolders: string[] = [...(executionResult.operation.settings?.outputFolderNames ?? [])];
if (executionResult.metadataFolderPath) {
outputFolders.push(executionResult.metadataFolderPath);
}
const buildCacheOptions: IProjectBuildCacheOptions = {
buildCacheConfiguration: options.buildCacheConfiguration,
terminal: options.terminal,
project: operation.associatedProject,
phaseName: operation.associatedPhase.name,
project: executionResult.operation.associatedProject,
phaseName: executionResult.operation.associatedPhase.name,
projectOutputFolderNames: outputFolders,
operationStateHash: operation.getStateHash()
operationStateHash: executionResult.getStateHash()
};
const cacheId: string | undefined = ProjectBuildCache._getCacheId(buildCacheOptions);
return new ProjectBuildCache(cacheId, buildCacheOptions);
const cacheId: string | undefined = OperationBuildCache._getCacheId(buildCacheOptions);
return new OperationBuildCache(cacheId, buildCacheOptions);
}

public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean> {
Expand Down Expand Up @@ -175,7 +184,7 @@ export class ProjectBuildCache {
)
);

const tarUtility: TarExecutable | undefined = await ProjectBuildCache._tryGetTarUtility(terminal);
const tarUtility: TarExecutable | undefined = await OperationBuildCache._tryGetTarUtility(terminal);
let restoreSuccess: boolean = false;
if (tarUtility && localCacheEntryPath) {
const logFilePath: string = this._getTarLogFilePath(cacheId, 'untar');
Expand Down Expand Up @@ -225,7 +234,7 @@ export class ProjectBuildCache {

let localCacheEntryPath: string | undefined;

const tarUtility: TarExecutable | undefined = await ProjectBuildCache._tryGetTarUtility(terminal);
const tarUtility: TarExecutable | undefined = await OperationBuildCache._tryGetTarUtility(terminal);
if (tarUtility) {
const finalLocalCacheEntryPath: string = this._localBuildCacheProvider.getCacheEntryPath(cacheId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,19 @@ import type { RushConfigurationProject } from '../../../api/RushConfigurationPro
import type { IGenerateCacheEntryIdOptions } from '../CacheEntryId';
import type { FileSystemBuildCacheProvider } from '../FileSystemBuildCacheProvider';

import { ProjectBuildCache } from '../ProjectBuildCache';
import { OperationBuildCache } from '../OperationBuildCache';

interface ITestOptions {
enabled: boolean;
writeAllowed: boolean;
trackedProjectFiles: string[] | undefined;
}

describe(ProjectBuildCache.name, () => {
function prepareSubject(options: Partial<ITestOptions>): ProjectBuildCache {
describe(OperationBuildCache.name, () => {
function prepareSubject(options: Partial<ITestOptions>): OperationBuildCache {
const terminal: Terminal = new Terminal(new StringBufferTerminalProvider());

const subject: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({
const subject: OperationBuildCache = OperationBuildCache.getOperationBuildCache({
buildCacheConfiguration: {
buildCacheEnabled: options.hasOwnProperty('enabled') ? options.enabled : true,
getCacheEntryId: (opts: IGenerateCacheEntryIdOptions) =>
Expand All @@ -46,9 +46,9 @@ describe(ProjectBuildCache.name, () => {
return subject;
}

describe(ProjectBuildCache.getProjectBuildCache.name, () => {
it('returns a ProjectBuildCache with a calculated cacheId value', () => {
const subject: ProjectBuildCache = prepareSubject({});
describe(OperationBuildCache.getOperationBuildCache.name, () => {
it('returns an OperationBuildCache with a calculated cacheId value', () => {
const subject: OperationBuildCache = prepareSubject({});
expect(subject['_cacheId']).toMatchInlineSnapshot(
`"acme-wizard/1926f30e8ed24cb47be89aea39e7efd70fcda075"`
);
Expand Down
12 changes: 6 additions & 6 deletions libraries/rush-lib/src/logic/cobuild/CobuildLock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { InternalError } from '@rushstack/node-core-library';
import type { CobuildConfiguration } from '../../api/CobuildConfiguration';
import type { OperationStatus } from '../operations/OperationStatus';
import type { ICobuildContext } from './ICobuildLockProvider';
import type { ProjectBuildCache } from '../buildCache/ProjectBuildCache';
import type { OperationBuildCache } from '../buildCache/OperationBuildCache';

const KEY_SEPARATOR: ':' = ':';

Expand All @@ -27,7 +27,7 @@ export interface ICobuildLockOptions {
* {@inheritdoc ICobuildContext.phaseName}
*/
phaseName: string;
projectBuildCache: ProjectBuildCache;
operationBuildCache: OperationBuildCache;
/**
* The expire time of the lock in seconds.
*/
Expand All @@ -41,23 +41,23 @@ export interface ICobuildCompletedState {

export class CobuildLock {
public readonly cobuildConfiguration: CobuildConfiguration;
public readonly projectBuildCache: ProjectBuildCache;
public readonly operationBuildCache: OperationBuildCache;

private _cobuildContext: ICobuildContext;

public constructor(options: ICobuildLockOptions) {
const {
cobuildConfiguration,
projectBuildCache,
operationBuildCache,
cobuildClusterId: clusterId,
lockExpireTimeInSeconds,
packageName,
phaseName
} = options;
const { cobuildContextId: contextId, cobuildRunnerId: runnerId } = cobuildConfiguration;
const { cacheId } = projectBuildCache;
const { cacheId } = operationBuildCache;
this.cobuildConfiguration = cobuildConfiguration;
this.projectBuildCache = projectBuildCache;
this.operationBuildCache = operationBuildCache;

if (!cacheId) {
// This should never happen
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import { CobuildLock, type ICobuildLockOptions } from '../CobuildLock';

import type { CobuildConfiguration } from '../../../api/CobuildConfiguration';
import type { ProjectBuildCache } from '../../buildCache/ProjectBuildCache';
import type { OperationBuildCache } from '../../buildCache/OperationBuildCache';
import type { ICobuildContext } from '../ICobuildLockProvider';

describe(CobuildLock.name, () => {
Expand All @@ -14,9 +14,9 @@ describe(CobuildLock.name, () => {
cobuildContextId: 'context_id',
cobuildRunnerId: 'runner_id'
} as unknown as CobuildConfiguration,
projectBuildCache: {
operationBuildCache: {
cacheId: 'cache_id'
} as unknown as ProjectBuildCache,
} as unknown as OperationBuildCache,
cobuildClusterId: 'cluster_id',
lockExpireTimeInSeconds: 30,
packageName: 'package_name',
Expand Down
Loading