diff --git a/common/changes/@microsoft/rush/omit-apple-doubles-from-cache_2026-02-16-05-38.json b/common/changes/@microsoft/rush/omit-apple-doubles-from-cache_2026-02-16-05-38.json new file mode 100644 index 00000000000..ba599debc38 --- /dev/null +++ b/common/changes/@microsoft/rush/omit-apple-doubles-from-cache_2026-02-16-05-38.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Add a new \"omitAppleDoubleFilesFromBuildCache\" experiment. When enabled, the Rush build cache will omit macOS AppleDouble metadata files (._*) from cache archives when a companion file exists in the same directory. This prevents platform-specific metadata files from polluting the shared build cache. The filtering only applies when running on macOS.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 471a2e19f6f..e761e6c29a2 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -471,6 +471,7 @@ export interface IExperimentsJson { forbidPhantomResolvableNodeModulesFolders?: boolean; generateProjectImpactGraphDuringRushUpdate?: boolean; noChmodFieldInTarHeaderNormalization?: boolean; + omitAppleDoubleFilesFromBuildCache?: boolean; omitImportersFromPreventManualShrinkwrapChanges?: boolean; printEventHooksOutputToConsole?: boolean; rushAlerts?: boolean; @@ -583,6 +584,7 @@ export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase { // @internal (undocumented) export interface _IOperationBuildCacheOptions { buildCacheConfiguration: BuildCacheConfiguration; + filterAppleDoubleFiles: boolean; terminal: ITerminal; } diff --git a/libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json b/libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json index 605e320a4c9..ab57492884b 100644 --- a/libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json +++ b/libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json @@ -117,5 +117,13 @@ * subspace. This is useful for large product groups who work in separate subspaces and generally prefer to consume * each other's packages via the NPM registry. */ - /*[LINE "HYPOTHETICAL"]*/ "exemptDecoupledDependenciesBetweenSubspaces": false + /*[LINE "HYPOTHETICAL"]*/ "exemptDecoupledDependenciesBetweenSubspaces": false, + + /** + * If true, when running on macOS, Rush will omit AppleDouble files (._*) from build cache archives + * when a companion file exists in the same directory. AppleDouble files are automatically created by + * macOS to store extended attributes on filesystems that don't support them, and should generally not + * be included in the shared build cache. + */ + /*[LINE "HYPOTHETICAL"]*/ "omitAppleDoubleFilesFromBuildCache": true } diff --git a/libraries/rush-lib/src/api/ExperimentsConfiguration.ts b/libraries/rush-lib/src/api/ExperimentsConfiguration.ts index 505aaf8a044..f8a9ae66e34 100644 --- a/libraries/rush-lib/src/api/ExperimentsConfiguration.ts +++ b/libraries/rush-lib/src/api/ExperimentsConfiguration.ts @@ -128,6 +128,14 @@ export interface IExperimentsJson { * each other's packages via the NPM registry. */ exemptDecoupledDependenciesBetweenSubspaces?: boolean; + + /** + * If true, when running on macOS, Rush will omit AppleDouble files (`._*`) from build cache archives + * when a companion file exists in the same directory. AppleDouble files are automatically created by + * macOS to store extended attributes on filesystems that don't support them, and should generally not + * be included in the shared build cache. + */ + omitAppleDoubleFilesFromBuildCache?: boolean; } const _EXPERIMENTS_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson); diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 6bf22cd9c18..76e5415fbae 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -508,7 +508,10 @@ export class PhasedScriptAction extends BaseScriptAction i .buildCacheWithAllowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration, - terminal + terminal, + filterAppleDoubleFiles: + !!this.rushConfiguration.experimentsConfiguration.configuration + .omitAppleDoubleFilesFromBuildCache }).apply(this.hooks); if (this._debugBuildCacheIdsParameter.value) { diff --git a/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts index 7fbc065881d..7c52597a76a 100644 --- a/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts @@ -27,6 +27,11 @@ export interface IOperationBuildCacheOptions { * The terminal to use for logging. */ terminal: ITerminal; + /** + * If true, omit AppleDouble (`._*`) files from cache archives when running on macOS + * and a companion file exists in the same directory. + */ + filterAppleDoubleFiles: boolean; } /** @@ -69,6 +74,7 @@ export class OperationBuildCache { private readonly _cacheWriteEnabled: boolean; private readonly _projectOutputFolderNames: ReadonlyArray; private readonly _cacheId: string | undefined; + private readonly _filterAppleDoubleFiles: boolean; private constructor(cacheId: string | undefined, options: IProjectBuildCacheOptions) { const { @@ -79,7 +85,8 @@ export class OperationBuildCache { cacheWriteEnabled }, project, - projectOutputFolderNames + projectOutputFolderNames, + filterAppleDoubleFiles } = options; this._project = project; this._localBuildCacheProvider = localCacheProvider; @@ -88,6 +95,7 @@ export class OperationBuildCache { this._cacheWriteEnabled = cacheWriteEnabled; this._projectOutputFolderNames = projectOutputFolderNames || []; this._cacheId = cacheId; + this._filterAppleDoubleFiles = filterAppleDoubleFiles && process.platform === 'darwin'; } private static _tryGetTarUtility(terminal: ITerminal): Promise { @@ -111,17 +119,20 @@ export class OperationBuildCache { executionResult: IOperationExecutionResult, options: IOperationBuildCacheOptions ): OperationBuildCache { + const { buildCacheConfiguration, terminal, filterAppleDoubleFiles } = options; const outputFolders: string[] = [...(executionResult.operation.settings?.outputFolderNames ?? [])]; if (executionResult.metadataFolderPath) { outputFolders.push(executionResult.metadataFolderPath); } + const buildCacheOptions: IProjectBuildCacheOptions = { - buildCacheConfiguration: options.buildCacheConfiguration, - terminal: options.terminal, + buildCacheConfiguration, + terminal, project: executionResult.operation.associatedProject, phaseName: executionResult.operation.associatedPhase.name, projectOutputFolderNames: outputFolders, - operationStateHash: executionResult.getStateHash() + operationStateHash: executionResult.getStateHash(), + filterAppleDoubleFiles }; const cacheId: string | undefined = OperationBuildCache._getCacheId(buildCacheOptions); return new OperationBuildCache(cacheId, buildCacheOptions); @@ -341,11 +352,20 @@ export class OperationBuildCache { const filteredOutputFolderNames: string[] = []; let hasSymbolicLinks: boolean = false; + const filterAppleDoubleFiles: boolean = this._filterAppleDoubleFiles; // Adds child directories to the queue, files to the path list, and bails on symlinks function processChildren(relativePath: string, diskPath: string, children: FolderItem[]): void { + // When filtering AppleDouble files, build a set of sibling names so we can check + // whether a companion file exists for each ._X file. + let childNameSet: Set | undefined; + if (filterAppleDoubleFiles) { + childNameSet = new Set(children.map(({ name }) => name)); + } + for (const child of children) { - const childRelativePath: string = `${relativePath}/${child.name}`; + const childName: string = child.name; + const childRelativePath: string = `${relativePath}/${childName}`; if (child.isSymbolicLink()) { terminal.writeError( `Unable to include "${childRelativePath}" in build cache. It is a symbolic link.` @@ -354,6 +374,15 @@ export class OperationBuildCache { } else if (child.isDirectory()) { queue.push([childRelativePath, `${diskPath}/${child.name}`]); } else { + // Check for macOS AppleDouble files (._X pattern) that have a companion file + if (childNameSet && childName.length > 2 && childName.startsWith('._')) { + const companionName: string = childName.substring(2); + if (childNameSet.has(companionName)) { + terminal.writeVerboseLine(`Omitting AppleDouble file "${childRelativePath}" from build cache.`); + continue; + } + } + outputFilePaths.push(childRelativePath); } } diff --git a/libraries/rush-lib/src/logic/buildCache/test/OperationBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/OperationBuildCache.test.ts index 82606c11784..319f4875389 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/OperationBuildCache.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/OperationBuildCache.test.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { FileSystem, type FolderItem } from '@rushstack/node-core-library'; import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; import type { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; @@ -14,6 +15,22 @@ interface ITestOptions { enabled: boolean; writeAllowed: boolean; trackedProjectFiles: string[] | undefined; + filterAppleDoubleFiles: boolean; +} + +function createFolderItem(name: string, type: 'file' | 'directory' | 'symlink'): FolderItem { + return { + name, + isSymbolicLink: () => type === 'symlink', + isDirectory: () => type === 'directory', + isFile: () => type === 'file', + isBlockDevice: () => false, + isCharacterDevice: () => false, + isFIFO: () => false, + isSocket: () => false, + parentPath: '', + path: name + } as unknown as FolderItem; } describe(OperationBuildCache.name, () => { @@ -34,13 +51,15 @@ describe(OperationBuildCache.name, () => { project: { packageName: 'acme-wizard', projectRelativeFolder: 'apps/acme-wizard', + projectFolder: '/repo/apps/acme-wizard', dependencyProjects: [] } as unknown as RushConfigurationProject, // Value from past tests, for consistency. // The project build cache is not responsible for calculating this value. operationStateHash: '1926f30e8ed24cb47be89aea39e7efd70fcda075', terminal, - phaseName: 'build' + phaseName: 'build', + filterAppleDoubleFiles: !!options.filterAppleDoubleFiles }); return subject; @@ -54,4 +73,146 @@ describe(OperationBuildCache.name, () => { ); }); }); + + describe('AppleDouble file filtering', () => { + const originalPlatform: NodeJS.Platform = process.platform; + + afterEach(() => { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + jest.restoreAllMocks(); + }); + + it('omits AppleDouble files with companions when enabled on macOS', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin' }); + + const subject: OperationBuildCache = prepareSubject({ filterAppleDoubleFiles: true }); + + jest + .spyOn(FileSystem, 'readFolderItemsAsync') + .mockResolvedValue([ + createFolderItem('foo.txt', 'file'), + createFolderItem('._foo.txt', 'file'), + createFolderItem('bar.js', 'file'), + createFolderItem('._bar.js', 'file') + ]); + + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + const terminal: Terminal = new Terminal(terminalProvider); + + const result: { outputFilePaths: string[]; filteredOutputFolderNames: string[] } | undefined = + await subject['_tryCollectPathsToCacheAsync'](terminal); + + expect(result).toBeDefined(); + expect(result!.outputFilePaths).toEqual(['dist/bar.js', 'dist/foo.txt']); + expect(result!.outputFilePaths).not.toContain('dist/._foo.txt'); + expect(result!.outputFilePaths).not.toContain('dist/._bar.js'); + }); + + it('keeps AppleDouble files without companion files', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin' }); + + const subject: OperationBuildCache = prepareSubject({ filterAppleDoubleFiles: true }); + + jest + .spyOn(FileSystem, 'readFolderItemsAsync') + .mockResolvedValue([createFolderItem('._orphan.txt', 'file'), createFolderItem('other.js', 'file')]); + + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + const terminal: Terminal = new Terminal(terminalProvider); + + const result: { outputFilePaths: string[]; filteredOutputFolderNames: string[] } | undefined = + await subject['_tryCollectPathsToCacheAsync'](terminal); + + expect(result).toBeDefined(); + expect(result!.outputFilePaths).toEqual(['dist/._orphan.txt', 'dist/other.js']); + }); + + it('does not filter AppleDouble files when the experiment is disabled', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin' }); + + const subject: OperationBuildCache = prepareSubject({ filterAppleDoubleFiles: false }); + + jest + .spyOn(FileSystem, 'readFolderItemsAsync') + .mockResolvedValue([createFolderItem('foo.txt', 'file'), createFolderItem('._foo.txt', 'file')]); + + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + const terminal: Terminal = new Terminal(terminalProvider); + + const result: { outputFilePaths: string[]; filteredOutputFolderNames: string[] } | undefined = + await subject['_tryCollectPathsToCacheAsync'](terminal); + + expect(result).toBeDefined(); + expect(result!.outputFilePaths).toEqual(['dist/._foo.txt', 'dist/foo.txt']); + }); + + it('does not filter AppleDouble files on non-macOS platforms', async () => { + Object.defineProperty(process, 'platform', { value: 'win32' }); + + const subject: OperationBuildCache = prepareSubject({ filterAppleDoubleFiles: true }); + + jest + .spyOn(FileSystem, 'readFolderItemsAsync') + .mockResolvedValue([createFolderItem('foo.txt', 'file'), createFolderItem('._foo.txt', 'file')]); + + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + const terminal: Terminal = new Terminal(terminalProvider); + + const result: { outputFilePaths: string[]; filteredOutputFolderNames: string[] } | undefined = + await subject['_tryCollectPathsToCacheAsync'](terminal); + + expect(result).toBeDefined(); + expect(result!.outputFilePaths).toEqual(['dist/._foo.txt', 'dist/foo.txt']); + }); + + it('does not filter files named exactly "._"', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin' }); + + const subject: OperationBuildCache = prepareSubject({ filterAppleDoubleFiles: true }); + + jest + .spyOn(FileSystem, 'readFolderItemsAsync') + .mockResolvedValue([createFolderItem('._', 'file'), createFolderItem('other.txt', 'file')]); + + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + const terminal: Terminal = new Terminal(terminalProvider); + + const result: { outputFilePaths: string[]; filteredOutputFolderNames: string[] } | undefined = + await subject['_tryCollectPathsToCacheAsync'](terminal); + + expect(result).toBeDefined(); + expect(result!.outputFilePaths).toEqual(['dist/._', 'dist/other.txt']); + }); + + it('filters AppleDouble files in nested directories', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin' }); + + const subject: OperationBuildCache = prepareSubject({ filterAppleDoubleFiles: true }); + + // First call returns the top-level dist/ contents with a subdirectory + // Second call returns the subdirectory contents + jest + .spyOn(FileSystem, 'readFolderItemsAsync') + .mockResolvedValueOnce([ + createFolderItem('index.js', 'file'), + createFolderItem('._index.js', 'file'), + createFolderItem('sub', 'directory') + ]) + .mockResolvedValueOnce([ + createFolderItem('nested.js', 'file'), + createFolderItem('._nested.js', 'file') + ]); + + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + const terminal: Terminal = new Terminal(terminalProvider); + + const result: { outputFilePaths: string[]; filteredOutputFolderNames: string[] } | undefined = + await subject['_tryCollectPathsToCacheAsync'](terminal); + + expect(result).toBeDefined(); + expect(result!.outputFilePaths).toEqual(['dist/index.js', 'dist/sub/nested.js']); + expect(result!.outputFilePaths).not.toContain('dist/._index.js'); + expect(result!.outputFilePaths).not.toContain('dist/sub/._nested.js'); + }); + }); }); diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index bedb37d64c4..32cc5913ba3 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -76,6 +76,22 @@ export interface ICacheableOperationPluginOptions { buildCacheConfiguration: BuildCacheConfiguration; cobuildConfiguration: CobuildConfiguration | undefined; terminal: ITerminal; + filterAppleDoubleFiles: boolean; +} + +interface ITryGetOperationBuildCacheOptionsBase { + buildCacheContext: IOperationBuildCacheContext; + buildCacheConfiguration: BuildCacheConfiguration | undefined; + terminal: ITerminal; + filterAppleDoubleFiles: boolean; + record: TRecord; +} + +type ITryGetOperationBuildCacheOptions = ITryGetOperationBuildCacheOptionsBase; + +interface ITryGetLogOnlyOperationBuildCacheOptions + extends ITryGetOperationBuildCacheOptionsBase { + cobuildConfiguration: CobuildConfiguration; } export class CacheableOperationPlugin implements IPhasedCommandPlugin { @@ -88,7 +104,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } public apply(hooks: PhasedCommandHooks): void { - const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration } = this._options; + const { + allowWarningsInSuccessfulBuild, + buildCacheConfiguration, + cobuildConfiguration, + filterAppleDoubleFiles + } = this._options; hooks.beforeExecuteOperations.tap( PLUGIN_NAME, @@ -250,7 +271,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheContext, buildCacheConfiguration, terminal: buildCacheTerminal, - record + record, + filterAppleDoubleFiles }); // Try to acquire the cobuild lock @@ -268,7 +290,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { cobuildConfiguration, buildCacheContext, record, - terminal: buildCacheTerminal + terminal: buildCacheTerminal, + filterAppleDoubleFiles }); if (operationBuildCache) { buildCacheTerminal.writeVerboseLine( @@ -559,17 +582,10 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext; } - private _tryGetOperationBuildCache({ - buildCacheConfiguration, - buildCacheContext, - terminal, - record - }: { - buildCacheContext: IOperationBuildCacheContext; - buildCacheConfiguration: BuildCacheConfiguration | undefined; - terminal: ITerminal; - record: OperationExecutionRecord; - }): OperationBuildCache | undefined { + private _tryGetOperationBuildCache( + options: ITryGetOperationBuildCacheOptions + ): OperationBuildCache | undefined { + const { buildCacheConfiguration, buildCacheContext, terminal, record, filterAppleDoubleFiles } = options; if (!buildCacheContext.operationBuildCache) { const { cacheDisabledReason } = buildCacheContext; if (cacheDisabledReason && !record.operation.settings?.allowCobuildWithoutCache) { @@ -584,7 +600,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheContext.operationBuildCache = OperationBuildCache.forOperation(record, { buildCacheConfiguration, - terminal + terminal, + filterAppleDoubleFiles }); } @@ -592,14 +609,17 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } // Get an OperationBuildCache only cache/restore log files - private async _tryGetLogOnlyOperationBuildCacheAsync(options: { - buildCacheContext: IOperationBuildCacheContext; - buildCacheConfiguration: BuildCacheConfiguration | undefined; - cobuildConfiguration: CobuildConfiguration; - record: IOperationRunnerContext & IOperationExecutionResult; - terminal: ITerminal; - }): Promise { - const { buildCacheContext, buildCacheConfiguration, cobuildConfiguration, record, terminal } = options; + private async _tryGetLogOnlyOperationBuildCacheAsync( + options: ITryGetLogOnlyOperationBuildCacheOptions + ): Promise { + const { + buildCacheContext, + buildCacheConfiguration, + cobuildConfiguration, + record, + terminal, + filterAppleDoubleFiles + } = options; if (!buildCacheConfiguration?.buildCacheEnabled) { return; @@ -628,7 +648,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheConfiguration, terminal, operationStateHash, - phaseName: associatedPhase.name + phaseName: associatedPhase.name, + filterAppleDoubleFiles }); buildCacheContext.operationBuildCache = operationBuildCache; diff --git a/libraries/rush-lib/src/schemas/experiments.schema.json b/libraries/rush-lib/src/schemas/experiments.schema.json index 35f173444b4..8a92fa9ee67 100644 --- a/libraries/rush-lib/src/schemas/experiments.schema.json +++ b/libraries/rush-lib/src/schemas/experiments.schema.json @@ -81,6 +81,10 @@ "exemptDecoupledDependenciesBetweenSubspaces": { "description": "Rush has a policy that normally requires Rush projects to specify `workspace:*` in package.json when depending on other projects in the workspace, unless they are explicitly declared as `decoupledLocalDependencies in rush.json. Enabling this experiment will remove that requirement for dependencies belonging to a different subspace. This is useful for large product groups who work in separate subspaces and generally prefer to consume each other's packages via the NPM registry.", "type": "boolean" + }, + "omitAppleDoubleFilesFromBuildCache": { + "description": "If true, when running on macOS, Rush will omit AppleDouble files (._*) from build cache archives when a companion file exists in the same directory. AppleDouble files are automatically created by macOS to store extended attributes on filesystems that don't support them, and should generally not be included in the shared build cache.", + "type": "boolean" } }, "additionalProperties": false diff --git a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts index ad3c810a19a..30303dae463 100644 --- a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts +++ b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts @@ -82,7 +82,14 @@ export class BridgeCachePlugin implements IRushPlugin { recordByOperation: Map, context: IExecuteOperationsContext ): Promise => { - const { buildCacheConfiguration } = context; + const { + buildCacheConfiguration, + rushConfiguration: { + experimentsConfiguration: { + configuration: { omitAppleDoubleFilesFromBuildCache } + } + } + } = context; const { terminal } = logger; if (cacheAction === undefined) { @@ -111,7 +118,8 @@ export class BridgeCachePlugin implements IRushPlugin { operationExecutionResult, { buildCacheConfiguration, - terminal + terminal, + filterAppleDoubleFiles: !!omitAppleDoubleFilesFromBuildCache } );