Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions components/legacy/scope/scope.ts
Original file line number Diff line number Diff line change
Expand Up @@ -743,8 +743,11 @@ once done, to continue working, please run "bit cc"`
this.objects.scopeJson = scopeJson;
}

public async getDependenciesGraphByComponentIds(componentIds: ComponentID[]): Promise<DependenciesGraph | undefined> {
if (!isFeatureEnabled(DEPS_GRAPH)) return undefined;
public async getDependenciesGraphByComponentIds(
componentIds: ComponentID[],
options?: { ignoreFeatureToggle?: boolean }
): Promise<DependenciesGraph | undefined> {
if (!options?.ignoreFeatureToggle && !isFeatureEnabled(DEPS_GRAPH)) return undefined;
let allGraph: DependenciesGraph | undefined;
await pMapPool(
componentIds,
Expand Down
126 changes: 126 additions & 0 deletions e2e/harmony/deps-graph.e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -543,4 +543,130 @@ chai.use(chaiFs);
expect(lockfile.packages).to.have.property('@pnpm.e2e/bar@100.0.0');
});
});
// `bit install --restore` seeds the lockfile from the dependency graphs stored on
// every bitmap entry, the same way `bit import` does for the components it writes.
// This lets a user recover from a deleted pnpm-lock.yaml without re-resolving from
// manifest specifiers (which would drift to whatever the registry considers latest).
describe('bit install --restore rebuilds the lockfile from workspace component graphs', function () {
let randomStr: string;
let lockfileAfterRestore: any;
before(async () => {
randomStr = generateRandomStr(4);
const name = `@ci/${randomStr}.{name}`;
helper.scopeHelper.setWorkspaceWithRemoteScope();
npmCiRegistry = new NpmCiRegistry(helper);
npmCiRegistry.configureCustomNameInPackageJsonHarmony(name);
await npmCiRegistry.init();
helper.command.setConfig('registry', npmCiRegistry.getRegistryUrl());
helper.env.setCustomNewEnv(
undefined,
undefined,
{ policy: { peers: [] } },
false,
'custom-env/env',
'custom-env/env'
);
helper.fs.createFile('comp1', 'comp1.js', 'require("@pnpm.e2e/foo"); // eslint-disable-line');
helper.command.addComponent('comp1');
helper.extensions.addExtensionToVariant('comp1', `${helper.scopes.remote}/custom-env/env`, {});
helper.fs.createFile('comp2', 'comp2.js', 'require("@pnpm.e2e/bar"); // eslint-disable-line');
helper.command.addComponent('comp2');
helper.extensions.addExtensionToVariant('comp2', `${helper.scopes.remote}/custom-env/env`, {});
helper.extensions.workspaceJsonc.addKeyValToDependencyResolver('rootComponents', true);
await addDistTag({ package: '@pnpm.e2e/foo', version: '100.0.0', distTag: 'latest' });
await addDistTag({ package: '@pnpm.e2e/bar', version: '100.0.0', distTag: 'latest' });
helper.command.install('--add-missing-deps');
helper.command.tagAllComponents('--skip-tests');
helper.command.export();

helper.scopeHelper.reInitWorkspace();
helper.scopeHelper.addRemoteScope();
helper.extensions.workspaceJsonc.addKeyValToDependencyResolver('rootComponents', true);
helper.command.import(`${helper.scopes.remote}/comp1@latest ${helper.scopes.remote}/comp2@latest`);

// bump registry and blow away the lockfile + node_modules, then restore from graphs.
await addDistTag({ package: '@pnpm.e2e/foo', version: '100.1.0', distTag: 'latest' });
await addDistTag({ package: '@pnpm.e2e/bar', version: '100.1.0', distTag: 'latest' });
helper.fs.deletePath('pnpm-lock.yaml');
helper.fs.deletePath('node_modules');
helper.command.runCmd('bit install --restore');
lockfileAfterRestore = yaml.load(fs.readFileSync(path.join(helper.scopes.localPath, 'pnpm-lock.yaml'), 'utf8'));
});
after(() => {
npmCiRegistry.destroy();
helper.command.delConfig('registry');
helper.scopeHelper.destroy();
});
it('should mark the regenerated lockfile as restoredFromModel', () => {
expect(lockfileAfterRestore.bit.restoredFromModel).to.eq(true);
});
it('should keep both components locked to the versions stored in their graphs', () => {
expect(lockfileAfterRestore.packages).to.have.property('@pnpm.e2e/foo@100.0.0');
expect(lockfileAfterRestore.packages).to.have.property('@pnpm.e2e/bar@100.0.0');
expect(lockfileAfterRestore.packages).to.not.have.property('@pnpm.e2e/foo@100.1.0');
expect(lockfileAfterRestore.packages).to.not.have.property('@pnpm.e2e/bar@100.1.0');
});
});
// --restore is an explicit opt-in, so it has to bypass the DEPS_GRAPH feature toggle
// and work on workspaces that never enabled the flag. The graph itself was authored on
// a scope that had the flag on at tag time, but the consumer shouldn't need to flip
// the flag just to restore from it.
describe('bit install --restore works when the DEPS_GRAPH feature toggle is disabled', function () {
let randomStr: string;
let lockfileAfterRestore: any;
before(async () => {
randomStr = generateRandomStr(4);
const name = `@ci/${randomStr}.{name}`;
helper.scopeHelper.setWorkspaceWithRemoteScope();
npmCiRegistry = new NpmCiRegistry(helper);
npmCiRegistry.configureCustomNameInPackageJsonHarmony(name);
await npmCiRegistry.init();
helper.command.setConfig('registry', npmCiRegistry.getRegistryUrl());
helper.env.setCustomNewEnv(
undefined,
undefined,
{ policy: { peers: [] } },
false,
'custom-env/env',
'custom-env/env'
);
helper.fs.createFile('comp1', 'comp1.js', 'require("@pnpm.e2e/foo"); // eslint-disable-line');
helper.command.addComponent('comp1');
helper.extensions.addExtensionToVariant('comp1', `${helper.scopes.remote}/custom-env/env`, {});
helper.extensions.workspaceJsonc.addKeyValToDependencyResolver('rootComponents', true);
await addDistTag({ package: '@pnpm.e2e/foo', version: '100.0.0', distTag: 'latest' });
helper.command.install('--add-missing-deps');
helper.command.tagAllComponents('--skip-tests');
helper.command.export();

helper.scopeHelper.reInitWorkspace();
helper.scopeHelper.addRemoteScope();
helper.extensions.workspaceJsonc.addKeyValToDependencyResolver('rootComponents', true);

// Turn the DEPS_GRAPH feature toggle off for the remainder of the test. The outer
// describe's after() hook will restore it via resetFeatures, but we also restore it
// here so subsequent describes in this suite aren't affected.
helper.command.resetFeatures();
try {
helper.command.import(`${helper.scopes.remote}/comp1@latest`);
await addDistTag({ package: '@pnpm.e2e/foo', version: '100.1.0', distTag: 'latest' });
helper.fs.deletePath('pnpm-lock.yaml');
helper.fs.deletePath('node_modules');
helper.command.runCmd('bit install --restore');
lockfileAfterRestore = yaml.load(fs.readFileSync(path.join(helper.scopes.localPath, 'pnpm-lock.yaml'), 'utf8'));
} finally {
helper.command.setFeatures(DEPS_GRAPH);
}
});
after(() => {
npmCiRegistry.destroy();
helper.command.delConfig('registry');
helper.scopeHelper.destroy();
});
it('should still restore the lockfile from the stored graph', () => {
expect(lockfileAfterRestore.bit.restoredFromModel).to.eq(true);
expect(lockfileAfterRestore.packages).to.have.property('@pnpm.e2e/foo@100.0.0');
expect(lockfileAfterRestore.packages).to.not.have.property('@pnpm.e2e/foo@100.1.0');
});
});
});
6 changes: 4 additions & 2 deletions scopes/dependencies/pnpm/pnpm.package-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import type {
CalcDepsGraphOptions,
} from '@teambit/dependency-resolver';
import { Registries, Registry } from '@teambit/pkg.entities.registry';
import { DEPS_GRAPH, isFeatureEnabled } from '@teambit/harmony.modules.feature-toggle';
import type { Logger } from '@teambit/logger';
import { type LockfileFile } from '@pnpm/lockfile.types';
import fs from 'fs';
Expand Down Expand Up @@ -131,9 +130,12 @@ export class PnpmPackageManager implements PackageManager {
const proxyConfig = await this.depResolver.getProxyConfig();
const networkConfig = await this.depResolver.getNetworkConfig();
const { config } = await this.readConfig(installOptions.packageManagerConfigRootDir);
// When dependenciesGraph is explicitly supplied (by the component writer on import, or
// by `bit install --restore`), honor it regardless of the DEPS_GRAPH feature toggle —
// the flag gates the *providers* that fetch graphs, so the presence of one here means
// the caller already decided this install should be seeded from stored graphs.
if (
installOptions.dependenciesGraph &&
isFeatureEnabled(DEPS_GRAPH) &&
(installOptions.rootComponents || installOptions.rootComponentsForCapsules)
) {
try {
Expand Down
7 changes: 5 additions & 2 deletions scopes/scope/scope/scope.main.runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1487,8 +1487,11 @@ export class ScopeMain implements ComponentFactory {
return scope;
}

public getDependenciesGraphByComponentIds(componentIds: ComponentID[]): Promise<DependenciesGraph | undefined> {
return this.legacyScope.getDependenciesGraphByComponentIds(componentIds);
public getDependenciesGraphByComponentIds(
componentIds: ComponentID[],
options?: { ignoreFeatureToggle?: boolean }
): Promise<DependenciesGraph | undefined> {
return this.legacyScope.getDependenciesGraphByComponentIds(componentIds, options);
}
}

Expand Down
7 changes: 7 additions & 0 deletions scopes/workspace/install/install.cmd.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type InstallCmdOptions = {
noOptional: boolean;
recurringInstall: boolean;
lockfileOnly: boolean;
restore: boolean;
allowScripts?: string;
disallowScripts?: string;
};
Expand Down Expand Up @@ -69,6 +70,11 @@ automatically imports components, compiles components, links to node_modules, an
],
['', 'no-optional [noOptional]', 'do not install optional dependencies (works with pnpm only)'],
['', 'lockfile-only', 'dependencies are not written to node_modules. Only the lockfile is updated'],
[
'',
'restore',
'reconstruct the lockfile from each workspace component\'s stored dependency graph before installing',
],
['', 'allow-scripts [pkgNames]', 'a comma separated list of package names that are allowed to run installation scripts'],
['', 'disallow-scripts [pkgNames]', 'a comma separated list of package names that are NOT allowed to run installation scripts'],
] as CommandOptions;
Expand Down Expand Up @@ -134,6 +140,7 @@ automatically imports components, compiles components, links to node_modules, an
updateAll: options.update,
recurringInstall: options.recurringInstall,
lockfileOnly: options.lockfileOnly,
restoreFromDependenciesGraph: options.restore,
showExternalPackageManagerPrompt: true,
allowScripts: this._parseAllowScriptsFlags(options.allowScripts, options.disallowScripts),
};
Expand Down
43 changes: 42 additions & 1 deletion scopes/workspace/install/install.main.runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,13 @@ export type WorkspaceInstallOptions = {
writeConfigFiles?: boolean;
skipPrune?: boolean;
dependenciesGraph?: DependenciesGraph;
/**
* When true, attempt to reconstruct the lockfile from each workspace component's
* stored dependency graph before running the package manager. Graphs are fetched for
* every component listed in the bitmap, merged, and handed to the package manager the
* same way `bit import` does for the components it writes.
*/
restoreFromDependenciesGraph?: boolean;
allowScripts?: Record<string, boolean>;
};

Expand Down Expand Up @@ -392,11 +399,12 @@ export class InstallMain {
}
);

const dependenciesGraph = await this.resolveDependenciesGraph(options, { hasRootComponents });
const pmInstallOptions: PackageManagerInstallOptions = {
...calcManifestsOpts,
autoInstallPeers: this.dependencyResolver.config.autoInstallPeers,
dedupePeers: this.dependencyResolver.config.dedupePeers,
dependenciesGraph: options?.dependenciesGraph,
dependenciesGraph,
includeOptionalDeps: options?.includeOptionalDeps,
neverBuiltDependencies: this.dependencyResolver.config.neverBuiltDependencies,
allowScripts: this.dependencyResolver.getAllowedScripts(),
Expand Down Expand Up @@ -516,6 +524,39 @@ export class InstallMain {
return nonLoadedEnvs.length > 0;
}

private async resolveDependenciesGraph(
options: ModulesInstallOptions | undefined,
context: { hasRootComponents: boolean }
): Promise<DependenciesGraph | undefined> {
if (options?.dependenciesGraph) return options.dependenciesGraph;
if (!options?.restoreFromDependenciesGraph) return undefined;
// The package manager only seeds the lockfile from a supplied graph when the workspace
// has `rootComponents` enabled (see PnpmPackageManager.install). Without that, --restore
// would silently no-op and re-resolve everything from manifest specifiers, so we bail
// out explicitly instead of letting the user think the graph was applied.
if (!context.hasRootComponents) {
this.logger.console(
chalk.yellow(
'--restore requires "rootComponents: true" in the dependency-resolver config; falling back to a regular install.'
)
);
return undefined;
}
// --restore is an explicit opt-in, so bypass the DEPS_GRAPH feature toggle that would
// otherwise make this return undefined on workspaces that haven't enabled the flag.
const graph = await this.workspace.scope.getDependenciesGraphByComponentIds(this.workspace.listIds(), {
ignoreFeatureToggle: true,
});
if (!graph) {
this.logger.console(
chalk.yellow(
'--restore was requested but no workspace component has a stored dependency graph. Falling back to a regular install.'
Comment thread
zkochan marked this conversation as resolved.
)
);
}
return graph;
}

/**
* This function is very important to fix some issues that might happen during the installation process.
* The case is the following:
Expand Down