From 2474a9219d413a5909ab2a4fadbcfa8b2540e765 Mon Sep 17 00:00:00 2001 From: Mihai Toader Date: Wed, 28 Aug 2024 13:57:02 -0700 Subject: [PATCH] [#6665] Add a `ExternalWorkspaceDataProvider` and wire inside the sync - 3/n --- base/src/META-INF/blaze-base.xml | 2 + .../base/command/mod/BlazeModRunnerImpl.java | 2 +- .../model/ExternalWorkspaceDataProvider.java | 129 ++++++++++++++++++ .../blaze/base/sync/ProjectStateSyncTask.java | 98 +++++++++---- .../base/sync/ProjectUpdateSyncTask.java | 3 +- .../blaze/base/sync/SyncProjectState.java | 5 + .../blaze/base/sync/BlazeSyncManagerTest.java | 2 + .../sync/BlazeSyncIntegrationTestCase.java | 28 ++++ .../idea/blaze/cpp/CppAspectArgsProvider.java | 11 +- 9 files changed, 249 insertions(+), 31 deletions(-) create mode 100644 base/src/com/google/idea/blaze/base/model/ExternalWorkspaceDataProvider.java diff --git a/base/src/META-INF/blaze-base.xml b/base/src/META-INF/blaze-base.xml index 313f805dac1..e45364e215b 100644 --- a/base/src/META-INF/blaze-base.xml +++ b/base/src/META-INF/blaze-base.xml @@ -263,6 +263,7 @@ + @@ -604,6 +605,7 @@ + diff --git a/base/src/com/google/idea/blaze/base/command/mod/BlazeModRunnerImpl.java b/base/src/com/google/idea/blaze/base/command/mod/BlazeModRunnerImpl.java index 12622f271ef..f85fa9ce9fb 100644 --- a/base/src/com/google/idea/blaze/base/command/mod/BlazeModRunnerImpl.java +++ b/base/src/com/google/idea/blaze/base/command/mod/BlazeModRunnerImpl.java @@ -63,7 +63,7 @@ public ListenableFuture dumpRepoMapping( } @Override - public ListenableFuture runBlazeModGetBytes( + protected ListenableFuture runBlazeModGetBytes( Project project, BuildSystem.BuildInvoker invoker, BlazeContext context, diff --git a/base/src/com/google/idea/blaze/base/model/ExternalWorkspaceDataProvider.java b/base/src/com/google/idea/blaze/base/model/ExternalWorkspaceDataProvider.java new file mode 100644 index 00000000000..ebf12613c18 --- /dev/null +++ b/base/src/com/google/idea/blaze/base/model/ExternalWorkspaceDataProvider.java @@ -0,0 +1,129 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.base.model; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.idea.blaze.base.async.executor.BlazeExecutor; +import com.google.idea.blaze.base.bazel.BazelVersion; +import com.google.idea.blaze.base.bazel.BuildSystem; +import com.google.idea.blaze.base.command.mod.BlazeModException; +import com.google.idea.blaze.base.command.mod.BlazeModRunner; +import com.google.idea.blaze.base.scope.BlazeContext; +import com.google.idea.blaze.base.scope.output.StatusOutput; +import com.google.idea.blaze.base.settings.Blaze; +import com.google.idea.blaze.base.settings.BlazeImportSettings; +import com.google.idea.blaze.base.settings.BlazeImportSettingsManager; +import com.google.idea.blaze.base.sync.SyncListener; +import com.google.idea.blaze.base.sync.SyncMode; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.project.Project; +import org.jetbrains.annotations.Nullable; + +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class ExternalWorkspaceDataProvider { + + // bazel mod dump_repo_mapping was added in bazel 7.1.0 + private static final BazelVersion MINIMUM_BLAZE_VERSION = new BazelVersion(7, 1, 0); + + private static final Logger logger = Logger.getInstance(ExternalWorkspaceDataProvider.class); + private final Project project; + + private volatile ExternalWorkspaceData externalWorkspaceData; + + public ExternalWorkspaceDataProvider(Project project) { + this.project = project; + } + + public static ExternalWorkspaceDataProvider getInstance(Project project) { + return project.getService(ExternalWorkspaceDataProvider.class); + } + + static Boolean isEnabled(BlazeVersionData blazeVersionData) { + return blazeVersionData.bazelIsAtLeastVersion(MINIMUM_BLAZE_VERSION); + } + + public ListenableFuture getExternalWorkspaceData( + BlazeContext context, + List blazeFlags, + BlazeVersionData blazeVersionData) { + if (!isEnabled(blazeVersionData)) { + return Futures.immediateFuture(ExternalWorkspaceData.EMPTY); + } + + return BlazeExecutor.getInstance().submit(() -> { + ExternalWorkspaceData mapping = getCachedExternalWorkspaceData(context, blazeFlags); + if (mapping == null) { + throw new BlazeModException("Unable to get module mapping"); + } + return mapping; + }); + } + + private @Nullable ExternalWorkspaceData getCachedExternalWorkspaceData( + BlazeContext context, + List blazeFlags) { + if (externalWorkspaceData != null) { + logger.info("Using cached External Repository Mapping"); + return externalWorkspaceData; + } + try { + BlazeImportSettings importSettings = BlazeImportSettingsManager.getInstance(project).getImportSettings(); + if (importSettings == null) { + return null; + } + BuildSystem.BuildInvoker buildInvoker = + Blaze.getBuildSystemProvider(project) + .getBuildSystem() + .getDefaultInvoker(project, context); + + externalWorkspaceData = BlazeModRunner.getInstance() + .dumpRepoMapping( + project, + buildInvoker, + context, + importSettings.getBuildSystem(), + blazeFlags + ).get(); + return externalWorkspaceData; + } catch (InterruptedException | ExecutionException e) { + logger.warn("Unable to run blaze mod dump_repo_mapping", e); + return null; + } + } + + public void invalidate(BlazeContext context, SyncMode syncMode) { + context.output(new StatusOutput(String.format("Invalidating External Repository Mapping info (%s)", syncMode))); + externalWorkspaceData = null; + } + + public static final class Invalidator implements SyncListener { + @Override + public void onSyncStart( + Project project, + BlazeContext context, + SyncMode syncMode) { + if (syncMode == SyncMode.NO_BUILD || syncMode == SyncMode.FULL) { + ExternalWorkspaceDataProvider provider = ExternalWorkspaceDataProvider.getInstance(project); + if (provider != null) { + provider.invalidate(context, syncMode); + } + } + } + } +} diff --git a/base/src/com/google/idea/blaze/base/sync/ProjectStateSyncTask.java b/base/src/com/google/idea/blaze/base/sync/ProjectStateSyncTask.java index b62a8514e6f..cc49b7f8f2a 100644 --- a/base/src/com/google/idea/blaze/base/sync/ProjectStateSyncTask.java +++ b/base/src/com/google/idea/blaze/base/sync/ProjectStateSyncTask.java @@ -29,9 +29,11 @@ import com.google.idea.blaze.base.command.info.BlazeInfo; import com.google.idea.blaze.base.command.info.BlazeInfoProvider; import com.google.idea.blaze.base.command.info.BlazeInfoRunner; +import com.google.idea.blaze.base.model.ExternalWorkspaceDataProvider; import com.google.idea.blaze.base.execution.ExecutionDeniedException; import com.google.idea.blaze.base.io.FileOperationProvider; import com.google.idea.blaze.base.model.BlazeVersionData; +import com.google.idea.blaze.base.model.ExternalWorkspaceData; import com.google.idea.blaze.base.model.primitives.WorkspaceRoot; import com.google.idea.blaze.base.plugin.BuildSystemVersionChecker; import com.google.idea.blaze.base.projectview.ProjectViewManager; @@ -58,6 +60,7 @@ import com.google.idea.blaze.common.PrintOutput; import com.google.idea.blaze.exception.BuildException; import com.intellij.openapi.project.Project; + import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -119,10 +122,10 @@ private SyncProjectState getProjectState(BlazeContext context, BlazeSyncParams p BlazeInvocationContext.SYNC_CONTEXT); ListenableFuture blazeInfoFuture = - createBazelInfoFuture(context, syncFlags, params.syncMode()); + createBazelInfoFuture(context, syncFlags, params.syncMode()); ListenableFuture workingSetFuture; - if(params.addWorkingSet() || params.syncMode() == SyncMode.FULL) { + if (params.addWorkingSet() || params.syncMode() == SyncMode.FULL) { workingSetFuture = vcsHandler.getWorkingSet(context, executor); } else { workingSetFuture = Futures.immediateFuture(null); @@ -142,12 +145,13 @@ private SyncProjectState getProjectState(BlazeContext context, BlazeSyncParams p if (exception != null) { Throwable cause = exception.getCause(); if (cause instanceof BuildException - && cause.getCause() instanceof ExecutionDeniedException) { + && cause.getCause() instanceof ExecutionDeniedException) { throw new SyncCanceledException(); } } throw new SyncFailedException(); } + BlazeVersionData blazeVersionData = BlazeVersionData.build( Blaze.getBuildSystemProvider(project).getBuildSystem(), workspaceRoot, blazeInfo); @@ -156,6 +160,9 @@ private SyncProjectState getProjectState(BlazeContext context, BlazeSyncParams p throw new SyncFailedException(); } + ExternalWorkspaceData externalWorkspaceData = + getExternalWorkspaceData(context, projectViewSet, blazeVersionData, params.syncMode()); + WorkspacePathResolver workspacePathResolver = workspacePathResolverAndProjectView.workspacePathResolver; WorkspaceLanguageSettings workspaceLanguageSettings = @@ -186,35 +193,78 @@ private SyncProjectState getProjectState(BlazeContext context, BlazeSyncParams p printWorkingSet(context, workingSet); } return SyncProjectState.builder() - .setProjectViewSet(projectViewSet) - .setLanguageSettings(workspaceLanguageSettings) - .setBlazeVersionData(blazeVersionData) - .setWorkingSet(workingSet) - .setWorkspacePathResolver(workspacePathResolver) - .build(); + .setProjectViewSet(projectViewSet) + .setLanguageSettings(workspaceLanguageSettings) + .setBlazeVersionData(blazeVersionData) + .setWorkingSet(workingSet) + .setWorkspacePathResolver(workspacePathResolver) + .setExternalWorkspaceData(externalWorkspaceData) + .build(); } private ListenableFuture createBazelInfoFuture( - BlazeContext context, - List syncFlags, - SyncMode syncMode) { + BlazeContext context, + List syncFlags, + SyncMode syncMode) { boolean useBazelInfoRunner = !BlazeInfoProvider.isEnabled() || syncMode == SyncMode.FULL; if (useBazelInfoRunner) { return BlazeInfoRunner.getInstance() - .runBlazeInfo( - project, - Blaze.getBuildSystemProvider(project) - .getBuildSystem() - .getDefaultInvoker(project, - context), - context, - importSettings.getBuildSystem(), - syncFlags); + .runBlazeInfo( + project, + Blaze.getBuildSystemProvider(project) + .getBuildSystem() + .getDefaultInvoker(project, + context), + context, + importSettings.getBuildSystem(), + syncFlags); } return BlazeInfoProvider.getInstance(project) - .getBlazeInfo( - context, - syncFlags); + .getBlazeInfo( + context, + syncFlags); + } + + private ExternalWorkspaceData getExternalWorkspaceData( + BlazeContext context, + ProjectViewSet projectViewSet, + BlazeVersionData blazeVersionData, + SyncMode syncMode) + throws SyncCanceledException, SyncFailedException { + + List syncFlags = + BlazeFlags.blazeFlags( + project, + projectViewSet, + BlazeCommandName.MOD, + context, + BlazeInvocationContext.SYNC_CONTEXT); + + ListenableFuture externalWorkspaceDataFuture = + ExternalWorkspaceDataProvider.getInstance(project) + .getExternalWorkspaceData(context, syncFlags, blazeVersionData); + + FutureResult externalWorkspaceDataResult = + FutureUtil.waitForFuture(context, externalWorkspaceDataFuture) + .timed(Blaze.buildSystemName(project) + "Mod", EventType.BlazeInvocation) + .withProgressMessage("Resolving module repository mapping...") + .onError(String.format("Could not run %s mod dump_repo_mapping", Blaze.buildSystemName(project))) + .run(); + + ExternalWorkspaceData externalWorkspaceData = externalWorkspaceDataResult.result(); + if (externalWorkspaceData == null) { + Exception exception = externalWorkspaceDataResult.exception(); + if (exception != null) { + Throwable cause = exception.getCause(); + if (cause instanceof BuildException + && cause.getCause() instanceof ExecutionDeniedException) { + throw new SyncCanceledException(); + } + } + throw new SyncFailedException(); + } + + return externalWorkspaceData; } private static class WorkspacePathResolverAndProjectView { diff --git a/base/src/com/google/idea/blaze/base/sync/ProjectUpdateSyncTask.java b/base/src/com/google/idea/blaze/base/sync/ProjectUpdateSyncTask.java index 681e1e0ec47..174d9679903 100644 --- a/base/src/com/google/idea/blaze/base/sync/ProjectUpdateSyncTask.java +++ b/base/src/com/google/idea/blaze/base/sync/ProjectUpdateSyncTask.java @@ -28,7 +28,6 @@ import com.google.idea.blaze.base.model.BlazeLibrary; import com.google.idea.blaze.base.model.BlazeProjectData; import com.google.idea.blaze.base.model.BlazeVersionData; -import com.google.idea.blaze.base.model.ExternalWorkspaceData; import com.google.idea.blaze.base.model.ProjectTargetData; import com.google.idea.blaze.base.model.RemoteOutputArtifacts; import com.google.idea.blaze.base.model.SyncState; @@ -228,7 +227,7 @@ private void run(BlazeContext context) throws SyncCanceledException, SyncFailedE projectState.getWorkspacePathResolver(), artifactLocationDecoder, projectState.getLanguageSettings(), - ExternalWorkspaceData.EMPTY, + projectState.getExternalWorkspaceData(), syncStateBuilder.build()); FileCaches.onSync( diff --git a/base/src/com/google/idea/blaze/base/sync/SyncProjectState.java b/base/src/com/google/idea/blaze/base/sync/SyncProjectState.java index 45e421e9f6c..8eea0fe41c5 100644 --- a/base/src/com/google/idea/blaze/base/sync/SyncProjectState.java +++ b/base/src/com/google/idea/blaze/base/sync/SyncProjectState.java @@ -17,6 +17,7 @@ import com.google.auto.value.AutoValue; import com.google.idea.blaze.base.model.BlazeVersionData; +import com.google.idea.blaze.base.model.ExternalWorkspaceData; import com.google.idea.blaze.base.projectview.ProjectViewSet; import com.google.idea.blaze.base.sync.projectview.WorkspaceLanguageSettings; import com.google.idea.blaze.base.sync.workspace.WorkingSet; @@ -38,6 +39,8 @@ public abstract class SyncProjectState { public abstract WorkspacePathResolver getWorkspacePathResolver(); + public abstract ExternalWorkspaceData getExternalWorkspaceData(); + public static Builder builder() { return new AutoValue_SyncProjectState.Builder(); } @@ -55,6 +58,8 @@ public abstract static class Builder { public abstract Builder setWorkspacePathResolver(WorkspacePathResolver pathResolver); + public abstract Builder setExternalWorkspaceData(ExternalWorkspaceData externalWorkspaceData); + public abstract SyncProjectState build(); } } diff --git a/base/tests/unittests/com/google/idea/blaze/base/sync/BlazeSyncManagerTest.java b/base/tests/unittests/com/google/idea/blaze/base/sync/BlazeSyncManagerTest.java index b0348c2d865..796172c5d7f 100644 --- a/base/tests/unittests/com/google/idea/blaze/base/sync/BlazeSyncManagerTest.java +++ b/base/tests/unittests/com/google/idea/blaze/base/sync/BlazeSyncManagerTest.java @@ -28,6 +28,7 @@ import com.google.idea.blaze.base.bazel.BuildSystemProvider; import com.google.idea.blaze.base.model.BlazeProjectData; import com.google.idea.blaze.base.model.BlazeVersionData; +import com.google.idea.blaze.base.model.ExternalWorkspaceData; import com.google.idea.blaze.base.model.MockBlazeProjectDataBuilder; import com.google.idea.blaze.base.model.primitives.LanguageClass; import com.google.idea.blaze.base.model.primitives.TargetExpression; @@ -192,6 +193,7 @@ private SyncProjectState getMockProjectStateWithLanguages( .setBlazeVersionData(BlazeVersionData.builder().build()) .setWorkspacePathResolver( new WorkspacePathResolverImpl(WorkspaceRoot.fromProjectSafe(project))) + .setExternalWorkspaceData(ExternalWorkspaceData.EMPTY) .build(); } diff --git a/base/tests/utils/integration/com/google/idea/blaze/base/sync/BlazeSyncIntegrationTestCase.java b/base/tests/utils/integration/com/google/idea/blaze/base/sync/BlazeSyncIntegrationTestCase.java index 0fac21b2f7a..4a791df1978 100644 --- a/base/tests/utils/integration/com/google/idea/blaze/base/sync/BlazeSyncIntegrationTestCase.java +++ b/base/tests/utils/integration/com/google/idea/blaze/base/sync/BlazeSyncIntegrationTestCase.java @@ -31,11 +31,13 @@ import com.google.idea.blaze.base.command.BlazeInvocationContext; import com.google.idea.blaze.base.command.info.BlazeInfo; import com.google.idea.blaze.base.command.info.BlazeInfoRunner; +import com.google.idea.blaze.base.command.mod.BlazeModRunner; import com.google.idea.blaze.base.ideinfo.ArtifactLocation; import com.google.idea.blaze.base.ideinfo.TargetMap; import com.google.idea.blaze.base.logging.utils.SyncStats; import com.google.idea.blaze.base.model.AspectSyncProjectData; import com.google.idea.blaze.base.model.BlazeVersionData; +import com.google.idea.blaze.base.model.ExternalWorkspaceData; import com.google.idea.blaze.base.model.ProjectTargetData; import com.google.idea.blaze.base.model.RemoteOutputArtifacts; import com.google.idea.blaze.base.model.primitives.WorkspaceRoot; @@ -101,6 +103,7 @@ public abstract class BlazeSyncIntegrationTestCase extends BlazeIntegrationTestC private Disposable thisClassDisposable; // disposed prior to calling parent class's @After methods private MockProjectViewManager projectViewManager; private MockBlazeInfoRunner blazeInfoData; + private MockBlazeModRunner blazeModData; private MockBlazeIdeInterface blazeIdeInterface; private MockEventLoggingService eventLogger; @Nullable private ProjectModuleMocker moduleMocker; // this will be null for heavy test cases @@ -115,12 +118,14 @@ public void doSetup() throws Throwable { ServiceHelper.registerExtension( BlazeVcsHandlerProvider.EP_NAME, new MockBlazeVcsHandlerProvider(), thisClassDisposable); blazeInfoData = new MockBlazeInfoRunner(); + blazeModData = new MockBlazeModRunner(); blazeIdeInterface = new MockBlazeIdeInterface(); eventLogger = new MockEventLoggingService(thisClassDisposable); if (isLightTestCase()) { moduleMocker = new ProjectModuleMocker(getProject(), thisClassDisposable); } registerApplicationService(BlazeInfoRunner.class, blazeInfoData); + registerApplicationService(BlazeModRunner.class, blazeModData); registerApplicationService(BlazeIdeInterface.class, blazeIdeInterface); errorCollector = new ErrorCollector(); @@ -312,6 +317,7 @@ public void setResults(Map results) { this.results.putAll(results); } } + private static class MockBlazeIdeInterface implements BlazeIdeInterface { private TargetMap targetMap = new TargetMap(ImmutableMap.of()); @@ -343,4 +349,26 @@ public BlazeBuildOutputs build( return BlazeBuildOutputs.noOutputs(BuildResult.SUCCESS); } } + + private static class MockBlazeModRunner extends BlazeModRunner { + @Override + public ListenableFuture dumpRepoMapping( + Project project, + BuildInvoker invoker, + BlazeContext context, + BuildSystemName buildSystemName, + List flags) { + return Futures.immediateFuture(ExternalWorkspaceData.EMPTY); + } + + @Override + protected ListenableFuture runBlazeModGetBytes( + Project project, + BuildInvoker invoker, + BlazeContext context, + List args, + List flags) { + return Futures.immediateFuture(null); + } + } } diff --git a/cpp/src/com/google/idea/blaze/cpp/CppAspectArgsProvider.java b/cpp/src/com/google/idea/blaze/cpp/CppAspectArgsProvider.java index 7335e72b4d4..3a4e6ba30eb 100644 --- a/cpp/src/com/google/idea/blaze/cpp/CppAspectArgsProvider.java +++ b/cpp/src/com/google/idea/blaze/cpp/CppAspectArgsProvider.java @@ -46,9 +46,12 @@ public void addSyncFlags( BlazeContext context, BlazeInvocationContext invocationContext, List flags) { - flags.add(String.format( - "--define=%s=%b", CPP_USE_GET_TOOL_FOR_ACTION, - Registry.is(REGISTRY_PREFIX + CPP_USE_GET_TOOL_FOR_ACTION) - )); + // bazel mod does not support this flag + if (command != BlazeCommandName.MOD) { + flags.add(String.format( + "--define=%s=%b", CPP_USE_GET_TOOL_FOR_ACTION, + Registry.is(REGISTRY_PREFIX + CPP_USE_GET_TOOL_FOR_ACTION) + )); + } } }