Merge pull request #9083 from lassewesth/reuseofprocedurefacades

reuseofprocedurefacades

executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java

@@ -25,12 +25,12 @@
 import org.neo4j.gds.BaseTest;
 import org.neo4j.gds.GdsCypher;
 import org.neo4j.gds.NodeProjections;
-import org.neo4j.gds.ProcedureCallContextReturnColumns;
 import org.neo4j.gds.RelationshipProjections;
 import org.neo4j.gds.api.AlgorithmMetaDataSetter;
 import org.neo4j.gds.api.CloseableResourceRegistry;
 import org.neo4j.gds.api.DatabaseId;
 import org.neo4j.gds.api.NodeLookup;
+import org.neo4j.gds.api.ProcedureReturnColumns;
 import org.neo4j.gds.catalog.GraphProjectProc;
 import org.neo4j.gds.compat.GraphDatabaseApiProxy;
 import org.neo4j.gds.compat.Neo4jProxy;
@@ -75,7 +75,7 @@ void setup() throws Exception {
             .builder()
             .databaseId(DatabaseId.of(db.databaseName()))
             .dependencyResolver(GraphDatabaseApiProxy.dependencyResolver(db))
-            .returnColumns(ProcedureCallContextReturnColumns.EMPTY)
+            .returnColumns(ProcedureReturnColumns.EMPTY)
             .log(Neo4jProxy.testLog())
             .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE)
             .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE)

executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java

@@ -22,11 +22,11 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.neo4j.gds.ProcedureCallContextReturnColumns;
 import org.neo4j.gds.api.AlgorithmMetaDataSetter;
 import org.neo4j.gds.api.CloseableResourceRegistry;
 import org.neo4j.gds.api.GraphStore;
 import org.neo4j.gds.api.NodeLookup;
+import org.neo4j.gds.api.ProcedureReturnColumns;
 import org.neo4j.gds.compat.Neo4jProxy;
 import org.neo4j.gds.config.GraphProjectConfig;
 import org.neo4j.gds.core.loading.GraphStoreCatalog;
@@ -133,7 +133,7 @@ private ExecutionContext executionContext(TaskStore taskStore) {
             .builder()
             .databaseId(graphStore.databaseInfo().databaseId())
             .log(Neo4jProxy.testLog())
-            .returnColumns(ProcedureCallContextReturnColumns.EMPTY)
+            .returnColumns(ProcedureReturnColumns.EMPTY)
             .taskRegistryFactory(jobId -> new TaskRegistry("", taskStore, jobId))
             .username("")
             .terminationMonitor(TerminationMonitor.EMPTY)

proc/common/src/main/java/org/neo4j/gds/BaseProc.java

@@ -33,6 +33,9 @@
 import org.neo4j.gds.executor.ImmutableExecutionContext;
 import org.neo4j.gds.metrics.MetricsFacade;
 import org.neo4j.gds.procedures.GraphDataScienceProcedures;
+import org.neo4j.gds.procedures.ProcedureCallContextReturnColumns;
+import org.neo4j.gds.procedures.TransactionCloseableResourceRegistry;
+import org.neo4j.gds.procedures.TransactionNodeLookup;
 import org.neo4j.gds.procedures.integration.TransactionTerminationMonitor;
 import org.neo4j.gds.transaction.DatabaseTransactionContext;
 import org.neo4j.gds.transaction.EmptyTransactionContext;

proc/test/build.gradle

@@ -34,6 +34,7 @@ dependencies {
     implementation project(':neo4j-api')
     implementation project(':opengds-procedure-facade')
     implementation project(':open-licensing')
+    implementation project(':open-model-catalog')
     implementation project(':pregel')
     implementation project(':procedure-integration')
     implementation project(':progress-tracking')

proc/test/src/main/java/org/neo4j/gds/ProcedureRunner.java

@@ -19,12 +19,11 @@
  */
 package org.neo4j.gds;
 
-import org.neo4j.gds.algorithms.similarity.WriteRelationshipService;
+import org.jetbrains.annotations.Nullable;
 import org.neo4j.gds.api.AlgorithmMetaDataSetter;
+import org.neo4j.gds.api.DatabaseId;
 import org.neo4j.gds.api.GraphLoaderContext;
 import org.neo4j.gds.api.User;
-import org.neo4j.gds.applications.ApplicationsFacade;
-import org.neo4j.gds.applications.algorithms.machinery.AlgorithmEstimationTemplate;
 import org.neo4j.gds.applications.algorithms.machinery.DefaultAlgorithmProcessingTemplate;
 import org.neo4j.gds.applications.algorithms.machinery.MemoryGuard;
 import org.neo4j.gds.applications.algorithms.machinery.RequestScopedDependencies;
@@ -33,27 +32,32 @@
 import org.neo4j.gds.configuration.LimitsConfiguration;
 import org.neo4j.gds.core.Username;
 import org.neo4j.gds.core.loading.GraphStoreCatalogService;
+import org.neo4j.gds.core.model.OpenModelCatalog;
 import org.neo4j.gds.core.utils.progress.TaskRegistryFactory;
 import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory;
 import org.neo4j.gds.core.utils.warnings.UserLogRegistryFactory;
-import org.neo4j.gds.memest.DatabaseGraphStoreEstimationService;
+import org.neo4j.gds.core.write.ExporterContext;
+import org.neo4j.gds.core.write.NativeExportBuildersProvider;
 import org.neo4j.gds.metrics.MetricsFacade;
+import org.neo4j.gds.metrics.PassthroughExecutionMetricRegistrar;
+import org.neo4j.gds.metrics.algorithms.AlgorithmMetricsService;
+import org.neo4j.gds.metrics.procedures.DeprecatedProceduresMetricService;
+import org.neo4j.gds.modelcatalogservices.ModelCatalogServiceProvider;
+import org.neo4j.gds.procedures.AlgorithmFacadeBuilderFactory;
+import org.neo4j.gds.procedures.CatalogProcedureFacadeFactory;
 import org.neo4j.gds.procedures.GraphDataScienceProcedures;
-import org.neo4j.gds.procedures.GraphDataScienceProceduresBuilder;
-import org.neo4j.gds.procedures.algorithms.centrality.CentralityProcedureFacade;
-import org.neo4j.gds.procedures.algorithms.configuration.ConfigurationCreator;
-import org.neo4j.gds.procedures.algorithms.configuration.ConfigurationParser;
-import org.neo4j.gds.procedures.algorithms.runners.EstimationModeRunner;
-import org.neo4j.gds.procedures.algorithms.runners.StatsModeAlgorithmRunner;
-import org.neo4j.gds.procedures.algorithms.runners.StreamModeAlgorithmRunner;
-import org.neo4j.gds.procedures.algorithms.runners.WriteModeAlgorithmRunner;
-import org.neo4j.gds.procedures.algorithms.stubs.GenericStub;
+import org.neo4j.gds.procedures.TaskRegistryFactoryService;
 import org.neo4j.gds.procedures.integration.LogAdapter;
-import org.neo4j.gds.services.DatabaseIdAccessor;
+import org.neo4j.gds.procedures.DatabaseIdAccessor;
+import org.neo4j.gds.procedures.UserLogServices;
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.internal.kernel.api.procs.ProcedureCallContext;
+import org.neo4j.internal.kernel.api.security.AuthSubject;
+import org.neo4j.internal.kernel.api.security.AuthenticationResult;
+import org.neo4j.internal.kernel.api.security.SecurityContext;
 import org.neo4j.kernel.api.KernelTransaction;
+import org.neo4j.kernel.impl.coreapi.InternalTransaction;
 import org.neo4j.logging.Log;
 
 import java.util.Optional;
@@ -117,6 +121,7 @@ public static <P extends BaseProc> P applyOnProcedure(
             databaseService,
             procedureCallContext,
             taskRegistryFactory,
+            tx,
             username
         );
 
@@ -154,70 +159,103 @@ private static GraphDataScienceProcedures createGraphDataScienceProcedures(
         GraphDatabaseService graphDatabaseService,
         ProcedureCallContext procedureCallContext,
         TaskRegistryFactory taskRegistryFactory,
+        Transaction procedureTransaction,
         Username username
     ) {
         var gdsLog = new LogAdapter(log);
 
-        var configurationParser = new ConfigurationParser(DefaultsConfiguration.Instance, LimitsConfiguration.Instance);
-        var graphStoreCatalogService = new GraphStoreCatalogService();
         var requestScopedDependencies = RequestScopedDependencies.builder()
             .with(new DatabaseIdAccessor().getDatabaseId(graphDatabaseService))
             .with(taskRegistryFactory)
             .with(new User(username.username(), false))
             .with(EmptyUserLogRegistryFactory.INSTANCE)
             .build();
+        var graphStoreCatalogService = new GraphStoreCatalogService();
 
-        var configurationCreator = new ConfigurationCreator(
-            configurationParser,
-            AlgorithmMetaDataSetter.EMPTY,
-            requestScopedDependencies.getUser()
-        );
-
-        var algorithmEstimationTemplate = new AlgorithmEstimationTemplate(
+        var algorithmProcessingTemplate = new DefaultAlgorithmProcessingTemplate(
+            gdsLog,
+            MetricsFacade.PASSTHROUGH_METRICS_FACADE.algorithmMetrics(),
             graphStoreCatalogService,
-            new DatabaseGraphStoreEstimationService(
-                GraphLoaderContext.NULL_CONTEXT,
-                requestScopedDependencies.getUser()
-            ),
+            MemoryGuard.DISABLED,
             requestScopedDependencies
         );
 
-        var closeableResourceRegistry = new TransactionCloseableResourceRegistry(kernelTransaction);
-
-        var centralityProcedureFacade = CentralityProcedureFacade.create(
-            new GenericStub(
-                DefaultsConfiguration.Instance,
-                LimitsConfiguration.Instance,
-                configurationCreator,
-                configurationParser,
-                requestScopedDependencies.getUser(),
-                algorithmEstimationTemplate
-            ),
-            ApplicationsFacade.create(
-                gdsLog,
-                Optional.empty(),
-                graphStoreCatalogService,
-                MetricsFacade.PASSTHROUGH_METRICS_FACADE.projectionMetrics(),
-                algorithmEstimationTemplate,
-                new DefaultAlgorithmProcessingTemplate(
-                    gdsLog,
-                    MetricsFacade.PASSTHROUGH_METRICS_FACADE.algorithmMetrics(),
-                    graphStoreCatalogService,
-                    MemoryGuard.DISABLED,
-                    requestScopedDependencies
-                ),
-                requestScopedDependencies,
-                new WriteRelationshipService(gdsLog, requestScopedDependencies)
-            ),
-            new ProcedureCallContextReturnColumns(procedureCallContext),
-            new EstimationModeRunner(configurationCreator),
-            new StatsModeAlgorithmRunner(configurationCreator),
-            new StreamModeAlgorithmRunner(closeableResourceRegistry, configurationCreator),
-            new WriteModeAlgorithmRunner(configurationCreator)
+        var catalogProcedureFacadeFactory = new CatalogProcedureFacadeFactory(
+            gdsLog,
+            __ -> new NativeExportBuildersProvider(), // procedure runner is OpenGDS
+            new TaskRegistryFactoryService(false, null) {
+                @Override
+                public TaskRegistryFactory getTaskRegistryFactory(DatabaseId databaseId, User user) {
+                    return taskRegistryFactory;
+                }
+            },
+            new UserLogServices()
         );
 
-        return new GraphDataScienceProceduresBuilder(gdsLog)
-            .with(centralityProcedureFacade)
-            .build();
+        var securityContext = new SecurityContext(new AuthSubject() {
+            @Override
+            public AuthenticationResult getAuthenticationResult() {
+                throw new UnsupportedOperationException("TODO");
+            }
+
+            @Override
+            public boolean hasUsername(String username) {
+                throw new UnsupportedOperationException("TODO");
+            }
+
+            @Override
+            public String executingUser() {
+                return username.username();
+            }
+        }, null, null, null);
+
+        var exporterContext = new ExporterContext() {
+            @Override
+            public GraphDatabaseService graphDatabaseAPI() {
+                return graphDatabaseService;
+            }
+
+            @Override
+            public @Nullable InternalTransaction internalTransaction() {
+                return (InternalTransaction) procedureTransaction;
+            }
+
+            @Override
+            public SecurityContext securityContext() {
+                return securityContext;
+            }
+        };
+
+        var modelCatalog = new OpenModelCatalog();
+
+        var algorithmFacadeBuilderFactory = new AlgorithmFacadeBuilderFactory(
+            gdsLog,
+            graphStoreCatalogService,
+            false,
+            new AlgorithmMetricsService(new PassthroughExecutionMetricRegistrar()),
+            new ModelCatalogServiceProvider(modelCatalog)
+        );
+
+        return GraphDataScienceProcedures.create(
+            gdsLog,
+            DefaultsConfiguration.Instance,
+            LimitsConfiguration.Instance,
+            Optional.empty(),
+            graphStoreCatalogService,
+            MetricsFacade.PASSTHROUGH_METRICS_FACADE.projectionMetrics(),
+            AlgorithmMetaDataSetter.EMPTY,
+            algorithmProcessingTemplate,
+            kernelTransaction,
+            GraphLoaderContext.NULL_CONTEXT,
+            procedureCallContext,
+            requestScopedDependencies,
+            catalogProcedureFacadeFactory,
+            securityContext,
+            exporterContext,
+            graphDatabaseService,
+            procedureTransaction,
+            algorithmFacadeBuilderFactory,
+            DeprecatedProceduresMetricService.PASSTHROUGH
+        );
     }
 }

procedures/extension/src/main/java/org/neo4j/gds/extension/OpenGraphDataScienceExtension.java

@@ -24,7 +24,7 @@
 import org.neo4j.gds.core.model.OpenModelCatalogProvider;
 import org.neo4j.gds.core.write.NativeExportBuildersProvider;
 import org.neo4j.gds.metrics.MetricsFacade;
-import org.neo4j.gds.procedures.integration.ExporterBuildersProviderService;
+import org.neo4j.gds.procedures.ExporterBuildersProviderService;
 import org.neo4j.gds.procedures.integration.GraphDataScienceExtensionBuilder;
 import org.neo4j.gds.procedures.integration.LogAccessor;
 import org.neo4j.kernel.api.procedure.GlobalProcedures;

procedures/facade/build.gradle

@@ -32,6 +32,7 @@ dependencies {
     implementation project(':graph-store-catalog-applications')
     implementation project(':legacy-cypher-projection')
     implementation project(':logging')
+    implementation project(':memory-estimation')
     implementation project(':memory-usage')
     implementation project(':metrics-api')
     implementation project(':ml-algo')

procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmFacadeFactory.java → procedures/facade/src/main/java/org/neo4j/gds/procedures/AlgorithmFacadeBuilder.java

@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-package org.neo4j.gds.procedures.integration;
+package org.neo4j.gds.procedures;
 
 import org.neo4j.gds.algorithms.centrality.CentralityAlgorithmsEstimateBusinessFacade;
 import org.neo4j.gds.algorithms.centrality.CentralityAlgorithmsFacade;
@@ -65,7 +65,7 @@
 import org.neo4j.gds.procedures.embeddings.NodeEmbeddingsProcedureFacade;
 import org.neo4j.gds.procedures.misc.MiscAlgorithmsProcedureFacade;
 
-class AlgorithmFacadeFactory {
+class AlgorithmFacadeBuilder {
     // Request scoped parameters
     private final ConfigurationCreator configurationCreator;
     private final NodeLookup nodeLookup;
@@ -82,7 +82,7 @@ class AlgorithmFacadeFactory {
     private final StatsModeAlgorithmRunner statsModeAlgorithmRunner;
     private final WriteModeAlgorithmRunner writeModeAlgorithmRunner;
 
-    AlgorithmFacadeFactory(
+    AlgorithmFacadeBuilder(
         ConfigurationCreator configurationCreator,
         NodeLookup nodeLookup,
         ProcedureReturnColumns procedureReturnColumns,

procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmFacadeFactoryProvider.java → procedures/facade/src/main/java/org/neo4j/gds/procedures/AlgorithmFacadeBuilderFactory.java

@@ -17,11 +17,8 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-package org.neo4j.gds.procedures.integration;
+package org.neo4j.gds.procedures;
 
-import org.neo4j.gds.ProcedureCallContextReturnColumns;
-import org.neo4j.gds.TransactionCloseableResourceRegistry;
-import org.neo4j.gds.TransactionNodeLookup;
 import org.neo4j.gds.algorithms.AlgorithmMemoryValidationService;
 import org.neo4j.gds.algorithms.estimation.AlgorithmEstimator;
 import org.neo4j.gds.algorithms.mutateservices.MutateNodePropertyService;
@@ -42,10 +39,10 @@
 import org.neo4j.gds.procedures.algorithms.runners.WriteModeAlgorithmRunner;
 import org.neo4j.gds.procedures.algorithms.stubs.GenericStub;
 import org.neo4j.graphdb.GraphDatabaseService;
+import org.neo4j.internal.kernel.api.procs.ProcedureCallContext;
 import org.neo4j.kernel.api.KernelTransaction;
-import org.neo4j.kernel.api.procedure.Context;
 
-class AlgorithmFacadeFactoryProvider {
+public class AlgorithmFacadeBuilderFactory {
     // dull utilities
     private final FictitiousGraphStoreEstimationService fictitiousGraphStoreEstimationService = new FictitiousGraphStoreEstimationService();
 
@@ -58,7 +55,7 @@ class AlgorithmFacadeFactoryProvider {
     private final AlgorithmMetricsService algorithmMetricsService;
     private final ModelCatalogServiceProvider modelCatalogServiceProvider;
 
-    AlgorithmFacadeFactoryProvider(
+    public AlgorithmFacadeBuilderFactory(
         Log log,
         GraphStoreCatalogService graphStoreCatalogService,
         boolean useMaxMemoryEstimation,
@@ -73,15 +70,15 @@ class AlgorithmFacadeFactoryProvider {
         this.modelCatalogServiceProvider = modelCatalogServiceProvider;
     }
 
-    AlgorithmFacadeFactory createAlgorithmFacadeFactory(
-        Context context,
+    AlgorithmFacadeBuilder create(
         ConfigurationCreator configurationCreator,
         RequestScopedDependencies requestScopedDependencies,
         KernelTransaction kernelTransaction,
         GraphDatabaseService graphDatabaseService,
         DatabaseGraphStoreEstimationService databaseGraphStoreEstimationService,
         ApplicationsFacade applicationsFacade,
-        GenericStub genericStub
+        GenericStub genericStub,
+        ProcedureCallContext procedureCallContext
     ) {
         /*
          * GDS services derived from Procedure Context.
@@ -92,7 +89,7 @@ AlgorithmFacadeFactory createAlgorithmFacadeFactory(
         var algorithmMemoryValidationService = new AlgorithmMemoryValidationService(log, useMaxMemoryEstimation);
         var mutateNodePropertyService = new MutateNodePropertyService(log);
         var nodeLookup = new TransactionNodeLookup(kernelTransaction);
-        var procedureReturnColumns = new ProcedureCallContextReturnColumns(context.procedureCallContext());
+        var procedureReturnColumns = new ProcedureCallContextReturnColumns(procedureCallContext);
 
         // Second layer
         var writeNodePropertyService = new WriteNodePropertyService(log, requestScopedDependencies);
@@ -119,7 +116,7 @@ AlgorithmFacadeFactory createAlgorithmFacadeFactory(
         var writeModeAlgorithmRunner = new WriteModeAlgorithmRunner(configurationCreator);
 
         // procedure facade
-        return new AlgorithmFacadeFactory(
+        return new AlgorithmFacadeBuilder(
             configurationCreator,
             nodeLookup,
             procedureReturnColumns,