feat(backend): structured properties and forms (datahub-project#9626)

Co-authored-by: Chris Collins <[email protected]> Co-authored-by: RyanHolstien <[email protected]>
acryldata · Jan 22, 2024 · 943bb57 · 943bb57
1 parent 1d16e42
commit 943bb57
Show file tree

Hide file tree

Showing 492 changed files with 15,378 additions and 1,931 deletions.
diff --git a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java
@@ -183,6 +183,7 @@ private void generateSchema(final File file) {
     final String fileBaseName;
     try {
       final JsonNode schema = JsonLoader.fromFile(file);
+
       final JsonNode result = buildResult(schema.toString());
       String prettySchema = JacksonUtils.prettyPrint(result);
       Path absolutePath = file.getAbsoluteFile().toPath();
@@ -195,11 +196,21 @@ private void generateSchema(final File file) {
       } else {
         fileBaseName = getBaseName(file.getName());
       }
-      Files.write(Paths.get(jsonDirectory + sep + fileBaseName + ".json"),
+
+      final String targetName;
+      if (schema.has("Aspect") && schema.get("Aspect").has("name") &&
+              !schema.get("Aspect").get("name").asText().equalsIgnoreCase(fileBaseName)) {
+        targetName = OpenApiEntities.toUpperFirst(schema.get("Aspect").get("name").asText());
+        prettySchema = prettySchema.replaceAll(fileBaseName, targetName);
+      } else {
+        targetName = fileBaseName;
+      }
+
+      Files.write(Paths.get(jsonDirectory + sep + targetName + ".json"),
           prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
           StandardOpenOption.TRUNCATE_EXISTING);
       if (schema.has("Aspect")) {
-        aspectType.add(NODE_FACTORY.objectNode().put("$ref", "#/definitions/" + getBaseName(file.getName())));
+        aspectType.add(NODE_FACTORY.objectNode().put("$ref", "#/definitions/" + targetName));
       }
     } catch (IOException | ProcessingException e) {
       throw new RuntimeException(e);

diff --git a/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java b/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java
@@ -6,6 +6,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.metadata.models.registry.config.Entities;
 import com.linkedin.metadata.models.registry.config.Entity;
@@ -58,8 +59,12 @@ public class OpenApiEntities {
                 .add("notebookInfo").add("editableNotebookProperties")
                 .add("dataProductProperties")
                 .add("institutionalMemory")
+                .add("forms").add("formInfo").add("dynamicFormAssignment")
                 .build();
 
+    private final static ImmutableSet<String> ENTITY_EXCLUSIONS = ImmutableSet.<String>builder()
+            .add("structuredProperty")
+            .build();
 
     public OpenApiEntities(JsonNodeFactory NODE_FACTORY) {
         this.NODE_FACTORY = NODE_FACTORY;
@@ -117,14 +122,27 @@ public ObjectNode entityExtension(List<ObjectNode> nodesList, ObjectNode schemas
         return componentsNode;
     }
 
-    private static String toUpperFirst(String s) {
-        return s.substring(0, 1).toUpperCase() + s.substring(1);
+    /**
+     * Convert the pdl model names to desired class names. Upper case first letter unless the 3rd character is upper case.
+     * i.e. mlModel -> MLModel
+     *      dataset -> Dataset
+     *      dataProduct -> DataProduct
+     * @param s input string
+     * @return class name
+     */
+    public static String toUpperFirst(String s) {
+        if (s.length() > 2 && s.substring(2, 3).equals(s.substring(2, 3).toUpperCase())) {
+            return s.substring(0, 2).toUpperCase() + s.substring(2);
+        } else {
+            return s.substring(0, 1).toUpperCase() + s.substring(1);
+        }
     }
 
     private Set<String> withEntitySchema(ObjectNode schemasNode, Set<String> definitions) {
         return entityMap.values().stream()
                 // Make sure the primary key is defined
                 .filter(entity -> definitions.contains(toUpperFirst(entity.getKeyAspect())))
+                .filter(entity -> !ENTITY_EXCLUSIONS.contains(entity.getName()))
                 .map(entity -> {
                     final String upperName = toUpperFirst(entity.getName());
 
@@ -547,7 +565,7 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) {
 
         ObjectNode getMethod = NODE_FACTORY.objectNode()
                 .put("summary", String.format("Get %s for %s.", aspect, entity.getName()))
-                .put("operationId", String.format("get%s", upperFirstAspect, upperFirstEntity));
+                .put("operationId", String.format("get%s", upperFirstAspect));
         getMethod.set("tags", tagsNode);
         ArrayNode singlePathParametersNode = NODE_FACTORY.arrayNode();
         getMethod.set("parameters", singlePathParametersNode);
@@ -575,13 +593,13 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) {
                         .set("application/json", NODE_FACTORY.objectNode())));
         ObjectNode headMethod = NODE_FACTORY.objectNode()
                 .put("summary", String.format("%s on %s existence.", aspect, upperFirstEntity))
-                .put("operationId", String.format("head%s", upperFirstAspect, upperFirstEntity))
+                .put("operationId", String.format("head%s", upperFirstAspect))
                 .set("responses", headResponses);
         headMethod.set("tags", tagsNode);
 
         ObjectNode deleteMethod = NODE_FACTORY.objectNode()
                 .put("summary", String.format("Delete %s on entity %s", aspect, upperFirstEntity))
-                .put("operationId", String.format("delete%s", upperFirstAspect, upperFirstEntity))
+                .put("operationId", String.format("delete%s", upperFirstAspect))
                 .set("responses", NODE_FACTORY.objectNode()
                         .set("200", NODE_FACTORY.objectNode()
                                 .put("description", String.format("Delete %s on %s entity.", aspect, upperFirstEntity))
@@ -591,7 +609,7 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) {
 
         ObjectNode postMethod = NODE_FACTORY.objectNode()
                 .put("summary", String.format("Create aspect %s on %s ", aspect, upperFirstEntity))
-                .put("operationId", String.format("create%s", upperFirstAspect, upperFirstEntity));
+                .put("operationId", String.format("create%s", upperFirstAspect));
         postMethod.set("requestBody", NODE_FACTORY.objectNode()
                 .put("description", String.format("Create aspect %s on %s entity.", aspect, upperFirstEntity))
                 .put("required", true).set("content", NODE_FACTORY.objectNode()

diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java
@@ -1,5 +1,6 @@
 package config;
 
+import com.linkedin.metadata.config.VisualConfiguration;
 import com.linkedin.metadata.config.cache.CacheConfiguration;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
@@ -22,4 +23,7 @@ public class ConfigurationProvider {
 
   /** Configuration for caching */
   private CacheConfiguration cache;
+
+  /** Configuration for the view layer */
+  private VisualConfiguration visualConfig;
 }
diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java
@@ -13,6 +13,7 @@
 import com.linkedin.util.Pair;
 import com.typesafe.config.Config;
 import java.io.InputStream;
+import java.net.URI;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
@@ -125,6 +126,12 @@ public CompletableFuture<Result> proxy(String path, Http.Request request)
       headers.put(Http.HeaderNames.X_FORWARDED_HOST, headers.get(Http.HeaderNames.HOST));
     }
 
+    if (!headers.containsKey(Http.HeaderNames.X_FORWARDED_PROTO)) {
+      final String schema =
+          Optional.ofNullable(URI.create(request.uri()).getScheme()).orElse("http");
+      headers.put(Http.HeaderNames.X_FORWARDED_PROTO, List.of(schema));
+    }
+
     return _ws.url(
             String.format(
                 "%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri))

diff --git a/datahub-frontend/app/controllers/RedirectController.java b/datahub-frontend/app/controllers/RedirectController.java
@@ -0,0 +1,25 @@
+package controllers;
+
+import config.ConfigurationProvider;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import play.mvc.Controller;
+import play.mvc.Http;
+import play.mvc.Result;
+
+@Singleton
+public class RedirectController extends Controller {
+
+  @Inject ConfigurationProvider config;
+
+  public Result favicon(Http.Request request) {
+    if (config.getVisualConfig().getAssets().getFaviconUrl().startsWith("http")) {
+      return permanentRedirect(config.getVisualConfig().getAssets().getFaviconUrl());
+    } else {
+      final String prefix = config.getVisualConfig().getAssets().getFaviconUrl().startsWith("/") ? "/public" : "/public/";
+      return ok(Application.class.getResourceAsStream(
+              prefix + config.getVisualConfig().getAssets().getFaviconUrl()))
+          .as("image/x-icon");
+    }
+  }
+}
diff --git a/datahub-frontend/conf/routes b/datahub-frontend/conf/routes
@@ -36,9 +36,13 @@ PUT           /openapi/*path                                                   c
 HEAD          /openapi/*path                                                   controllers.Application.proxy(path: String, request: Request)
 PATCH         /openapi/*path                                                   controllers.Application.proxy(path: String, request: Request)
 
+
 # Analytics route
 POST          /track                                                           controllers.TrackingController.track(request: Request)
 
+# Map static resources from the /public folder to the /assets URL path
+GET           /assets/icons/favicon.ico                                        controllers.RedirectController.favicon(request: Request)
+
 # Known React asset routes
 GET           /assets/*file                                                    controllers.Assets.at(path="/public/assets", file)
 GET           /node_modules/*file                                              controllers.Assets.at(path="/public/node_modules", file)

diff --git a/datahub-frontend/public b/datahub-frontend/public
@@ -0,0 +1 @@
+../datahub-web-react/public
diff --git a/datahub-frontend/test/resources/public/logos/datahub-logo.png b/datahub-frontend/test/resources/public/logos/datahub-logo.png
diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle
@@ -12,6 +12,7 @@ dependencies {
     implementation project(':metadata-service:services')
     implementation project(':metadata-io')
     implementation project(':metadata-utils')
+    implementation project(':metadata-models')
 
     implementation externalDependency.graphqlJava
     implementation externalDependency.graphqlJavaScalars
@@ -40,8 +41,10 @@ graphqlCodegen {
         "$projectDir/src/main/resources/auth.graphql".toString(),
         "$projectDir/src/main/resources/timeline.graphql".toString(),
         "$projectDir/src/main/resources/tests.graphql".toString(),
+        "$projectDir/src/main/resources/properties.graphql".toString(),
         "$projectDir/src/main/resources/step.graphql".toString(),
         "$projectDir/src/main/resources/lineage.graphql".toString(),
+        "$projectDir/src/main/resources/forms.graphql".toString()
     ]
     outputDir = new File("$projectDir/src/mainGeneratedGraphQL/java")
     packageName = "com.linkedin.datahub.graphql.generated"

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java
@@ -19,9 +19,10 @@ private Constants() {}
   public static final String TESTS_SCHEMA_FILE = "tests.graphql";
   public static final String STEPS_SCHEMA_FILE = "step.graphql";
   public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql";
+  public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql";
+  public static final String FORMS_SCHEMA_FILE = "forms.graphql";
   public static final String BROWSE_PATH_DELIMITER = "/";
   public static final String BROWSE_PATH_V2_DELIMITER = "␟";
   public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";
-
   public static final String ENTITY_FILTER_NAME = "_entityType";
 }