[persistence] add namespace to persistence tables (#1475)

* [persistence] add namespace to tables except task_entity

* [persistence] add namespace to task_entity table

* add indexes and uniqueness constraints

thirdeye-persistence/src/main/java/ai/startree/thirdeye/datalayer/entity/AbstractIndexEntity.java

@@ -13,15 +13,29 @@
  */
 package ai.startree.thirdeye.datalayer.entity;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
+
 public abstract class AbstractIndexEntity extends AbstractEntity {
 
   protected Long baseId;
+
+  protected String namespace;
 
   public Long getBaseId() {
     return baseId;
   }
 
-  public void setBaseId(Long baseId) {
+  public AbstractIndexEntity setBaseId(final Long baseId) {
     this.baseId = baseId;
+    return this;
+  }
+
+  public String getNamespace() {
+    return namespace;
+  }
+
+  public AbstractIndexEntity setNamespace(final @Nullable String namespace) {
+    this.namespace = namespace;
+    return this;
   }
 }

thirdeye-persistence/src/main/java/ai/startree/thirdeye/datalayer/entity/TaskEntity.java

@@ -14,6 +14,7 @@
 package ai.startree.thirdeye.datalayer.entity;
 
 import java.sql.Timestamp;
+import org.checkerframework.checker.nullness.qual.Nullable;
 
 public class TaskEntity extends AbstractEntity implements HasJsonVal<TaskEntity> {
 
@@ -28,6 +29,7 @@ public class TaskEntity extends AbstractEntity implements HasJsonVal<TaskEntity>
   private String jsonVal;
 
   private Long refId;
+  private @Nullable String namespace; 
 
   @Override
   public String getJsonVal() {
@@ -120,4 +122,13 @@ public TaskEntity setRefId(final Long refId) {
     this.refId = refId;
     return this;
   }
+
+  public @Nullable String getNamespace() {
+    return namespace;
+  }
+
+  public TaskEntity setNamespace(final @Nullable String namespace) {
+    this.namespace = namespace;
+    return this;
+  }
 }

thirdeye-persistence/src/main/java/ai/startree/thirdeye/datalayer/mapper/DtoIndexMapper.java

@@ -13,6 +13,8 @@
  */
 package ai.startree.thirdeye.datalayer.mapper;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 import ai.startree.thirdeye.datalayer.entity.AbstractIndexEntity;
 import ai.startree.thirdeye.datalayer.entity.HasJsonVal;
 import ai.startree.thirdeye.spi.datalayer.dto.AbstractDTO;
@@ -41,7 +43,10 @@ public static <E extends AbstractDTO> AbstractIndexEntity toAbstractIndexEntity(
     }
     abstractIndexEntity.setBaseId(pojo.getId());
     abstractIndexEntity.setUpdateTime(pojo.getUpdateTime());
-    // todo cyril add namespace here and add namespace to all index tables + maybe add to generic main table?
+    // todo cyril authz - namespace not empty string should be tested sooner - it's not trivial because it can come from jackson by reflection, so doing a precondition check in the setter is not enough - also in new mode maybe we will want to prevent the null namespace  
+    //  for the moment we at least ensure it fails at db write time
+    checkArgument(pojo.namespace() == null || !pojo.namespace().isEmpty(), "Namespace cannot be an empty string. Null namespace is allowed");
+    abstractIndexEntity.setNamespace(pojo.namespace());
 
     return abstractIndexEntity;
   }

thirdeye-persistence/src/main/java/ai/startree/thirdeye/datalayer/mapper/TaskEntityMapper.java

@@ -26,5 +26,6 @@ public interface TaskEntityMapper {
   @Mapping(source = "jobName", target = "name")
   @Mapping(source = "taskType", target = "type")
   @Mapping(target = "jsonVal", ignore = true)
+  @Mapping(source = "auth.namespace", target = "namespace")
   TaskEntity toTaskEntity(TaskDTO dto);
 }

thirdeye-persistence/src/main/resources/db/migration/mysql/README.md

@@ -1,31 +1,15 @@
 
 ## [V1_293_0__remove_name_unique_constraints_datasource.sql](V1_293_0__remove_name_unique_constraints_datasource.sql)
-Datasource:
-TODO CYRIL authz best would be to have a constraint on (name, namespace) but namespace is not added to the index tables yet
-for the moment we just remove the name unique constraints
-indexes should be revisited once the namespace migration is done
+Datasource: DONE
 
 ## [V1_293_1__remove_name_unique_constraints_dataset.sql](V1_293_1__remove_name_unique_constraints_dataset.sql)
-Dataset:
-TODO CYRIL authz best would be to have a constraint on (name, namespace) but namespace is not added to the index tables yet
-for the moment we just remove the name unique constraints
-indexes should be revisited once the namespace migration is done
+Dataset: DONE
 
 ## [V1_293_2__remove_name_dataset_unique_constraints_metrics.sql](V1_293_2__remove_name_dataset_unique_constraints_metrics.sql)
-Metrics:
-TODO CYRIL authz best would be to have a constraint on (name, dataset, namespace) but namespace is not added to the index tables yet
-for the moment we just remove the name unique constraints
-indexes should be revisited once the namespace migration is done
+Metrics: DONE
 
 ## [V1_293_4__remove_name_unique_constraints_alert.sql](V1_293_4__remove_name_unique_constraints_alert.sql)
-Alert
-TODO CYRIL authz best would be to have a constraint on (name, namespace) but namespace is not added to the index tables yet
-for the moment we just remove the name unique constraints
-indexes should be revisited once the namespace migration is done
+Alert: DONE
 
 ## [V1_293_5__remove_name_unique_constraints_subscription_group.sql](V1_293_5__remove_name_unique_constraints_subscription_group.sql)
-Subscription Group
-TODO CYRIL authz best would be to have a constraint on (name, namespace) but namespace is not added to the index tables yet
-for the moment we just remove the name unique constraints
-indexes should be revisited once the namespace migration is done
-
+Subscription Group: DONE

thirdeye-persistence/src/main/resources/db/migration/mysql/V1_302_0__add_namespace.sql

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 StarTree Inc
+ *
+ * Licensed under the StarTree Community License (the "License"); you may not use
+ * this file except in compliance with the License. You may obtain a copy of the
+ * License at http://www.startree.ai/legal/startree-community-license
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OF ANY KIND,
+ * either express or implied.
+ * See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+ALTER TABLE alert_template_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON alert_template_index (namespace);
+ALTER TABLE alert_template_index ADD UNIQUE unique_name_namespace (`name`, namespace);
+
+ALTER TABLE anomaly_feedback_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON anomaly_feedback_index (namespace);
+
+ALTER TABLE data_source_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON data_source_index (namespace);
+ALTER TABLE data_source_index ADD UNIQUE unique_name_namespace (`name`, namespace);
+
+ALTER TABLE dataset_config_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON dataset_config_index (namespace);
+ALTER TABLE dataset_config_index ADD UNIQUE unique_name_namespace (dataset, namespace);
+
+ALTER TABLE detection_alert_config_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON detection_alert_config_index (namespace);
+ALTER TABLE detection_alert_config_index ADD UNIQUE unique_name_namespace (`name`, namespace);
+
+ALTER TABLE detection_config_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON detection_config_index (namespace);
+ALTER TABLE detection_config_index ADD UNIQUE unique_name_namespace (`name`, namespace);
+
+ALTER TABLE enumeration_item_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON enumeration_item_index (namespace);
+
+ALTER TABLE event_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON event_index (namespace);
+
+ALTER TABLE merged_anomaly_result_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON merged_anomaly_result_index (namespace);
+
+ALTER TABLE metric_config_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON metric_config_index (namespace);
+ALTER TABLE metric_config_index ADD UNIQUE unique_name_dataset_namespace (`name`, dataset, namespace);
+
+ALTER TABLE rca_investigation_index ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON rca_investigation_index (namespace);
+
+ALTER TABLE task_entity ADD COLUMN namespace varchar(255) DEFAULT NULL;
+CREATE INDEX namespace_idx ON task_entity (namespace);

thirdeye-server/src/main/java/ai/startree/thirdeye/service/DatasetService.java

@@ -22,6 +22,7 @@
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
+// fixme deleting a dataset should delete related metrics - longer term I feel like this metrics entity does not help - could be kept inside datasets imo
 @Singleton
 public class DatasetService extends CrudService<DatasetApi, DatasetConfigDTO> {
 

thirdeye-spi/src/main/java/ai/startree/thirdeye/spi/api/AuthorizationConfigurationApi.java

@@ -13,15 +13,17 @@
  */
 package ai.startree.thirdeye.spi.api;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
+
 public class AuthorizationConfigurationApi {
 
-  private String namespace;
+  private @Nullable String namespace;
 
   public String getNamespace() {
     return namespace;
   }
 
-  public AuthorizationConfigurationApi setNamespace(final String namespace) {
+  public AuthorizationConfigurationApi setNamespace(final @Nullable String namespace) {
     this.namespace = namespace;
     return this;
   }

thirdeye-spi/src/main/java/ai/startree/thirdeye/spi/datalayer/dto/AuthorizationConfigurationDTO.java

@@ -14,16 +14,17 @@
 package ai.startree.thirdeye.spi.datalayer.dto;
 
 import java.util.Objects;
+import org.checkerframework.checker.nullness.qual.Nullable;
 
 public class AuthorizationConfigurationDTO {
 
-  private String namespace;
+  private @Nullable String namespace;
 
   public String getNamespace() {
     return namespace;
   }
 
-  public AuthorizationConfigurationDTO setNamespace(final String namespace) {
+  public AuthorizationConfigurationDTO setNamespace(final @Nullable String namespace) {
     this.namespace = namespace;
     return this;
   }