diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexChartInfoConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexChartInfoConfig.java new file mode 100644 index 0000000000000..a9668da4329a0 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexChartInfoConfig.java @@ -0,0 +1,30 @@ +package com.linkedin.datahub.upgrade.config.restoreindices; + +import com.linkedin.datahub.upgrade.conditions.SystemUpdateCondition; +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.datahub.upgrade.system.restoreindices.chartinfo.ReindexChartInfo; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class) +public class ReindexChartInfoConfig { + + @Bean + public NonBlockingSystemUpgrade reindexChartInfo( + final OperationContext opContext, + final EntityService entityService, + final AspectDao aspectDao, + @Value("${systemUpdate.chartInfo.enabled}") final boolean enabled, + @Value("${systemUpdate.chartInfo.batchSize}") final Integer batchSize, + @Value("${systemUpdate.chartInfo.delayMs}") final Integer delayMs, + @Value("${systemUpdate.chartInfo.limit}") final Integer limit) { + return new ReindexChartInfo( + opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/chartinfo/ReindexChartInfo.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/chartinfo/ReindexChartInfo.java new file mode 100644 index 0000000000000..d519587ef8dfe --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/chartinfo/ReindexChartInfo.java @@ -0,0 +1,46 @@ +package com.linkedin.datahub.upgrade.system.restoreindices.chartinfo; + +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import java.util.List; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +/** A job that reindexes all chart info aspects as part of reindexing charts lastModifiedAt */ +@Slf4j +public class ReindexChartInfo implements NonBlockingSystemUpgrade { + + private final List _steps; + + public ReindexChartInfo( + @Nonnull OperationContext opContext, + EntityService entityService, + AspectDao aspectDao, + boolean enabled, + Integer batchSize, + Integer batchDelayMs, + Integer limit) { + if (enabled) { + _steps = + ImmutableList.of( + new ReindexChartInfoStep( + opContext, entityService, aspectDao, batchSize, batchDelayMs, limit)); + } else { + _steps = ImmutableList.of(); + } + } + + @Override + public String id() { + return this.getClass().getName(); + } + + @Override + public List steps() { + return _steps; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/chartinfo/ReindexChartInfoStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/chartinfo/ReindexChartInfoStep.java new file mode 100644 index 0000000000000..a9b10b9cb3c82 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/chartinfo/ReindexChartInfoStep.java @@ -0,0 +1,42 @@ +package com.linkedin.datahub.upgrade.system.restoreindices.chartinfo; + +import static com.linkedin.metadata.Constants.*; + +import com.linkedin.datahub.upgrade.system.AbstractMCLStep; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; + +@Slf4j +public class ReindexChartInfoStep extends AbstractMCLStep { + + public ReindexChartInfoStep( + OperationContext opContext, + EntityService entityService, + AspectDao aspectDao, + Integer batchSize, + Integer batchDelayMs, + Integer limit) { + super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit); + } + + @Override + public String id() { + return "chart-info-v1"; + } + + @Nonnull + @Override + protected String getAspectName() { + return CHART_INFO_ASPECT_NAME; + } + + @Nullable + @Override + protected String getUrnLike() { + return "urn:li:" + CHART_ENTITY_NAME + ":%"; + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java b/metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java index 2a6e709fc0e08..29676b02f4756 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java @@ -118,6 +118,8 @@ public PropertiesCollector propertiesCollector(Environment environment) { // System update properties "systemUpdate.*.enabled", "systemUpdate.*.batchSize", + "systemUpdate.*.limit", + "systemUpdate.*.delayMs", // Kafka topic Configs "kafka.topics.*.name", diff --git a/metadata-models/src/main/pegasus/com/linkedin/chart/ChartInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/chart/ChartInfo.pdl index f0077dc1f2ac5..eae534704e518 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/chart/ChartInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/chart/ChartInfo.pdl @@ -40,6 +40,12 @@ record ChartInfo includes CustomProperties, ExternalReference { /** * Captures information about who created/last modified/deleted this chart and when */ + @Searchable = { + "/lastModified/time": { + "fieldName": "lastModifiedAt", + "fieldType": "DATETIME" + } + } lastModified: ChangeAuditStamps /** diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index d1532edff8f36..fa5fbf73be3f5 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -744,6 +744,11 @@ systemUpdate: batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_BATCH_SIZE:1000} delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_DELAY_MS:30000} limit: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_CLL_LIMIT:0} + chartInfo: + enabled: ${BOOTSTRAP_SYSTEM_UPDATE_CHART_INFO_ENABLED:true} + batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_CHART_INFO_BATCH_SIZE:1000} + delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_CHART_INFO_DELAY_MS:30000} + limit: ${BOOTSTRAP_SYSTEM_UPDATE_CHART_INFO_CLL_LIMIT:0} dashboardInfo: enabled: ${BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_ENABLED:true} batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_BATCH_SIZE:1000}