Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package com.linkedin.datahub.upgrade.config.restoreindices;

import com.linkedin.datahub.upgrade.conditions.SystemUpdateCondition;
import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
import com.linkedin.datahub.upgrade.system.restoreindices.chartinfo.ReindexChartInfo;
import com.linkedin.metadata.entity.AspectDao;
import com.linkedin.metadata.entity.EntityService;
import io.datahubproject.metadata.context.OperationContext;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Configuration;

@Configuration
@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class)
public class ReindexChartInfoConfig {

@Bean
public NonBlockingSystemUpgrade reindexChartInfo(
final OperationContext opContext,
final EntityService<?> entityService,
final AspectDao aspectDao,
@Value("${systemUpdate.chartInfo.enabled}") final boolean enabled,
@Value("${systemUpdate.chartInfo.batchSize}") final Integer batchSize,
@Value("${systemUpdate.chartInfo.delayMs}") final Integer delayMs,
@Value("${systemUpdate.chartInfo.limit}") final Integer limit) {
return new ReindexChartInfo(
opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package com.linkedin.datahub.upgrade.system.restoreindices.chartinfo;

import com.google.common.collect.ImmutableList;
import com.linkedin.datahub.upgrade.UpgradeStep;
import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
import com.linkedin.metadata.entity.AspectDao;
import com.linkedin.metadata.entity.EntityService;
import io.datahubproject.metadata.context.OperationContext;
import java.util.List;
import javax.annotation.Nonnull;
import lombok.extern.slf4j.Slf4j;

/** A job that reindexes all chart info aspects as part of reindexing charts lastModifiedAt */
@Slf4j
public class ReindexChartInfo implements NonBlockingSystemUpgrade {

private final List<UpgradeStep> _steps;

public ReindexChartInfo(
@Nonnull OperationContext opContext,
EntityService<?> entityService,
AspectDao aspectDao,
boolean enabled,
Integer batchSize,
Integer batchDelayMs,
Integer limit) {
if (enabled) {
_steps =
ImmutableList.of(
new ReindexChartInfoStep(
opContext, entityService, aspectDao, batchSize, batchDelayMs, limit));
} else {
_steps = ImmutableList.of();
}
}

@Override
public String id() {
return this.getClass().getName();
}

@Override
public List<UpgradeStep> steps() {
return _steps;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package com.linkedin.datahub.upgrade.system.restoreindices.chartinfo;

import static com.linkedin.metadata.Constants.*;

import com.linkedin.datahub.upgrade.system.AbstractMCLStep;
import com.linkedin.metadata.entity.AspectDao;
import com.linkedin.metadata.entity.EntityService;
import io.datahubproject.metadata.context.OperationContext;
import javax.annotation.Nonnull;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.Nullable;

@Slf4j
public class ReindexChartInfoStep extends AbstractMCLStep {

public ReindexChartInfoStep(
OperationContext opContext,
EntityService<?> entityService,
AspectDao aspectDao,
Integer batchSize,
Integer batchDelayMs,
Integer limit) {
super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit);
}

@Override
public String id() {
return "chart-info-v1";
}

@Nonnull
@Override
protected String getAspectName() {
return CHART_INFO_ASPECT_NAME;
}

@Nullable
@Override
protected String getUrnLike() {
return "urn:li:" + CHART_ENTITY_NAME + ":%";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ public PropertiesCollector propertiesCollector(Environment environment) {
// System update properties
"systemUpdate.*.enabled",
"systemUpdate.*.batchSize",
"systemUpdate.*.limit",
"systemUpdate.*.delayMs",

// Kafka topic Configs
"kafka.topics.*.name",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ record ChartInfo includes CustomProperties, ExternalReference {
/**
* Captures information about who created/last modified/deleted this chart and when
*/
@Searchable = {
"/lastModified/time": {
"fieldName": "lastModifiedAt",
"fieldType": "DATETIME"
}
}
lastModified: ChangeAuditStamps

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,11 @@ systemUpdate:
batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_BATCH_SIZE:1000}
delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_DELAY_MS:30000}
limit: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_CLL_LIMIT:0}
chartInfo:
enabled: ${BOOTSTRAP_SYSTEM_UPDATE_CHART_INFO_ENABLED:true}
batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_CHART_INFO_BATCH_SIZE:1000}
delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_CHART_INFO_DELAY_MS:30000}
limit: ${BOOTSTRAP_SYSTEM_UPDATE_CHART_INFO_CLL_LIMIT:0}
dashboardInfo:
enabled: ${BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_ENABLED:true}
batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_BATCH_SIZE:1000}
Expand Down
Loading