Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

W-13497368 #537

Merged
merged 3 commits into from
Jun 4, 2023
Merged

W-13497368 #537

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d9cdfb8
W-13497368
or-koren Jun 4, 2023
857a606
Merge remote-tracking branch 'origin/master' into W-13497368-final
or-koren Jun 4, 2023
e8a686b
W-13497368
or-koren Jun 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .../timbermill-local/src/main/java/com/datorama/oss/timbermill/common/cache/CacheConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,23 @@
public class CacheConfig {
private RedisService redisService;
private int cacheRedisTtlInSeconds;
private int orphansCacheRedisTtlInSeconds;
private int eventsCacheRedisTtlInSeconds;
private long maximumTasksCacheWeight;
private long maximumOrphansCacheWeight;

public CacheConfig(RedisService redisService, int cacheRedisTtlInSeconds, long maximumTasksCacheWeight, long maximumOrphansCacheWeight) {
this(redisService, cacheRedisTtlInSeconds, maximumTasksCacheWeight, maximumOrphansCacheWeight, cacheRedisTtlInSeconds, cacheRedisTtlInSeconds);
}
public CacheConfig(RedisService redisService, int cacheRedisTtlInSeconds, long maximumTasksCacheWeight, long maximumOrphansCacheWeight, int orphansCacheRedisTtlInSeconds,
int eventsCacheRedisTtlInSeconds) {
this.redisService = redisService;
this.cacheRedisTtlInSeconds = cacheRedisTtlInSeconds;
this.maximumTasksCacheWeight = maximumTasksCacheWeight;
this.maximumOrphansCacheWeight = maximumOrphansCacheWeight;
this.eventsCacheRedisTtlInSeconds = eventsCacheRedisTtlInSeconds;
this.orphansCacheRedisTtlInSeconds = orphansCacheRedisTtlInSeconds;

}

RedisService getRedisService() {
Expand All @@ -31,4 +40,12 @@ long getMaximumOrphansCacheWeight() {
return maximumOrphansCacheWeight;
}

public int getOrphansCacheRedisTtlInSeconds() {
return orphansCacheRedisTtlInSeconds;
}

public int getEventsCacheRedisTtlInSeconds() {
return eventsCacheRedisTtlInSeconds;
}

}
2 changes: 1 addition & 1 deletion ...ermill-local/src/main/java/com/datorama/oss/timbermill/common/cache/CacheHandlerUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
public class CacheHandlerUtil {
public static AbstractCacheHandler getCacheHandler(String strategy, CacheConfig cacheParams) {
if (strategy.compareToIgnoreCase("redis") == 0){
return new RedisCacheHandler(cacheParams.getRedisService(), cacheParams.getCacheRedisTtlInSeconds());
return new RedisCacheHandler(cacheParams.getRedisService(), cacheParams.getCacheRedisTtlInSeconds(), cacheParams.getOrphansCacheRedisTtlInSeconds(), cacheParams.getEventsCacheRedisTtlInSeconds());
}
else {
return new LocalCacheHandler(cacheParams.getMaximumTasksCacheWeight(), cacheParams.getMaximumOrphansCacheWeight());
Expand Down
21 changes: 12 additions & 9 deletions ...rmill-local/src/main/java/com/datorama/oss/timbermill/common/cache/RedisCacheHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,24 @@ public class RedisCacheHandler extends AbstractCacheHandler {
private JedisLock lock;
private final RedisService redisService;
private final int redisTtlInSeconds;

private int orphanRedisTTLRedisTTLInSeconds;
private int cacheRedisTTLRedisTTLInSeconds;

private int cacheOrphansRedisTtlInSeconds;
private int cacheEventsRedisTtlInSeconds;

RedisCacheHandler(RedisService redisService, int cacheRedisTtlInSeconds) {
this(redisService, cacheRedisTtlInSeconds, cacheRedisTtlInSeconds, cacheRedisTtlInSeconds);
}

RedisCacheHandler(RedisService redisService, int cacheRedisTtlInSeconds, int cacheOrphansRedisTtlInSeconds, int cacheEventsRedisTtlInSeconds) {
if (redisService == null){
throw new RuntimeException("Redis cache used but no redis host defined");
}
this.redisService = redisService;
this.redisTtlInSeconds = cacheRedisTtlInSeconds;
orphanRedisTTLRedisTTLInSeconds = System.getenv("ORPHANS_REDIS_TTL_SECONDS") != null ? Integer.parseInt(System.getenv("ORPHANS_REDIS_TTL_SECONDS")) : cacheRedisTtlInSeconds;
cacheRedisTTLRedisTTLInSeconds = System.getenv("CACHE_REDIS_TTL_SECONDS") != null ? Integer.parseInt(System.getenv("CACHE_REDIS_TTL_SECONDS")) : cacheRedisTtlInSeconds;
LOG.info("Initializing Timbermill Redis Cache with orphanRedisTTLRedisTTLInSeconds {} seconds and cacheRedisTTLRedisTTLInSeconds {}", orphanRedisTTLRedisTTLInSeconds, cacheRedisTTLRedisTTLInSeconds);
this.cacheEventsRedisTtlInSeconds = cacheEventsRedisTtlInSeconds;
this.cacheOrphansRedisTtlInSeconds = cacheOrphansRedisTtlInSeconds;
LOG.info("Initializing Timbermill Redis Cache with cacheEventsRedisTtlInSeconds: {}, cacheOrphansRedisTtlInSeconds: {}, cacheRedisTtlInSeconds:{} ", cacheEventsRedisTtlInSeconds,
cacheOrphansRedisTtlInSeconds, cacheRedisTtlInSeconds);
}

@Override
Expand All @@ -59,8 +63,7 @@ public void pushToOrphanCache(Map<String, List<String>> orphansMap) {
String orphanCacheKey = ORPHAN_PREFIX + entry.getKey();
newOrphansMap.put(orphanCacheKey, entry.getValue());
}

if (!redisService.pushToRedis(newOrphansMap, orphanRedisTTLRedisTTLInSeconds)){
if (!redisService.pushToRedis(newOrphansMap, cacheOrphansRedisTtlInSeconds)){
LOG.error("Failed to push some ids to Redis orphans cache.");
KamonConstants.ORPHAN_CACHE_FAILED_COUNTER.withoutTags().increment();
}
Expand All @@ -73,7 +76,7 @@ public Map<String, LocalTask> getFromTasksCache(Collection<String> idsList) {

@Override
public void pushToTasksCache(Map<String, LocalTask> idsToMap) {
boolean allPushed = redisService.pushToRedis(idsToMap, cacheRedisTTLRedisTTLInSeconds);
boolean allPushed = redisService.pushToRedis(idsToMap, cacheEventsRedisTtlInSeconds);
if (!allPushed){
LOG.error("Failed to push some ids to Redis tasks cache.");
}
Expand Down
4 changes: 3 additions & 1 deletion ...ermill-server/src/main/java/com/datorama/timbermill/server/service/TimbermillService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ public TimbermillService(@Value("${INDEX_BULK_SIZE:200000}") Integer indexBulkSi
@Value("${MAXIMUM_ORPHANS_CACHE_WEIGHT:1000000000}") long maximumOrphansCacheWeight,
@Value("${CACHE_STRATEGY:}") String cacheStrategy,
@Value("${CACHE_TTL_IN_SECONDS:604800}") int cacheRedisTtlInSeconds,
@Value("${ORPHAN_CACHE_TTL_IN_SECONDS:86400}") int orphansCacheRedisTtlInSeconds, //One day
@Value("${EVENTS_CACHE_TTL_IN_SECONDS:86400}") int eventsCacheRedisTtlInSeconds, //One day
@Value("${REDIS_MAX_MEMORY:}") String redisMaxMemory,
@Value("${REDIS_MAX_MEMORY_POLICY:}") String redisMaxMemoryPolicy,
@Value("${REDIS_HOST:}") String redisHost,
Expand Down Expand Up @@ -137,7 +139,7 @@ public TimbermillService(@Value("${INDEX_BULK_SIZE:200000}") Integer indexBulkSi
elasticPassword, maxIndexAge, maxIndexSizeInGB, maxIndexDocs, numOfElasticSearchActionsTries, maxBulkIndexFetches, searchMaxSize, persistenceHandler, numberOfShards, numberOfReplicas,
maxTotalFields, null, scrollLimitation, scrollTimeoutSeconds, fetchByIdsPartitions, expiredMaxIndicesToDeleteInParallel);

CacheConfig cacheParams = new CacheConfig(redisService, cacheRedisTtlInSeconds, maximumTasksCacheWeight, maximumOrphansCacheWeight);
CacheConfig cacheParams = new CacheConfig(redisService, cacheRedisTtlInSeconds, maximumTasksCacheWeight, maximumOrphansCacheWeight, orphansCacheRedisTtlInSeconds, eventsCacheRedisTtlInSeconds);
AbstractCacheHandler cacheHandler = CacheHandlerUtil.getCacheHandler(cacheStrategy, cacheParams);
this.eventsMaxElement = eventsMaxElement;
taskIndexer = new TaskIndexer(pluginsJson, daysRotation, es, timbermillVersion, cacheHandler);
Expand Down