Skip to content

Commit

Permalink
Revert "make larger segments"
Browse files Browse the repository at this point in the history
This reverts commit f289e21.
  • Loading branch information
vthacker committed Jun 19, 2023
1 parent 0c7193a commit 034541a
Showing 1 changed file with 0 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,6 @@ public class LuceneIndexStoreImpl implements LogStore<LogMessage> {
private final io.micrometer.core.instrument.Timer commitsTimer;
private final io.micrometer.core.instrument.Timer refreshesTimer;

private final String MAX_RAM_BUFFER_SIZE_MB = "maxRamBufferSizeMb";

// TODO: Set the policy via a lucene config file.
public static LuceneIndexStoreImpl makeLogStore(
File dataDirectory, KaldbConfigs.LuceneConfig luceneConfig, MeterRegistry metricsRegistry)
Expand Down Expand Up @@ -152,14 +150,10 @@ private IndexWriterConfig buildIndexWriterConfig(
SnapshotDeletionPolicy snapshotDeletionPolicy,
LuceneIndexStoreConfig config,
MeterRegistry metricsRegistry) {
int ramBufferSizeMb = Integer.getInteger(MAX_RAM_BUFFER_SIZE_MB, 1024);
boolean useCFSFiles = ramBufferSizeMb <= 128;
final IndexWriterConfig indexWriterCfg =
new IndexWriterConfig(analyzer)
.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
.setMergeScheduler(new KalDBMergeScheduler(metricsRegistry, config.throttleMerges))
.setRAMBufferSizeMB(ramBufferSizeMb)
.setUseCompoundFile(useCFSFiles)
// we sort by timestamp descending, as that is the order we expect to return results the
// majority of the time
.setIndexSort(
Expand All @@ -169,13 +163,6 @@ private IndexWriterConfig buildIndexWriterConfig(
SortField.Type.LONG,
true)))
.setIndexDeletionPolicy(snapshotDeletionPolicy);

// See
// https://lucene.apache.org/core/9_5_0/core/org/apache/lucene/index/IndexWriterConfig.html#setUseCompoundFile(boolean)
if (ramBufferSizeMb >= 128) {
indexWriterCfg.getMergePolicy().setNoCFSRatio(0.0);
}

if (config.enableTracing) {
indexWriterCfg.setInfoStream(System.out);
}
Expand Down

0 comments on commit 034541a

Please sign in to comment.