Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tvlist feat new #14616

Merged
merged 56 commits into from
Feb 7, 2025
Merged
Changes from 1 commit
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
ef5de53
null bitmap for int tvlist
shizy818 Nov 27, 2024
7a15073
update min/max timestamp and sequential part of tvlist during insert
shizy818 Nov 30, 2024
c28fe3f
mutable & immutable tvlists in writable memchunk
shizy818 Dec 1, 2024
6df2e10
copy-on-write array list
shizy818 Dec 1, 2024
435d5da
review comments part 1
shizy818 Dec 3, 2024
d5fad22
fix unit test errors
shizy818 Dec 3, 2024
402c869
review comments part 2
shizy818 Dec 4, 2024
5aaa59b
push down global time filter
shizy818 Dec 5, 2024
cfc9e74
fix MemPageReaderTest case
shizy818 Dec 5, 2024
cd3f7d9
fix memory page offsets error
shizy818 Dec 5, 2024
e3d2d5e
synchronized sort & MergeSortTvListIterator bug
shizy818 Dec 6, 2024
73e9f83
tvlist_sort_threshold config property
shizy818 Dec 7, 2024
0545ff1
bug fix:
shizy818 Dec 11, 2024
2fcdcdb
optimize TVListIterator & MergeSortTvListIterator
shizy818 Dec 11, 2024
45db409
retrofit encode when tvlist_sort_threshold is zero
shizy818 Dec 13, 2024
9411fea
delay sort & statistic generation to query execution
shizy818 Dec 11, 2024
e3346d3
fix: skip deleted data during encode
shizy818 Dec 13, 2024
8e0f8e5
aligned time series part
shizy818 Dec 16, 2024
6eedcf6
fix: MemAlignedChunkReader page offset
shizy818 Dec 18, 2024
0a9c4ed
performance issue:
shizy818 Dec 17, 2024
9b0a6f8
fix: memory chunk reader may read more points than expected in one page
shizy818 Dec 23, 2024
23bf1f7
update chunk & page statistic for aligend memchunk by column
shizy818 Dec 21, 2024
298e16c
revert: getAlignedValueForQuery
shizy818 Dec 24, 2024
359929d
fix: * CopyOnWriteArrayList for AlignedTVList bitmaps
shizy818 Dec 25, 2024
6817370
refactor: Tim/Quick/Backward TVList
shizy818 Dec 25, 2024
a8beea5
refactor: synchronized tvlist method: sort, putXXX
shizy818 Dec 25, 2024
6369293
refactor: change list to array in AlignedTVList iterator
shizy818 Dec 25, 2024
a80bcc0
revert: remove CopyOnWriteArrayList
shizy818 Dec 26, 2024
3e0b904
refactor: clone MergeSort iterator from ReadOnlyChunk
shizy818 Dec 27, 2024
ca08605
fix: clone working tvlist during flush if there is query on it
shizy818 Dec 28, 2024
f23a72b
fix: writable mem chunk flush conditions
shizy818 Dec 30, 2024
8c7db55
refactor: add annotation and variable/function rename
shizy818 Dec 31, 2024
086f7be
merge force_ci/split_chunk
shizy818 Jan 1, 2025
36aa04d
fix: * remove delete method in BinaryTVList
shizy818 Jan 2, 2025
809ebd5
fix: remove getSortedTvListForQuery in SeriesRegionScan
shizy818 Jan 2, 2025
ab17aea
fix: TsFileProcessorTest unit test
shizy818 Jan 2, 2025
044d1ab
fix: IoTDBNullIdQueryIT.noMeasurementColumnsSelectTest
shizy818 Jan 2, 2025
1d2a840
fix: delete column of aligned time series
shizy818 Jan 5, 2025
93b9eeb
fix: aligned timeseries encode bug
shizy818 Jan 5, 2025
46775f0
fix: IoTDBGroupByNaturalMonthIT
shizy818 Jan 5, 2025
3501b8f
remove avgSeriesPointNumberThreshold setting
shizy818 Jan 5, 2025
062adc7
fix: IoTDBDeleteAlignedTimeseriesIT & AlignedTVListTest
shizy818 Jan 6, 2025
7c2c11d
fix: Copy globalTimeFilter due to GroupByMonthFilter
shizy818 Jan 6, 2025
edf7952
reset tmpLength for backward sort
shizy818 Jan 6, 2025
d6bf701
* fix TVList clear
shizy818 Jan 10, 2025
89ad4d5
hot-load TVLIST_SORT_THRESHOLD
shizy818 Jan 10, 2025
a3feeee
fix: isNullValue caller
shizy818 Jan 10, 2025
cb7e13a
fix unit test
shizy818 Jan 11, 2025
1dc4192
refactor: abstract prepareTvListMapForQuery method
shizy818 Jan 11, 2025
bab27fd
refactor: clear/clone/expand indices and bitmap
shizy818 Jan 12, 2025
8f78b21
merge sort using min heap
shizy818 Jan 12, 2025
11f376c
fix: WritableMemChunk deserialize
shizy818 Jan 13, 2025
736e596
feat: add index mem cost for TVList
shizy818 Jan 14, 2025
7fe1f85
fix: hot-load tvlist_sort_threshold setting
shizy818 Jan 14, 2025
b09904e
merge force_ci/split_chunk
shizy818 Jan 22, 2025
0900011
remove needless line in property template
shizy818 Jan 23, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: aligned timeseries encode bug
shizy818 committed Jan 5, 2025

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 93b9eeb1a0f8db367b2412c49c9cb326ef01cda2
Original file line number Diff line number Diff line change
@@ -734,7 +734,8 @@ private void writePageValuesIntoWriter(
int[] accessInfo = pageAccessInfo.get(index);
TsPrimitiveType value =
timeValuePairIterator.getPrimitiveObject(accessInfo, columnIndex);
valueChunkWriter.write(times[index], value.getBoolean(), value.getValue() == null);
valueChunkWriter.write(
times[index], value != null && value.getBoolean(), value == null);
}
break;
case INT32:
@@ -743,7 +744,7 @@ private void writePageValuesIntoWriter(
int[] accessInfo = pageAccessInfo.get(index);
TsPrimitiveType value =
timeValuePairIterator.getPrimitiveObject(accessInfo, columnIndex);
valueChunkWriter.write(times[index], value.getInt(), value.getValue() == null);
valueChunkWriter.write(times[index], value == null ? 0 : value.getInt(), value == null);
}
break;
case INT64:
@@ -752,23 +753,26 @@ private void writePageValuesIntoWriter(
int[] accessInfo = pageAccessInfo.get(index);
TsPrimitiveType value =
timeValuePairIterator.getPrimitiveObject(accessInfo, columnIndex);
valueChunkWriter.write(times[index], value.getLong(), value.getValue() == null);
valueChunkWriter.write(
times[index], value == null ? 0L : value.getLong(), value == null);
}
break;
case FLOAT:
for (int index = 0; index < pageAccessInfo.count(); index++) {
int[] accessInfo = pageAccessInfo.get(index);
TsPrimitiveType value =
timeValuePairIterator.getPrimitiveObject(accessInfo, columnIndex);
valueChunkWriter.write(times[index], value.getFloat(), value.getValue() == null);
valueChunkWriter.write(
times[index], value == null ? 0f : value.getFloat(), value == null);
}
break;
case DOUBLE:
for (int index = 0; index < pageAccessInfo.count(); index++) {
int[] accessInfo = pageAccessInfo.get(index);
TsPrimitiveType value =
timeValuePairIterator.getPrimitiveObject(accessInfo, columnIndex);
valueChunkWriter.write(times[index], value.getDouble(), value.getValue() == null);
valueChunkWriter.write(
times[index], value == null ? 0d : value.getDouble(), value == null);
}
break;
case TEXT:
@@ -778,7 +782,10 @@ private void writePageValuesIntoWriter(
int[] accessInfo = pageAccessInfo.get(index);
TsPrimitiveType value =
timeValuePairIterator.getPrimitiveObject(accessInfo, columnIndex);
valueChunkWriter.write(times[index], value.getBinary(), value.getValue() == null);
valueChunkWriter.write(
times[index],
value == null ? Binary.EMPTY_VALUE : value.getBinary(),
value == null);
}
break;
default:
@@ -815,8 +822,8 @@ public synchronized void encode(BlockingQueue<Object> ioTaskQueue) {
while (timeValuePairIterator.hasNextTimeValuePair()) {
// prepare column access info for current page
int[][] accessInfo = timeValuePairIterator.getColumnAccessInfo();
times[pointNumInPage] = timeValuePairIterator.getTime();
for (int i = 0; i < dataTypes.size(); i++) {
times[pointNumInPage] = timeValuePairIterator.getTime();
pageColumnAccessInfo[i].add(accessInfo[i]);
}
timeValuePairIterator.step();
@@ -848,12 +855,14 @@ public synchronized void encode(BlockingQueue<Object> ioTaskQueue) {
}

// last batch of points
if (pointNumInPage > 0) {
writePageValuesIntoWriter(
alignedChunkWriter, times, pageColumnAccessInfo, timeValuePairIterator);
alignedChunkWriter.write(times, pointNumInPage, 0);
alignedChunkWriter.sealCurrentPage();
alignedChunkWriter.clearPageWriter();
if (pointNumInChunk > 0) {
if (pointNumInPage > 0) {
writePageValuesIntoWriter(
alignedChunkWriter, times, pageColumnAccessInfo, timeValuePairIterator);
alignedChunkWriter.write(times, pointNumInPage, 0);
alignedChunkWriter.sealCurrentPage();
alignedChunkWriter.clearPageWriter();
}
try {
ioTaskQueue.put(alignedChunkWriter);
} catch (InterruptedException e) {