From 70dc71148639cdcaa8298003d24c72539be13125 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 17 Jul 2025 16:44:28 +0800 Subject: [PATCH 01/18] add method --- .../file/metadata/MetadataIndexNode.java | 73 +++++++++++++++++++ .../tsfile/read/TsFileSequenceReader.java | 62 ++++++++++++++++ 2 files changed, 135 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 693b4ffb1..5bc7e7ed9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -33,6 +33,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class MetadataIndexNode { @@ -127,6 +128,10 @@ public static MetadataIndexNode deserializeFrom( public Pair getChildIndexEntry(Comparable key, boolean exactSearch) { int index = binarySearchInChildren(key, exactSearch); + return getChildIndexEntry(index); + } + + private Pair getChildIndexEntry(int index) { if (index == -1) { return null; } @@ -165,6 +170,74 @@ int binarySearchInChildren(Comparable key, boolean exactSearch) { } } + public List> getChildIndexEntry( + List keys, boolean exactSearch) { + int[] indexArr = binarySearchInChildren(keys, exactSearch, 0, children.size() - 1); + List> pairs = new ArrayList<>(); + int previousIndex = -1; + Pair previousPair = null; + for (int idx : indexArr) { + if (previousIndex == idx) { + pairs.add(previousPair); + } else { + Pair current = getChildIndexEntry(idx); + pairs.add(current); + previousIndex = idx; + previousPair = current; + } + } + return pairs; + } + + int[] binarySearchInChildren( + List keys, boolean exactSearch, int low, int high) { + int[] results = new int[keys.size()]; + Arrays.fill(results, -1); + int currentLow = low; + + for (int i = 0; i < keys.size(); i++) { + Comparable key = keys.get(i); + if (currentLow > high) { + Arrays.fill( + results, i, keys.size(), exactSearch ? -1 : (currentLow == 0 ? 0 : currentLow - 1)); + return results; + } + + int foundIndex = -1; + int start = currentLow; + int end = high; + + while (start <= end) { + int mid = (start + end) >>> 1; + IMetadataIndexEntry midVal = children.get(mid); + int cmp = midVal.getCompareKey().compareTo(key); + + if (cmp < 0) { + start = mid + 1; + } else if (cmp > 0) { + end = mid - 1; + } else { + foundIndex = mid; + break; + } + } + + if (foundIndex >= 0) { + results[i] = foundIndex; + currentLow = foundIndex + 1; + } else { + if (exactSearch) { + results[i] = -1; + } else { + int insertPos = (start == 0) ? 0 : start - 1; + results[i] = insertPos; + currentLow = start; + } + } + } + return results; + } + public boolean isDeviceLevel() { return this.nodeType == MetadataIndexNodeType.INTERNAL_DEVICE || this.nodeType == MetadataIndexNodeType.LEAF_DEVICE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 033e3d044..978e45ede 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -535,6 +535,68 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev return deviceMetadata; } + public long[][] getDeviceMetadataIndexNodeOffsetList( + String table, List devices, boolean isSorted) throws IOException { + readFileMetadata(); + MetadataIndexNode tableMetadataIndexNode = getTableRootNode(table); + if (tableMetadataIndexNode == null) { + throw new IllegalArgumentException(""); + } + int[] deviceIdx = isSorted ? null : new int[devices.size()]; + if (!isSorted) { + devices = devices.stream().sorted(IDeviceID::compareTo).collect(Collectors.toList()); + } + long[][] results = new long[devices.size()][]; + getDeviceMetadataIndexNodeOffsetList( + results, devices, 0, devices.size(), tableMetadataIndexNode); + return results; + } + + private void getDeviceMetadataIndexNodeOffsetList( + long[][] results, + List devices, + int deviceStartIdx, + int deviceEndIdx, + MetadataIndexNode startNode) + throws IOException { + MetadataIndexNodeType metadataIndexNodeType = startNode.getNodeType(); + boolean exactSearch = metadataIndexNodeType == MetadataIndexNodeType.LEAF_DEVICE; + List> entries = + startNode.getChildIndexEntry(devices.subList(deviceStartIdx, deviceEndIdx), exactSearch); + Iterator> metadataIndexEntriesIterator = entries.iterator(); + int startIdxOfChild = 0; + Pair previousPair = null; + for (int i = deviceStartIdx; i < deviceEndIdx; i++) { + Pair pair = metadataIndexEntriesIterator.next(); + if (exactSearch) { + results[i] = pair == null ? null : new long[] {pair.getLeft().getOffset(), pair.getRight()}; + continue; + } + if (previousPair == null) { + previousPair = pair; + continue; + } + if (previousPair == pair) { + continue; + } + IMetadataIndexEntry entry = previousPair.getLeft(); + ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight()); + MetadataIndexNode lastNode = + MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); + getDeviceMetadataIndexNodeOffsetList(results, devices, startIdxOfChild, i, lastNode); + previousPair = pair; + startIdxOfChild = i; + } + if (exactSearch || previousPair == null) { + return; + } + // for last entry + IMetadataIndexEntry entry = previousPair.getLeft(); + ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight()); + MetadataIndexNode lastNode = MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); + getDeviceMetadataIndexNodeOffsetList(results, devices, startIdxOfChild, deviceEndIdx, lastNode); + } + public TimeseriesMetadata readTimeseriesMetadata( IDeviceID device, String measurement, boolean ignoreNotExists) throws IOException { return readTimeseriesMetadata(device, measurement, ignoreNotExists, null); From dde9fd20bca1366d30ae9fcb7e4b3593afd6ac96 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 18 Jul 2025 11:43:13 +0800 Subject: [PATCH 02/18] add ut --- .../tsfile/read/TsFileSequenceReader.java | 20 +++---- .../tsfile/read/TsFileSequenceReaderTest.java | 60 +++++++++++++++++++ 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 978e45ede..54e0d03ab 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -535,24 +535,20 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev return deviceMetadata; } - public long[][] getDeviceMetadataIndexNodeOffsetList( - String table, List devices, boolean isSorted) throws IOException { + public long[][] getDeviceMetadataIndexNodeOffsets(String table, List sortedDevices) + throws IOException { readFileMetadata(); MetadataIndexNode tableMetadataIndexNode = getTableRootNode(table); if (tableMetadataIndexNode == null) { throw new IllegalArgumentException(""); } - int[] deviceIdx = isSorted ? null : new int[devices.size()]; - if (!isSorted) { - devices = devices.stream().sorted(IDeviceID::compareTo).collect(Collectors.toList()); - } - long[][] results = new long[devices.size()][]; - getDeviceMetadataIndexNodeOffsetList( - results, devices, 0, devices.size(), tableMetadataIndexNode); + long[][] results = new long[sortedDevices.size()][]; + getDeviceMetadataIndexNodeOffsets( + results, sortedDevices, 0, sortedDevices.size(), tableMetadataIndexNode); return results; } - private void getDeviceMetadataIndexNodeOffsetList( + private void getDeviceMetadataIndexNodeOffsets( long[][] results, List devices, int deviceStartIdx, @@ -583,7 +579,7 @@ private void getDeviceMetadataIndexNodeOffsetList( ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight()); MetadataIndexNode lastNode = MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); - getDeviceMetadataIndexNodeOffsetList(results, devices, startIdxOfChild, i, lastNode); + getDeviceMetadataIndexNodeOffsets(results, devices, startIdxOfChild, i, lastNode); previousPair = pair; startIdxOfChild = i; } @@ -594,7 +590,7 @@ private void getDeviceMetadataIndexNodeOffsetList( IMetadataIndexEntry entry = previousPair.getLeft(); ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight()); MetadataIndexNode lastNode = MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); - getDeviceMetadataIndexNodeOffsetList(results, devices, startIdxOfChild, deviceEndIdx, lastNode); + getDeviceMetadataIndexNodeOffsets(results, devices, startIdxOfChild, deviceEndIdx, lastNode); } public TimeseriesMetadata readTimeseriesMetadata( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java index 8daa25277..e2588d9c0 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java @@ -29,10 +29,13 @@ import org.apache.tsfile.file.header.ChunkGroupHeader; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.file.metadata.MetadataIndexNode; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Path; @@ -60,6 +63,7 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -263,4 +267,60 @@ public void testGetTableSchemaMap() throws IOException, WriteProcessException { Assert.assertTrue(reader.readFileMetadata().hasTableSchemaMapCache()); } } + + @Test + public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WriteProcessException { + File file = new File(FILE_PATH); + try { + tsFile.close(); + Files.deleteIfExists(file.toPath()); + } catch (IOException ignored) { + } + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new MeasurementSchema("device", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT32)), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + try (ITsFileWriter writer = + new TsFileWriterBuilder().tableSchema(tableSchema).file(file).build()) { + Tablet tablet = + new Tablet( + Arrays.asList("device", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + 10000); + for (int i = 0; i < 10000; i++) { + tablet.addTimestamp(i, i); + tablet.addValue("device", i, "d" + i); + tablet.addValue("s1", i, i); + } + writer.write(tablet); + } + + List queriedDevices = new ArrayList<>(); + for (int i = 0; i < 20000; i++) { + queriedDevices.add(new StringArrayDeviceID("t1.d" + i)); + } + queriedDevices.sort(IDeviceID::compareTo); + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + long[][] offsets = reader.getDeviceMetadataIndexNodeOffsets("t1", queriedDevices); + Assert.assertEquals(20000, offsets.length); + for (int i = 0; i < offsets.length; i++) { + IDeviceID deviceID = queriedDevices.get(i); + int deviceNumber = Integer.parseInt(deviceID.toString().substring("t1.d".length())); + if (deviceNumber >= 10000) { + Assert.assertNull(offsets[i]); + continue; + } + MetadataIndexNode metadataIndexNode = + reader.readMetadataIndexNode(offsets[i][0], offsets[i][1], false); + List alignedChunkMetadataList = + reader.getAlignedChunkMetadataByMetadataIndexNode(deviceID, metadataIndexNode, true); + Assert.assertEquals(1, alignedChunkMetadataList.size()); + + Assert.assertEquals(deviceNumber, alignedChunkMetadataList.get(0).getStartTime()); + } + } + } } From f802f20eaa79749db357fb1e8b17c29d017aecf7 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 18 Jul 2025 11:58:41 +0800 Subject: [PATCH 03/18] add comment --- .../org/apache/tsfile/read/TsFileSequenceReader.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 54e0d03ab..e8ff30bcc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -535,12 +535,21 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev return deviceMetadata; } + /** + * Find the offset of MetadataIndexNode corresponding to every device to avoid repeated reading of + * internal MetadataIndexNode + * + * @param table table name, or "" for tree model + * @param sortedDevices devices should be sorted + * @throws IOException io error + */ public long[][] getDeviceMetadataIndexNodeOffsets(String table, List sortedDevices) throws IOException { readFileMetadata(); MetadataIndexNode tableMetadataIndexNode = getTableRootNode(table); if (tableMetadataIndexNode == null) { - throw new IllegalArgumentException(""); + throw new IllegalArgumentException( + "table {" + table + "} is not in tsFileMetaData of " + file); } long[][] results = new long[sortedDevices.size()][]; getDeviceMetadataIndexNodeOffsets( From 64cf6375f750a58a3e6f55cb8a8372c03a6eff2b Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 21 Jul 2025 14:30:04 +0800 Subject: [PATCH 04/18] add merge search --- .../file/metadata/MetadataIndexNode.java | 41 ++++++++++++++++--- .../tsfile/read/TsFileSequenceReader.java | 2 +- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 5bc7e7ed9..f734b66b4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -170,9 +170,12 @@ int binarySearchInChildren(Comparable key, boolean exactSearch) { } } - public List> getChildIndexEntry( + public List> getChildIndexEntries( List keys, boolean exactSearch) { - int[] indexArr = binarySearchInChildren(keys, exactSearch, 0, children.size() - 1); + int[] indexArr = + keys.size() >= children.size() + ? mergeSearchInChildren(keys, exactSearch) + : binarySearchInChildren(keys, exactSearch); List> pairs = new ArrayList<>(); int previousIndex = -1; Pair previousPair = null; @@ -189,11 +192,11 @@ public List> getChildIndexEntry( return pairs; } - int[] binarySearchInChildren( - List keys, boolean exactSearch, int low, int high) { + int[] binarySearchInChildren(List keys, boolean exactSearch) { int[] results = new int[keys.size()]; Arrays.fill(results, -1); - int currentLow = low; + int currentLow = 0; + int high = children.size() - 1; for (int i = 0; i < keys.size(); i++) { Comparable key = keys.get(i); @@ -238,6 +241,34 @@ int[] binarySearchInChildren( return results; } + int[] mergeSearchInChildren(List keys, boolean exactSearch) { + int[] results = new int[keys.size()]; + int i = 0; + int j = 0; + while (i < keys.size() && j < children.size()) { + Comparable currentKey = keys.get(i); + Comparable currentChild = children.get(j).getCompareKey(); + int cmp = currentKey.compareTo(currentChild); + if (cmp == 0) { + results[i] = j; + i++; + j++; + } else if (cmp > 0) { + j++; + } else { + if (exactSearch) { + results[i] = -1; + } else { + results[i] = j == 0 ? 0 : j - 1; + } + i++; + } + } + Arrays.fill( + results, i, keys.size(), (exactSearch || children.isEmpty()) ? -1 : children.size() - 1); + return results; + } + public boolean isDeviceLevel() { return this.nodeType == MetadataIndexNodeType.INTERNAL_DEVICE || this.nodeType == MetadataIndexNodeType.LEAF_DEVICE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index e8ff30bcc..4c6e4f47c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -567,7 +567,7 @@ private void getDeviceMetadataIndexNodeOffsets( MetadataIndexNodeType metadataIndexNodeType = startNode.getNodeType(); boolean exactSearch = metadataIndexNodeType == MetadataIndexNodeType.LEAF_DEVICE; List> entries = - startNode.getChildIndexEntry(devices.subList(deviceStartIdx, deviceEndIdx), exactSearch); + startNode.getChildIndexEntries(devices.subList(deviceStartIdx, deviceEndIdx), exactSearch); Iterator> metadataIndexEntriesIterator = entries.iterator(); int startIdxOfChild = 0; Pair previousPair = null; From 6abc0760d64dfb3987eb0ec421f2df611b75af85 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 21 Jul 2025 15:00:12 +0800 Subject: [PATCH 05/18] modify MetadataIndexNode --- .../java/org/apache/tsfile/file/metadata/MetadataIndexNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index f734b66b4..401a2249b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -265,7 +265,7 @@ int[] mergeSearchInChildren(List keys, boolean exactSearch } } Arrays.fill( - results, i, keys.size(), (exactSearch || children.isEmpty()) ? -1 : children.size() - 1); + results, i, keys.size(), exactSearch ? -1 : children.size() - 1); return results; } From 32f102455e00b5d7a7c80d8b0cfbfe369b89eed3 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 21 Jul 2025 15:27:53 +0800 Subject: [PATCH 06/18] add comment --- .../apache/tsfile/file/metadata/MetadataIndexNode.java | 3 +-- .../java/org/apache/tsfile/read/TsFileSequenceReader.java | 5 +++++ .../org/apache/tsfile/read/TsFileSequenceReaderTest.java | 8 ++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 401a2249b..84abbe495 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -264,8 +264,7 @@ int[] mergeSearchInChildren(List keys, boolean exactSearch i++; } } - Arrays.fill( - results, i, keys.size(), exactSearch ? -1 : children.size() - 1); + Arrays.fill(results, i, keys.size(), exactSearch ? -1 : children.size() - 1); return results; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 4c6e4f47c..afe5f666d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -541,6 +541,8 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev * * @param table table name, or "" for tree model * @param sortedDevices devices should be sorted + * @return Each element of the outer array corresponds to the device at this index. The inner + * array size is 2, the first element is the start offset, and the second is the end offset * @throws IOException io error */ public long[][] getDeviceMetadataIndexNodeOffsets(String table, List sortedDevices) @@ -552,6 +554,9 @@ public long[][] getDeviceMetadataIndexNodeOffsets(String table, List "table {" + table + "} is not in tsFileMetaData of " + file); } long[][] results = new long[sortedDevices.size()][]; + if (sortedDevices.isEmpty()) { + return results; + } getDeviceMetadataIndexNodeOffsets( results, sortedDevices, 0, sortedDevices.size(), tableMetadataIndexNode); return results; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java index e2588d9c0..a711b6201 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java @@ -321,6 +321,14 @@ public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WritePro Assert.assertEquals(deviceNumber, alignedChunkMetadataList.get(0).getStartTime()); } + + Assert.assertEquals( + 0, reader.getDeviceMetadataIndexNodeOffsets("t1", Collections.emptyList()).length); + offsets = + reader.getDeviceMetadataIndexNodeOffsets( + "t1", Collections.singletonList(new StringArrayDeviceID("t1.d"))); + Assert.assertEquals(1, offsets.length); + Assert.assertNull(offsets[0]); } } } From b81dfeff324f60f540ca6d688848300a75c16485 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 21 Jul 2025 15:58:14 +0800 Subject: [PATCH 07/18] modify MetadataIndexNode --- .../org/apache/tsfile/file/metadata/MetadataIndexNode.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 84abbe495..870399ddb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -38,6 +38,7 @@ public class MetadataIndexNode { + private static final double LOG2 = Math.log(2); protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); protected final List children; protected long endOffset; @@ -174,6 +175,8 @@ public List> getChildIndexEntries( List keys, boolean exactSearch) { int[] indexArr = keys.size() >= children.size() + || (keys.size() * Math.log(children.size()) / LOG2) + > (keys.size() + children.size()) ? mergeSearchInChildren(keys, exactSearch) : binarySearchInChildren(keys, exactSearch); List> pairs = new ArrayList<>(); From f551d3d5e45ce21f821676546290bca7cc3529e1 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 21 Jul 2025 16:43:10 +0800 Subject: [PATCH 08/18] fix bug --- .../main/java/org/apache/tsfile/read/TsFileSequenceReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index afe5f666d..6bc0ad9d3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -574,7 +574,7 @@ private void getDeviceMetadataIndexNodeOffsets( List> entries = startNode.getChildIndexEntries(devices.subList(deviceStartIdx, deviceEndIdx), exactSearch); Iterator> metadataIndexEntriesIterator = entries.iterator(); - int startIdxOfChild = 0; + int startIdxOfChild = deviceStartIdx; Pair previousPair = null; for (int i = deviceStartIdx; i < deviceEndIdx; i++) { Pair pair = metadataIndexEntriesIterator.next(); From 938df9b7cd7d4fed205c7948fec93d57439f8ebb Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 21 Jul 2025 17:20:03 +0800 Subject: [PATCH 09/18] add ioSizeRecorder --- .../tsfile/read/TsFileSequenceReader.java | 20 +++++++++++-------- .../tsfile/read/TsFileSequenceReaderTest.java | 6 +++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 6bc0ad9d3..160fcd3fa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -541,12 +541,13 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev * * @param table table name, or "" for tree model * @param sortedDevices devices should be sorted + * @param ioSizeRecorder can be null * @return Each element of the outer array corresponds to the device at this index. The inner * array size is 2, the first element is the start offset, and the second is the end offset * @throws IOException io error */ - public long[][] getDeviceMetadataIndexNodeOffsets(String table, List sortedDevices) - throws IOException { + public long[][] getDeviceMetadataIndexNodeOffsets( + String table, List sortedDevices, LongConsumer ioSizeRecorder) throws IOException { readFileMetadata(); MetadataIndexNode tableMetadataIndexNode = getTableRootNode(table); if (tableMetadataIndexNode == null) { @@ -558,7 +559,7 @@ public long[][] getDeviceMetadataIndexNodeOffsets(String table, List return results; } getDeviceMetadataIndexNodeOffsets( - results, sortedDevices, 0, sortedDevices.size(), tableMetadataIndexNode); + results, sortedDevices, 0, sortedDevices.size(), tableMetadataIndexNode, ioSizeRecorder); return results; } @@ -567,7 +568,8 @@ private void getDeviceMetadataIndexNodeOffsets( List devices, int deviceStartIdx, int deviceEndIdx, - MetadataIndexNode startNode) + MetadataIndexNode startNode, + LongConsumer ioSizeRecorder) throws IOException { MetadataIndexNodeType metadataIndexNodeType = startNode.getNodeType(); boolean exactSearch = metadataIndexNodeType == MetadataIndexNodeType.LEAF_DEVICE; @@ -590,10 +592,11 @@ private void getDeviceMetadataIndexNodeOffsets( continue; } IMetadataIndexEntry entry = previousPair.getLeft(); - ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight()); + ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight(), ioSizeRecorder); MetadataIndexNode lastNode = MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); - getDeviceMetadataIndexNodeOffsets(results, devices, startIdxOfChild, i, lastNode); + getDeviceMetadataIndexNodeOffsets( + results, devices, startIdxOfChild, i, lastNode, ioSizeRecorder); previousPair = pair; startIdxOfChild = i; } @@ -602,9 +605,10 @@ private void getDeviceMetadataIndexNodeOffsets( } // for last entry IMetadataIndexEntry entry = previousPair.getLeft(); - ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight()); + ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight(), ioSizeRecorder); MetadataIndexNode lastNode = MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); - getDeviceMetadataIndexNodeOffsets(results, devices, startIdxOfChild, deviceEndIdx, lastNode); + getDeviceMetadataIndexNodeOffsets( + results, devices, startIdxOfChild, deviceEndIdx, lastNode, ioSizeRecorder); } public TimeseriesMetadata readTimeseriesMetadata( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java index a711b6201..09f7dde0b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java @@ -304,7 +304,7 @@ public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WritePro } queriedDevices.sort(IDeviceID::compareTo); try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { - long[][] offsets = reader.getDeviceMetadataIndexNodeOffsets("t1", queriedDevices); + long[][] offsets = reader.getDeviceMetadataIndexNodeOffsets("t1", queriedDevices, null); Assert.assertEquals(20000, offsets.length); for (int i = 0; i < offsets.length; i++) { IDeviceID deviceID = queriedDevices.get(i); @@ -323,10 +323,10 @@ public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WritePro } Assert.assertEquals( - 0, reader.getDeviceMetadataIndexNodeOffsets("t1", Collections.emptyList()).length); + 0, reader.getDeviceMetadataIndexNodeOffsets("t1", Collections.emptyList(), null).length); offsets = reader.getDeviceMetadataIndexNodeOffsets( - "t1", Collections.singletonList(new StringArrayDeviceID("t1.d"))); + "t1", Collections.singletonList(new StringArrayDeviceID("t1.d")), null); Assert.assertEquals(1, offsets.length); Assert.assertNull(offsets[0]); } From 1aff2e270a6ea3edc6d4e702e3202c0c95212819 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 21 Jul 2025 17:51:43 +0800 Subject: [PATCH 10/18] modify MetadataIndexNode --- .../org/apache/tsfile/file/metadata/MetadataIndexNode.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 870399ddb..7f043c076 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -204,8 +204,7 @@ int[] binarySearchInChildren(List keys, boolean exactSearc for (int i = 0; i < keys.size(); i++) { Comparable key = keys.get(i); if (currentLow > high) { - Arrays.fill( - results, i, keys.size(), exactSearch ? -1 : (currentLow == 0 ? 0 : currentLow - 1)); + Arrays.fill(results, i, keys.size(), exactSearch ? -1 : currentLow - 1); return results; } @@ -235,7 +234,7 @@ int[] binarySearchInChildren(List keys, boolean exactSearc if (exactSearch) { results[i] = -1; } else { - int insertPos = (start == 0) ? 0 : start - 1; + int insertPos = start - 1; results[i] = insertPos; currentLow = start; } @@ -262,7 +261,7 @@ int[] mergeSearchInChildren(List keys, boolean exactSearch if (exactSearch) { results[i] = -1; } else { - results[i] = j == 0 ? 0 : j - 1; + results[i] = j - 1; } i++; } From e59da8e82157388dd1c0631007993058aca11068 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 25 Jul 2025 16:22:14 +0800 Subject: [PATCH 11/18] modify TsFileSequenceReader --- .../tsfile/read/TsFileSequenceReader.java | 66 +++++++++++++++---- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 160fcd3fa..d638d6061 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -89,6 +89,7 @@ import java.nio.ByteBuffer; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Deque; @@ -546,7 +547,7 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev * array size is 2, the first element is the start offset, and the second is the end offset * @throws IOException io error */ - public long[][] getDeviceMetadataIndexNodeOffsets( + public long[] getDeviceMetadataIndexNodeOffsets( String table, List sortedDevices, LongConsumer ioSizeRecorder) throws IOException { readFileMetadata(); MetadataIndexNode tableMetadataIndexNode = getTableRootNode(table); @@ -554,7 +555,8 @@ public long[][] getDeviceMetadataIndexNodeOffsets( throw new IllegalArgumentException( "table {" + table + "} is not in tsFileMetaData of " + file); } - long[][] results = new long[sortedDevices.size()][]; + long[] results = new long[2 * sortedDevices.size()]; + Arrays.fill(results, -1); if (sortedDevices.isEmpty()) { return results; } @@ -564,7 +566,7 @@ public long[][] getDeviceMetadataIndexNodeOffsets( } private void getDeviceMetadataIndexNodeOffsets( - long[][] results, + long[] results, List devices, int deviceStartIdx, int deviceEndIdx, @@ -581,7 +583,10 @@ private void getDeviceMetadataIndexNodeOffsets( for (int i = deviceStartIdx; i < deviceEndIdx; i++) { Pair pair = metadataIndexEntriesIterator.next(); if (exactSearch) { - results[i] = pair == null ? null : new long[] {pair.getLeft().getOffset(), pair.getRight()}; + if (pair != null) { + results[2 * i] = pair.getLeft().getOffset(); + results[2 * i + 1] = pair.getRight(); + } continue; } if (previousPair == null) { @@ -622,11 +627,31 @@ public TimeseriesMetadata readTimeseriesMetadata( boolean ignoreNotExistDevice, LongConsumer ioSizeConsumer) throws IOException { + return readTimeseriesMetadata(device, null, measurement, ignoreNotExistDevice, ioSizeConsumer); + } + + public TimeseriesMetadata readTimeseriesMetadata( + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + String measurement, + boolean ignoreNotExistDevice, + LongConsumer ioSizeConsumer) + throws IOException { readFileMetadata(ioSizeConsumer); MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); - Pair metadataIndexPair = - getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true, ioSizeConsumer); + Pair metadataIndexPair; + if (deviceMetadataIndexNodeOffset == null) { + metadataIndexPair = + getMetadataAndEndOffsetOfDeviceNode( + deviceMetadataIndexNode, device, true, ioSizeConsumer); + } else { + metadataIndexPair = + new Pair<>( + new DeviceMetadataIndexEntry(device, deviceMetadataIndexNodeOffset[0]), + deviceMetadataIndexNodeOffset[1]); + } + if (metadataIndexPair == null) { if (ignoreNotExistDevice) { return null; @@ -759,13 +784,15 @@ public ITimeSeriesMetadata readITimeseriesMetadata(Path path, boolean ignoreNotE */ public List readTimeseriesMetadata( IDeviceID device, + long[] deviceMetadataIndexNodeOffset, String measurement, Set allSensors, boolean ignoreNotExistDevice, LongConsumer ioSizeRecorder) throws IOException { Pair metadataIndexPair = - getLeafMetadataIndexPair(device, measurement, ioSizeRecorder); + getLeafMetadataIndexPair( + device, deviceMetadataIndexNodeOffset, measurement, ioSizeRecorder); if (metadataIndexPair == null) { if (ignoreNotExistDevice) { return Collections.emptyList(); @@ -820,17 +847,28 @@ public List readTimeseriesMetadata( /* Get leaf MetadataIndexPair which contains path */ private Pair getLeafMetadataIndexPair( - IDeviceID device, String measurement, LongConsumer ioSizeRecorder) throws IOException { + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + String measurement, + LongConsumer ioSizeRecorder) + throws IOException { readFileMetadata(ioSizeRecorder); MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); - Pair metadataIndexPair = - getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true, ioSizeRecorder); - if (metadataIndexPair == null) { - return null; + Pair metadataIndexPair = null; + ByteBuffer buffer; + if (deviceMetadataIndexNodeOffset == null) { + metadataIndexPair = + getMetadataAndEndOffsetOfDeviceNode( + deviceMetadataIndexNode, device, true, ioSizeRecorder); + if (metadataIndexPair == null) { + return null; + } + buffer = + readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right, ioSizeRecorder); + } else { + buffer = readData(deviceMetadataIndexNodeOffset[0], deviceMetadataIndexNodeOffset[1]); } - ByteBuffer buffer = - readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right, ioSizeRecorder); MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode; if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { try { From c44f690bc6c956556ca1cefed8225cad1a3a58b9 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 25 Jul 2025 16:25:16 +0800 Subject: [PATCH 12/18] update ut --- .../tsfile/read/TimeSeriesMetadataReadTest.java | 6 ++++-- .../tsfile/read/TsFileSequenceReaderTest.java | 14 +++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index e4bfbb44d..750c53feb 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -73,7 +73,8 @@ public void testReadTimeseriesMetadata() throws IOException { // s4 should not be returned as result set.add("s4"); List timeseriesMetadataList = - reader.readTimeseriesMetadata(path.getIDeviceID(), path.getMeasurement(), set, false, null); + reader.readTimeseriesMetadata( + path.getIDeviceID(), null, path.getMeasurement(), set, false, null); Assert.assertEquals(3, timeseriesMetadataList.size()); for (int i = 1; i <= timeseriesMetadataList.size(); i++) { Assert.assertEquals("s" + i, timeseriesMetadataList.get(i - 1).getMeasurementId()); @@ -87,7 +88,8 @@ public void testReadTimeseriesMetadata() throws IOException { // so the result is not supposed to contain this measurement's timeseries metadata set.add("s8"); timeseriesMetadataList = - reader.readTimeseriesMetadata(path.getIDeviceID(), path.getMeasurement(), set, false, null); + reader.readTimeseriesMetadata( + path.getIDeviceID(), null, path.getMeasurement(), set, false, null); Assert.assertEquals(2, timeseriesMetadataList.size()); for (int i = 5; i < 7; i++) { Assert.assertEquals("s" + i, timeseriesMetadataList.get(i - 5).getMeasurementId()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java index 09f7dde0b..f4225d2c4 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java @@ -304,17 +304,17 @@ public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WritePro } queriedDevices.sort(IDeviceID::compareTo); try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { - long[][] offsets = reader.getDeviceMetadataIndexNodeOffsets("t1", queriedDevices, null); - Assert.assertEquals(20000, offsets.length); - for (int i = 0; i < offsets.length; i++) { + long[] offsets = reader.getDeviceMetadataIndexNodeOffsets("t1", queriedDevices, null); + Assert.assertEquals(20000 * 2, offsets.length); + for (int i = 0; i < queriedDevices.size(); i++) { IDeviceID deviceID = queriedDevices.get(i); int deviceNumber = Integer.parseInt(deviceID.toString().substring("t1.d".length())); if (deviceNumber >= 10000) { - Assert.assertNull(offsets[i]); + Assert.assertEquals(-1, offsets[2 * i]); continue; } MetadataIndexNode metadataIndexNode = - reader.readMetadataIndexNode(offsets[i][0], offsets[i][1], false); + reader.readMetadataIndexNode(offsets[2 * i], offsets[2 * i + 1], false); List alignedChunkMetadataList = reader.getAlignedChunkMetadataByMetadataIndexNode(deviceID, metadataIndexNode, true); Assert.assertEquals(1, alignedChunkMetadataList.size()); @@ -327,8 +327,8 @@ public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WritePro offsets = reader.getDeviceMetadataIndexNodeOffsets( "t1", Collections.singletonList(new StringArrayDeviceID("t1.d")), null); - Assert.assertEquals(1, offsets.length); - Assert.assertNull(offsets[0]); + Assert.assertEquals(2, offsets.length); + Assert.assertEquals(-1, offsets[0]); } } } From 91bbc5155f7a34def4c8ce549de65a02e810c1d9 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 29 Jul 2025 15:35:06 +0800 Subject: [PATCH 13/18] support search devices from different tables --- .../tsfile/read/TsFileSequenceReader.java | 58 +++++++++++++++---- .../tsfile/read/TsFileSequenceReaderTest.java | 8 ++- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index d638d6061..b399fd744 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -540,7 +540,7 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev * Find the offset of MetadataIndexNode corresponding to every device to avoid repeated reading of * internal MetadataIndexNode * - * @param table table name, or "" for tree model + * @param table table name. ("" or null) for cases with different tables * @param sortedDevices devices should be sorted * @param ioSizeRecorder can be null * @return Each element of the outer array corresponds to the device at this index. The inner @@ -549,22 +549,60 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev */ public long[] getDeviceMetadataIndexNodeOffsets( String table, List sortedDevices, LongConsumer ioSizeRecorder) throws IOException { - readFileMetadata(); - MetadataIndexNode tableMetadataIndexNode = getTableRootNode(table); - if (tableMetadataIndexNode == null) { - throw new IllegalArgumentException( - "table {" + table + "} is not in tsFileMetaData of " + file); - } long[] results = new long[2 * sortedDevices.size()]; - Arrays.fill(results, -1); if (sortedDevices.isEmpty()) { return results; } - getDeviceMetadataIndexNodeOffsets( - results, sortedDevices, 0, sortedDevices.size(), tableMetadataIndexNode, ioSizeRecorder); + Arrays.fill(results, -1); + readFileMetadata(); + // for many table + if (table == null || table.isEmpty()) { + int tableStartIndex = 0; + String previousTable = null; + for (int i = 0; i < sortedDevices.size(); i++) { + String currentTable = sortedDevices.get(i).getTableName(); + if (i == 0) { + previousTable = currentTable; + continue; + } + if (!previousTable.equals(currentTable)) { + getDeviceMetadataIndexNodeOffsets( + results, sortedDevices, tableStartIndex, i, previousTable, ioSizeRecorder); + tableStartIndex = i; + previousTable = currentTable; + } + } + // for last table + getDeviceMetadataIndexNodeOffsets( + results, + sortedDevices, + tableStartIndex, + sortedDevices.size(), + previousTable, + ioSizeRecorder); + } else { + getDeviceMetadataIndexNodeOffsets( + results, sortedDevices, 0, sortedDevices.size(), table, ioSizeRecorder); + } return results; } + private void getDeviceMetadataIndexNodeOffsets( + long[] results, + List devices, + int deviceStartIdx, + int deviceEndIdx, + String table, + LongConsumer ioSizeRecorder) + throws IOException { + MetadataIndexNode tableMetadataIndexNode = getTableRootNode(table); + if (tableMetadataIndexNode == null) { + return; + } + getDeviceMetadataIndexNodeOffsets( + results, devices, deviceStartIdx, deviceEndIdx, tableMetadataIndexNode, ioSizeRecorder); + } + private void getDeviceMetadataIndexNodeOffsets( long[] results, List devices, diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java index f4225d2c4..25a45a618 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java @@ -300,11 +300,15 @@ public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WritePro List queriedDevices = new ArrayList<>(); for (int i = 0; i < 20000; i++) { - queriedDevices.add(new StringArrayDeviceID("t1.d" + i)); + if (i >= 15000) { + queriedDevices.add(new StringArrayDeviceID("t2.d" + i)); + } else { + queriedDevices.add(new StringArrayDeviceID("t1.d" + i)); + } } queriedDevices.sort(IDeviceID::compareTo); try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { - long[] offsets = reader.getDeviceMetadataIndexNodeOffsets("t1", queriedDevices, null); + long[] offsets = reader.getDeviceMetadataIndexNodeOffsets(null, queriedDevices, null); Assert.assertEquals(20000 * 2, offsets.length); for (int i = 0; i < queriedDevices.size(); i++) { IDeviceID deviceID = queriedDevices.get(i); From 1525b7582daf0cf123d7b82f29c5752e0b19e181 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 31 Jul 2025 18:24:19 +0800 Subject: [PATCH 14/18] compact array to map --- .../tsfile/read/TsFileSequenceReader.java | 32 +- ...DeviceMetadataIndexEntriesQueryResult.java | 142 ++++++++ ...eMetadataIndexNodeOffsetsQueryContext.java | 321 ++++++++++++++++++ .../tsfile/read/TsFileSequenceReaderTest.java | 19 +- 4 files changed, 491 insertions(+), 23 deletions(-) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index b399fd744..71af9fd74 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -67,6 +67,8 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl; +import org.apache.tsfile.read.query.DeviceMetadataIndexEntriesQueryResult; +import org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext; import org.apache.tsfile.read.reader.TsFileInput; import org.apache.tsfile.read.reader.page.PageReader; import org.apache.tsfile.read.reader.page.TimePageReader; @@ -89,7 +91,6 @@ import java.nio.ByteBuffer; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Deque; @@ -547,13 +548,13 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev * array size is 2, the first element is the start offset, and the second is the end offset * @throws IOException io error */ - public long[] getDeviceMetadataIndexNodeOffsets( + public DeviceMetadataIndexEntriesQueryResult getDeviceMetadataIndexNodeOffsets( String table, List sortedDevices, LongConsumer ioSizeRecorder) throws IOException { - long[] results = new long[2 * sortedDevices.size()]; + DeviceMetadataIndexNodeOffsetsQueryContext context = + new DeviceMetadataIndexNodeOffsetsQueryContext(sortedDevices.size()); if (sortedDevices.isEmpty()) { - return results; + return context.compact(); } - Arrays.fill(results, -1); readFileMetadata(); // for many table if (table == null || table.isEmpty()) { @@ -567,14 +568,14 @@ public long[] getDeviceMetadataIndexNodeOffsets( } if (!previousTable.equals(currentTable)) { getDeviceMetadataIndexNodeOffsets( - results, sortedDevices, tableStartIndex, i, previousTable, ioSizeRecorder); + context, sortedDevices, tableStartIndex, i, previousTable, ioSizeRecorder); tableStartIndex = i; previousTable = currentTable; } } // for last table getDeviceMetadataIndexNodeOffsets( - results, + context, sortedDevices, tableStartIndex, sortedDevices.size(), @@ -582,13 +583,13 @@ public long[] getDeviceMetadataIndexNodeOffsets( ioSizeRecorder); } else { getDeviceMetadataIndexNodeOffsets( - results, sortedDevices, 0, sortedDevices.size(), table, ioSizeRecorder); + context, sortedDevices, 0, sortedDevices.size(), table, ioSizeRecorder); } - return results; + return context.compact(); } private void getDeviceMetadataIndexNodeOffsets( - long[] results, + DeviceMetadataIndexNodeOffsetsQueryContext context, List devices, int deviceStartIdx, int deviceEndIdx, @@ -600,11 +601,11 @@ private void getDeviceMetadataIndexNodeOffsets( return; } getDeviceMetadataIndexNodeOffsets( - results, devices, deviceStartIdx, deviceEndIdx, tableMetadataIndexNode, ioSizeRecorder); + context, devices, deviceStartIdx, deviceEndIdx, tableMetadataIndexNode, ioSizeRecorder); } private void getDeviceMetadataIndexNodeOffsets( - long[] results, + DeviceMetadataIndexNodeOffsetsQueryContext context, List devices, int deviceStartIdx, int deviceEndIdx, @@ -622,8 +623,7 @@ private void getDeviceMetadataIndexNodeOffsets( Pair pair = metadataIndexEntriesIterator.next(); if (exactSearch) { if (pair != null) { - results[2 * i] = pair.getLeft().getOffset(); - results[2 * i + 1] = pair.getRight(); + context.addDeviceMetadataIndexNodeOffset(i, pair.getLeft().getOffset(), pair.getRight()); } continue; } @@ -639,7 +639,7 @@ private void getDeviceMetadataIndexNodeOffsets( MetadataIndexNode lastNode = MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); getDeviceMetadataIndexNodeOffsets( - results, devices, startIdxOfChild, i, lastNode, ioSizeRecorder); + context, devices, startIdxOfChild, i, lastNode, ioSizeRecorder); previousPair = pair; startIdxOfChild = i; } @@ -651,7 +651,7 @@ private void getDeviceMetadataIndexNodeOffsets( ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight(), ioSizeRecorder); MetadataIndexNode lastNode = MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); getDeviceMetadataIndexNodeOffsets( - results, devices, startIdxOfChild, deviceEndIdx, lastNode, ioSizeRecorder); + context, devices, startIdxOfChild, deviceEndIdx, lastNode, ioSizeRecorder); } public TimeseriesMetadata readTimeseriesMetadata( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java new file mode 100644 index 000000000..d1c5e6323 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.query; + +import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Arrays; + +public interface DeviceMetadataIndexEntriesQueryResult extends Accountable { + long[] getDeviceMetadataIndexNodeOffset(int deviceIndex); + + int length(); +} + +class ArrDeviceMetadataIndexEntriesQueryResult implements DeviceMetadataIndexEntriesQueryResult { + private static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ArrDeviceMetadataIndexEntriesQueryResult.class); + + private final Object offsetDeltaArr; + private final Object nodeSizeArr; + private final long standardOffset; + private final int length; + + ArrDeviceMetadataIndexEntriesQueryResult( + Object offsetDeltaArr, Object nodeSizeArr, long standardOffset, int length) { + this.offsetDeltaArr = offsetDeltaArr; + this.nodeSizeArr = nodeSizeArr; + this.standardOffset = standardOffset; + this.length = length; + } + + @Override + public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { + if (deviceIndex >= length) { + return null; + } + long nodeSize; + long startOffset; + if (nodeSizeArr instanceof byte[]) { + nodeSize = (((byte[]) nodeSizeArr)[deviceIndex]) & 0XFFL; + } else if (nodeSizeArr instanceof short[]) { + nodeSize = (((short[]) nodeSizeArr)[deviceIndex]) & 0XFFFFL; + } else if (nodeSizeArr instanceof int[]) { + nodeSize = (((int[]) nodeSizeArr)[deviceIndex]) & 0XFFFFFFFFL; + } else { + nodeSize = ((long[]) nodeSizeArr)[deviceIndex]; + } + if (nodeSize <= 0) { + return null; + } + if (offsetDeltaArr instanceof short[]) { + startOffset = standardOffset + ((short[]) offsetDeltaArr)[deviceIndex]; + } else if (offsetDeltaArr instanceof int[]) { + startOffset = standardOffset + ((int[]) offsetDeltaArr)[deviceIndex]; + } else { + startOffset = standardOffset + ((long[]) offsetDeltaArr)[deviceIndex]; + } + + return new long[] {startOffset, startOffset + nodeSize}; + } + + @Override + public int length() { + return length; + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + getRamBytesUsedOfArr(nodeSizeArr) + getRamBytesUsedOfArr(offsetDeltaArr); + } + + private long getRamBytesUsedOfArr(Object arr) { + if (arr instanceof long[]) { + return RamUsageEstimator.sizeOfLongArray(length); + } else if (arr instanceof int[]) { + return RamUsageEstimator.sizeOfIntArray(length); + } else if (arr instanceof short[]) { + return RamUsageEstimator.sizeOfShortArray(length); + } else { + return RamUsageEstimator.sizeOfByteArray(length); + } + } +} + +class MapDeviceMetadataIndexEntriesQueryResult implements DeviceMetadataIndexEntriesQueryResult { + + private static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(MapDeviceMetadataIndexEntriesQueryResult.class); + private final int[] indexMap; + private final ArrDeviceMetadataIndexEntriesQueryResult arrResult; + + public MapDeviceMetadataIndexEntriesQueryResult( + int[] indexMap, Object offsetDeltaArr, Object nodeSizeArr, long standardOffset, int length) { + this.indexMap = indexMap; + this.arrResult = + new ArrDeviceMetadataIndexEntriesQueryResult( + offsetDeltaArr, nodeSizeArr, standardOffset, length); + } + + @Override + public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { + if (indexMap.length == 0) { + return null; + } + int idx = + Arrays.binarySearch(indexMap, 0, Math.min(deviceIndex + 1, indexMap.length), deviceIndex); + if (idx < 0) { + return null; + } + return arrResult.getDeviceMetadataIndexNodeOffset(idx); + } + + @Override + public int length() { + return indexMap.length; + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + + arrResult.ramBytesUsed() + + RamUsageEstimator.sizeOfIntArray(indexMap.length); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java new file mode 100644 index 000000000..a10fcc5eb --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.query; + +public class DeviceMetadataIndexNodeOffsetsQueryContext { + + private long standardStartOffset; + private long minStartOffset = Long.MAX_VALUE; + private long maxStartOffset = Long.MIN_VALUE; + private long maxNodeSize; + private final int length; + private int used; + + private long[] longStartOffsetArr; + private long[] longNodeSizeArr; + private int[] intNodeSizeArr; + private short[] shortNodeSizeArr; + private byte[] byteNodeSizeArr; + + private Object startOffsetDeltaArr; + private Object nodeSizeArr; + + public DeviceMetadataIndexNodeOffsetsQueryContext(int length) { + this(length, -1); + } + + public DeviceMetadataIndexNodeOffsetsQueryContext(int length, long metadataSize) { + this.length = length; + this.used = 0; + longStartOffsetArr = new long[length]; + if (metadataSize <= 0 || metadataSize > 0XFFFFFFFFL) { + longNodeSizeArr = new long[length]; + } else if (metadataSize > 0XFFFFL) { + intNodeSizeArr = new int[length]; + } else if (metadataSize > 0XFFL) { + shortNodeSizeArr = new short[length]; + } else { + byteNodeSizeArr = new byte[length]; + } + } + + public void addDeviceMetadataIndexNodeOffset(int i, long startOffset, long endOffset) { + minStartOffset = Math.min(startOffset, minStartOffset); + maxStartOffset = Math.max(startOffset, maxStartOffset); + longStartOffsetArr[i] = startOffset; + long nodeSize = endOffset - startOffset; + maxNodeSize = Math.max(nodeSize, maxNodeSize); + if (longNodeSizeArr != null) { + longNodeSizeArr[i] = nodeSize; + } else if (intNodeSizeArr != null) { + intNodeSizeArr[i] = (int) (nodeSize & 0XFFFFFFFFL); + } else if (shortNodeSizeArr != null) { + shortNodeSizeArr[i] = (short) (nodeSize & 0XFFFFL); + } else { + byteNodeSizeArr[i] = (byte) (nodeSize & 0XFFL); + } + used++; + } + + public DeviceMetadataIndexEntriesQueryResult compact() { + boolean compactToMap = true; + return compactToMap ? compactToMap() : compactToArr(); + } + + private DeviceMetadataIndexEntriesQueryResult compactToArr() { + long maxStartOffsetDelta = (maxStartOffset - minStartOffset) / 2; + standardStartOffset = minStartOffset + maxStartOffsetDelta; + if (maxStartOffsetDelta < Short.MAX_VALUE) { + short[] shortStartOffsetDeltaArr = new short[length]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + shortStartOffsetDeltaArr[i] = (short) (longStartOffsetArr[i] - standardStartOffset); + } + startOffsetDeltaArr = shortStartOffsetDeltaArr; + } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { + int[] intStartOffsetDeltaArr = new int[length]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + intStartOffsetDeltaArr[i] = (int) (longStartOffsetArr[i] - standardStartOffset); + } + startOffsetDeltaArr = intStartOffsetDeltaArr; + } else { + standardStartOffset = 0; + startOffsetDeltaArr = longStartOffsetArr; + } + if (maxNodeSize <= 0XFFL && byteNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToByteArr(); + } else if (maxNodeSize <= 0XFFFFL && shortNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToShortArr(); + } else if (maxNodeSize <= 0XFFFFFFFFL && intNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToIntArr(); + } else { + nodeSizeArr = longNodeSizeArr; + } + + longStartOffsetArr = null; + longNodeSizeArr = null; + return new ArrDeviceMetadataIndexEntriesQueryResult( + startOffsetDeltaArr, nodeSizeArr, standardStartOffset, length); + } + + private DeviceMetadataIndexEntriesQueryResult compactToMap() { + int[] map = new int[used]; + int mapSize = 0; + long maxStartOffsetDelta = (maxStartOffset - minStartOffset) / 2; + standardStartOffset = minStartOffset + maxStartOffsetDelta; + if (maxStartOffsetDelta < Short.MAX_VALUE) { + short[] shortStartOffsetDeltaArr = new short[length]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + shortStartOffsetDeltaArr[mapSize] = (short) (longStartOffsetArr[i] - standardStartOffset); + map[mapSize++] = i; + } + startOffsetDeltaArr = shortStartOffsetDeltaArr; + } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { + int[] intStartOffsetDeltaArr = new int[length]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + intStartOffsetDeltaArr[mapSize] = (int) (longStartOffsetArr[i] - standardStartOffset); + map[mapSize++] = i; + } + startOffsetDeltaArr = intStartOffsetDeltaArr; + } else { + long[] newLongStartOffsetDeltaArr = new long[length]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + newLongStartOffsetDeltaArr[mapSize] = longStartOffsetArr[i]; + map[mapSize++] = i; + } + startOffsetDeltaArr = newLongStartOffsetDeltaArr; + standardStartOffset = 0; + } + + if (maxNodeSize <= 0XFFL && byteNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToByteArr(used); + } else if (maxNodeSize <= 0XFFFFL && shortNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToShortArr(used); + } else if (maxNodeSize <= 0XFFFFFFFFL && intNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToIntArr(used); + } else { + nodeSizeArr = compactNodeSizeToLongArr(used); + } + + return new MapDeviceMetadataIndexEntriesQueryResult( + map, startOffsetDeltaArr, nodeSizeArr, standardStartOffset, used); + } + + private int[] compactNodeSizeArrToIntArr() { + if (intNodeSizeArr != null) { + return intNodeSizeArr; + } + int[] intNodeSizeArr = new int[length]; + for (int i = 0; i < length; i++) { + intNodeSizeArr[i] = (int) (longNodeSizeArr[i] & 0XFFFFFFFFL); + } + return intNodeSizeArr; + } + + private short[] compactNodeSizeArrToShortArr() { + shortNodeSizeArr = shortNodeSizeArr == null ? new short[length] : shortNodeSizeArr; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + shortNodeSizeArr[i] = (short) (longNodeSizeArr[i] & 0XFFFFL); + } + longNodeSizeArr = null; + } else if (intNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + shortNodeSizeArr[i] = (short) (intNodeSizeArr[i] & 0XFFFFL); + } + } + return shortNodeSizeArr; + } + + private byte[] compactNodeSizeArrToByteArr() { + byteNodeSizeArr = byteNodeSizeArr == null ? new byte[length] : byteNodeSizeArr; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + byteNodeSizeArr[i] = (byte) (longNodeSizeArr[i] & 0XFFL); + } + longNodeSizeArr = null; + } else if (intNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + byteNodeSizeArr[i] = (byte) (intNodeSizeArr[i] & 0XFFL); + } + intNodeSizeArr = null; + } else if (shortNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + byteNodeSizeArr[i] = (byte) (shortNodeSizeArr[i] & 0XFFL); + } + shortNodeSizeArr = null; + } + return byteNodeSizeArr; + } + + private long[] compactNodeSizeToLongArr(int newLength) { + long[] newLongNodeSizeArr = new long[newLength]; + int j = 0; + for (int i = 0; i < length; i++) { + if (longNodeSizeArr[i] == 0) { + continue; + } + newLongNodeSizeArr[j++] = longNodeSizeArr[i]; + } + return newLongNodeSizeArr; + } + + private int[] compactNodeSizeArrToIntArr(int newLength) { + int[] intNodeSizeArr = new int[newLength]; + int j = 0; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (longNodeSizeArr[i] == 0) { + continue; + } + intNodeSizeArr[j++] = (int) (longNodeSizeArr[i] & 0XFFFFFFFFL); + } + } else { + for (int i = 0; i < length; i++) { + if (intNodeSizeArr[i] == 0) { + continue; + } + intNodeSizeArr[j++] = intNodeSizeArr[i]; + } + } + return intNodeSizeArr; + } + + private short[] compactNodeSizeArrToShortArr(int newLength) { + short[] newShortNodeSizeArr = new short[newLength]; + int j = 0; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (longNodeSizeArr[i] == 0) { + continue; + } + newShortNodeSizeArr[j++] = (short) (longNodeSizeArr[i] & 0XFFFFL); + } + longNodeSizeArr = null; + } else if (intNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (intNodeSizeArr[i] == 0) { + continue; + } + newShortNodeSizeArr[j++] = (short) (intNodeSizeArr[i] & 0XFFFFL); + } + } else { + for (int i = 0; i < length; i++) { + if (shortNodeSizeArr[i] == 0) { + continue; + } + newShortNodeSizeArr[j++] = shortNodeSizeArr[i]; + } + } + return newShortNodeSizeArr; + } + + private byte[] compactNodeSizeArrToByteArr(int newLength) { + byte[] newByteNodeSizeArr = new byte[newLength]; + int j = 0; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (longNodeSizeArr[i] == 0) { + continue; + } + newByteNodeSizeArr[j++] = (byte) (longNodeSizeArr[i] & 0XFFL); + } + longNodeSizeArr = null; + } else if (intNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (intNodeSizeArr[i] == 0) { + continue; + } + newByteNodeSizeArr[j++] = (byte) (intNodeSizeArr[i] & 0XFFL); + } + intNodeSizeArr = null; + } else if (shortNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (shortNodeSizeArr[i] == 0) { + continue; + } + newByteNodeSizeArr[j++] = (byte) (shortNodeSizeArr[i] & 0XFFL); + } + shortNodeSizeArr = null; + } else { + for (int i = 0; i < length; i++) { + if (byteNodeSizeArr[i] == 0) { + continue; + } + newByteNodeSizeArr[j++] = newByteNodeSizeArr[i]; + } + } + return newByteNodeSizeArr; + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java index 25a45a618..f6800a2da 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java @@ -39,6 +39,7 @@ import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.query.DeviceMetadataIndexEntriesQueryResult; import org.apache.tsfile.utils.BloomFilter; import org.apache.tsfile.utils.FileGenerator; import org.apache.tsfile.utils.Pair; @@ -308,17 +309,21 @@ public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WritePro } queriedDevices.sort(IDeviceID::compareTo); try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { - long[] offsets = reader.getDeviceMetadataIndexNodeOffsets(null, queriedDevices, null); - Assert.assertEquals(20000 * 2, offsets.length); + DeviceMetadataIndexEntriesQueryResult offsets = + reader.getDeviceMetadataIndexNodeOffsets(null, queriedDevices, null); for (int i = 0; i < queriedDevices.size(); i++) { IDeviceID deviceID = queriedDevices.get(i); int deviceNumber = Integer.parseInt(deviceID.toString().substring("t1.d".length())); + long[] metadataIndexNodeOffsetOfCurDevice = offsets.getDeviceMetadataIndexNodeOffset(i); if (deviceNumber >= 10000) { - Assert.assertEquals(-1, offsets[2 * i]); + Assert.assertNull(metadataIndexNodeOffsetOfCurDevice); continue; } MetadataIndexNode metadataIndexNode = - reader.readMetadataIndexNode(offsets[2 * i], offsets[2 * i + 1], false); + reader.readMetadataIndexNode( + metadataIndexNodeOffsetOfCurDevice[0], + metadataIndexNodeOffsetOfCurDevice[1], + false); List alignedChunkMetadataList = reader.getAlignedChunkMetadataByMetadataIndexNode(deviceID, metadataIndexNode, true); Assert.assertEquals(1, alignedChunkMetadataList.size()); @@ -327,12 +332,12 @@ public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WritePro } Assert.assertEquals( - 0, reader.getDeviceMetadataIndexNodeOffsets("t1", Collections.emptyList(), null).length); + 0, + reader.getDeviceMetadataIndexNodeOffsets("t1", Collections.emptyList(), null).length()); offsets = reader.getDeviceMetadataIndexNodeOffsets( "t1", Collections.singletonList(new StringArrayDeviceID("t1.d")), null); - Assert.assertEquals(2, offsets.length); - Assert.assertEquals(-1, offsets[0]); + Assert.assertNull(offsets.getDeviceMetadataIndexNodeOffset(0)); } } } From 3d275805896dcfa410564e8dde8546013390dfeb Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 31 Jul 2025 19:02:01 +0800 Subject: [PATCH 15/18] fix bug --- ...eMetadataIndexNodeOffsetsQueryContext.java | 54 +++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java index a10fcc5eb..12205a93d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java @@ -19,11 +19,14 @@ package org.apache.tsfile.read.query; +import org.apache.tsfile.utils.RamUsageEstimator; + public class DeviceMetadataIndexNodeOffsetsQueryContext { private long standardStartOffset; private long minStartOffset = Long.MAX_VALUE; private long maxStartOffset = Long.MIN_VALUE; + private long maxStartOffsetDelta; private long maxNodeSize; private final int length; private int used; @@ -75,12 +78,37 @@ public void addDeviceMetadataIndexNodeOffset(int i, long startOffset, long endOf } public DeviceMetadataIndexEntriesQueryResult compact() { - boolean compactToMap = true; + maxStartOffsetDelta = (maxStartOffset - minStartOffset) / 2; + boolean compactToMap = estimateMapRamBytes() < estimateArrRamBytes(length); return compactToMap ? compactToMap() : compactToArr(); } + private long estimateMapRamBytes() { + return estimateArrRamBytes(used) + RamUsageEstimator.sizeOfIntArray(used); + } + + private long estimateArrRamBytes(int arrLength) { + long cost = 0; + if (maxStartOffsetDelta < Short.MAX_VALUE) { + cost += RamUsageEstimator.sizeOfShortArray(arrLength); + } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { + cost += RamUsageEstimator.sizeOfIntArray(arrLength); + } else { + cost += RamUsageEstimator.sizeOfLongArray(arrLength); + } + if (maxNodeSize < Byte.MAX_VALUE) { + cost += RamUsageEstimator.sizeOfByteArray(arrLength); + } else if (maxNodeSize < Short.MAX_VALUE) { + cost += RamUsageEstimator.sizeOfShortArray(arrLength); + } else if (maxNodeSize < Integer.MAX_VALUE) { + cost += RamUsageEstimator.sizeOfIntArray(arrLength); + } else { + cost += RamUsageEstimator.sizeOfLongArray(arrLength); + } + return cost; + } + private DeviceMetadataIndexEntriesQueryResult compactToArr() { - long maxStartOffsetDelta = (maxStartOffset - minStartOffset) / 2; standardStartOffset = minStartOffset + maxStartOffsetDelta; if (maxStartOffsetDelta < Short.MAX_VALUE) { short[] shortStartOffsetDeltaArr = new short[length]; @@ -104,6 +132,7 @@ private DeviceMetadataIndexEntriesQueryResult compactToArr() { standardStartOffset = 0; startOffsetDeltaArr = longStartOffsetArr; } + longStartOffsetArr = null; if (maxNodeSize <= 0XFFL && byteNodeSizeArr == null) { nodeSizeArr = compactNodeSizeArrToByteArr(); } else if (maxNodeSize <= 0XFFFFL && shortNodeSizeArr == null) { @@ -113,9 +142,8 @@ private DeviceMetadataIndexEntriesQueryResult compactToArr() { } else { nodeSizeArr = longNodeSizeArr; } + clearDeprecatedNodeSizeArr(); - longStartOffsetArr = null; - longNodeSizeArr = null; return new ArrDeviceMetadataIndexEntriesQueryResult( startOffsetDeltaArr, nodeSizeArr, standardStartOffset, length); } @@ -123,10 +151,9 @@ private DeviceMetadataIndexEntriesQueryResult compactToArr() { private DeviceMetadataIndexEntriesQueryResult compactToMap() { int[] map = new int[used]; int mapSize = 0; - long maxStartOffsetDelta = (maxStartOffset - minStartOffset) / 2; standardStartOffset = minStartOffset + maxStartOffsetDelta; if (maxStartOffsetDelta < Short.MAX_VALUE) { - short[] shortStartOffsetDeltaArr = new short[length]; + short[] shortStartOffsetDeltaArr = new short[used]; for (int i = 0; i < length; i++) { if (longStartOffsetArr[i] <= 0) { continue; @@ -136,7 +163,7 @@ private DeviceMetadataIndexEntriesQueryResult compactToMap() { } startOffsetDeltaArr = shortStartOffsetDeltaArr; } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { - int[] intStartOffsetDeltaArr = new int[length]; + int[] intStartOffsetDeltaArr = new int[used]; for (int i = 0; i < length; i++) { if (longStartOffsetArr[i] <= 0) { continue; @@ -146,7 +173,7 @@ private DeviceMetadataIndexEntriesQueryResult compactToMap() { } startOffsetDeltaArr = intStartOffsetDeltaArr; } else { - long[] newLongStartOffsetDeltaArr = new long[length]; + long[] newLongStartOffsetDeltaArr = new long[used]; for (int i = 0; i < length; i++) { if (longStartOffsetArr[i] <= 0) { continue; @@ -157,6 +184,7 @@ private DeviceMetadataIndexEntriesQueryResult compactToMap() { startOffsetDeltaArr = newLongStartOffsetDeltaArr; standardStartOffset = 0; } + longStartOffsetArr = null; if (maxNodeSize <= 0XFFL && byteNodeSizeArr == null) { nodeSizeArr = compactNodeSizeArrToByteArr(used); @@ -167,11 +195,19 @@ private DeviceMetadataIndexEntriesQueryResult compactToMap() { } else { nodeSizeArr = compactNodeSizeToLongArr(used); } + clearDeprecatedNodeSizeArr(); return new MapDeviceMetadataIndexEntriesQueryResult( map, startOffsetDeltaArr, nodeSizeArr, standardStartOffset, used); } + private void clearDeprecatedNodeSizeArr() { + longNodeSizeArr = null; + intNodeSizeArr = null; + shortNodeSizeArr = null; + byteNodeSizeArr = null; + } + private int[] compactNodeSizeArrToIntArr() { if (intNodeSizeArr != null) { return intNodeSizeArr; @@ -313,7 +349,7 @@ private byte[] compactNodeSizeArrToByteArr(int newLength) { if (byteNodeSizeArr[i] == 0) { continue; } - newByteNodeSizeArr[j++] = newByteNodeSizeArr[i]; + newByteNodeSizeArr[j++] = byteNodeSizeArr[i]; } } return newByteNodeSizeArr; From 73cae618df7cb801407cfc26543247d501ab8aa8 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 1 Aug 2025 14:48:29 +0800 Subject: [PATCH 16/18] fix --- .../tsfile/read/TsFileSequenceReader.java | 2 +- ...DeviceMetadataIndexEntriesQueryResult.java | 25 +++++++--- ...eMetadataIndexNodeOffsetsQueryContext.java | 46 ++++++++++++++----- 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 71af9fd74..cf60f7097 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -551,7 +551,7 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev public DeviceMetadataIndexEntriesQueryResult getDeviceMetadataIndexNodeOffsets( String table, List sortedDevices, LongConsumer ioSizeRecorder) throws IOException { DeviceMetadataIndexNodeOffsetsQueryContext context = - new DeviceMetadataIndexNodeOffsetsQueryContext(sortedDevices.size()); + new DeviceMetadataIndexNodeOffsetsQueryContext(sortedDevices.size(), getAllMetadataSize()); if (sortedDevices.isEmpty()) { return context.compact(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java index d1c5e6323..e0f2c6b57 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java @@ -104,11 +104,11 @@ class MapDeviceMetadataIndexEntriesQueryResult implements DeviceMetadataIndexEnt private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MapDeviceMetadataIndexEntriesQueryResult.class); - private final int[] indexMap; + private final Object indexMap; private final ArrDeviceMetadataIndexEntriesQueryResult arrResult; public MapDeviceMetadataIndexEntriesQueryResult( - int[] indexMap, Object offsetDeltaArr, Object nodeSizeArr, long standardOffset, int length) { + Object indexMap, Object offsetDeltaArr, Object nodeSizeArr, long standardOffset, int length) { this.indexMap = indexMap; this.arrResult = new ArrDeviceMetadataIndexEntriesQueryResult( @@ -117,11 +117,22 @@ public MapDeviceMetadataIndexEntriesQueryResult( @Override public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { - if (indexMap.length == 0) { + if (arrResult.length() == 0) { return null; } - int idx = - Arrays.binarySearch(indexMap, 0, Math.min(deviceIndex + 1, indexMap.length), deviceIndex); + int idx; + if (indexMap instanceof int[]) { + idx = + Arrays.binarySearch( + (int[]) indexMap, 0, Math.min(deviceIndex + 1, arrResult.length()), deviceIndex); + } else { + idx = + Arrays.binarySearch( + (short[]) indexMap, + 0, + Math.min(deviceIndex + 1, arrResult.length()), + (short) deviceIndex); + } if (idx < 0) { return null; } @@ -130,13 +141,13 @@ public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { @Override public int length() { - return indexMap.length; + return arrResult.length(); } @Override public long ramBytesUsed() { return SHALLOW_SIZE + arrResult.ramBytesUsed() - + RamUsageEstimator.sizeOfIntArray(indexMap.length); + + RamUsageEstimator.sizeOfIntArray(arrResult.length()); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java index 12205a93d..8a114447b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java @@ -80,11 +80,13 @@ public void addDeviceMetadataIndexNodeOffset(int i, long startOffset, long endOf public DeviceMetadataIndexEntriesQueryResult compact() { maxStartOffsetDelta = (maxStartOffset - minStartOffset) / 2; boolean compactToMap = estimateMapRamBytes() < estimateArrRamBytes(length); - return compactToMap ? compactToMap() : compactToArr(); + return compactToMap ? compactToIntMap() : compactToArr(); } private long estimateMapRamBytes() { - return estimateArrRamBytes(used) + RamUsageEstimator.sizeOfIntArray(used); + return estimateArrRamBytes(used) + length < Short.MAX_VALUE + ? RamUsageEstimator.sizeOfShortArray(used) + : RamUsageEstimator.sizeOfIntArray(used); } private long estimateArrRamBytes(int arrLength) { @@ -148,8 +150,14 @@ private DeviceMetadataIndexEntriesQueryResult compactToArr() { startOffsetDeltaArr, nodeSizeArr, standardStartOffset, length); } - private DeviceMetadataIndexEntriesQueryResult compactToMap() { - int[] map = new int[used]; + private DeviceMetadataIndexEntriesQueryResult compactToIntMap() { + int[] intIndexMap = null; + short[] shortIndexMap = null; + if (length <= Short.MAX_VALUE) { + shortIndexMap = new short[used]; + } else { + intIndexMap = new int[used]; + } int mapSize = 0; standardStartOffset = minStartOffset + maxStartOffsetDelta; if (maxStartOffsetDelta < Short.MAX_VALUE) { @@ -159,7 +167,11 @@ private DeviceMetadataIndexEntriesQueryResult compactToMap() { continue; } shortStartOffsetDeltaArr[mapSize] = (short) (longStartOffsetArr[i] - standardStartOffset); - map[mapSize++] = i; + if (intIndexMap != null) { + intIndexMap[mapSize++] = i; + } else { + shortIndexMap[mapSize++] = (short) i; + } } startOffsetDeltaArr = shortStartOffsetDeltaArr; } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { @@ -169,7 +181,11 @@ private DeviceMetadataIndexEntriesQueryResult compactToMap() { continue; } intStartOffsetDeltaArr[mapSize] = (int) (longStartOffsetArr[i] - standardStartOffset); - map[mapSize++] = i; + if (intIndexMap != null) { + intIndexMap[mapSize++] = i; + } else { + shortIndexMap[mapSize++] = (short) i; + } } startOffsetDeltaArr = intStartOffsetDeltaArr; } else { @@ -179,7 +195,11 @@ private DeviceMetadataIndexEntriesQueryResult compactToMap() { continue; } newLongStartOffsetDeltaArr[mapSize] = longStartOffsetArr[i]; - map[mapSize++] = i; + if (intIndexMap != null) { + intIndexMap[mapSize++] = i; + } else { + shortIndexMap[mapSize++] = (short) i; + } } startOffsetDeltaArr = newLongStartOffsetDeltaArr; standardStartOffset = 0; @@ -198,7 +218,11 @@ private DeviceMetadataIndexEntriesQueryResult compactToMap() { clearDeprecatedNodeSizeArr(); return new MapDeviceMetadataIndexEntriesQueryResult( - map, startOffsetDeltaArr, nodeSizeArr, standardStartOffset, used); + intIndexMap == null ? shortIndexMap : intIndexMap, + startOffsetDeltaArr, + nodeSizeArr, + standardStartOffset, + used); } private void clearDeprecatedNodeSizeArr() { @@ -212,11 +236,11 @@ private int[] compactNodeSizeArrToIntArr() { if (intNodeSizeArr != null) { return intNodeSizeArr; } - int[] intNodeSizeArr = new int[length]; + int[] newIntNodeSizeArr = new int[length]; for (int i = 0; i < length; i++) { - intNodeSizeArr[i] = (int) (longNodeSizeArr[i] & 0XFFFFFFFFL); + newIntNodeSizeArr[i] = (int) (longNodeSizeArr[i] & 0XFFFFFFFFL); } - return intNodeSizeArr; + return newIntNodeSizeArr; } private short[] compactNodeSizeArrToShortArr() { From 55c04190d2dfe01b06e71bfb1981ef7dfd22c1b4 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 5 Aug 2025 16:06:31 +0800 Subject: [PATCH 17/18] add ut --- ...DeviceMetadataIndexEntriesQueryResult.java | 40 +++- ...eMetadataIndexNodeOffsetsQueryContext.java | 98 +++++----- .../DeviceMetadataIndexEntriesQueryTest.java | 180 ++++++++++++++++++ .../tsfile/read/TsFileSequenceReaderTest.java | 77 -------- 4 files changed, 263 insertions(+), 132 deletions(-) create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/read/DeviceMetadataIndexEntriesQueryTest.java diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java index e0f2c6b57..5d962f5b8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java @@ -24,6 +24,10 @@ import java.util.Arrays; +import static org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext.MAX_UNSIGNED_BYTE; +import static org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext.MAX_UNSIGNED_INTEGER; +import static org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext.MAX_UNSIGNED_SHORT; + public interface DeviceMetadataIndexEntriesQueryResult extends Accountable { long[] getDeviceMetadataIndexNodeOffset(int deviceIndex); @@ -55,11 +59,11 @@ public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { long nodeSize; long startOffset; if (nodeSizeArr instanceof byte[]) { - nodeSize = (((byte[]) nodeSizeArr)[deviceIndex]) & 0XFFL; + nodeSize = (((byte[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_BYTE; } else if (nodeSizeArr instanceof short[]) { - nodeSize = (((short[]) nodeSizeArr)[deviceIndex]) & 0XFFFFL; + nodeSize = (((short[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_BYTE; } else if (nodeSizeArr instanceof int[]) { - nodeSize = (((int[]) nodeSizeArr)[deviceIndex]) & 0XFFFFFFFFL; + nodeSize = (((int[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_INTEGER; } else { nodeSize = ((long[]) nodeSizeArr)[deviceIndex]; } @@ -67,9 +71,9 @@ public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { return null; } if (offsetDeltaArr instanceof short[]) { - startOffset = standardOffset + ((short[]) offsetDeltaArr)[deviceIndex]; + startOffset = standardOffset + (((short[]) offsetDeltaArr)[deviceIndex] & MAX_UNSIGNED_SHORT); } else if (offsetDeltaArr instanceof int[]) { - startOffset = standardOffset + ((int[]) offsetDeltaArr)[deviceIndex]; + startOffset = standardOffset + (((int[]) offsetDeltaArr)[deviceIndex] & MAX_UNSIGNED_INTEGER); } else { startOffset = standardOffset + ((long[]) offsetDeltaArr)[deviceIndex]; } @@ -127,11 +131,8 @@ public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { (int[]) indexMap, 0, Math.min(deviceIndex + 1, arrResult.length()), deviceIndex); } else { idx = - Arrays.binarySearch( - (short[]) indexMap, - 0, - Math.min(deviceIndex + 1, arrResult.length()), - (short) deviceIndex); + unsignedBinarySearch( + (short[]) indexMap, 0, Math.min(deviceIndex + 1, arrResult.length()), deviceIndex); } if (idx < 0) { return null; @@ -139,6 +140,25 @@ public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { return arrResult.getDeviceMetadataIndexNodeOffset(idx); } + private int unsignedBinarySearch(short[] arr, int fromIndex, int toIndex, int key) { + int low = fromIndex; + int high = toIndex - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + int midVal = arr[mid] & (int) MAX_UNSIGNED_SHORT; + + if (midVal < key) { + low = mid + 1; + } else if (midVal > key) { + high = mid - 1; + } else { + return mid; + } // key found + } + return -(low + 1); // key not found. + } + @Override public int length() { return arrResult.length(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java index 8a114447b..43cf4cee5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java @@ -22,6 +22,9 @@ import org.apache.tsfile.utils.RamUsageEstimator; public class DeviceMetadataIndexNodeOffsetsQueryContext { + public static final long MAX_UNSIGNED_INTEGER = 0XFFFFFFFFL; + public static final long MAX_UNSIGNED_SHORT = 0XFFFFL; + public static final long MAX_UNSIGNED_BYTE = 0XFFL; private long standardStartOffset; private long minStartOffset = Long.MAX_VALUE; @@ -48,11 +51,11 @@ public DeviceMetadataIndexNodeOffsetsQueryContext(int length, long metadataSize) this.length = length; this.used = 0; longStartOffsetArr = new long[length]; - if (metadataSize <= 0 || metadataSize > 0XFFFFFFFFL) { + if (metadataSize <= 0 || metadataSize > MAX_UNSIGNED_INTEGER) { longNodeSizeArr = new long[length]; - } else if (metadataSize > 0XFFFFL) { + } else if (metadataSize > MAX_UNSIGNED_SHORT) { intNodeSizeArr = new int[length]; - } else if (metadataSize > 0XFFL) { + } else if (metadataSize > MAX_UNSIGNED_BYTE) { shortNodeSizeArr = new short[length]; } else { byteNodeSizeArr = new byte[length]; @@ -68,41 +71,42 @@ public void addDeviceMetadataIndexNodeOffset(int i, long startOffset, long endOf if (longNodeSizeArr != null) { longNodeSizeArr[i] = nodeSize; } else if (intNodeSizeArr != null) { - intNodeSizeArr[i] = (int) (nodeSize & 0XFFFFFFFFL); + intNodeSizeArr[i] = (int) (nodeSize & MAX_UNSIGNED_INTEGER); } else if (shortNodeSizeArr != null) { - shortNodeSizeArr[i] = (short) (nodeSize & 0XFFFFL); + shortNodeSizeArr[i] = (short) (nodeSize & MAX_UNSIGNED_SHORT); } else { - byteNodeSizeArr[i] = (byte) (nodeSize & 0XFFL); + byteNodeSizeArr[i] = (byte) (nodeSize & MAX_UNSIGNED_BYTE); } used++; } public DeviceMetadataIndexEntriesQueryResult compact() { - maxStartOffsetDelta = (maxStartOffset - minStartOffset) / 2; + maxStartOffsetDelta = maxStartOffset - minStartOffset; boolean compactToMap = estimateMapRamBytes() < estimateArrRamBytes(length); return compactToMap ? compactToIntMap() : compactToArr(); } private long estimateMapRamBytes() { - return estimateArrRamBytes(used) + length < Short.MAX_VALUE - ? RamUsageEstimator.sizeOfShortArray(used) - : RamUsageEstimator.sizeOfIntArray(used); + return estimateArrRamBytes(used) + + (length <= MAX_UNSIGNED_SHORT + ? RamUsageEstimator.sizeOfShortArray(used) + : RamUsageEstimator.sizeOfIntArray(used)); } private long estimateArrRamBytes(int arrLength) { long cost = 0; - if (maxStartOffsetDelta < Short.MAX_VALUE) { + if (maxStartOffsetDelta <= MAX_UNSIGNED_SHORT) { cost += RamUsageEstimator.sizeOfShortArray(arrLength); - } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { + } else if (maxStartOffsetDelta <= MAX_UNSIGNED_INTEGER) { cost += RamUsageEstimator.sizeOfIntArray(arrLength); } else { cost += RamUsageEstimator.sizeOfLongArray(arrLength); } - if (maxNodeSize < Byte.MAX_VALUE) { + if (maxNodeSize <= MAX_UNSIGNED_BYTE) { cost += RamUsageEstimator.sizeOfByteArray(arrLength); - } else if (maxNodeSize < Short.MAX_VALUE) { + } else if (maxNodeSize <= MAX_UNSIGNED_SHORT) { cost += RamUsageEstimator.sizeOfShortArray(arrLength); - } else if (maxNodeSize < Integer.MAX_VALUE) { + } else if (maxNodeSize <= MAX_UNSIGNED_INTEGER) { cost += RamUsageEstimator.sizeOfIntArray(arrLength); } else { cost += RamUsageEstimator.sizeOfLongArray(arrLength); @@ -111,14 +115,15 @@ private long estimateArrRamBytes(int arrLength) { } private DeviceMetadataIndexEntriesQueryResult compactToArr() { - standardStartOffset = minStartOffset + maxStartOffsetDelta; - if (maxStartOffsetDelta < Short.MAX_VALUE) { + standardStartOffset = minStartOffset; + if (maxStartOffsetDelta <= MAX_UNSIGNED_SHORT) { short[] shortStartOffsetDeltaArr = new short[length]; for (int i = 0; i < length; i++) { if (longStartOffsetArr[i] <= 0) { continue; } - shortStartOffsetDeltaArr[i] = (short) (longStartOffsetArr[i] - standardStartOffset); + shortStartOffsetDeltaArr[i] = + (short) (MAX_UNSIGNED_SHORT & (longStartOffsetArr[i] - standardStartOffset)); } startOffsetDeltaArr = shortStartOffsetDeltaArr; } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { @@ -127,7 +132,8 @@ private DeviceMetadataIndexEntriesQueryResult compactToArr() { if (longStartOffsetArr[i] <= 0) { continue; } - intStartOffsetDeltaArr[i] = (int) (longStartOffsetArr[i] - standardStartOffset); + intStartOffsetDeltaArr[i] = + (int) (MAX_UNSIGNED_INTEGER & (longStartOffsetArr[i] - standardStartOffset)); } startOffsetDeltaArr = intStartOffsetDeltaArr; } else { @@ -135,11 +141,11 @@ private DeviceMetadataIndexEntriesQueryResult compactToArr() { startOffsetDeltaArr = longStartOffsetArr; } longStartOffsetArr = null; - if (maxNodeSize <= 0XFFL && byteNodeSizeArr == null) { + if (maxNodeSize <= MAX_UNSIGNED_BYTE && byteNodeSizeArr == null) { nodeSizeArr = compactNodeSizeArrToByteArr(); - } else if (maxNodeSize <= 0XFFFFL && shortNodeSizeArr == null) { + } else if (maxNodeSize <= MAX_UNSIGNED_SHORT && shortNodeSizeArr == null) { nodeSizeArr = compactNodeSizeArrToShortArr(); - } else if (maxNodeSize <= 0XFFFFFFFFL && intNodeSizeArr == null) { + } else if (maxNodeSize <= MAX_UNSIGNED_INTEGER && intNodeSizeArr == null) { nodeSizeArr = compactNodeSizeArrToIntArr(); } else { nodeSizeArr = longNodeSizeArr; @@ -159,32 +165,34 @@ private DeviceMetadataIndexEntriesQueryResult compactToIntMap() { intIndexMap = new int[used]; } int mapSize = 0; - standardStartOffset = minStartOffset + maxStartOffsetDelta; - if (maxStartOffsetDelta < Short.MAX_VALUE) { + standardStartOffset = minStartOffset; + if (maxStartOffsetDelta <= MAX_UNSIGNED_SHORT) { short[] shortStartOffsetDeltaArr = new short[used]; for (int i = 0; i < length; i++) { if (longStartOffsetArr[i] <= 0) { continue; } - shortStartOffsetDeltaArr[mapSize] = (short) (longStartOffsetArr[i] - standardStartOffset); + shortStartOffsetDeltaArr[mapSize] = + (short) (MAX_UNSIGNED_SHORT & (longStartOffsetArr[i] - standardStartOffset)); if (intIndexMap != null) { intIndexMap[mapSize++] = i; } else { - shortIndexMap[mapSize++] = (short) i; + shortIndexMap[mapSize++] = (short) (i & MAX_UNSIGNED_SHORT); } } startOffsetDeltaArr = shortStartOffsetDeltaArr; - } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { + } else if (maxStartOffsetDelta <= MAX_UNSIGNED_INTEGER) { int[] intStartOffsetDeltaArr = new int[used]; for (int i = 0; i < length; i++) { if (longStartOffsetArr[i] <= 0) { continue; } - intStartOffsetDeltaArr[mapSize] = (int) (longStartOffsetArr[i] - standardStartOffset); + intStartOffsetDeltaArr[mapSize] = + (int) (MAX_UNSIGNED_INTEGER & (longStartOffsetArr[i] - standardStartOffset)); if (intIndexMap != null) { intIndexMap[mapSize++] = i; } else { - shortIndexMap[mapSize++] = (short) i; + shortIndexMap[mapSize++] = (short) (i & MAX_UNSIGNED_SHORT); } } startOffsetDeltaArr = intStartOffsetDeltaArr; @@ -198,7 +206,7 @@ private DeviceMetadataIndexEntriesQueryResult compactToIntMap() { if (intIndexMap != null) { intIndexMap[mapSize++] = i; } else { - shortIndexMap[mapSize++] = (short) i; + shortIndexMap[mapSize++] = (short) (i & MAX_UNSIGNED_SHORT); } } startOffsetDeltaArr = newLongStartOffsetDeltaArr; @@ -206,11 +214,11 @@ private DeviceMetadataIndexEntriesQueryResult compactToIntMap() { } longStartOffsetArr = null; - if (maxNodeSize <= 0XFFL && byteNodeSizeArr == null) { + if (maxNodeSize <= MAX_UNSIGNED_BYTE && byteNodeSizeArr == null) { nodeSizeArr = compactNodeSizeArrToByteArr(used); - } else if (maxNodeSize <= 0XFFFFL && shortNodeSizeArr == null) { + } else if (maxNodeSize <= MAX_UNSIGNED_SHORT && shortNodeSizeArr == null) { nodeSizeArr = compactNodeSizeArrToShortArr(used); - } else if (maxNodeSize <= 0XFFFFFFFFL && intNodeSizeArr == null) { + } else if (maxNodeSize <= MAX_UNSIGNED_INTEGER && intNodeSizeArr == null) { nodeSizeArr = compactNodeSizeArrToIntArr(used); } else { nodeSizeArr = compactNodeSizeToLongArr(used); @@ -238,7 +246,7 @@ private int[] compactNodeSizeArrToIntArr() { } int[] newIntNodeSizeArr = new int[length]; for (int i = 0; i < length; i++) { - newIntNodeSizeArr[i] = (int) (longNodeSizeArr[i] & 0XFFFFFFFFL); + newIntNodeSizeArr[i] = (int) (longNodeSizeArr[i] & MAX_UNSIGNED_INTEGER); } return newIntNodeSizeArr; } @@ -247,12 +255,12 @@ private short[] compactNodeSizeArrToShortArr() { shortNodeSizeArr = shortNodeSizeArr == null ? new short[length] : shortNodeSizeArr; if (longNodeSizeArr != null) { for (int i = 0; i < length; i++) { - shortNodeSizeArr[i] = (short) (longNodeSizeArr[i] & 0XFFFFL); + shortNodeSizeArr[i] = (short) (longNodeSizeArr[i] & MAX_UNSIGNED_SHORT); } longNodeSizeArr = null; } else if (intNodeSizeArr != null) { for (int i = 0; i < length; i++) { - shortNodeSizeArr[i] = (short) (intNodeSizeArr[i] & 0XFFFFL); + shortNodeSizeArr[i] = (short) (intNodeSizeArr[i] & MAX_UNSIGNED_SHORT); } } return shortNodeSizeArr; @@ -262,17 +270,17 @@ private byte[] compactNodeSizeArrToByteArr() { byteNodeSizeArr = byteNodeSizeArr == null ? new byte[length] : byteNodeSizeArr; if (longNodeSizeArr != null) { for (int i = 0; i < length; i++) { - byteNodeSizeArr[i] = (byte) (longNodeSizeArr[i] & 0XFFL); + byteNodeSizeArr[i] = (byte) (longNodeSizeArr[i] & MAX_UNSIGNED_BYTE); } longNodeSizeArr = null; } else if (intNodeSizeArr != null) { for (int i = 0; i < length; i++) { - byteNodeSizeArr[i] = (byte) (intNodeSizeArr[i] & 0XFFL); + byteNodeSizeArr[i] = (byte) (intNodeSizeArr[i] & MAX_UNSIGNED_BYTE); } intNodeSizeArr = null; } else if (shortNodeSizeArr != null) { for (int i = 0; i < length; i++) { - byteNodeSizeArr[i] = (byte) (shortNodeSizeArr[i] & 0XFFL); + byteNodeSizeArr[i] = (byte) (shortNodeSizeArr[i] & MAX_UNSIGNED_BYTE); } shortNodeSizeArr = null; } @@ -299,7 +307,7 @@ private int[] compactNodeSizeArrToIntArr(int newLength) { if (longNodeSizeArr[i] == 0) { continue; } - intNodeSizeArr[j++] = (int) (longNodeSizeArr[i] & 0XFFFFFFFFL); + intNodeSizeArr[j++] = (int) (longNodeSizeArr[i] & MAX_UNSIGNED_INTEGER); } } else { for (int i = 0; i < length; i++) { @@ -320,7 +328,7 @@ private short[] compactNodeSizeArrToShortArr(int newLength) { if (longNodeSizeArr[i] == 0) { continue; } - newShortNodeSizeArr[j++] = (short) (longNodeSizeArr[i] & 0XFFFFL); + newShortNodeSizeArr[j++] = (short) (longNodeSizeArr[i] & MAX_UNSIGNED_SHORT); } longNodeSizeArr = null; } else if (intNodeSizeArr != null) { @@ -328,7 +336,7 @@ private short[] compactNodeSizeArrToShortArr(int newLength) { if (intNodeSizeArr[i] == 0) { continue; } - newShortNodeSizeArr[j++] = (short) (intNodeSizeArr[i] & 0XFFFFL); + newShortNodeSizeArr[j++] = (short) (intNodeSizeArr[i] & MAX_UNSIGNED_SHORT); } } else { for (int i = 0; i < length; i++) { @@ -349,7 +357,7 @@ private byte[] compactNodeSizeArrToByteArr(int newLength) { if (longNodeSizeArr[i] == 0) { continue; } - newByteNodeSizeArr[j++] = (byte) (longNodeSizeArr[i] & 0XFFL); + newByteNodeSizeArr[j++] = (byte) (longNodeSizeArr[i] & MAX_UNSIGNED_BYTE); } longNodeSizeArr = null; } else if (intNodeSizeArr != null) { @@ -357,7 +365,7 @@ private byte[] compactNodeSizeArrToByteArr(int newLength) { if (intNodeSizeArr[i] == 0) { continue; } - newByteNodeSizeArr[j++] = (byte) (intNodeSizeArr[i] & 0XFFL); + newByteNodeSizeArr[j++] = (byte) (intNodeSizeArr[i] & MAX_UNSIGNED_BYTE); } intNodeSizeArr = null; } else if (shortNodeSizeArr != null) { @@ -365,7 +373,7 @@ private byte[] compactNodeSizeArrToByteArr(int newLength) { if (shortNodeSizeArr[i] == 0) { continue; } - newByteNodeSizeArr[j++] = (byte) (shortNodeSizeArr[i] & 0XFFL); + newByteNodeSizeArr[j++] = (byte) (shortNodeSizeArr[i] & MAX_UNSIGNED_BYTE); } shortNodeSizeArr = null; } else { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/DeviceMetadataIndexEntriesQueryTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/DeviceMetadataIndexEntriesQueryTest.java new file mode 100644 index 000000000..51e019f97 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/DeviceMetadataIndexEntriesQueryTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.MetadataIndexNode; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.query.DeviceMetadataIndexEntriesQueryResult; +import org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext; +import org.apache.tsfile.utils.FileGenerator; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class DeviceMetadataIndexEntriesQueryTest { + private static final String FILE_PATH = FileGenerator.outputDataFile; + + @Before + public void before() throws IOException {} + + @After + public void after() throws IOException { + Files.deleteIfExists(new File(FILE_PATH).toPath()); + } + + @Test + public void test1() throws IOException, WriteProcessException { + File file = new File(FILE_PATH); + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new MeasurementSchema("device", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT32)), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + try (ITsFileWriter writer = + new TsFileWriterBuilder().tableSchema(tableSchema).file(file).build()) { + Tablet tablet = + new Tablet( + Arrays.asList("device", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + 10000); + for (int i = 0; i < 10000; i++) { + tablet.addTimestamp(i, i); + tablet.addValue("device", i, "d" + i); + tablet.addValue("s1", i, i); + } + writer.write(tablet); + } + + List queriedDevices = new ArrayList<>(); + for (int i = 0; i < 20000; i++) { + if (i >= 15000) { + queriedDevices.add(new StringArrayDeviceID("t2.d" + i)); + } else { + queriedDevices.add(new StringArrayDeviceID("t1.d" + i)); + } + } + queriedDevices.sort(IDeviceID::compareTo); + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + DeviceMetadataIndexEntriesQueryResult offsets = + reader.getDeviceMetadataIndexNodeOffsets(null, queriedDevices, null); + for (int i = 0; i < queriedDevices.size(); i++) { + IDeviceID deviceID = queriedDevices.get(i); + int deviceNumber = Integer.parseInt(deviceID.toString().substring("t1.d".length())); + long[] metadataIndexNodeOffsetOfCurDevice = offsets.getDeviceMetadataIndexNodeOffset(i); + if (deviceNumber >= 10000) { + Assert.assertNull(metadataIndexNodeOffsetOfCurDevice); + continue; + } + MetadataIndexNode metadataIndexNode = + reader.readMetadataIndexNode( + metadataIndexNodeOffsetOfCurDevice[0], + metadataIndexNodeOffsetOfCurDevice[1], + false); + List alignedChunkMetadataList = + reader.getAlignedChunkMetadataByMetadataIndexNode(deviceID, metadataIndexNode, true); + Assert.assertEquals(1, alignedChunkMetadataList.size()); + + Assert.assertEquals(deviceNumber, alignedChunkMetadataList.get(0).getStartTime()); + } + + Assert.assertEquals( + 0, + reader.getDeviceMetadataIndexNodeOffsets("t1", Collections.emptyList(), null).length()); + offsets = + reader.getDeviceMetadataIndexNodeOffsets( + "t1", Collections.singletonList(new StringArrayDeviceID("t1.d")), null); + Assert.assertNull(offsets.getDeviceMetadataIndexNodeOffset(0)); + } + } + + @Test + public void test2() { + DeviceMetadataIndexNodeOffsetsQueryContext queryContext = + new DeviceMetadataIndexNodeOffsetsQueryContext(1); + queryContext.addDeviceMetadataIndexNodeOffset(0, 1, (long) Integer.MAX_VALUE + 10); + DeviceMetadataIndexEntriesQueryResult result1 = queryContext.compact(); + long[] offsets = result1.getDeviceMetadataIndexNodeOffset(0); + Assert.assertEquals(1, offsets[0]); + Assert.assertEquals((long) Integer.MAX_VALUE + 10, offsets[1]); + } + + @Test + public void test3() { + DeviceMetadataIndexNodeOffsetsQueryContext queryContext = + new DeviceMetadataIndexNodeOffsetsQueryContext(10); + for (int i = 0; i < 10; i++) { + queryContext.addDeviceMetadataIndexNodeOffset(i, i + 1, 0XFFFFFFFFL + 10 + i); + } + DeviceMetadataIndexEntriesQueryResult result1 = queryContext.compact(); + for (int i = 0; i < 10; i++) { + Assert.assertEquals(i + 1, result1.getDeviceMetadataIndexNodeOffset(i)[0]); + Assert.assertEquals(0XFFFFFFFFL + 10 + i, result1.getDeviceMetadataIndexNodeOffset(i)[1]); + } + queryContext = new DeviceMetadataIndexNodeOffsetsQueryContext(10); + queryContext.addDeviceMetadataIndexNodeOffset(0, 1, 0XFFFFFFFFL + 10 + 10); + DeviceMetadataIndexEntriesQueryResult result2 = queryContext.compact(); + Assert.assertTrue(result1.ramBytesUsed() > result2.ramBytesUsed()); + Assert.assertEquals(1, result2.getDeviceMetadataIndexNodeOffset(0)[0]); + Assert.assertEquals(0XFFFFFFFFL + 10 + 10, result2.getDeviceMetadataIndexNodeOffset(0)[1]); + } + + @Test + public void test4() { + int length = (int) Short.MAX_VALUE * 2; + DeviceMetadataIndexNodeOffsetsQueryContext queryContext = + new DeviceMetadataIndexNodeOffsetsQueryContext(length); + for (int i = 0; i < length; i++) { + if (i % 2 == 0) { + queryContext.addDeviceMetadataIndexNodeOffset(i, i + 1, 0XFFFFFFFFL + 10 + i); + } + } + DeviceMetadataIndexEntriesQueryResult result1 = queryContext.compact(); + for (int i = 0; i < length; i++) { + if (i % 2 != 0) { + Assert.assertNull(result1.getDeviceMetadataIndexNodeOffset(i)); + continue; + } + Assert.assertEquals(i + 1, result1.getDeviceMetadataIndexNodeOffset(i)[0]); + Assert.assertEquals(0XFFFFFFFFL + 10 + i, result1.getDeviceMetadataIndexNodeOffset(i)[1]); + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java index f6800a2da..8daa25277 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java @@ -29,17 +29,13 @@ import org.apache.tsfile.file.header.ChunkGroupHeader; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; -import org.apache.tsfile.file.metadata.MetadataIndexNode; -import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Path; -import org.apache.tsfile.read.query.DeviceMetadataIndexEntriesQueryResult; import org.apache.tsfile.utils.BloomFilter; import org.apache.tsfile.utils.FileGenerator; import org.apache.tsfile.utils.Pair; @@ -64,7 +60,6 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -268,76 +263,4 @@ public void testGetTableSchemaMap() throws IOException, WriteProcessException { Assert.assertTrue(reader.readFileMetadata().hasTableSchemaMapCache()); } } - - @Test - public void testGetDeviceMetadataIndexNodeOffsets() throws IOException, WriteProcessException { - File file = new File(FILE_PATH); - try { - tsFile.close(); - Files.deleteIfExists(file.toPath()); - } catch (IOException ignored) { - } - TableSchema tableSchema = - new TableSchema( - "t1", - Arrays.asList( - new MeasurementSchema("device", TSDataType.STRING), - new MeasurementSchema("s1", TSDataType.INT32)), - Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); - try (ITsFileWriter writer = - new TsFileWriterBuilder().tableSchema(tableSchema).file(file).build()) { - Tablet tablet = - new Tablet( - Arrays.asList("device", "s1"), - Arrays.asList(TSDataType.STRING, TSDataType.INT32), - 10000); - for (int i = 0; i < 10000; i++) { - tablet.addTimestamp(i, i); - tablet.addValue("device", i, "d" + i); - tablet.addValue("s1", i, i); - } - writer.write(tablet); - } - - List queriedDevices = new ArrayList<>(); - for (int i = 0; i < 20000; i++) { - if (i >= 15000) { - queriedDevices.add(new StringArrayDeviceID("t2.d" + i)); - } else { - queriedDevices.add(new StringArrayDeviceID("t1.d" + i)); - } - } - queriedDevices.sort(IDeviceID::compareTo); - try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { - DeviceMetadataIndexEntriesQueryResult offsets = - reader.getDeviceMetadataIndexNodeOffsets(null, queriedDevices, null); - for (int i = 0; i < queriedDevices.size(); i++) { - IDeviceID deviceID = queriedDevices.get(i); - int deviceNumber = Integer.parseInt(deviceID.toString().substring("t1.d".length())); - long[] metadataIndexNodeOffsetOfCurDevice = offsets.getDeviceMetadataIndexNodeOffset(i); - if (deviceNumber >= 10000) { - Assert.assertNull(metadataIndexNodeOffsetOfCurDevice); - continue; - } - MetadataIndexNode metadataIndexNode = - reader.readMetadataIndexNode( - metadataIndexNodeOffsetOfCurDevice[0], - metadataIndexNodeOffsetOfCurDevice[1], - false); - List alignedChunkMetadataList = - reader.getAlignedChunkMetadataByMetadataIndexNode(deviceID, metadataIndexNode, true); - Assert.assertEquals(1, alignedChunkMetadataList.size()); - - Assert.assertEquals(deviceNumber, alignedChunkMetadataList.get(0).getStartTime()); - } - - Assert.assertEquals( - 0, - reader.getDeviceMetadataIndexNodeOffsets("t1", Collections.emptyList(), null).length()); - offsets = - reader.getDeviceMetadataIndexNodeOffsets( - "t1", Collections.singletonList(new StringArrayDeviceID("t1.d")), null); - Assert.assertNull(offsets.getDeviceMetadataIndexNodeOffset(0)); - } - } } From 50049531ff486690d8d2dec55e7cbeb32bbba6bb Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 5 Aug 2025 16:14:34 +0800 Subject: [PATCH 18/18] fix bug --- .../read/query/DeviceMetadataIndexEntriesQueryResult.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java index 5d962f5b8..0786c504a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java @@ -61,7 +61,7 @@ public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { if (nodeSizeArr instanceof byte[]) { nodeSize = (((byte[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_BYTE; } else if (nodeSizeArr instanceof short[]) { - nodeSize = (((short[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_BYTE; + nodeSize = (((short[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_SHORT; } else if (nodeSizeArr instanceof int[]) { nodeSize = (((int[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_INTEGER; } else { @@ -168,6 +168,8 @@ public int length() { public long ramBytesUsed() { return SHALLOW_SIZE + arrResult.ramBytesUsed() - + RamUsageEstimator.sizeOfIntArray(arrResult.length()); + + (indexMap instanceof int[] + ? RamUsageEstimator.sizeOfIntArray(arrResult.length()) + : RamUsageEstimator.sizeOfShortArray(arrResult.length())); } }