diff --git a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java index 0538f1b39..11437f5d7 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java @@ -37,8 +37,9 @@ public static TsPrimitiveType getByType(TSDataType dataType) { case BOOLEAN: return new TsPrimitiveType.TsBoolean(); case INT32: - case DATE: return new TsPrimitiveType.TsInt(); + case DATE: + return new TsPrimitiveType.TsInt(TSDataType.DATE); case INT64: case TIMESTAMP: return new TsPrimitiveType.TsLong(); @@ -69,8 +70,9 @@ public static TsPrimitiveType getByType(TSDataType dataType, Object v) { case BOOLEAN: return new TsPrimitiveType.TsBoolean((boolean) v); case INT32: - case DATE: return new TsPrimitiveType.TsInt((int) v); + case DATE: + return new TsPrimitiveType.TsInt((int) v, TSDataType.DATE); case INT64: case TIMESTAMP: return new TsPrimitiveType.TsLong((long) v); @@ -260,12 +262,23 @@ public static class TsInt extends TsPrimitiveType { private int value; + private TSDataType dataType = TSDataType.INT32; + public TsInt() {} public TsInt(int value) { this.value = value; } + public TsInt(TSDataType dataType) { + this.dataType = dataType; + } + + public TsInt(int value, TSDataType dataType) { + this.value = value; + this.dataType = dataType; + } + @Override public int getInt() { return value; @@ -327,7 +340,7 @@ public String getStringValue() { @Override public TSDataType getDataType() { - return TSDataType.INT32; + return dataType; } @Override diff --git a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl index 83538cf12..fcd3bb01b 100644 --- a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl +++ b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl @@ -146,9 +146,9 @@ public final class ${className} { @Override public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return valueSatisfy((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return valueSatisfy((${filter.dataType}) value); } @@ -173,7 +173,19 @@ public final class ${className} { @SuppressWarnings("unchecked") public boolean canSkip(Statistics statistics) { <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> - <#if filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.javaBoxName == "String"> + if(statistics.isEmpty()){ + return false; + } + if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){ + return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0 + || constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) < 0 + || constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) > 0; + } + <#elseif filter.javaBoxName == "Tag"> if(statistics.isEmpty()){ return false; } @@ -216,8 +228,14 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMinValue()) == 0 - && constant.compareTo((${filter.dataType}) statistics.getMaxValue()) == 0; + if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){ + return constant.compareTo((${filter.dataType}) statistics.getMinValue()) == 0 + && constant.compareTo((${filter.dataType}) statistics.getMaxValue()) == 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) == 0 + && constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) == 0; + } <#else> return false; @@ -270,9 +288,9 @@ public final class ${className} { @Override public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return valueSatisfy((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return valueSatisfy((${filter.dataType}) value); } else { @@ -300,8 +318,14 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMinValue()) == 0 - && constant.compareTo((${filter.dataType}) statistics.getMaxValue()) == 0; + if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){ + return constant.compareTo((${filter.dataType}) statistics.getMinValue()) == 0 + && constant.compareTo((${filter.dataType}) statistics.getMaxValue()) == 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) == 0 + && constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) == 0; + } <#else> return false; @@ -340,8 +364,14 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0 - || constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0; + if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){ + return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0 + || constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) < 0 + || constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) > 0; + } <#else> return false; @@ -394,9 +424,9 @@ public final class ${className} { @Override @SuppressWarnings("unchecked") public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return valueSatisfy((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return valueSatisfy((${filter.dataType}) value); } else { @@ -426,7 +456,12 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMinValue()) <= 0; + if(statistics.getMinValue() instanceof Binary){ + return constant.compareTo((${filter.dataType}) statistics.getMinValue()) <= 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) <= 0; + } <#else> return false; @@ -462,7 +497,12 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0; + if(statistics.getMaxValue() instanceof Binary){ + return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) > 0; + } <#else> return false; @@ -512,9 +552,9 @@ public final class ${className} { @Override @SuppressWarnings("unchecked") public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return valueSatisfy((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return valueSatisfy((${filter.dataType}) value); } else { @@ -544,7 +584,12 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0; + if(statistics.getMinValue() instanceof Binary){ + return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) < 0; + } <#else> return false; @@ -580,7 +625,12 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) >= 0; + if(statistics.getMaxValue() instanceof Binary){ + return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) >= 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) >= 0; + } <#else> return false; @@ -630,9 +680,9 @@ public final class ${className} { @Override @SuppressWarnings("unchecked") public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return valueSatisfy((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return valueSatisfy((${filter.dataType}) value); } else { @@ -662,7 +712,12 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) >= 0; + if(statistics.getMaxValue() instanceof Binary){ + return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) >= 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) >= 0; + } <#else> return false; @@ -698,7 +753,12 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0; + if(statistics.getMinValue() instanceof Binary){ + return constant.compareTo((${filter.dataType}) statistics.getMinValue()) < 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) < 0; + } <#else> return false; @@ -748,9 +808,9 @@ public final class ${className} { @Override @SuppressWarnings("unchecked") public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return valueSatisfy((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return valueSatisfy((${filter.dataType}) value); } else { @@ -780,7 +840,12 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0; + if(statistics.getMaxValue() instanceof Binary){ + return constant.compareTo((${filter.dataType}) statistics.getMaxValue()) > 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)) > 0; + } <#else> return false; @@ -816,7 +881,12 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return constant.compareTo((${filter.dataType}) statistics.getMinValue()) <= 0; + if(statistics.getMinValue() instanceof Binary){ + return constant.compareTo((${filter.dataType}) statistics.getMinValue()) <= 0; + } + else{ + return constant.compareTo(new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)) <= 0; + } <#else> return false; @@ -934,9 +1004,9 @@ public final class ${className} { @Override @SuppressWarnings("unchecked") public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return valueSatisfy((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return valueSatisfy((${filter.dataType}) value); } else { @@ -968,8 +1038,14 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return ((${filter.dataType}) statistics.getMaxValue()).compareTo(min) < 0 - || ((${filter.dataType}) statistics.getMinValue()).compareTo(max) > 0; + if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){ + return ((${filter.dataType}) statistics.getMaxValue()).compareTo(min) < 0 + || ((${filter.dataType}) statistics.getMinValue()).compareTo(max) > 0; + } + else{ + return (new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)).compareTo(min) < 0 + || (new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)).compareTo(max) > 0; + } <#else> return false; @@ -1007,8 +1083,14 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return ((${filter.dataType}) statistics.getMinValue()).compareTo(min) >= 0 - && ((${filter.dataType}) statistics.getMaxValue()).compareTo(max) <= 0; + if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){ + return ((${filter.dataType}) statistics.getMinValue()).compareTo(min) >= 0 + && ((${filter.dataType}) statistics.getMaxValue()).compareTo(max) <= 0; + } + else{ + return (new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)).compareTo(min) >= 0 + && (new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)).compareTo(max) <= 0; + } <#else> return false; @@ -1061,9 +1143,9 @@ public final class ${className} { @Override @SuppressWarnings("unchecked") public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return valueSatisfy((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return valueSatisfy((${filter.dataType}) value); } else { @@ -1094,8 +1176,14 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return ((${filter.dataType}) statistics.getMinValue()).compareTo(min) >= 0 - && ((${filter.dataType}) statistics.getMaxValue()).compareTo(max) <= 0; + if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){ + return ((${filter.dataType}) statistics.getMinValue()).compareTo(min) >= 0 + && ((${filter.dataType}) statistics.getMaxValue()).compareTo(max) <= 0; + } + else{ + return (new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)).compareTo(min) >= 0 + && (new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)).compareTo(max) <= 0; + } <#else> return false; @@ -1133,8 +1221,14 @@ public final class ${className} { if(statistics.isEmpty()){ return false; } - return ((${filter.dataType}) statistics.getMinValue()).compareTo(max) > 0 - || ((${filter.dataType}) statistics.getMaxValue()).compareTo(min) < 0; + if((statistics.getMinValue() instanceof Binary) && (statistics.getMaxValue() instanceof Binary)){ + return ((${filter.dataType}) statistics.getMinValue()).compareTo(max) > 0 + || ((${filter.dataType}) statistics.getMaxValue()).compareTo(min) < 0; + } + else{ + return (new ${filter.dataType}(String.valueOf(statistics.getMinValue()), StandardCharsets.UTF_8)).compareTo(max) > 0 + || (new ${filter.dataType}(String.valueOf(statistics.getMaxValue()), StandardCharsets.UTF_8)).compareTo(min) < 0; + } <#else> return false; @@ -1292,9 +1386,9 @@ public final class ${className} { @Override public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return candidates.contains((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return candidates.contains((${filter.dataType}) value); } else { @@ -1330,7 +1424,22 @@ public final class ${className} { if (statistics.isPresent()) { Statistics stat = statistics.get(); - <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")> + <#if filter.dataType == "Binary" && filter.javaBoxName == "String"> + ${filter.dataType} valuesMin; + ${filter.dataType} valuesMax; + if(stat.getMinValue() instanceof Binary){ + valuesMin = (${filter.dataType}) stat.getMinValue(); + } + else{ + valuesMin = new ${filter.dataType}(String.valueOf(stat.getMinValue()), StandardCharsets.UTF_8); + } + if(stat.getMaxValue() instanceof Binary){ + valuesMax = (${filter.dataType}) stat.getMaxValue(); + } + else{ + valuesMax = new ${filter.dataType}(String.valueOf(stat.getMaxValue()), StandardCharsets.UTF_8); + } + <#elseif filter.dataType == "String" && filter.javaBoxName == "Tag"> ${filter.dataType} valuesMin = (${filter.dataType}) stat.getMinValue(); ${filter.dataType} valuesMax = (${filter.dataType}) stat.getMaxValue(); <#else> @@ -1384,7 +1493,22 @@ public final class ${className} { // All values are same if (statistics.isPresent()) { Statistics stat = statistics.get(); - <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")> + <#if filter.dataType == "Binary" && filter.javaBoxName == "String"> + ${filter.dataType} valuesMin; + ${filter.dataType} valuesMax; + if(stat.getMinValue() instanceof Binary){ + valuesMin = (${filter.dataType}) stat.getMinValue(); + } + else{ + valuesMin = new ${filter.dataType}(String.valueOf(stat.getMinValue()), StandardCharsets.UTF_8); + } + if(stat.getMaxValue() instanceof Binary){ + valuesMax = (${filter.dataType}) stat.getMaxValue(); + } + else{ + valuesMax = new ${filter.dataType}(String.valueOf(stat.getMaxValue()), StandardCharsets.UTF_8); + } + <#elseif filter.dataType == "String" && filter.javaBoxName == "Tag"> ${filter.dataType} valuesMin = (${filter.dataType}) stat.getMinValue(); ${filter.dataType} valuesMax = (${filter.dataType}) stat.getMaxValue(); <#else> @@ -1446,9 +1570,9 @@ public final class ${className} { @Override public boolean valueSatisfy(Object value){ - <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> + <#if filter.dataType == "boolean" || filter.javaBoxName == "Tag"> return !candidates.contains((${filter.dataType}) value); - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.javaBoxName == "String"> if(value instanceof Binary){ return !candidates.contains((${filter.dataType}) value); } else { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java index e06f8c377..edd35e726 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java @@ -111,10 +111,18 @@ private TsBlockBuilder(int initialExpectedEntries, int maxTsBlockBytes, List types) { tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); break; case INT32: + valueColumnBuilders[i] = + new IntColumnBuilder( + tsBlockBuilderStatus.createColumnBuilderStatus(), + initialExpectedEntries, + TSDataType.INT32); + break; case DATE: valueColumnBuilders[i] = new IntColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); + tsBlockBuilderStatus.createColumnBuilderStatus(), + initialExpectedEntries, + TSDataType.DATE); break; case INT64: case TIMESTAMP: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java index fc9774a15..4b5c468ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java @@ -38,8 +38,9 @@ public static Column create(TSDataType dataType, int initialCapacity) { case OBJECT: return new BinaryColumn(initialCapacity); case INT32: + return new IntColumn(initialCapacity, TSDataType.INT32); case DATE: - return new IntColumn(initialCapacity); + return new IntColumn(initialCapacity, TSDataType.DATE); case BOOLEAN: return new BooleanColumn(initialCapacity); default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index a5ea68519..fdaa4e7e6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -50,14 +50,27 @@ public class IntColumn implements Column { private final long retainedSizeInBytes; + private TSDataType dataType = TSDataType.INT32; + public IntColumn(int initialCapacity) { this(0, 0, null, new int[initialCapacity]); } + public IntColumn(int initialCapacity, TSDataType dataType) { + this(initialCapacity); + this.dataType = dataType; + } + public IntColumn(int positionCount, Optional valueIsNull, int[] values) { this(0, positionCount, valueIsNull.orElse(null), values); } + public IntColumn( + int positionCount, Optional valueIsNull, int[] values, TSDataType dataType) { + this(positionCount, valueIsNull, values); + this.dataType = dataType; + } + IntColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, int[] values) { if (arrayOffset < 0) { throw new IllegalArgumentException("arrayOffset is negative"); @@ -82,9 +95,19 @@ public IntColumn(int positionCount, Optional valueIsNull, int[] value INSTANCE_SIZE + sizeOfIntArray(positionCount) + sizeOfBooleanArray(positionCount); } + IntColumn( + int arrayOffset, + int positionCount, + boolean[] valueIsNull, + int[] values, + TSDataType dataType) { + this(arrayOffset, positionCount, valueIsNull, values); + this.dataType = dataType; + } + @Override public TSDataType getDataType() { - return TSDataType.INT32; + return dataType; } @Override @@ -165,7 +188,7 @@ public Object getObject(int position) { @Override public TsPrimitiveType getTsPrimitiveType(int position) { - return new TsPrimitiveType.TsInt(getInt(position)); + return new TsPrimitiveType.TsInt(getInt(position), dataType); } @Override @@ -206,7 +229,7 @@ public long getSizeInBytes() { @Override public Column getRegion(int positionOffset, int length) { checkValidRegion(getPositionCount(), positionOffset, length); - return new IntColumn(positionOffset + arrayOffset, length, valueIsNull, values); + return new IntColumn(positionOffset + arrayOffset, length, valueIsNull, values, dataType); } @Override @@ -219,7 +242,7 @@ public Column getRegionCopy(int positionOffset, int length) { valueIsNull != null ? Arrays.copyOfRange(valueIsNull, from, to) : null; int[] valuesCopy = Arrays.copyOfRange(values, from, to); - return new IntColumn(0, length, valueIsNullCopy, valuesCopy); + return new IntColumn(0, length, valueIsNullCopy, valuesCopy, dataType); } @Override @@ -227,7 +250,8 @@ public Column subColumn(int fromIndex) { if (fromIndex > positionCount) { throw new IllegalArgumentException("fromIndex is not valid"); } - return new IntColumn(arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values); + return new IntColumn( + arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values, dataType); } @Override @@ -242,7 +266,7 @@ public Column subColumnCopy(int fromIndex) { int[] valuesCopy = Arrays.copyOfRange(values, from, positionCount); int length = positionCount - fromIndex; - return new IntColumn(0, length, valueIsNullCopy, valuesCopy); + return new IntColumn(0, length, valueIsNullCopy, valuesCopy, dataType); } @Override @@ -286,7 +310,7 @@ public Column copyPositions(int[] positions, int offset, int length) { } newValues[i] = values[position + arrayOffset]; } - return new IntColumn(0, length, newValueIsNull, newValues); + return new IntColumn(0, length, newValueIsNull, newValues, dataType); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumnBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumnBuilder.java index a0fc7a5b4..742e17767 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumnBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumnBuilder.java @@ -56,6 +56,8 @@ public class IntColumnBuilder implements ColumnBuilder { private long retainedSizeInBytes; + private TSDataType dataType = TSDataType.INT32; + public IntColumnBuilder(ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { this.columnBuilderStatus = columnBuilderStatus; this.initialEntryCount = max(expectedEntries, 1); @@ -63,6 +65,12 @@ public IntColumnBuilder(ColumnBuilderStatus columnBuilderStatus, int expectedEnt updateDataSize(); } + public IntColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries, TSDataType dataType) { + this(columnBuilderStatus, expectedEntries); + this.dataType = dataType; + } + @Override public int getPositionCount() { return positionCount; @@ -128,12 +136,13 @@ public Column build() { if (!hasNonNullValue) { return new RunLengthEncodedColumn(NULL_VALUE_BLOCK, positionCount); } - return new IntColumn(0, positionCount, hasNullValue ? valueIsNull : null, values); + return new IntColumn( + 0, positionCount, hasNullValue ? valueIsNull : null, values, getDataType()); } @Override public TSDataType getDataType() { - return TSDataType.INT32; + return dataType; } @Override @@ -143,7 +152,8 @@ public long getRetainedSizeInBytes() { @Override public ColumnBuilder newColumnBuilderLike(ColumnBuilderStatus columnBuilderStatus) { - return new IntColumnBuilder(columnBuilderStatus, calculateBlockResetSize(positionCount)); + return new IntColumnBuilder( + columnBuilderStatus, calculateBlockResetSize(positionCount), dataType); } private void growCapacity() { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index ff34c98b2..50835f5f8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -190,7 +190,10 @@ public TsPrimitiveType nextValue(long timestamp, int timeIndex) throws IOExcepti case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsInt(anInt); + resultValue = + dataType.equals(TSDataType.INT32) + ? new TsPrimitiveType.TsInt(anInt) + : new TsPrimitiveType.TsInt(anInt, TSDataType.DATE); } break; case INT64: @@ -253,7 +256,10 @@ public TsPrimitiveType[] nextValueBatch(long[] timeBatch) throws IOException { case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsInt(anInt); + valueBatch[i] = + dataType.equals(TSDataType.INT32) + ? new TsPrimitiveType.TsInt(anInt) + : new TsPrimitiveType.TsInt(anInt, TSDataType.DATE); } break; case INT64: