From 0ed23defed4e13a8485669279718de3f33f90ceb Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 15 Oct 2025 22:08:58 -0700 Subject: [PATCH 01/41] PPL tostring() implementation issue #4492 Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 7 + .../function/PPLBuiltinOperators.java | 2 + .../expression/function/PPLFuncImpTable.java | 3 + .../function/udf/ToStringFunction.java | 162 +++++++++++++++ .../function/udf/ToStringFunctionTest.java | 149 ++++++++++++++ docs/user/ppl/functions/conversion.rst | 92 +++++++++ docs/user/ppl/functions/string.rst | 3 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 6 +- .../sql/ppl/parser/AstExpressionBuilder.java | 11 +- .../calcite/CalcitePPLStringFunctionTest.java | 185 ++++++++++++++++++ sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 + 13 files changed, 621 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index 12a9a297542..69e1492538c 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -112,6 +112,13 @@ private PPLOperandTypes() {} SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)); + public static final UDFOperandMetadata BOOLEAN_OR_NUMERIC_STRING_OR_STRING_STRING = + UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + OperandTypes.family(SqlTypeFamily.BOOLEAN) + .or(OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) + .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); + public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 76f625ebd38..984f398b1ee 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -66,6 +66,7 @@ import org.opensearch.sql.expression.function.udf.RexExtractMultiFunction; import org.opensearch.sql.expression.function.udf.RexOffsetFunction; import org.opensearch.sql.expression.function.udf.SpanFunction; +import org.opensearch.sql.expression.function.udf.ToStringFunction; import org.opensearch.sql.expression.function.udf.condition.EarliestFunction; import org.opensearch.sql.expression.function.udf.condition.EnhancedCoalesceFunction; import org.opensearch.sql.expression.function.udf.condition.LatestFunction; @@ -411,6 +412,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("multi_match", false); public static final SqlOperator NUMBER_TO_STRING = new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); + public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); public static final SqlOperator WIDTH_BUCKET = new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction() .toUDF("WIDTH_BUCKET"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 8297ecf73ce..de89df383d2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -210,6 +210,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_FORMAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_TO_SEC; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TOSTRING; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_DAYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_SECONDS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRANSFORM; @@ -887,6 +888,7 @@ void populate() { registerOperator(WEEKOFYEAR, PPLBuiltinOperators.WEEK); registerOperator(INTERNAL_PATTERN_PARSER, PPLBuiltinOperators.PATTERN_PARSER); + registerOperator(TOSTRING, PPLBuiltinOperators.TOSTRING); // Register MVJOIN to use Calcite's ARRAY_JOIN register( @@ -1058,6 +1060,7 @@ void populate() { SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)), false)); + register( LOG, (FunctionImp2) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java new file mode 100644 index 00000000000..70d2a82b13c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -0,0 +1,162 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.text.NumberFormat; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.function.Strict; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.calcite.utils.PPLReturnTypes; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * A custom implementation of number/boolean to string . + * + *

This operator is necessary because tostring has following requirements "binary" Converts a + * number to a binary value. "hex" Converts the number to a hexadecimal value. "commas" Formats the + * number with commas. If the number includes a decimal, the function rounds the number to nearest + * two decimal places. "duration" Converts the value in seconds to the readable time format + * HH:MM:SS. if not format parameter provided, then consider value as boolean + */ +public class ToStringFunction extends ImplementorUDF { + public ToStringFunction() { + super( + new org.opensearch.sql.expression.function.udf.ToStringFunction.ToStringImplementor(), + NullPolicy.ANY); + } + + public static final String DURATION_FORMAT = "duration"; + public static final String HEX_FORMAT = "hex"; + public static final String COMMAS_FORMAT = "commas"; + public static final String BINARY_FORMAT = "binary"; + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return PPLReturnTypes.STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.BOOLEAN_OR_NUMERIC_STRING_OR_STRING_STRING; + } + + public static class ToStringImplementor implements NotNullImplementor { + + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + Expression fieldValue = translatedOperands.get(0); + if (translatedOperands.size() > 1) { + Expression format = translatedOperands.get(1); + return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); + } else { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } + } + } + + @Strict + public static String toString(boolean fieldValue) { + if (fieldValue) { + return "True"; + } else { + return "False"; + } + } + + @Strict + public static String toString(String fieldValue) { + return toString(Boolean.parseBoolean(fieldValue)); + } + + @Strict + public static String toString(BigDecimal num, String format) { + if (format.equals(DURATION_FORMAT)) { + Duration d = Duration.ofSeconds(num.toBigInteger().longValue()); + long hours = d.toHours(); + int minutes = d.toMinutesPart(); + int remainingSeconds = d.toSecondsPart(); + + String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return num.toBigInteger().toString(16); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + nf.setMinimumFractionDigits(0); + nf.setMaximumFractionDigits(2); + return nf.format(num); + + } else if (format.equals(BINARY_FORMAT)) { + BigInteger integerPart = num.toBigInteger(); // 42 + return integerPart.toString(2); + } + return num.toString(); + } + + @Strict + public static String toString(double num, String format) { + if (format.equals(DURATION_FORMAT)) { + Duration d = Duration.ofSeconds(Math.round(num)); + long hours = d.toHours(); + int minutes = d.toMinutesPart(); + int remainingSeconds = d.toSecondsPart(); + String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return Double.toHexString(num); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + return nf.format(num); + } else if (format.equals(BINARY_FORMAT)) { + return Long.toBinaryString(Double.doubleToLongBits(num)); + } + return Double.toString(num); + } + + @Strict + public static String toString(int num, String format) { + + if (format.equals(DURATION_FORMAT)) { + + int hours = num / 3600; + int minutes = (num % 3600) / 60; + int seconds = num % 60; + + String time_str = String.format("%02d:%02d:%02d", hours, minutes, seconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return Integer.toHexString(num); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + return nf.format(num); + } else if (format.equals(BINARY_FORMAT)) { + return Integer.toBinaryString(num); + } + return Integer.toString(num); + } + + @Strict + public static String toString(String str, String format) { + if (str.contains(".") || (str.length() > 10)) { + return toString(Double.parseDouble(str), format); + } else { + return toString(Integer.parseInt(str), format); + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java new file mode 100644 index 00000000000..cfd25796b23 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Locale; + +import static org.junit.jupiter.api.Assertions.*; + +public class ToStringFunctionTest { + + private final ToStringFunction function = new ToStringFunction(); + + @Test + void testBooleanToString() { + assertEquals("True", ToStringFunction.toString(true)); + assertEquals("False", ToStringFunction.toString(false)); + } + + @Test + void testStringBooleanToString() { + assertEquals("True", ToStringFunction.toString("true")); + assertEquals("False", ToStringFunction.toString("false")); + assertEquals("False", ToStringFunction.toString("anythingElse")); + } + + @Test + void testBigDecimalToStringDurationFormat() { + BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testBigDecimalToStringHexFormat() { + BigDecimal num = new BigDecimal("255"); + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertEquals("ff", result); + } + + @Test + void testBigDecimalToStringCommasFormat() { + Locale.setDefault(Locale.US); // Ensure predictable comma placement + BigDecimal num = new BigDecimal("1234567.891"); + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testBigDecimalToStringBinaryFormat() { + BigDecimal num = new BigDecimal("10"); + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertEquals("1010", result); + } + + @Test + void testBigDecimalToStringDefault() { + BigDecimal num = new BigDecimal("123.45"); + assertEquals("123.45", ToStringFunction.toString(num, "unknown")); + } + + @Test + void testDoubleToStringDurationFormat() { + double num = 3661.4; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testDoubleToStringHexFormat() { + double num = 10.5; + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertTrue(result.startsWith("0x")); + } + + @Test + void testDoubleToStringCommasFormat() { + Locale.setDefault(Locale.US); + double num = 12345.678; + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testDoubleToStringBinaryFormat() { + double num = 10.0; + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + assertFalse(result.isEmpty()); + } + + @Test + void testDoubleToStringDefault() { + assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); + } + + @Test + void testIntToStringDurationFormat() { + int num = 3661; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testIntToStringHexFormat() { + assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); + } + + @Test + void testIntToStringCommasFormat() { + Locale.setDefault(Locale.US); + String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testIntToStringBinaryFormat() { + assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); + } + + @Test + void testIntToStringDefault() { + assertEquals("123", ToStringFunction.toString(123, "unknown")); + } + + @Test + void testStringNumericToStringIntFormat() { + String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); + assertEquals("2a", result); + } + + @Test + void testStringNumericToStringDoubleFormat() { + String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains("42")); + } + + @Test + void testStringLargeNumberAsDouble() { + String largeNum = "1234567890123"; + String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + } +} diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index dbe4403540c..21124c27edc 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -78,3 +78,95 @@ Cast function can be chained:: |-------| | True | +-------+ + +TOSTRING +----- + +Description +>>>>>>>>>>> +There are two available usage based on paraemter types and number of parameters. +Usage with format type: tostring(number|string, string) converts the number in first argument to provided format type string in second argument. + Return type: string +Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. + Return type: string + +You can use this function with the eval commands and as part of eval expressions. +The first argument can be a number, number as string or boolean. +If first argument is a a number or number as string , second argument need to be format name. +If first argument is boolean, then second argument is not needed. + +format types: +a) "binary" Converts a number to a binary value. +b) "hex" Converts the number to a hexadecimal value. +c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. +d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. +The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. + +Binary conversion +You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: +eval result = tostring(9, "binary") + +For information about bitwise functions that you can use with the tostring function, see Bitwise functions. + +Basic examples +The following example returns "True 0xF 12,345.68". +... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") +The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. + +... | eval foo=615 | eval foo2 = tostring(foo, "duration") +The following example formats the column totalSales to display values with a currency symbol and commas. You must use a period between the currency value and the tostring function. + +Example:: + + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` + fetched rows / total rows = 1/1 + +---------------------+ + | boolean_str | + |---------------------+ + | True | + +---------------------+ + os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_binary | SAL | + |---------------+------------------+------------+ + | SMITH | 1001110001000000 | 80000.00 | + +---------------+------------------+------------+ + os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_hex | SAL | + |---------------+------------------+------------+ + | SMITH | 13880 | 80000.00 | + +---------------+---------------+---------------+ + + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_commas | SAL | + |---------------+------------------+------------+ + | SMITH | 80,000 | 80000.00 | + +---------------+------------------+------------+ + + + duration + + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" + fetched rows / total rows = 1/1 + +---------------+-------------+ + | ENAME | duration | + |---------------+-------------+ + | SMITH | 01:48:20 | + +---------------+-------------+ + +Usage for boolean parameter without format type:: + +Example:: + + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` + fetched rows / total rows = 1/1 + +---------------------+ + | boolean_str | + |---------------------+ + | True | + +---------------------+ \ No newline at end of file diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 24efa1434f5..43efed3c470 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -397,3 +397,6 @@ Example:: |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | +---------------------+---------------------+ + + + diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ba1e4960bb2..e528bb553ab 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -392,6 +392,7 @@ STRFTIME: 'STRFTIME'; // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; SUBSTRING: 'SUBSTRING'; +TOSTRING: 'TOSTRING'; LTRIM: 'LTRIM'; RTRIM: 'RTRIM'; TRIM: 'TRIM'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index e13447b68e9..852c55c863c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -847,11 +847,14 @@ evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS ; -// cast function + +// cast, tostring function dataTypeFunctionCall : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS + | TOSTRING LT_PRTHS functionArgs RT_PRTHS ; + convertedDataType : typeName = DATE | typeName = TIME @@ -1434,6 +1437,7 @@ searchableKeyWord | USING | VALUE | CAST + | TOSTRING | GET_FORMAT | EXTRACT | INTERVAL diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index f037376f5c2..1460a3efc9a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -414,7 +414,16 @@ private Function buildFunction( /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + ParseTree rootNode = ctx.getChild(0); + String functionName = rootNode.getText(); + final String mappedName = + FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); + + if (mappedName.equals("cast")) { + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + } else { + return buildFunction(mappedName, ctx.functionArgs().functionArg()); + } } @Override diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index 1e97052dea0..d41b2c22453 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -46,6 +46,191 @@ public void testLower() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testToStringBoolean() { + String ppl = "source=EMP | eval boolean_value = tostring(1==1) | fields boolean_value |head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(boolean_value=[TOSTRING(true)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "boolean_value=True\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TOSTRING`(TRUE) `boolean_value`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringBin() { + String ppl = + "source=EMP | eval salary_binary = tostring(SAL, \"binary\") | fields ENAME," + + " salary_binary, SAL"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_binary=[TOSTRING($5, 'binary':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_binary=1100100000; SAL=800.00\n" + + "ENAME=ALLEN; salary_binary=11001000000; SAL=1600.00\n" + + "ENAME=WARD; salary_binary=10011100010; SAL=1250.00\n" + + "ENAME=JONES; salary_binary=101110011111; SAL=2975.00\n" + + "ENAME=MARTIN; salary_binary=10011100010; SAL=1250.00\n" + + "ENAME=BLAKE; salary_binary=101100100010; SAL=2850.00\n" + + "ENAME=CLARK; salary_binary=100110010010; SAL=2450.00\n" + + "ENAME=SCOTT; salary_binary=101110111000; SAL=3000.00\n" + + "ENAME=KING; salary_binary=1001110001000; SAL=5000.00\n" + + "ENAME=TURNER; salary_binary=10111011100; SAL=1500.00\n" + + "ENAME=ADAMS; salary_binary=10001001100; SAL=1100.00\n" + + "ENAME=JAMES; salary_binary=1110110110; SAL=950.00\n" + + "ENAME=FORD; salary_binary=101110111000; SAL=3000.00\n" + + "ENAME=MILLER; salary_binary=10100010100; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'binary') `salary_binary`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringHex() { + String ppl = + "source=EMP | eval salary_hex = tostring(SAL, \"hex\") | fields ENAME, salary_hex, SAL"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_hex=[TOSTRING($5, 'hex':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_hex=320; SAL=800.00\n" + + "ENAME=ALLEN; salary_hex=640; SAL=1600.00\n" + + "ENAME=WARD; salary_hex=4e2; SAL=1250.00\n" + + "ENAME=JONES; salary_hex=b9f; SAL=2975.00\n" + + "ENAME=MARTIN; salary_hex=4e2; SAL=1250.00\n" + + "ENAME=BLAKE; salary_hex=b22; SAL=2850.00\n" + + "ENAME=CLARK; salary_hex=992; SAL=2450.00\n" + + "ENAME=SCOTT; salary_hex=bb8; SAL=3000.00\n" + + "ENAME=KING; salary_hex=1388; SAL=5000.00\n" + + "ENAME=TURNER; salary_hex=5dc; SAL=1500.00\n" + + "ENAME=ADAMS; salary_hex=44c; SAL=1100.00\n" + + "ENAME=JAMES; salary_hex=3b6; SAL=950.00\n" + + "ENAME=FORD; salary_hex=bb8; SAL=3000.00\n" + + "ENAME=MILLER; salary_hex=514; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'hex') `salary_hex`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringHexFromNumberAsString() { + String ppl = + "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_hex=640\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringCommaFromNumberAsString() { + String ppl = + "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME, salary_comma| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR, 'commas':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_comma=160,040,222\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test + public void testToStringBinaryFromNumberAsString() { + String ppl = + "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME, salary_binary| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR, 'binary':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test + public void testToStringCommas() { + String ppl = + "source=EMP | eval salary_commas = tostring(SAL, \"commas\") | fields ENAME," + + " salary_commas, SAL"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_commas=[TOSTRING($5, 'commas':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_commas=800; SAL=800.00\n" + + "ENAME=ALLEN; salary_commas=1,600; SAL=1600.00\n" + + "ENAME=WARD; salary_commas=1,250; SAL=1250.00\n" + + "ENAME=JONES; salary_commas=2,975; SAL=2975.00\n" + + "ENAME=MARTIN; salary_commas=1,250; SAL=1250.00\n" + + "ENAME=BLAKE; salary_commas=2,850; SAL=2850.00\n" + + "ENAME=CLARK; salary_commas=2,450; SAL=2450.00\n" + + "ENAME=SCOTT; salary_commas=3,000; SAL=3000.00\n" + + "ENAME=KING; salary_commas=5,000; SAL=5000.00\n" + + "ENAME=TURNER; salary_commas=1,500; SAL=1500.00\n" + + "ENAME=ADAMS; salary_commas=1,100; SAL=1100.00\n" + + "ENAME=JAMES; salary_commas=950; SAL=950.00\n" + + "ENAME=FORD; salary_commas=3,000; SAL=3000.00\n" + + "ENAME=MILLER; salary_commas=1,300; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'commas') `salary_commas`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringDuration() { + String ppl = + "source=EMP | eval duration_commas = tostring(6500, \"duration\") | fields ENAME," + + " duration_commas|HEAD 1"; + + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], duration_commas=[TOSTRING(6500, 'duration':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; duration_commas=01:48:20\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(6500, 'duration') `duration_commas`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testLike() { String ppl = "source=EMP | where like(JOB, 'SALE%') | stats count() as cnt"; diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index ba7c5be85ab..6465c692da3 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -133,6 +133,7 @@ STDDEV_SAMP: 'STDDEV_SAMP'; SUBSTRING: 'SUBSTRING'; TRIM: 'TRIM'; +TOSTRING: 'TOSTRING'; // Keywords, but can be ID // Common Keywords, but can be ID diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 5f7361160b3..fbaef12fb98 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -417,6 +417,7 @@ specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CAST '(' expression AS convertedDataType ')' # dataTypeFunctionCall + | TOSTRING '(' functionArg ')' # dataTypeFunctionCall ; relevanceFunction From 5a5b778cdf83230ff8a68a259a628e3ee0b3f318 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:33:26 -0700 Subject: [PATCH 02/41] removed sql changes Signed-off-by: Asif Bashar --- sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 - sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 - 2 files changed, 2 deletions(-) diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 6465c692da3..ba7c5be85ab 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -133,7 +133,6 @@ STDDEV_SAMP: 'STDDEV_SAMP'; SUBSTRING: 'SUBSTRING'; TRIM: 'TRIM'; -TOSTRING: 'TOSTRING'; // Keywords, but can be ID // Common Keywords, but can be ID diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index fbaef12fb98..5f7361160b3 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -417,7 +417,6 @@ specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CAST '(' expression AS convertedDataType ')' # dataTypeFunctionCall - | TOSTRING '(' functionArg ')' # dataTypeFunctionCall ; relevanceFunction From 9d71a95f8487a7c5fa0e4cea8d11b4a3893efcaf Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:50:53 -0700 Subject: [PATCH 03/41] doc changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 4 ++-- docs/user/ppl/functions/string.rst | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 21124c27edc..33af0075431 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -95,14 +95,14 @@ The first argument can be a number, number as string or boolean. If first argument is a a number or number as string , second argument need to be format name. If first argument is boolean, then second argument is not needed. -format types: +Format types: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -Binary conversion +Binary conversion: You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: eval result = tostring(9, "binary") diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 43efed3c470..c1dd52a5d89 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -397,6 +397,4 @@ Example:: |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | +---------------------+---------------------+ - - - + From be2c2e264a8cc337940f3f1ae2152a15672ce2cb Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:52:09 -0700 Subject: [PATCH 04/41] docs changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 12 ++++++------ docs/user/ppl/functions/string.rst | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 33af0075431..9d0bf6eab1b 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -85,15 +85,15 @@ TOSTRING Description >>>>>>>>>>> There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(number|string, string) converts the number in first argument to provided format type string in second argument. - Return type: string +Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. + Return type: string Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. - Return type: string + Return type: string You can use this function with the eval commands and as part of eval expressions. The first argument can be a number, number as string or boolean. -If first argument is a a number or number as string , second argument need to be format name. -If first argument is boolean, then second argument is not needed. +If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. +If first argument is boolean, then second argument is not used even if its provided. Format types: a) "binary" Converts a number to a binary value. @@ -108,7 +108,7 @@ eval result = tostring(9, "binary") For information about bitwise functions that you can use with the tostring function, see Bitwise functions. -Basic examples +Basic examples: The following example returns "True 0xF 12,345.68". ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index c1dd52a5d89..01b9b85b882 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -396,5 +396,4 @@ Example:: | UPPER('helloworld') | UPPER('HELLOWORLD') | |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | - +---------------------+---------------------+ - + +---------------------+---------------------+ \ No newline at end of file From fc763a431cbbdcad3fe35ab3f0e70f334d0b12a6 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:00:16 -0700 Subject: [PATCH 05/41] reverted string doc changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/string.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 01b9b85b882..24efa1434f5 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -396,4 +396,4 @@ Example:: | UPPER('helloworld') | UPPER('HELLOWORLD') | |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | - +---------------------+---------------------+ \ No newline at end of file + +---------------------+---------------------+ From a04eb14529029cf51d020e75ba5e7e6f921fb07c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:06:46 -0700 Subject: [PATCH 06/41] removed extra word Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 9d0bf6eab1b..58171d7cc72 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -91,7 +91,7 @@ Usage for boolean parameter without format type: tostring(boolean) converts the Return type: string You can use this function with the eval commands and as part of eval expressions. -The first argument can be a number, number as string or boolean. +The first argument can be a number, number as string or boolean. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. If first argument is boolean, then second argument is not used even if its provided. @@ -148,9 +148,6 @@ Example:: | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ - - duration - os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ From 590a8e63add13b75fa90047cd2eca1ab4d802936 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:24:25 -0700 Subject: [PATCH 07/41] added any type Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 38 +++++++++++--------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 58171d7cc72..c4ebe0505de 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -85,27 +85,22 @@ TOSTRING Description >>>>>>>>>>> There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. +Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. Return type: string Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. Return type: string - You can use this function with the eval commands and as part of eval expressions. -The first argument can be a number, number as string or boolean. -If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. + +If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. -Format types: +Format types:: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -Binary conversion: -You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: -eval result = tostring(9, "binary") - For information about bitwise functions that you can use with the tostring function, see Bitwise functions. Basic examples: @@ -114,17 +109,11 @@ The following example returns "True 0xF 12,345.68". The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. ... | eval foo=615 | eval foo2 = tostring(foo, "duration") -The following example formats the column totalSales to display values with a currency symbol and commas. You must use a period between the currency value and the tostring function. -Example:: - os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` - fetched rows / total rows = 1/1 - +---------------------+ - | boolean_str | - |---------------------+ - | True | - +---------------------+ + +You can use this function to convert a number to a string of its binary representation. +Example:: os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -132,6 +121,10 @@ Example:: |---------------+------------------+------------+ | SMITH | 1001110001000000 | 80000.00 | +---------------+------------------+------------+ + + +You can use this function to convert a number to a string of its hex representation. +Example:: os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -140,6 +133,8 @@ Example:: | SMITH | 13880 | 80000.00 | +---------------+---------------+---------------+ +The following example formats the column totalSales to display values with commas. +Example:: os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -147,7 +142,8 @@ Example:: |---------------+------------------+------------+ | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ - +The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. +Example:: os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ @@ -156,10 +152,8 @@ Example:: | SMITH | 01:48:20 | +---------------+-------------+ -Usage for boolean parameter without format type:: - +Example for boolean parameter. Example:: - os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` fetched rows / total rows = 1/1 +---------------------+ From 6938e2c08dfef04119f1e37802d0a3a9554a0d21 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 11:00:43 -0700 Subject: [PATCH 08/41] doc formatting fixes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 38 ++++++++++++++++---------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index c4ebe0505de..280befee0aa 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -80,40 +80,44 @@ Cast function can be chained:: +-------+ TOSTRING ------ +----------- Description >>>>>>>>>>> -There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. - Return type: string -Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. - Return type: string -You can use this function with the eval commands and as part of eval expressions. +The following usage options are available, depending on the parameter types and the number of parameters. -If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. -If first argument is boolean, then second argument is not used even if its provided. +Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. +Return type: string + +Usage for boolean parameter without format type tostring(boolean): Converts the string to 'True' or 'False'. +Return type: string + +You can use this function with the eval commands and as part of eval expressions. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. + +Format types: -Format types:: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. -The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -For information about bitwise functions that you can use with the tostring function, see Bitwise functions. +The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. Basic examples: + The following example returns "True 0xF 12,345.68". -... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") + + ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") + The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. -... | eval foo=615 | eval foo2 = tostring(foo, "duration") + ... | eval foo=615 | eval foo2 = tostring(foo, "duration") You can use this function to convert a number to a string of its binary representation. Example:: + os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -125,6 +129,7 @@ Example:: You can use this function to convert a number to a string of its hex representation. Example:: + os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -135,6 +140,7 @@ Example:: The following example formats the column totalSales to display values with commas. Example:: + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -142,8 +148,10 @@ Example:: |---------------+------------------+------------+ | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ + The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. Example:: + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ @@ -154,8 +162,10 @@ Example:: Example for boolean parameter. Example:: + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` fetched rows / total rows = 1/1 + +---------------------+ | boolean_str | |---------------------+ From 314fccdaee8ab28ede0eab2f1da4e2b83cc24e96 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 11:04:06 -0700 Subject: [PATCH 09/41] description for boolean example Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 280befee0aa..036a89daf00 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -160,7 +160,7 @@ Example:: | SMITH | 01:48:20 | +---------------+-------------+ -Example for boolean parameter. +The following example for converts boolean parameter to string. Example:: os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` From 0ee17b9a4a262a1d6070b98ae8a1162f6556f517 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:21:49 -0700 Subject: [PATCH 10/41] added format_time call from calcite , added duration_millis as splunk default duration is in seconds which will be used for duration format , added cast call for tostring with 1 argument Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 2 +- .../function/udf/ToStringFunction.java | 53 ++-- .../function/udf/ToStringFunctionTest.java | 265 +++++++++--------- .../sql/ppl/parser/AstExpressionBuilder.java | 52 +++- .../calcite/CalcitePPLStringFunctionTest.java | 157 +++++++---- 5 files changed, 312 insertions(+), 217 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index 69e1492538c..20811f1af48 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -117,7 +117,7 @@ private PPLOperandTypes() {} (CompositeOperandTypeChecker) OperandTypes.family(SqlTypeFamily.BOOLEAN) .or(OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) - .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); + .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = UDFOperandMetadata.wrap( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index 70d2a82b13c..be76100f38d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -8,7 +8,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.text.NumberFormat; -import java.time.Duration; import java.util.List; import java.util.Locale; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -18,6 +17,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -41,9 +41,13 @@ public ToStringFunction() { } public static final String DURATION_FORMAT = "duration"; + public static final String DURATION_MILLIS_FORMAT = "duration_millis"; public static final String HEX_FORMAT = "hex"; public static final String COMMAS_FORMAT = "commas"; public static final String BINARY_FORMAT = "binary"; + public static final SqlFunctions.DateFormatFunction dateTimeFormatter = + new SqlFunctions.DateFormatFunction(); + public static final String format24hour = "%H:%M:%S"; // 24-hour format @Override public SqlReturnTypeInference getReturnTypeInference() { @@ -65,7 +69,13 @@ public Expression implement( Expression format = translatedOperands.get(1); return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); } else { - return Expressions.call(ToStringFunction.class, "toString", fieldValue); + // autoboxes to Boolean + + if (!fieldValue.getType().getTypeName().equals("Boolean")) { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } else { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } } } } @@ -87,13 +97,13 @@ public static String toString(String fieldValue) { @Strict public static String toString(BigDecimal num, String format) { if (format.equals(DURATION_FORMAT)) { - Duration d = Duration.ofSeconds(num.toBigInteger().longValue()); - long hours = d.toHours(); - int minutes = d.toMinutesPart(); - int remainingSeconds = d.toSecondsPart(); - String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, num.toBigInteger().intValue() * 1000); + + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + + return dateTimeFormatter.formatTime(format24hour, num.toBigInteger().intValue()); + } else if (format.equals(HEX_FORMAT)) { return num.toBigInteger().toString(16); } else if (format.equals(COMMAS_FORMAT)) { @@ -112,12 +122,11 @@ public static String toString(BigDecimal num, String format) { @Strict public static String toString(double num, String format) { if (format.equals(DURATION_FORMAT)) { - Duration d = Duration.ofSeconds(Math.round(num)); - long hours = d.toHours(); - int minutes = d.toMinutesPart(); - int remainingSeconds = d.toSecondsPart(); - String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, ((int) Math.round(num)) * 1000); + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + + return dateTimeFormatter.formatTime(format24hour, ((int) Math.round(num))); + } else if (format.equals(HEX_FORMAT)) { return Double.toHexString(num); } else if (format.equals(COMMAS_FORMAT)) { @@ -129,17 +138,19 @@ public static String toString(double num, String format) { return Double.toString(num); } + @Strict + public static String toString(short num, String format) { + int i = (int) num; + return toString(i, format); + } + @Strict public static String toString(int num, String format) { if (format.equals(DURATION_FORMAT)) { - - int hours = num / 3600; - int minutes = (num % 3600) / 60; - int seconds = num % 60; - - String time_str = String.format("%02d:%02d:%02d", hours, minutes, seconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, num * 1000); + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + return dateTimeFormatter.formatTime(format24hour, num); } else if (format.equals(HEX_FORMAT)) { return Integer.toHexString(num); } else if (format.equals(COMMAS_FORMAT)) { diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java index cfd25796b23..f0867f93327 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java @@ -5,145 +5,144 @@ package org.opensearch.sql.expression.function.udf; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; import java.math.BigDecimal; import java.util.Locale; - -import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; public class ToStringFunctionTest { private final ToStringFunction function = new ToStringFunction(); - @Test - void testBooleanToString() { - assertEquals("True", ToStringFunction.toString(true)); - assertEquals("False", ToStringFunction.toString(false)); - } - - @Test - void testStringBooleanToString() { - assertEquals("True", ToStringFunction.toString("true")); - assertEquals("False", ToStringFunction.toString("false")); - assertEquals("False", ToStringFunction.toString("anythingElse")); - } - - @Test - void testBigDecimalToStringDurationFormat() { - BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testBigDecimalToStringHexFormat() { - BigDecimal num = new BigDecimal("255"); - String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); - assertEquals("ff", result); - } - - @Test - void testBigDecimalToStringCommasFormat() { - Locale.setDefault(Locale.US); // Ensure predictable comma placement - BigDecimal num = new BigDecimal("1234567.891"); - String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testBigDecimalToStringBinaryFormat() { - BigDecimal num = new BigDecimal("10"); - String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); - assertEquals("1010", result); - } - - @Test - void testBigDecimalToStringDefault() { - BigDecimal num = new BigDecimal("123.45"); - assertEquals("123.45", ToStringFunction.toString(num, "unknown")); - } - - @Test - void testDoubleToStringDurationFormat() { - double num = 3661.4; - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testDoubleToStringHexFormat() { - double num = 10.5; - String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); - assertTrue(result.startsWith("0x")); - } - - @Test - void testDoubleToStringCommasFormat() { - Locale.setDefault(Locale.US); - double num = 12345.678; - String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testDoubleToStringBinaryFormat() { - double num = 10.0; - String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); - assertNotNull(result); - assertFalse(result.isEmpty()); - } - - @Test - void testDoubleToStringDefault() { - assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); - } - - @Test - void testIntToStringDurationFormat() { - int num = 3661; - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testIntToStringHexFormat() { - assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); - } - - @Test - void testIntToStringCommasFormat() { - Locale.setDefault(Locale.US); - String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testIntToStringBinaryFormat() { - assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); - } - - @Test - void testIntToStringDefault() { - assertEquals("123", ToStringFunction.toString(123, "unknown")); - } - - @Test - void testStringNumericToStringIntFormat() { - String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); - assertEquals("2a", result); - } - - @Test - void testStringNumericToStringDoubleFormat() { - String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains("42")); - } - - @Test - void testStringLargeNumberAsDouble() { - String largeNum = "1234567890123"; - String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); - assertNotNull(result); - } + @Test + void testBooleanToString() { + assertEquals("True", ToStringFunction.toString(true)); + assertEquals("False", ToStringFunction.toString(false)); + } + + @Test + void testStringBooleanToString() { + assertEquals("True", ToStringFunction.toString("true")); + assertEquals("False", ToStringFunction.toString("false")); + assertEquals("False", ToStringFunction.toString("anythingElse")); + } + + @Test + void testBigDecimalToStringDurationFormat() { + BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testBigDecimalToStringHexFormat() { + BigDecimal num = new BigDecimal("255"); + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertEquals("ff", result); + } + + @Test + void testBigDecimalToStringCommasFormat() { + Locale.setDefault(Locale.US); // Ensure predictable comma placement + BigDecimal num = new BigDecimal("1234567.891"); + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testBigDecimalToStringBinaryFormat() { + BigDecimal num = new BigDecimal("10"); + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertEquals("1010", result); + } + + @Test + void testBigDecimalToStringDefault() { + BigDecimal num = new BigDecimal("123.45"); + assertEquals("123.45", ToStringFunction.toString(num, "unknown")); + } + + @Test + void testDoubleToStringDurationFormat() { + double num = 3661.4; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testDoubleToStringHexFormat() { + double num = 10.5; + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertTrue(result.startsWith("0x")); + } + + @Test + void testDoubleToStringCommasFormat() { + Locale.setDefault(Locale.US); + double num = 12345.678; + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testDoubleToStringBinaryFormat() { + double num = 10.0; + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + assertFalse(result.isEmpty()); + } + + @Test + void testDoubleToStringDefault() { + assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); + } + + @Test + void testIntToStringDurationFormat() { + int num = 3661; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testIntToStringHexFormat() { + assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); + } + + @Test + void testIntToStringCommasFormat() { + Locale.setDefault(Locale.US); + String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testIntToStringBinaryFormat() { + assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); + } + + @Test + void testIntToStringDefault() { + assertEquals("123", ToStringFunction.toString(123, "unknown")); + } + + @Test + void testStringNumericToStringIntFormat() { + String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); + assertEquals("2a", result); + } + + @Test + void testStringNumericToStringDoubleFormat() { + String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains("42")); + } + + @Test + void testStringLargeNumberAsDouble() { + String largeNum = "1234567890123"; + String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 1460a3efc9a..2112282f728 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; @@ -29,8 +30,10 @@ import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinaryArithmeticContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; @@ -411,18 +414,51 @@ private Function buildFunction( functionName, args.stream().map(this::visitFunctionArg).collect(Collectors.toList())); } + public DataTypeFunctionCallContext createDataTypeFunctionCallContext(String castExpression) { + // Create a case-insensitive character stream from the input + CaseInsensitiveCharStream charStream = new CaseInsensitiveCharStream(castExpression); + + // Create lexer and parser + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(charStream); + CommonTokenStream tokens = new CommonTokenStream(lexer); + OpenSearchPPLParser parser = new OpenSearchPPLParser(tokens); + + // Parse the expression - cast is part of evalFunctionCall + DataTypeFunctionCallContext evalContext = parser.dataTypeFunctionCall(); + return evalContext; + } + /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - ParseTree rootNode = ctx.getChild(0); - String functionName = rootNode.getText(); - final String mappedName = - FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); - - if (mappedName.equals("cast")) { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + if (ctx.functionArgs() != null) { + + ParseTree rootNode = ctx.getChild(0); + String functionName = rootNode.getText(); + final String mappedName = + FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); + System.out.println(mappedName); + if (mappedName != null && mappedName.equals("tostring")) { + if (ctx.functionArgs().functionArg().size() == 1) { + List functionArgs = + ctx.functionArgs().functionArg(); + + String castExpresstion = + String.format("cast( %s as String)", functionArgs.getFirst().getText()); + DataTypeFunctionCallContext toStringDataTypeConversionContext = + this.createDataTypeFunctionCallContext(castExpresstion); + return new Cast( + visit(toStringDataTypeConversionContext.logicalExpression()), + visit(toStringDataTypeConversionContext.convertedDataType())); + // + } else { + return buildFunction(mappedName, ctx.functionArgs().functionArg()); + } + } else { + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + } } else { - return buildFunction(mappedName, ctx.functionArgs().functionArg()); + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index d41b2c22453..b32ecd04e3f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -46,20 +46,55 @@ public void testLower() { verifyPPLToSparkSQL(root, expectedSparkSql); } + // This test evalutes tostring where it gets converted to cast call + + @Test + public void testToStringFormatNotSpecified() { + String ppl = + "source=EMP | eval string_value = tostring(MGR) | eval cast_value = cast(MGR as string)|" + + " fields string_value, cast_value"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(string_value=[SAFE_CAST($3)], cast_value=[SAFE_CAST($3)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "string_value=7902; cast_value=7902\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7566; cast_value=7566\n" + + "string_value=null; cast_value=null\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7788; cast_value=7788\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7566; cast_value=7566\n" + + "string_value=7782; cast_value=7782\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT SAFE_CAST(`MGR` AS STRING) `string_value`, SAFE_CAST(`MGR` AS STRING)" + + " `cast_value`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testToStringBoolean() { String ppl = "source=EMP | eval boolean_value = tostring(1==1) | fields boolean_value |head 1"; RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[1])\n" - + " LogicalProject(boolean_value=[TOSTRING(true)])\n" + + " LogicalProject(boolean_value=['TRUE':VARCHAR])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = "boolean_value=True\n"; + String expectedResult = "boolean_value=TRUE\n"; verifyLogical(root, expectedLogical); verifyResult(root, expectedResult); - String expectedSparkSql = - "SELECT `TOSTRING`(TRUE) `boolean_value`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; + String expectedSparkSql = "SELECT 'TRUE' `boolean_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -126,56 +161,70 @@ public void testToStringHex() { verifyPPLToSparkSQL(root, expectedSparkSql); } - @Test - public void testToStringHexFromNumberAsString() { - String ppl = - "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_hex=640\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - - @Test - public void testToStringCommaFromNumberAsString() { - String ppl = - "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME, salary_comma| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR, 'commas':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_comma=160,040,222\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - @Test - public void testToStringBinaryFromNumberAsString() { - String ppl = - "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME, salary_binary| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR, 'binary':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - @Test + @Test + public void testToStringHexFromNumberAsString() { + String ppl = + "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex|" + + " head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_hex=640\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringCommaFromNumberAsString() { + String ppl = + "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME," + + " salary_comma| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR," + + " 'commas':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_comma=160,040,222\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringBinaryFromNumberAsString() { + String ppl = + "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME," + + " salary_binary| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR," + + " 'binary':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test public void testToStringCommas() { String ppl = "source=EMP | eval salary_commas = tostring(SAL, \"commas\") | fields ENAME," From 6e24aa3c4b16f6a5734db18f210b8b5c1f5f245f Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:32:18 -0700 Subject: [PATCH 11/41] added doc update to specifically set 2nd argument as optional Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 036a89daf00..9f9646efef4 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -86,10 +86,10 @@ Description >>>>>>>>>>> The following usage options are available, depending on the parameter types and the number of parameters. -Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. +Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. Return type: string -Usage for boolean parameter without format type tostring(boolean): Converts the string to 'True' or 'False'. +Usage for boolean parameter without format type tostring(boolean): Converts the string to 'TRUE' or 'FALSE'. Return type: string You can use this function with the eval commands and as part of eval expressions. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. @@ -100,12 +100,13 @@ a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. +5) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. Basic examples: -The following example returns "True 0xF 12,345.68". +The following example returns "TRUE 0xF 12,345.68". ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") @@ -169,5 +170,5 @@ Example:: +---------------------+ | boolean_str | |---------------------+ - | True | + | TRUE | +---------------------+ \ No newline at end of file From 454cfc8db324866376f2452290b0fd0739008754 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:33:59 -0700 Subject: [PATCH 12/41] mentioned as value instead of number specifically Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 9f9646efef4..4f2d432c71a 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -86,7 +86,7 @@ Description >>>>>>>>>>> The following usage options are available, depending on the parameter types and the number of parameters. -Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. +Usage with format type: tostring(ANY, [format]): Converts the value in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. Return type: string Usage for boolean parameter without format type tostring(boolean): Converts the string to 'TRUE' or 'FALSE'. From b221e8c4f583b947198253ee90b5edb6f15b382a Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:46:56 -0700 Subject: [PATCH 13/41] fixed wrong bullet point Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 4f2d432c71a..708bde1fa11 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -100,7 +100,7 @@ a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. -5) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. +e) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. From ea9ba0fa8b5b63053df344930dc218e83dc75f91 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 20 Oct 2025 12:41:39 -0700 Subject: [PATCH 14/41] added more unit tests Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunctionTest.java | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java new file mode 100644 index 00000000000..d7f2b6fd685 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -0,0 +1,192 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import static org.junit.jupiter.api.Assertions.*; + +import java.math.BigDecimal; +import java.util.Locale; +import org.apache.calcite.sql.type.ReturnTypes; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; + +public class ToNumberFunctionTest { + + private final ToNumberFunction function = new ToNumberFunction(); + + @Test + void testGetReturnTypeInference() { + assertEquals(ReturnTypes.DOUBLE_FORCE_NULLABLE, function.getReturnTypeInference()); + } + + @Test + void testGetOperandMetadata() { + assertEquals(PPLOperandTypes.STRING_OR_STRING_INTEGER, function.getOperandMetadata()); + } + + @Test + void testToNumberWithDefaultBase() { + assertEquals(123, ToNumberFunction.toNumber("123")); + assertEquals(0, ToNumberFunction.toNumber("0")); + assertEquals(-456, ToNumberFunction.toNumber("-456")); + assertEquals(123.45, ToNumberFunction.toNumber("123.45")); + assertEquals(-123.45, ToNumberFunction.toNumber("-123.45")); + assertEquals(0.5, ToNumberFunction.toNumber("0.5")); + assertEquals(-0.5, ToNumberFunction.toNumber("-0.5")); + } + + @Test + void testToNumberWithBase10() { + assertEquals(123, ToNumberFunction.toNumber("123", 10)); + assertEquals(0, ToNumberFunction.toNumber("0", 10)); + assertEquals(-456, ToNumberFunction.toNumber("-456", 10)); + assertEquals(123.45, ToNumberFunction.toNumber("123.45", 10)); + assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); + } + + @Test + void testToNumberWithBase2() { + assertEquals(5, ToNumberFunction.toNumber("101", 2)); + assertEquals(0, ToNumberFunction.toNumber("0", 2)); + assertEquals(1, ToNumberFunction.toNumber("1", 2)); + assertEquals(7, ToNumberFunction.toNumber("111", 2)); + assertEquals(10, ToNumberFunction.toNumber("1010", 2)); + } + + @Test + void testToNumberWithBase8() { + assertEquals(64, ToNumberFunction.toNumber("100", 8)); + assertEquals(8, ToNumberFunction.toNumber("10", 8)); + assertEquals(83, ToNumberFunction.toNumber("123", 8)); + assertEquals(511, ToNumberFunction.toNumber("777", 8)); + } + + @Test + void testToNumberWithBase16() { + assertEquals(255, ToNumberFunction.toNumber("FF", 16)); + assertEquals(16, ToNumberFunction.toNumber("10", 16)); + assertEquals(171, ToNumberFunction.toNumber("AB", 16)); + assertEquals(291, ToNumberFunction.toNumber("123", 16)); + assertEquals(4095, ToNumberFunction.toNumber("FFF", 16)); + } + + @Test + void testToNumberWithBase36() { + assertEquals(35, ToNumberFunction.toNumber("Z", 36)); + assertEquals(1295, ToNumberFunction.toNumber("ZZ", 36)); + assertEquals(46655, ToNumberFunction.toNumber("ZZZ", 36)); + } + + @Test + void testToNumberWithDecimalBase2() { + assertEquals(2.5, ToNumberFunction.toNumber("10.1", 2)); + assertEquals(1.5, ToNumberFunction.toNumber("1.1", 2)); + assertEquals(3.75, ToNumberFunction.toNumber("11.11", 2)); + } + + @Test + void testToNumberWithDecimalBase16() { + assertEquals(255.5, ToNumberFunction.toNumber("FF.8", 16)); + assertEquals(16.25, ToNumberFunction.toNumber("10.4", 16)); + assertEquals(171.6875, ToNumberFunction.toNumber("AB.B", 16)); + } + + @Test + void testToNumberWithNegativeDecimal() { + assertEquals(-2.5, ToNumberFunction.toNumber("-10.1", 2)); + assertEquals(-255.5, ToNumberFunction.toNumber("-FF.8", 16)); + assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); + } + + @Test + void testToNumberWithEmptyFractionalPart() { + assertEquals(123.0, ToNumberFunction.toNumber("123.", 10)); + assertEquals(255.0, ToNumberFunction.toNumber("FF.", 16)); + assertEquals(5.0, ToNumberFunction.toNumber("101.", 2)); + } + + @Test + void testToNumberWithZeroIntegerPart() { + assertEquals(0.5, ToNumberFunction.toNumber("0.5", 10)); + assertEquals(0.5, ToNumberFunction.toNumber("0.1", 2)); + } + + @Test + void testToNumberInvalidBase() { + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123", 1); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123", 37); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123", 0); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123", -1); + }); + } + + @Test + void testToNumberInvalidDigits() { + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("12A", 10); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("102", 2); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("189", 8); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("GHI", 16); + }); + } + + @Test + void testToNumberInvalidFractionalDigits() { + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("10.2", 2); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("FF.G", 16); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123.ABC", 10); + }); + } + + @Test + void testToNumberEdgeCases() { + assertEquals(0, ToNumberFunction.toNumber("0", 2)); + assertEquals(0, ToNumberFunction.toNumber("0", 36)); + assertEquals(0.0, ToNumberFunction.toNumber("0.0", 10)); + assertEquals(0.0, ToNumberFunction.toNumber("0.000", 10)); + } + + @Test + void testToNumberLargeNumbers() { + assertEquals(Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); + assertEquals(Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); + } + + @Test + void testToNumberCaseInsensitivity() { + assertEquals(255, ToNumberFunction.toNumber("ff", 16)); + assertEquals(255, ToNumberFunction.toNumber("FF", 16)); + assertEquals(255, ToNumberFunction.toNumber("fF", 16)); + assertEquals(171, ToNumberFunction.toNumber("ab", 16)); + assertEquals(171, ToNumberFunction.toNumber("AB", 16)); + } +} From 370b9bc5d30ec90fbd7dc668ca5ee0939a720f87 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:03:42 -0700 Subject: [PATCH 15/41] Update docs/user/ppl/functions/conversion.rst Co-authored-by: ritvibhatt <53196324+ritvibhatt@users.noreply.github.com> Signed-off-by: Asif Bashar Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index fad37c0bf26..d9da1bf3d8e 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -86,7 +86,7 @@ Description >>>>>>>>>>> The following usage options are available, depending on the parameter types and the number of parameters. -Usage with format type: tonumber(string, [base]): Converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. +Usage: tonumber(string, [base]) converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. Return type: Number From f0706849c4446e47615a2228debbea69c069d0b0 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:20:00 -0700 Subject: [PATCH 16/41] fix per recommendation Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 34 ++++++++----------- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 4 +-- .../sql/ppl/parser/AstExpressionBuilder.java | 32 ----------------- 3 files changed, 16 insertions(+), 54 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index d9da1bf3d8e..3264d422ce1 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -84,39 +84,34 @@ TONUMBER Description >>>>>>>>>>> + The following usage options are available, depending on the parameter types and the number of parameters. Usage: tonumber(string, [base]) converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. + Return type: Number You can use this function with the eval commands and as part of eval expressions. Base values can be between 2 and 36. - - -Basic examples: You can use this function to convert a string representation of a binary number to return the corresponding number in base 10. -For example, the result of the following function is 5: -eval result = tonumber("0101", 2) - +Following example converts a string in binary to the number representation:: -Example:: -Following example converts a string in binary to the number representation. - os> source=EMP | eval int_value = tonumber('010101',2) | fields int_value|head 1 + os> source=EMP | eval int_value = tonumber('010101',2) | fields int_value | head 1 fetched rows / total rows = 1/1 - +--------------+ - | int_value | - |--------------+ - | 21.0 | - +--------------+ + +---------------+ + | int_value | + |---------------+ + | 21.0 | + +---------------+ + +Following example converts a string in hex to the number representation:: -Following example converts a string in hex to the number representation. -Example:: - os> source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1 + os> source=EMP | eval int_value = tonumber('FA34',16) | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | @@ -124,10 +119,9 @@ Example:: | 64052.0 | +---------------+ -Following example converts a string in decimal to the number representation. -Example:: +Following example converts a string in decimal to the number representation:: - os> source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1 + os> source=EMP | eval int_value = tonumber('4598') | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index e974c303375..a26b7381797 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -848,10 +848,9 @@ evalFunctionCall ; -// cast, tostring function +// cast function dataTypeFunctionCall : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS - | TONUMBER LT_PRTHS functionArgs RT_PRTHS ; @@ -1234,6 +1233,7 @@ textFunctionName | LOCATE | REPLACE | REVERSE + | TONUMBER ; positionFunctionName diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 244f287c77a..b70226d039a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -431,39 +431,7 @@ public DataTypeFunctionCallContext createDataTypeFunctionCallContext(String cast /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - if (ctx.functionArgs() != null) { - - ParseTree rootNode = ctx.getChild(0); - String functionName = rootNode.getText(); - final String mappedName = - FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); - System.out.println(mappedName); - if (mappedName != null && mappedName.equals("tostring")) { - if (ctx.functionArgs().functionArg().size() == 1) { - List functionArgs = - ctx.functionArgs().functionArg(); - - String castExpresstion = - String.format("cast( %s as String)", functionArgs.getFirst().getText()); - DataTypeFunctionCallContext toStringDataTypeConversionContext = - this.createDataTypeFunctionCallContext(castExpresstion); - return new Cast( - visit(toStringDataTypeConversionContext.logicalExpression()), - visit(toStringDataTypeConversionContext.convertedDataType())); - // - } else { - return buildFunction(mappedName, ctx.functionArgs().functionArg()); - } - } else if (mappedName != null && mappedName.equals("tonumber")) { - - return buildFunction(mappedName, ctx.functionArgs().functionArg()); - - } else { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); - } - } else { return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); - } } @Override From a2accf5bc8caa7ad804fba592cc15954dc8fbd28 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:33:51 -0700 Subject: [PATCH 17/41] updated recommended changes Signed-off-by: Asif Bashar --- .../function/PPLBuiltinOperators.java | 9 +- .../expression/function/PPLFuncImpTable.java | 226 +++++++++++++++++- 2 files changed, 233 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 14f82947c74..7fd768a8d8b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -60,7 +60,14 @@ import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; -import org.opensearch.sql.expression.function.udf.*; +import org.opensearch.sql.expression.function.udf.CryptographicFunction; +import org.opensearch.sql.expression.function.udf.ParseFunction; +import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction; +import org.opensearch.sql.expression.function.udf.RexExtractFunction; +import org.opensearch.sql.expression.function.udf.RexExtractMultiFunction; +import org.opensearch.sql.expression.function.udf.RexOffsetFunction; +import org.opensearch.sql.expression.function.udf.SpanFunction; +import org.opensearch.sql.expression.function.udf.ToNumberFunction; import org.opensearch.sql.expression.function.udf.condition.EarliestFunction; import org.opensearch.sql.expression.function.udf.condition.EnhancedCoalesceFunction; import org.opensearch.sql.expression.function.udf.condition.LatestFunction; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 6f61eb14b48..d9e594f089f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -8,7 +8,231 @@ import static org.apache.calcite.sql.SqlJsonConstructorNullClause.NULL_ON_NULL; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.getLegacyTypeName; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.*; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ABS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ACOS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.AND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY_LENGTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASCII; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.AVG; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CBRT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEIL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEILING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CIDRMATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COALESCE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONCAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONCAT_WS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONV; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONVERT_TZ; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COSH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COUNT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CRC32; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATEDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATETIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_SUB; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYNAME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFMONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFWEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFYEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_MONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_WEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DEGREES; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDEFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.E; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EARLIEST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EQUAL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXISTS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXPM1; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FILTER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FIRST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FLOOR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FORALL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_DAYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_UNIXTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GET_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GREATER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR_OF_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IFNULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_GROK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PARSE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PATTERN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PATTERN_PARSER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_3; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_5; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_PG_4; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_TRANSLATE3; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_BLANK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_EMPTY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_PRESENT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_APPEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY_LENGTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_DELETE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTRACT_ALL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_KEYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_OBJECT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_SET; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_VALID; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LAST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LAST_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LATEST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LEFT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LENGTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LESS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LIKE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LIST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCALTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCALTIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG10; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOWER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKEDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKETIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_APPEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_CONCAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_REMOVE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_BOOL_PREFIX; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE_PREFIX; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAX; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MD5; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MEDIAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MICROSECOND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_HOUR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MOD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUSFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTHNAME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVAPPEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVJOIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOW; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NULLIF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.OR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERCENTILE_APPROX; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERIOD_ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERIOD_DIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PI; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POSITION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POW; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POWER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUARTER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUERY_STRING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RADIANS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RAND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REDUCE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEXP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEX_MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REPLACE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REVERSE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REX_EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REX_EXTRACT_MULTI; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REX_OFFSET; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RIGHT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RINT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ROUND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RTRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND_OF_MINUTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SEC_TO_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA1; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGNUM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIMPLE_QUERY_STRING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SINH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SPAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SQRT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STDDEV_POP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STDDEV_SAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STRCMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STRFTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STR_TO_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBSTR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBSTRING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACTFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SYSDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TAKE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMEDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_TO_SEC; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_DAYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_SECONDS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRANSFORM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRUNCATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TYPEOF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UNIX_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UPPER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.VALUES; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.VARPOP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.VARSAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEKDAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEKOFYEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.XOR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEARWEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TONUMBER; import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; From 3adda5d37467116c9e8b1591832755281361ce9e Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:36:28 -0700 Subject: [PATCH 18/41] updated recommended changes Signed-off-by: Asif Bashar --- .../sql/ppl/parser/AstExpressionBuilder.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index b70226d039a..f7f6b3e914a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -414,24 +414,10 @@ private Function buildFunction( functionName, args.stream().map(this::visitFunctionArg).collect(Collectors.toList())); } - public DataTypeFunctionCallContext createDataTypeFunctionCallContext(String castExpression) { - // Create a case-insensitive character stream from the input - CaseInsensitiveCharStream charStream = new CaseInsensitiveCharStream(castExpression); - - // Create lexer and parser - OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(charStream); - CommonTokenStream tokens = new CommonTokenStream(lexer); - OpenSearchPPLParser parser = new OpenSearchPPLParser(tokens); - - // Parse the expression - cast is part of evalFunctionCall - DataTypeFunctionCallContext evalContext = parser.dataTypeFunctionCall(); - return evalContext; - } - /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); } @Override From 17b1d86c3098b1fa11171ffe05fdcaa47fbf53d8 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:38:11 -0700 Subject: [PATCH 19/41] updated recommended changes Signed-off-by: Asif Bashar --- .../org/opensearch/sql/ppl/parser/AstExpressionBuilder.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index f7f6b3e914a..f037376f5c2 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -19,7 +19,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; @@ -30,10 +29,8 @@ import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.calcite.plan.OpenSearchConstants; -import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinaryArithmeticContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; From eca7d139fce7d750e2be988facd8764dd6d1674c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:38:59 -0700 Subject: [PATCH 20/41] updated recommended changes Signed-off-by: Asif Bashar --- .../opensearch/sql/expression/function/BuiltinFunctionName.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index fe862adc322..df3ae06d5c8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -149,6 +149,7 @@ public enum BuiltinFunctionName { SYSDATE(FunctionName.of("sysdate")), /** Text Functions. */ + TOSTRING(FunctionName.of("tostring")), TONUMBER(FunctionName.of("tonumber")), /** IP Functions. */ From e780c156df19215bc03e4ba45988b32393f6ddcc Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:40:50 -0700 Subject: [PATCH 21/41] updated recommended changes Signed-off-by: Asif Bashar --- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 -- 1 file changed, 2 deletions(-) diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index a26b7381797..d386813fc96 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -847,13 +847,11 @@ evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS ; - // cast function dataTypeFunctionCall : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS ; - convertedDataType : typeName = DATE | typeName = TIME From e3884b0a5ac00d147268fe964dcf554739685c88 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:42:31 -0700 Subject: [PATCH 22/41] updated recommended changes Signed-off-by: Asif Bashar --- .../org/opensearch/sql/expression/function/PPLFuncImpTable.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index d9e594f089f..a5d0d1b8712 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -1062,7 +1062,6 @@ void populate() { SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)), false)); - register( LOG, (FunctionImp2) From 042d3e821a03a2c5a5438f449bfbd983d8e53057 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 22 Oct 2025 10:11:40 -0700 Subject: [PATCH 23/41] spotless apply Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 10 +- .../expression/function/PPLFuncImpTable.java | 2 +- .../function/udf/ToNumberFunction.java | 179 ++++++++---------- .../function/udf/ToNumberFunctionTest.java | 112 ++++++----- 4 files changed, 153 insertions(+), 150 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index b00fa607d55..c64fa6ef4d5 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -106,12 +106,12 @@ private PPLOperandTypes() {} SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER)); public static final UDFOperandMetadata STRING_OR_STRING_INTEGER = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.CHARACTER).or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER))); - + UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + OperandTypes.family(SqlTypeFamily.CHARACTER) + .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER))); - public static final UDFOperandMetadata STRING_STRING_INTEGER_INTEGER = + public static final UDFOperandMetadata STRING_STRING_INTEGER_INTEGER = UDFOperandMetadata.wrap( OperandTypes.family( SqlTypeFamily.CHARACTER, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index a5d0d1b8712..4339619275c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -211,6 +211,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_FORMAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_TO_SEC; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TONUMBER; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_DAYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_SECONDS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRANSFORM; @@ -232,7 +233,6 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.XOR; import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEAR; import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEARWEEK; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.TONUMBER; import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 95b37a0107c..0c5b576b297 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -5,12 +5,7 @@ package org.opensearch.sql.expression.function.udf; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.text.NumberFormat; import java.util.List; -import java.util.Locale; - import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -22,7 +17,6 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; -import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -36,105 +30,92 @@ * HH:MM:SS. if not format parameter provided, then consider value as boolean */ public class ToNumberFunction extends ImplementorUDF { - public ToNumberFunction() { - super( - new org.opensearch.sql.expression.function.udf.ToNumberFunction.ToNumberImplementor(), - NullPolicy.ANY); - } - - public static final String DURATION_FORMAT = "duration"; - public static final String DURATION_MILLIS_FORMAT = "duration_millis"; - public static final String HEX_FORMAT = "hex"; - public static final String COMMAS_FORMAT = "commas"; - public static final String BINARY_FORMAT = "binary"; - public static final SqlFunctions.DateFormatFunction dateTimeFormatter = - new SqlFunctions.DateFormatFunction(); - public static final String format24hour = "%H:%M:%S"; // 24-hour format + public ToNumberFunction() { + super( + new org.opensearch.sql.expression.function.udf.ToNumberFunction.ToNumberImplementor(), + NullPolicy.ANY); + } + + public static final String DURATION_FORMAT = "duration"; + public static final String DURATION_MILLIS_FORMAT = "duration_millis"; + public static final String HEX_FORMAT = "hex"; + public static final String COMMAS_FORMAT = "commas"; + public static final String BINARY_FORMAT = "binary"; + public static final SqlFunctions.DateFormatFunction dateTimeFormatter = + new SqlFunctions.DateFormatFunction(); + public static final String format24hour = "%H:%M:%S"; // 24-hour format + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.DOUBLE_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.STRING_OR_STRING_INTEGER; + } + + public static class ToNumberImplementor implements NotNullImplementor { @Override - public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.DOUBLE_FORCE_NULLABLE; + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + Expression fieldValue = translatedOperands.get(0); + int base = 10; + if (translatedOperands.size() > 1) { + Expression baseExpr = translatedOperands.get(1); + return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue, baseExpr); + } else { + return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue); + } } + } - @Override - public UDFOperandMetadata getOperandMetadata() { - return PPLOperandTypes.STRING_OR_STRING_INTEGER; - } - - public static class ToNumberImplementor implements NotNullImplementor { - - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - Expression fieldValue = translatedOperands.get(0); - int base = 10; - if (translatedOperands.size() > 1) { - Expression baseExpr = translatedOperands.get(1); - return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue, baseExpr); - } else { - return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue); - } - - + @Strict + public static Number toNumber(String numStr) { + return toNumber(numStr, 10); + } - - - } + @Strict + public static Number toNumber(String numStr, int base) { + if (base < 2 || base > 36) { + throw new IllegalArgumentException("Base must be between 2 and 36"); } - @Strict - public static Number toNumber(String numStr) { - return toNumber(numStr, 10); + if (numStr.contains(".")) { + + boolean isNegative = numStr.startsWith("-"); + if (isNegative) { + numStr = numStr.substring(1); + } + + // Split integer and fractional parts + String[] parts = numStr.split("\\."); + String intPart = parts[0]; + String fracPart = parts.length > 1 ? parts[1] : ""; + + // Convert integer part + double intValue = 0; + for (char c : intPart.toCharArray()) { + int digit = Character.digit(c, base); + if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); + intValue = intValue * base + digit; + } + + // Convert fractional part + double fracValue = 0; + double divisor = base; + for (char c : fracPart.toCharArray()) { + int digit = Character.digit(c, base); + if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); + fracValue += (double) digit / divisor; + divisor *= base; + } + + double result = intValue + fracValue; + return isNegative ? -result : result; + } else { + return Integer.parseInt(numStr, base); } - - @Strict - public static Number toNumber(String numStr, int base) { - if (base < 2 || base > 36) { - throw new IllegalArgumentException("Base must be between 2 and 36"); - } - - if (numStr.contains(".")) { - - - boolean isNegative = numStr.startsWith("-"); - if (isNegative) { - numStr = numStr.substring(1); - } - - // Split integer and fractional parts - String[] parts = numStr.split("\\."); - String intPart = parts[0]; - String fracPart = parts.length > 1 ? parts[1] : ""; - - // Convert integer part - double intValue = 0; - for (char c : intPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) - throw new IllegalArgumentException("Invalid digit: " + c); - intValue = intValue * base + digit; - } - - // Convert fractional part - double fracValue = 0; - double divisor = base; - for (char c : fracPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) - throw new IllegalArgumentException("Invalid digit: " + c); - fracValue += (double) digit / divisor; - divisor *= base; - } - - double result = intValue + fracValue; - return isNegative ? -result : result; - } else { - return Integer.parseInt(numStr, base); - } - } - + } } - - - - diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index d7f2b6fd685..7c29f75ba0a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -7,8 +7,6 @@ import static org.junit.jupiter.api.Assertions.*; -import java.math.BigDecimal; -import java.util.Locale; import org.apache.calcite.sql.type.ReturnTypes; import org.junit.jupiter.api.Test; import org.opensearch.sql.calcite.utils.PPLOperandTypes; @@ -116,55 +114,77 @@ void testToNumberWithZeroIntegerPart() { @Test void testToNumberInvalidBase() { - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123", 1); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123", 37); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123", 0); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123", -1); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123", 1); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123", 37); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123", 0); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123", -1); + }); } @Test void testToNumberInvalidDigits() { - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("12A", 10); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("102", 2); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("189", 8); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("GHI", 16); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("12A", 10); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("102", 2); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("189", 8); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("GHI", 16); + }); } @Test void testToNumberInvalidFractionalDigits() { - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("10.2", 2); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("FF.G", 16); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123.ABC", 10); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("10.2", 2); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("FF.G", 16); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123.ABC", 10); + }); } @Test @@ -177,8 +197,10 @@ void testToNumberEdgeCases() { @Test void testToNumberLargeNumbers() { - assertEquals(Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); - assertEquals(Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); + assertEquals( + Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); + assertEquals( + Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); } @Test From 58d8e1b5714d9f02f80b910da847741cc309f713 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 28 Oct 2025 10:53:26 -0700 Subject: [PATCH 24/41] merge conflict fix Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 849d2334e41..ffe98228c39 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -117,3 +117,54 @@ Use string in comparison operator example :: | True | False | True | False | True | True | null | +------+-------+------+-------+------+------+------+ + + +TONUMBER +----------- + +Description +>>>>>>>>>>> + +The following usage options are available, depending on the parameter types and the number of parameters. + +Usage: tonumber(string, [base]) converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. + +Return type: Number + + +You can use this function with the eval commands and as part of eval expressions. +Base values can be between 2 and 36. + +You can use this function to convert a string representation of a binary number to return the corresponding number in base 10. + +Following example converts a string in binary to the number representation:: + + os> source=EMP | eval int_value = tonumber('010101',2) | fields int_value | head 1 + fetched rows / total rows = 1/1 + +---------------+ + | int_value | + |---------------+ + | 21.0 | + +---------------+ + + +Following example converts a string in hex to the number representation:: + + + os> source=EMP | eval int_value = tonumber('FA34',16) | fields int_value | head 1 + fetched rows / total rows = 1/1 + +---------------+ + | int_value | + |---------------+ + | 64052.0 | + +---------------+ + +Following example converts a string in decimal to the number representation:: + + os> source=EMP | eval int_value = tonumber('4598') | fields int_value | head 1 + fetched rows / total rows = 1/1 + +---------------+ + | int_value | + |---------------+ + | 4598.0 | + +---------------+ From 34f86355d4da09a88b761ac3e1190f7a609b1156 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sat, 1 Nov 2025 22:48:31 -0700 Subject: [PATCH 25/41] added recommended changes Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 24 ++-- .../function/udf/ToNumberFunctionTest.java | 105 +++++++----------- docs/user/ppl/functions/conversion.rst | 6 +- .../CalcitePPLToNumberFunctionTest.java | 71 ++++++++++++ 4 files changed, 133 insertions(+), 73 deletions(-) create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 0c5b576b297..38f1734a8aa 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -104,18 +104,26 @@ public static Number toNumber(String numStr, int base) { // Convert fractional part double fracValue = 0; - double divisor = base; - for (char c : fracPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); - fracValue += (double) digit / divisor; - divisor *= base; + if (base == 10) { + double divisor = base; + for (char c : fracPart.toCharArray()) { + int digit = Character.digit(c, base); + if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); + fracValue += (double) digit / divisor; + divisor *= base; + } } double result = intValue + fracValue; - return isNegative ? -result : result; + result = isNegative ? -result : result; + if (base == 10) { + return result; + } + else { + return (long) result; + } } else { - return Integer.parseInt(numStr, base); + return Long.parseLong(numStr, base); } } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index 7c29f75ba0a..3a8c3185240 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -27,9 +27,9 @@ void testGetOperandMetadata() { @Test void testToNumberWithDefaultBase() { - assertEquals(123, ToNumberFunction.toNumber("123")); - assertEquals(0, ToNumberFunction.toNumber("0")); - assertEquals(-456, ToNumberFunction.toNumber("-456")); + assertEquals(123L, ToNumberFunction.toNumber("123")); + assertEquals(0L, ToNumberFunction.toNumber("0")); + assertEquals(-456L, ToNumberFunction.toNumber("-456")); assertEquals(123.45, ToNumberFunction.toNumber("123.45")); assertEquals(-123.45, ToNumberFunction.toNumber("-123.45")); assertEquals(0.5, ToNumberFunction.toNumber("0.5")); @@ -38,78 +38,78 @@ void testToNumberWithDefaultBase() { @Test void testToNumberWithBase10() { - assertEquals(123, ToNumberFunction.toNumber("123", 10)); - assertEquals(0, ToNumberFunction.toNumber("0", 10)); - assertEquals(-456, ToNumberFunction.toNumber("-456", 10)); + assertEquals(123L, ToNumberFunction.toNumber("123", 10)); + assertEquals(0L, ToNumberFunction.toNumber("0", 10)); + assertEquals(-456L, ToNumberFunction.toNumber("-456", 10)); assertEquals(123.45, ToNumberFunction.toNumber("123.45", 10)); assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); } @Test void testToNumberWithBase2() { - assertEquals(5, ToNumberFunction.toNumber("101", 2)); - assertEquals(0, ToNumberFunction.toNumber("0", 2)); - assertEquals(1, ToNumberFunction.toNumber("1", 2)); - assertEquals(7, ToNumberFunction.toNumber("111", 2)); - assertEquals(10, ToNumberFunction.toNumber("1010", 2)); + assertEquals(5L, ToNumberFunction.toNumber("101", 2)); + assertEquals(0L, ToNumberFunction.toNumber("0", 2)); + assertEquals(1L, ToNumberFunction.toNumber("1", 2)); + assertEquals(7L, ToNumberFunction.toNumber("111", 2)); + assertEquals(10L, ToNumberFunction.toNumber("1010", 2)); } @Test void testToNumberWithBase8() { - assertEquals(64, ToNumberFunction.toNumber("100", 8)); - assertEquals(8, ToNumberFunction.toNumber("10", 8)); - assertEquals(83, ToNumberFunction.toNumber("123", 8)); - assertEquals(511, ToNumberFunction.toNumber("777", 8)); + assertEquals(64L, ToNumberFunction.toNumber("100", 8)); + assertEquals(8L, ToNumberFunction.toNumber("10", 8)); + assertEquals(83L, ToNumberFunction.toNumber("123", 8)); + assertEquals(511L, ToNumberFunction.toNumber("777", 8)); } @Test void testToNumberWithBase16() { - assertEquals(255, ToNumberFunction.toNumber("FF", 16)); - assertEquals(16, ToNumberFunction.toNumber("10", 16)); - assertEquals(171, ToNumberFunction.toNumber("AB", 16)); - assertEquals(291, ToNumberFunction.toNumber("123", 16)); - assertEquals(4095, ToNumberFunction.toNumber("FFF", 16)); + assertEquals(255L, ToNumberFunction.toNumber("FF", 16)); + assertEquals(16L, ToNumberFunction.toNumber("10", 16)); + assertEquals(171L, ToNumberFunction.toNumber("AB", 16)); + assertEquals(291L, ToNumberFunction.toNumber("123", 16)); + assertEquals(4095L, ToNumberFunction.toNumber("FFF", 16)); } @Test void testToNumberWithBase36() { - assertEquals(35, ToNumberFunction.toNumber("Z", 36)); - assertEquals(1295, ToNumberFunction.toNumber("ZZ", 36)); - assertEquals(46655, ToNumberFunction.toNumber("ZZZ", 36)); + assertEquals(35L, ToNumberFunction.toNumber("Z", 36)); + assertEquals(1295L, ToNumberFunction.toNumber("ZZ", 36)); + assertEquals(46655L, ToNumberFunction.toNumber("ZZZ", 36)); } @Test void testToNumberWithDecimalBase2() { - assertEquals(2.5, ToNumberFunction.toNumber("10.1", 2)); - assertEquals(1.5, ToNumberFunction.toNumber("1.1", 2)); - assertEquals(3.75, ToNumberFunction.toNumber("11.11", 2)); + assertEquals(2L, ToNumberFunction.toNumber("10.1", 2)); + assertEquals(1L, ToNumberFunction.toNumber("1.1", 2)); + assertEquals(3L, ToNumberFunction.toNumber("11.11", 2)); } @Test void testToNumberWithDecimalBase16() { - assertEquals(255.5, ToNumberFunction.toNumber("FF.8", 16)); - assertEquals(16.25, ToNumberFunction.toNumber("10.4", 16)); - assertEquals(171.6875, ToNumberFunction.toNumber("AB.B", 16)); + assertEquals(255L, ToNumberFunction.toNumber("FF.8", 16)); + assertEquals(16L, ToNumberFunction.toNumber("10.4", 16)); + assertEquals(171L, ToNumberFunction.toNumber("AB.B", 16)); } @Test void testToNumberWithNegativeDecimal() { - assertEquals(-2.5, ToNumberFunction.toNumber("-10.1", 2)); - assertEquals(-255.5, ToNumberFunction.toNumber("-FF.8", 16)); + assertEquals(-2L, ToNumberFunction.toNumber("-10.1", 2)); + assertEquals(-255L, ToNumberFunction.toNumber("-FF.8", 16)); assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); } @Test void testToNumberWithEmptyFractionalPart() { assertEquals(123.0, ToNumberFunction.toNumber("123.", 10)); - assertEquals(255.0, ToNumberFunction.toNumber("FF.", 16)); - assertEquals(5.0, ToNumberFunction.toNumber("101.", 2)); + assertEquals(255L, ToNumberFunction.toNumber("FF.", 16)); + assertEquals(5L, ToNumberFunction.toNumber("101.", 2)); } @Test void testToNumberWithZeroIntegerPart() { assertEquals(0.5, ToNumberFunction.toNumber("0.5", 10)); - assertEquals(0.5, ToNumberFunction.toNumber("0.1", 2)); + assertEquals(0L, ToNumberFunction.toNumber("0.1", 2)); } @Test @@ -166,31 +166,12 @@ void testToNumberInvalidDigits() { }); } - @Test - void testToNumberInvalidFractionalDigits() { - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("10.2", 2); - }); - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("FF.G", 16); - }); - - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("123.ABC", 10); - }); - } @Test void testToNumberEdgeCases() { - assertEquals(0, ToNumberFunction.toNumber("0", 2)); - assertEquals(0, ToNumberFunction.toNumber("0", 36)); + assertEquals(0L, ToNumberFunction.toNumber("0", 2)); + assertEquals(0L, ToNumberFunction.toNumber("0", 36)); assertEquals(0.0, ToNumberFunction.toNumber("0.0", 10)); assertEquals(0.0, ToNumberFunction.toNumber("0.000", 10)); } @@ -198,17 +179,17 @@ void testToNumberEdgeCases() { @Test void testToNumberLargeNumbers() { assertEquals( - Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); + (long) Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); assertEquals( - Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); + (long) Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); } @Test void testToNumberCaseInsensitivity() { - assertEquals(255, ToNumberFunction.toNumber("ff", 16)); - assertEquals(255, ToNumberFunction.toNumber("FF", 16)); - assertEquals(255, ToNumberFunction.toNumber("fF", 16)); - assertEquals(171, ToNumberFunction.toNumber("ab", 16)); - assertEquals(171, ToNumberFunction.toNumber("AB", 16)); + assertEquals(255L, ToNumberFunction.toNumber("ff", 16)); + assertEquals(255L, ToNumberFunction.toNumber("FF", 16)); + assertEquals(255L, ToNumberFunction.toNumber("fF", 16)); + assertEquals(171L, ToNumberFunction.toNumber("ab", 16)); + assertEquals(171L, ToNumberFunction.toNumber("AB", 16)); } } diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index ffe98228c39..0176c6a6003 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -139,7 +139,7 @@ You can use this function to convert a string representation of a binary number Following example converts a string in binary to the number representation:: - os> source=EMP | eval int_value = tonumber('010101',2) | fields int_value | head 1 + os> source=people | eval int_value = tonumber('010101',2) | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | @@ -151,7 +151,7 @@ Following example converts a string in binary to the number representation:: Following example converts a string in hex to the number representation:: - os> source=EMP | eval int_value = tonumber('FA34',16) | fields int_value | head 1 + os> source=people | eval int_value = tonumber('FA34',16) | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | @@ -161,7 +161,7 @@ Following example converts a string in hex to the number representation:: Following example converts a string in decimal to the number representation:: - os> source=EMP | eval int_value = tonumber('4598') | fields int_value | head 1 + os> source=people | eval int_value = tonumber('4598') | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java new file mode 100644 index 00000000000..885812a9003 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; + +public class CalcitePPLToNumberFunctionTest extends CalcitePPLAbstractTest { + + public CalcitePPLToNumberFunctionTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testNumberBinary() { + String ppl = "source=EMP | eval int_value = tonumber('010101',2) | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('010101':VARCHAR, 2)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=21.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT `TONUMBER`('010101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + + @Test + public void testNumberHex() { + String ppl = "source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('FA34':VARCHAR, 16)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=64052.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT `TONUMBER`('FA34', 16) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumber() { + String ppl = "source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('4598':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=4598.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT `TONUMBER`('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumberDecimal() { + String ppl = "source=EMP | eval int_value = tonumber('4598.54922') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('4598.54922':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=4598.54922\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT `TONUMBER`('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } +} From 8279bf3c18889272ee8ac0266a5fdeaa3c4ab5b2 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 4 Nov 2025 18:13:10 -0800 Subject: [PATCH 26/41] fixed doctest for conversion.rst and applied spotless Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 23 +++-- .../function/udf/ToNumberFunctionTest.java | 6 +- docs/user/ppl/functions/conversion.rst | 78 ++++++++--------- .../CalcitePPLToNumberFunctionTest.java | 85 +++++++++++-------- 4 files changed, 101 insertions(+), 91 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 38f1734a8aa..479cbefaf87 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -105,22 +105,21 @@ public static Number toNumber(String numStr, int base) { // Convert fractional part double fracValue = 0; if (base == 10) { - double divisor = base; - for (char c : fracPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); - fracValue += (double) digit / divisor; - divisor *= base; - } + double divisor = base; + for (char c : fracPart.toCharArray()) { + int digit = Character.digit(c, base); + if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); + fracValue += (double) digit / divisor; + divisor *= base; + } } double result = intValue + fracValue; - result = isNegative ? -result : result; + result = isNegative ? -result : result; if (base == 10) { - return result; - } - else { - return (long) result; + return result; + } else { + return (long) result; } } else { return Long.parseLong(numStr, base); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index 3a8c3185240..cd976956408 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -166,8 +166,6 @@ void testToNumberInvalidDigits() { }); } - - @Test void testToNumberEdgeCases() { assertEquals(0L, ToNumberFunction.toNumber("0", 2)); @@ -179,9 +177,9 @@ void testToNumberEdgeCases() { @Test void testToNumberLargeNumbers() { assertEquals( - (long) Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); + (long) Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); assertEquals( - (long) Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); + (long) Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); } @Test diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 0176c6a6003..fd34681b03f 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -43,21 +43,21 @@ Cast to string example:: os> source=people | eval `cbool` = CAST(true as string), `cint` = CAST(1 as string), `cdate` = CAST(CAST('2012-08-07' as date) as string) | fields `cbool`, `cint`, `cdate` fetched rows / total rows = 1/1 - +-------+------+------------+ - | cbool | cint | cdate | - |-------+------+------------| - | TRUE | 1 | 2012-08-07 | - +-------+------+------------+ + +---------+--------+------------+ + | cbool | cint | cdate | + |---------+--------+------------| + | TRUE | 1 | 2012-08-07 | + +---------+--------+------------+ Cast to number example:: os> source=people | eval `cbool` = CAST(true as int), `cstring` = CAST('1' as int) | fields `cbool`, `cstring` fetched rows / total rows = 1/1 - +-------+---------+ - | cbool | cstring | - |-------+---------| - | 1 | 1 | - +-------+---------+ + +---------+-----------+ + | cbool | cstring | + |---------+-----------| + | 1 | 1 | + +---------+-----------+ Cast to date example:: @@ -73,11 +73,11 @@ Cast function can be chained:: os> source=people | eval `cbool` = CAST(CAST(true as string) as boolean) | fields `cbool` fetched rows / total rows = 1/1 - +-------+ - | cbool | - |-------| - | True | - +-------+ + +---------+ + | cbool | + |---------| + | True | + +---------+ IMPLICIT (AUTO) TYPE CONVERSION @@ -101,11 +101,11 @@ Use string in arithmetic operator example :: os> source=people | eval divide="5"/10, multiply="5" * 10, add="5" + 10, minus="5" - 10, concat="5" + "5" | fields divide, multiply, add, minus, concat fetched rows / total rows = 1/1 - +--------+----------+------+-------+--------+ - | divide | multiply | add | minus | concat | - |--------+----------+------+-------+--------| - | 0.5 | 50.0 | 15.0 | -5.0 | 55 | - +--------+----------+------+-------+--------+ + +----------+------------+-------+---------+----------+ + | divide | multiply | add | minus | concat | + |----------+------------+-------+---------+----------| + | 0.5 | 50.0 | 15.0 | -5.0 | 55 | + +----------+------------+-------+---------+----------+ Use string in comparison operator example :: @@ -120,7 +120,7 @@ Use string in comparison operator example :: TONUMBER ------------ +-------- Description >>>>>>>>>>> @@ -141,30 +141,30 @@ Following example converts a string in binary to the number representation:: os> source=people | eval int_value = tonumber('010101',2) | fields int_value | head 1 fetched rows / total rows = 1/1 - +---------------+ - | int_value | - |---------------+ - | 21.0 | - +---------------+ + +-------------+ + | int_value | + |-------------| + | 21.0 | + +-------------+ Following example converts a string in hex to the number representation:: - os> source=people | eval int_value = tonumber('FA34',16) | fields int_value | head 1 fetched rows / total rows = 1/1 - +---------------+ - | int_value | - |---------------+ - | 64052.0 | - +---------------+ + +-------------+ + | int_value | + |-------------| + | 64052.0 | + +-------------+ Following example converts a string in decimal to the number representation:: - os> source=people | eval int_value = tonumber('4598') | fields int_value | head 1 - fetched rows / total rows = 1/1 - +---------------+ - | int_value | - |---------------+ - | 4598.0 | - +---------------+ + os> source=people | eval int_value = tonumber('4598') | fields int_value | head 1 + fetched rows / total rows = 1/1 + +-------------+ + | int_value | + |-------------| + | 4598.0 | + +-------------+ + diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java index 885812a9003..37053e59f51 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -20,52 +20,65 @@ public void testNumberBinary() { String ppl = "source=EMP | eval int_value = tonumber('010101',2) | fields int_value|head 1"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('010101':VARCHAR, 2)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('010101':VARCHAR, 2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "int_value=21.0\n"; verifyResult(root, expectedResult); - String expectedSparkSql = "SELECT `TONUMBER`('010101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + String expectedSparkSql = + "SELECT `TONUMBER`('010101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testNumberHex() { + String ppl = "source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('FA34':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=64052.0\n"; + verifyResult(root, expectedResult); - @Test - public void testNumberHex() { - String ppl = "source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('FA34':VARCHAR, 16)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = "int_value=64052.0\n"; - verifyResult(root, expectedResult); - - String expectedSparkSql = "SELECT `TONUMBER`('FA34', 16) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } + String expectedSparkSql = + "SELECT `TONUMBER`('FA34', 16) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } - @Test - public void testNumber() { - String ppl = "source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('4598':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = "int_value=4598.0\n"; - verifyResult(root, expectedResult); + @Test + public void testNumber() { + String ppl = "source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('4598':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=4598.0\n"; + verifyResult(root, expectedResult); - String expectedSparkSql = "SELECT `TONUMBER`('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } + String expectedSparkSql = "SELECT `TONUMBER`('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } - @Test - public void testNumberDecimal() { - String ppl = "source=EMP | eval int_value = tonumber('4598.54922') | fields int_value|head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('4598.54922':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = "int_value=4598.54922\n"; - verifyResult(root, expectedResult); + @Test + public void testNumberDecimal() { + String ppl = "source=EMP | eval int_value = tonumber('4598.54922') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('4598.54922':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=4598.54922\n"; + verifyResult(root, expectedResult); - String expectedSparkSql = "SELECT `TONUMBER`('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } + String expectedSparkSql = + "SELECT `TONUMBER`('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } From b5346a6dcfb385f040cc103f8ad480678dc0156a Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 4 Nov 2025 18:20:04 -0800 Subject: [PATCH 27/41] removed decimal point from hex Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunctionTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index cd976956408..d457a8a36a9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -80,36 +80,36 @@ void testToNumberWithBase36() { @Test void testToNumberWithDecimalBase2() { - assertEquals(2L, ToNumberFunction.toNumber("10.1", 2)); - assertEquals(1L, ToNumberFunction.toNumber("1.1", 2)); - assertEquals(3L, ToNumberFunction.toNumber("11.11", 2)); + assertEquals(2L, ToNumberFunction.toNumber("10", 2)); + assertEquals(1L, ToNumberFunction.toNumber("1", 2)); + assertEquals(3L, ToNumberFunction.toNumber("11", 2)); } @Test void testToNumberWithDecimalBase16() { - assertEquals(255L, ToNumberFunction.toNumber("FF.8", 16)); - assertEquals(16L, ToNumberFunction.toNumber("10.4", 16)); - assertEquals(171L, ToNumberFunction.toNumber("AB.B", 16)); + assertEquals(255L, ToNumberFunction.toNumber("FF", 16)); + assertEquals(16L, ToNumberFunction.toNumber("10", 16)); + assertEquals(171L, ToNumberFunction.toNumber("AB", 16)); } @Test void testToNumberWithNegativeDecimal() { - assertEquals(-2L, ToNumberFunction.toNumber("-10.1", 2)); - assertEquals(-255L, ToNumberFunction.toNumber("-FF.8", 16)); + assertEquals(-2L, ToNumberFunction.toNumber("-10", 2)); + assertEquals(-255L, ToNumberFunction.toNumber("-FF", 16)); assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); } @Test void testToNumberWithEmptyFractionalPart() { assertEquals(123.0, ToNumberFunction.toNumber("123.", 10)); - assertEquals(255L, ToNumberFunction.toNumber("FF.", 16)); - assertEquals(5L, ToNumberFunction.toNumber("101.", 2)); + assertEquals(255L, ToNumberFunction.toNumber("FF", 16)); + assertEquals(5L, ToNumberFunction.toNumber("101", 2)); } @Test void testToNumberWithZeroIntegerPart() { assertEquals(0.5, ToNumberFunction.toNumber("0.5", 10)); - assertEquals(0L, ToNumberFunction.toNumber("0.1", 2)); + assertEquals(0L, ToNumberFunction.toNumber("0", 2)); } @Test From 7cf867b4fa7a1e170df483c3b84e992b62c771a0 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 5 Nov 2025 09:56:12 -0800 Subject: [PATCH 28/41] added doctest fixes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 70 +++++++++++++------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index fd34681b03f..52eb04b8e63 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -43,21 +43,21 @@ Cast to string example:: os> source=people | eval `cbool` = CAST(true as string), `cint` = CAST(1 as string), `cdate` = CAST(CAST('2012-08-07' as date) as string) | fields `cbool`, `cint`, `cdate` fetched rows / total rows = 1/1 - +---------+--------+------------+ - | cbool | cint | cdate | - |---------+--------+------------| - | TRUE | 1 | 2012-08-07 | - +---------+--------+------------+ + +-------+------+------------+ + | cbool | cint | cdate | + |-------+------+------------| + | TRUE | 1 | 2012-08-07 | + +-------+------+------------+ Cast to number example:: os> source=people | eval `cbool` = CAST(true as int), `cstring` = CAST('1' as int) | fields `cbool`, `cstring` fetched rows / total rows = 1/1 - +---------+-----------+ - | cbool | cstring | - |---------+-----------| - | 1 | 1 | - +---------+-----------+ + +-------+---------+ + | cbool | cstring | + |-------+---------| + | 1 | 1 | + +-------+---------+ Cast to date example:: @@ -73,11 +73,11 @@ Cast function can be chained:: os> source=people | eval `cbool` = CAST(CAST(true as string) as boolean) | fields `cbool` fetched rows / total rows = 1/1 - +---------+ - | cbool | - |---------| - | True | - +---------+ + +-------+ + | cbool | + |-------| + | True | + +-------+ IMPLICIT (AUTO) TYPE CONVERSION @@ -101,11 +101,11 @@ Use string in arithmetic operator example :: os> source=people | eval divide="5"/10, multiply="5" * 10, add="5" + 10, minus="5" - 10, concat="5" + "5" | fields divide, multiply, add, minus, concat fetched rows / total rows = 1/1 - +----------+------------+-------+---------+----------+ - | divide | multiply | add | minus | concat | - |----------+------------+-------+---------+----------| - | 0.5 | 50.0 | 15.0 | -5.0 | 55 | - +----------+------------+-------+---------+----------+ + +--------+----------+------+-------+--------+ + | divide | multiply | add | minus | concat | + |--------+----------+------+-------+--------| + | 0.5 | 50.0 | 15.0 | -5.0 | 55 | + +--------+----------+------+-------+--------+ Use string in comparison operator example :: @@ -141,30 +141,30 @@ Following example converts a string in binary to the number representation:: os> source=people | eval int_value = tonumber('010101',2) | fields int_value | head 1 fetched rows / total rows = 1/1 - +-------------+ - | int_value | - |-------------| - | 21.0 | - +-------------+ + +-----------+ + | int_value | + |-----------| + | 21.0 | + +-----------+ Following example converts a string in hex to the number representation:: os> source=people | eval int_value = tonumber('FA34',16) | fields int_value | head 1 fetched rows / total rows = 1/1 - +-------------+ - | int_value | - |-------------| - | 64052.0 | - +-------------+ + +-----------+ + | int_value | + |-----------| + | 64052.0 | + +-----------+ Following example converts a string in decimal to the number representation:: os> source=people | eval int_value = tonumber('4598') | fields int_value | head 1 fetched rows / total rows = 1/1 - +-------------+ - | int_value | - |-------------| - | 4598.0 | - +-------------+ + +-----------+ + | int_value | + |-----------| + | 4598.0 | + +-----------+ From 46f010bb62eca84723683347d56cf31323f9ff5b Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 5 Nov 2025 23:11:47 -0800 Subject: [PATCH 29/41] removed unused variables Signed-off-by: Asif Bashar --- .../sql/expression/function/udf/ToNumberFunction.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 479cbefaf87..ba30a7a7d1d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -36,14 +36,7 @@ public ToNumberFunction() { NullPolicy.ANY); } - public static final String DURATION_FORMAT = "duration"; - public static final String DURATION_MILLIS_FORMAT = "duration_millis"; - public static final String HEX_FORMAT = "hex"; - public static final String COMMAS_FORMAT = "commas"; - public static final String BINARY_FORMAT = "binary"; - public static final SqlFunctions.DateFormatFunction dateTimeFormatter = - new SqlFunctions.DateFormatFunction(); - public static final String format24hour = "%H:%M:%S"; // 24-hour format + @Override public SqlReturnTypeInference getReturnTypeInference() { From b7afa1713253d1adc49d343467cdda50d98869a4 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 6 Nov 2025 06:36:34 -0800 Subject: [PATCH 30/41] spotless Signed-off-by: Asif Bashar --- .../sql/expression/function/udf/ToNumberFunction.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index ba30a7a7d1d..5b3a49fe549 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -13,7 +13,6 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; @@ -36,8 +35,6 @@ public ToNumberFunction() { NullPolicy.ANY); } - - @Override public SqlReturnTypeInference getReturnTypeInference() { return ReturnTypes.DOUBLE_FORCE_NULLABLE; From 0f0125a127778b29f74f69aeb16119d9d3716074 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 6 Nov 2025 10:24:54 -0800 Subject: [PATCH 31/41] made recommended changes to simply parse to number, null when malformed Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 52 +++++-------------- .../function/udf/ToNumberFunctionTest.java | 29 +++-------- 2 files changed, 19 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 5b3a49fe549..2557121b0c8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function.udf; +import java.math.BigInteger; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -69,50 +70,23 @@ public static Number toNumber(String numStr) { @Strict public static Number toNumber(String numStr, int base) { if (base < 2 || base > 36) { - throw new IllegalArgumentException("Base must be between 2 and 36"); + throw new IllegalArgumentException("Base has to be between 2 and 36."); } - - if (numStr.contains(".")) { - - boolean isNegative = numStr.startsWith("-"); - if (isNegative) { - numStr = numStr.substring(1); - } - - // Split integer and fractional parts - String[] parts = numStr.split("\\."); - String intPart = parts[0]; - String fracPart = parts.length > 1 ? parts[1] : ""; - - // Convert integer part - double intValue = 0; - for (char c : intPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); - intValue = intValue * base + digit; - } - - // Convert fractional part - double fracValue = 0; + Number result = null; + try { if (base == 10) { - double divisor = base; - for (char c : fracPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); - fracValue += (double) digit / divisor; - divisor *= base; + if (numStr.contains(".")) { + result = Double.parseDouble(numStr); + } else { + result = Long.parseLong(numStr); } - } - - double result = intValue + fracValue; - result = isNegative ? -result : result; - if (base == 10) { - return result; } else { - return (long) result; + BigInteger bigInteger = new BigInteger(numStr, base); + result = bigInteger.longValue(); } - } else { - return Long.parseLong(numStr, base); + } catch (Exception e) { + } + return result; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index d457a8a36a9..34ed102ec5b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -141,29 +141,12 @@ void testToNumberInvalidBase() { @Test void testToNumberInvalidDigits() { - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("12A", 10); - }); - - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("102", 2); - }); - - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("189", 8); - }); - - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("GHI", 16); - }); + assertEquals(null, ToNumberFunction.toNumber("12A", 10)); + assertEquals(null, ToNumberFunction.toNumber("102", 2)); + assertEquals(null, ToNumberFunction.toNumber("101.101", 2)); + assertEquals(null, ToNumberFunction.toNumber("189", 8)); + assertEquals(null, ToNumberFunction.toNumber("GHI", 16)); + assertEquals(null, ToNumberFunction.toNumber("FF.8", 16)); } @Test From 804db79a7ad546937cece7a99eea0e84af32c66a Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sat, 8 Nov 2025 16:51:14 -0800 Subject: [PATCH 32/41] hex max limit doc and unit tests Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 3 +- .../CalcitePPLToNumberFunctionTest.java | 80 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index ca9297b1f0d..c55e868f2cd 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -133,7 +133,8 @@ Return type: Number You can use this function with the eval commands and as part of eval expressions. -Base values can be between 2 and 36. +Base values can be between 2 and 36. The maximum value supported for base 10 is +(2-2^-52)·2^1023 and minimum is -(2-2^-52)·2^1023. +The maximum for other supported bases is 2^63-1 (or 7FFFFFFFFFFFFFFF) and minimum is -2^63 (or -7FFFFFFFFFFFFFFF). You can use this function to convert a string representation of a binary number to return the corresponding number in base 10. diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java index 37053e59f51..448cddeefec 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -49,6 +49,86 @@ public void testNumberHex() { verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testNumberHexMinLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('-7FFFFFFFFFFFFFFF',16) | fields long_value|head" + + " 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('-7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=-9.223372036854776E18\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('-7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumberHexMaxLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('7FFFFFFFFFFFFFFF',16) | fields long_value|head" + + " 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=9.223372036854776E18\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumberHexOverNegativeMaxLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('-FFFFFFFFFFFFFFFF',16) | fields long_value|head" + + " 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('-FFFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=1.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('-FFFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumberHexOverPositiveMaxLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('FFFFFFFFFFFFFFFF',16) | fields long_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('FFFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=-1.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('FFFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testNumber() { String ppl = "source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1"; From 75883979380e8cad8b0f4e2318ecaf748fd6d0aa Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sun, 9 Nov 2025 10:51:54 -0800 Subject: [PATCH 33/41] fix to spotless Signed-off-by: Asif Bashar --- .../CalcitePPLToNumberFunctionTest.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java index 448cddeefec..bd4f2c293b6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -49,26 +49,25 @@ public void testNumberHex() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testNumberHexMinLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('-7FFFFFFFFFFFFFFF',16) | fields long_value|head" + + " 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('-7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=-9.223372036854776E18\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('-7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - @Test - public void testNumberHexMinLimit() { - String ppl = - "source=EMP | eval long_value = tonumber('-7FFFFFFFFFFFFFFF',16) | fields long_value|head" - + " 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n" - + " LogicalProject(long_value=[TONUMBER('-7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = "long_value=-9.223372036854776E18\n"; - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `TONUMBER`('-7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - - verifyPPLToSparkSQL(root, expectedSparkSql); - } + verifyPPLToSparkSQL(root, expectedSparkSql); + } @Test public void testNumberHexMaxLimit() { From d208779099e4993668ab9e8cb055132b8b05b375 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 18 Nov 2025 23:12:44 -0800 Subject: [PATCH 34/41] added IT , add null cases, fixed format for integer/big integer Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 65 ++++++++++- docs/user/ppl/functions/conversion.rst | 17 ++- .../sql/ppl/ConversionFunctionIT.java | 110 ++++++++++++++++++ .../CalcitePPLToNumberFunctionTest.java | 66 +++++++++-- 4 files changed, 243 insertions(+), 15 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/ConversionFunctionIT.java diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 2557121b0c8..bd95986cd27 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -14,8 +14,8 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -38,7 +38,65 @@ public ToNumberFunction() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.DOUBLE_FORCE_NULLABLE; + return (opBinding) -> { + // Try to determine if the result will be Long or Double based on the input + int base = 10; + if (opBinding.getOperandCount() > 1) { + + base = opBinding.getOperandLiteralValue(1, Integer.class); + } + + if (opBinding.getOperandCount() > 0 && opBinding.isOperandLiteral(0, false)) { + String literal = opBinding.getOperandLiteralValue(0, String.class); + if (literal != null) { + try { + // Check if it's a decimal number + if (base != 10) { + return opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(SqlTypeName.BIGINT), true); + } + if (literal.contains(".")) { + return opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding + .getTypeFactory() + .createSqlType(org.apache.calcite.sql.type.SqlTypeName.DOUBLE), + true); + } else { + // Check if it's an integer that fits in Long + Long.parseLong(literal); + return opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding + .getTypeFactory() + .createSqlType(org.apache.calcite.sql.type.SqlTypeName.BIGINT), + true); + } + } catch (NumberFormatException e) { + // If parsing fails, default to Double (matches the runtime behavior) + return opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding + .getTypeFactory() + .createSqlType(org.apache.calcite.sql.type.SqlTypeName.DOUBLE), + true); + } + } + } + // Default to Double when we can't determine the type at compile time + return opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding + .getTypeFactory() + .createSqlType(org.apache.calcite.sql.type.SqlTypeName.DOUBLE), + true); + }; } @Override @@ -52,7 +110,6 @@ public static class ToNumberImplementor implements NotNullImplementor { public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { Expression fieldValue = translatedOperands.get(0); - int base = 10; if (translatedOperands.size() > 1) { Expression baseExpr = translatedOperands.get(1); return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue, baseExpr); @@ -85,7 +142,7 @@ public static Number toNumber(String numStr, int base) { result = bigInteger.longValue(); } } catch (Exception e) { - + // Return null when parsing fails, matches function behavior } return result; } diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index c55e868f2cd..461a7d7fe1b 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -127,7 +127,7 @@ Description The following usage options are available, depending on the parameter types and the number of parameters. -Usage: tonumber(string, [base]) converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. +Usage: tonumber(string, [base]) converts the value in first argument. The second argument describe the base of first argument. If second argument is not provided, then it converts to base 10 number representation. Return type: Number @@ -145,7 +145,7 @@ Following example converts a string in binary to the number representation:: +-----------+ | int_value | |-----------| - | 21.0 | + | 21 | +-----------+ @@ -156,7 +156,7 @@ Following example converts a string in hex to the number representation:: +-----------+ | int_value | |-----------| - | 64052.0 | + | 64052 | +-----------+ Following example converts a string in decimal to the number representation:: @@ -166,9 +166,18 @@ Following example converts a string in decimal to the number representation:: +-----------+ | int_value | |-----------| - | 4598.0 | + | 4598 | +-----------+ +Following example converts a string in decimal with fraction to the number representation:: + + os> source=people | eval double_value = tonumber('4598.678') | fields double_value | head 1 + fetched rows / total rows = 1/1 + +--------------+ + | double_value | + |--------------| + | 4598.678 | + +--------------+ TOSTRING ----------- diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ConversionFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ConversionFunctionIT.java new file mode 100644 index 00000000000..95ff1a73cf0 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ConversionFunctionIT.java @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; + +public class ConversionFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void testDecimal() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s |head 1| eval a = tonumber('4598.678') | fields a", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("a", "double")); + + verifyDataRows(actual, rows(4598.678)); + } + + @Test + public void testHex() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s |head 1| eval a = tonumber('FF12CA',16) | fields a", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("a", "bigint")); + + verifyDataRows(actual, rows(16716490)); + } + + @Test + public void testBinary() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s |head 1| eval a = tonumber('0110111',2) | fields a", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("a", "bigint")); + + verifyDataRows(actual, rows(55)); + } + + @Test + public void testOctal() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s |head 1| eval a = tonumber('20415442',8) | fields a", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("a", "bigint")); + + verifyDataRows(actual, rows(4332322)); + } + + @Test + public void testOctalWithUnsupportedValue() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s |head 1| eval a = tonumber('20415.442',8) | fields a", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("a", "bigint")); + + assertEquals(actual.getJSONArray("datarows").getJSONArray(0).get(0), null); + } + + @Test + public void testBinaryWithUnsupportedValue() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s |head 1| eval a = tonumber('1010.11',2) | fields a", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("a", "bigint")); + + assertEquals(actual.getJSONArray("datarows").getJSONArray(0).get(0), null); + } + + @Test + public void testHexWithUnsupportedValue() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s |head 1| eval a = tonumber('A.B',16) | fields a", TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("a", "bigint")); + + assertEquals(actual.getJSONArray("datarows").getJSONArray(0).get(0), null); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java index bd4f2c293b6..08a4cd7f9a8 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -24,7 +24,7 @@ public void testNumberBinary() { + " LogicalProject(int_value=[TONUMBER('010101':VARCHAR, 2)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "int_value=21.0\n"; + String expectedResult = "int_value=21\n"; verifyResult(root, expectedResult); String expectedSparkSql = @@ -32,6 +32,23 @@ public void testNumberBinary() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testNumberBinaryUnsupportedResultNull() { + String ppl = "source=EMP | eval int_value = tonumber('010.101',2) | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('010.101':VARCHAR, 2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=null\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('010.101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testNumberHex() { String ppl = "source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1"; @@ -41,7 +58,7 @@ public void testNumberHex() { + " LogicalProject(int_value=[TONUMBER('FA34':VARCHAR, 16)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "int_value=64052.0\n"; + String expectedResult = "int_value=64052\n"; verifyResult(root, expectedResult); String expectedSparkSql = @@ -49,6 +66,24 @@ public void testNumberHex() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testNumberHexUnsupportedValuesResultNull() { + String ppl = + "source=EMP | eval double_value = tonumber('FA.34',16) | fields double_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(double_value=[TONUMBER('FA.34':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "double_value=null\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('FA.34', 16) `double_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testNumberHexMinLimit() { String ppl = @@ -60,7 +95,7 @@ public void testNumberHexMinLimit() { + " LogicalProject(long_value=[TONUMBER('-7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "long_value=-9.223372036854776E18\n"; + String expectedResult = "long_value=-9223372036854775807\n"; verifyResult(root, expectedResult); String expectedSparkSql = @@ -80,7 +115,7 @@ public void testNumberHexMaxLimit() { + " LogicalProject(long_value=[TONUMBER('7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "long_value=9.223372036854776E18\n"; + String expectedResult = "long_value=9223372036854775807\n"; verifyResult(root, expectedResult); String expectedSparkSql = @@ -100,7 +135,7 @@ public void testNumberHexOverNegativeMaxLimit() { + " LogicalProject(long_value=[TONUMBER('-FFFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "long_value=1.0\n"; + String expectedResult = "long_value=1\n"; verifyResult(root, expectedResult); String expectedSparkSql = @@ -119,7 +154,7 @@ public void testNumberHexOverPositiveMaxLimit() { + " LogicalProject(long_value=[TONUMBER('FFFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "long_value=-1.0\n"; + String expectedResult = "long_value=-1\n"; verifyResult(root, expectedResult); String expectedSparkSql = @@ -137,7 +172,7 @@ public void testNumber() { + " LogicalProject(int_value=[TONUMBER('4598':VARCHAR)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "int_value=4598.0\n"; + String expectedResult = "int_value=4598\n"; verifyResult(root, expectedResult); String expectedSparkSql = "SELECT `TONUMBER`('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; @@ -160,4 +195,21 @@ public void testNumberDecimal() { "SELECT `TONUMBER`('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testNumberUnsupportedResultNull() { + String ppl = "source=EMP | eval int_value = tonumber('4A598.54922') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('4A598.54922':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=null\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('4A598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } From 0f4149d950b871ae1692efe447d2962aa702a312 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 18 Nov 2025 23:41:56 -0800 Subject: [PATCH 35/41] added IT , add null cases, fixed format for integer/big integer Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 48 +++++-------------- .../function/udf/ToNumberFunctionTest.java | 6 --- 2 files changed, 11 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index bd95986cd27..8f2b5e5acab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -41,50 +41,24 @@ public SqlReturnTypeInference getReturnTypeInference() { return (opBinding) -> { // Try to determine if the result will be Long or Double based on the input int base = 10; - if (opBinding.getOperandCount() > 1) { - - base = opBinding.getOperandLiteralValue(1, Integer.class); + try { + base = + opBinding.getOperandCount() > 1 + ? opBinding.getOperandLiteralValue(1, Integer.class) + : 10; + } catch (NumberFormatException e) { + // If parsing fails, default to base 10 } - if (opBinding.getOperandCount() > 0 && opBinding.isOperandLiteral(0, false)) { String literal = opBinding.getOperandLiteralValue(0, String.class); if (literal != null) { - try { - // Check if it's a decimal number - if (base != 10) { - return opBinding - .getTypeFactory() - .createTypeWithNullability( - opBinding.getTypeFactory().createSqlType(SqlTypeName.BIGINT), true); - } - if (literal.contains(".")) { - return opBinding - .getTypeFactory() - .createTypeWithNullability( - opBinding - .getTypeFactory() - .createSqlType(org.apache.calcite.sql.type.SqlTypeName.DOUBLE), - true); - } else { - // Check if it's an integer that fits in Long - Long.parseLong(literal); - return opBinding - .getTypeFactory() - .createTypeWithNullability( - opBinding - .getTypeFactory() - .createSqlType(org.apache.calcite.sql.type.SqlTypeName.BIGINT), - true); - } - } catch (NumberFormatException e) { - // If parsing fails, default to Double (matches the runtime behavior) + + // Check if it's a decimal number + if (base != 10 || !(literal.contains("."))) { return opBinding .getTypeFactory() .createTypeWithNullability( - opBinding - .getTypeFactory() - .createSqlType(org.apache.calcite.sql.type.SqlTypeName.DOUBLE), - true); + opBinding.getTypeFactory().createSqlType(SqlTypeName.BIGINT), true); } } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index 34ed102ec5b..b941f8268e5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.*; -import org.apache.calcite.sql.type.ReturnTypes; import org.junit.jupiter.api.Test; import org.opensearch.sql.calcite.utils.PPLOperandTypes; @@ -15,11 +14,6 @@ public class ToNumberFunctionTest { private final ToNumberFunction function = new ToNumberFunction(); - @Test - void testGetReturnTypeInference() { - assertEquals(ReturnTypes.DOUBLE_FORCE_NULLABLE, function.getReturnTypeInference()); - } - @Test void testGetOperandMetadata() { assertEquals(PPLOperandTypes.STRING_OR_STRING_INTEGER, function.getOperandMetadata()); From 4a2338271b1332829c4b9bc87074c594e2ca77a3 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 18 Nov 2025 23:43:10 -0800 Subject: [PATCH 36/41] added IT , add null cases, fixed format for integer/big integer Signed-off-by: Asif Bashar --- .../sql/expression/function/udf/ToNumberFunction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 8f2b5e5acab..4a20c679e09 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -62,7 +62,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } } } - // Default to Double when we can't determine the type at compile time + // Default to Double when we can't determine the type at compile time or bigint type is confirmed return opBinding .getTypeFactory() .createTypeWithNullability( From 8532aeacfbd55f0f91144b153da2f2152bbdad09 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 20 Nov 2025 10:08:07 -0800 Subject: [PATCH 37/41] spotless apply Signed-off-by: Asif Bashar --- .../sql/expression/function/udf/ToNumberFunction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 4a20c679e09..d1604d53bd3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -62,7 +62,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } } } - // Default to Double when we can't determine the type at compile time or bigint type is confirmed + // Default to Double when we can't determine the type at compile time or bigint type is + // confirmed return opBinding .getTypeFactory() .createTypeWithNullability( From eda5fcd6fd34f26529adbcabecb3ffbb1b96708f Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 20 Nov 2025 12:30:34 -0800 Subject: [PATCH 38/41] spotless apply Signed-off-by: Asif Bashar --- .../CalcitePPLToNumberFunctionTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java index 08a4cd7f9a8..c30590fa268 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -28,7 +28,7 @@ public void testNumberBinary() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('010101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('010101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -45,7 +45,7 @@ public void testNumberBinaryUnsupportedResultNull() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('010.101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('010.101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -62,7 +62,7 @@ public void testNumberHex() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('FA34', 16) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('FA34', 16) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -80,7 +80,7 @@ public void testNumberHexUnsupportedValuesResultNull() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('FA.34', 16) `double_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('FA.34', 16) `double_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -99,7 +99,7 @@ public void testNumberHexMinLimit() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('-7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('-7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -119,7 +119,7 @@ public void testNumberHexMaxLimit() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -139,7 +139,7 @@ public void testNumberHexOverNegativeMaxLimit() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('-FFFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('-FFFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -158,7 +158,7 @@ public void testNumberHexOverPositiveMaxLimit() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('FFFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('FFFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -175,7 +175,7 @@ public void testNumber() { String expectedResult = "int_value=4598\n"; verifyResult(root, expectedResult); - String expectedSparkSql = "SELECT `TONUMBER`('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + String expectedSparkSql = "SELECT TONUMBER('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -192,7 +192,7 @@ public void testNumberDecimal() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -209,7 +209,7 @@ public void testNumberUnsupportedResultNull() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `TONUMBER`('4A598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + "SELECT TONUMBER('4A598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } } From 1bd2ca3bb32647af1f202cf2fa6913f122d52057 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 24 Nov 2025 19:14:35 -0600 Subject: [PATCH 39/41] fix javadoc Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index d1604d53bd3..dd6de40bdba 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -21,13 +21,23 @@ import org.opensearch.sql.expression.function.UDFOperandMetadata; /** - * A custom implementation of number/boolean to string . + * The following usage options are available, depending on the parameter types + * and the number of parameters. * - *

This operator is necessary because tostring has following requirements "binary" Converts a - * number to a binary value. "hex" Converts the number to a hexadecimal value. "commas" Formats the - * number with commas. If the number includes a decimal, the function rounds the number to nearest - * two decimal places. "duration" Converts the value in seconds to the readable time format - * HH:MM:SS. if not format parameter provided, then consider value as boolean + *

Usage: {@code tonumber(string, [base])} converts the value in the first + * argument. The second argument describes the base of the first argument. + * If the second argument is not provided, the value is converted using base 10. + * + *

Return type: Number + * + *

You can use this function with the eval commands and as part of eval expressions. + * + *

Base values can range from 2 to 36. + * The maximum value supported for base 10 is {@code +(2 − 2^-52) · 2^1023} + * and the minimum is {@code −(2 − 2^-52) · 2^1023}. + * + *

The maximum for other supported bases is {@code 2^63 − 1} (or {@code 7FFFFFFFFFFFFFFF}) + * and the minimum is {@code -2^63} (or {@code -7FFFFFFFFFFFFFFF}). */ public class ToNumberFunction extends ImplementorUDF { public ToNumberFunction() { From fc70233d858d51087324309fc4bb57e459cad98c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 24 Nov 2025 19:33:43 -0600 Subject: [PATCH 40/41] fix doc to add null value return condition details Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 461a7d7fe1b..9f40befd53a 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -135,7 +135,7 @@ Return type: Number You can use this function with the eval commands and as part of eval expressions. Base values can be between 2 and 36. The maximum value supported for base 10 is +(2-2^-52)·2^1023 and minimum is -(2-2^-52)·2^1023. The maximum for other supported bases is 2^63-1 (or 7FFFFFFFFFFFFFFF) and minimum is -2^63 (or -7FFFFFFFFFFFFFFF). - +If the tonumber function cannot parse a field value to a number, the function returns NULL. You can use this function to convert a string representation of a binary number to return the corresponding number in base 10. Following example converts a string in binary to the number representation:: From 77d53a74ee27bc0dfa2f01cd357c94c473f16f15 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 25 Nov 2025 13:07:54 -0600 Subject: [PATCH 41/41] spotless Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index dd6de40bdba..1c3a836cc84 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -21,23 +21,22 @@ import org.opensearch.sql.expression.function.UDFOperandMetadata; /** - * The following usage options are available, depending on the parameter types - * and the number of parameters. + * The following usage options are available, depending on the parameter types and the number of + * parameters. * - *

Usage: {@code tonumber(string, [base])} converts the value in the first - * argument. The second argument describes the base of the first argument. - * If the second argument is not provided, the value is converted using base 10. + *

Usage: {@code tonumber(string, [base])} converts the value in the first argument. The + * second argument describes the base of the first argument. If the second argument is not provided, + * the value is converted using base 10. * *

Return type: Number * *

You can use this function with the eval commands and as part of eval expressions. * - *

Base values can range from 2 to 36. - * The maximum value supported for base 10 is {@code +(2 − 2^-52) · 2^1023} - * and the minimum is {@code −(2 − 2^-52) · 2^1023}. + *

Base values can range from 2 to 36. The maximum value supported for base 10 is {@code +(2 − + * 2^-52) · 2^1023} and the minimum is {@code −(2 − 2^-52) · 2^1023}. * - *

The maximum for other supported bases is {@code 2^63 − 1} (or {@code 7FFFFFFFFFFFFFFF}) - * and the minimum is {@code -2^63} (or {@code -7FFFFFFFFFFFFFFF}). + *

The maximum for other supported bases is {@code 2^63 − 1} (or {@code 7FFFFFFFFFFFFFFF}) and + * the minimum is {@code -2^63} (or {@code -7FFFFFFFFFFFFFFF}). */ public class ToNumberFunction extends ImplementorUDF { public ToNumberFunction() {