Improve synthetic source for tdigest field (#138121)

Follow up to #137982 to support returning min, max, and sum in the synthetic source results. This also drops support for sending the count as a parameter (and thus doesn't include the total count in the synthetic source result), which matches the behavior of the exponential histogram field.

Author: not-napoleon

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/TDigestFieldMapper.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
@@ -69,7 +70,6 @@ public class TDigestFieldMapper extends FieldMapper {
     public static final String CENTROIDS_NAME = "centroids";
     public static final String COUNTS_NAME = "counts";
     public static final String SUM_FIELD_NAME = "sum";
-    public static final String TOTAL_COUNT_FIELD_NAME = "count";
     public static final String MIN_FIELD_NAME = "min";
     public static final String MAX_FIELD_NAME = "max";
     public static final String CONTENT_TYPE = "tdigest";
@@ -506,17 +506,41 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
     private class TDigestSyntheticFieldLoader implements CompositeSyntheticFieldLoader.DocValuesLayer {
         private final InternalTDigestValue value = new InternalTDigestValue();
         private BytesRef binaryValue;
+        private double min;
+        private double max;
+        private double sum;
 
         @Override
         public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
             BinaryDocValues docValues = leafReader.getBinaryDocValues(fieldType().name());
+            NumericDocValues minValues = leafReader.getNumericDocValues(valuesMinSubFieldName(fullPath()));
+            NumericDocValues maxValues = leafReader.getNumericDocValues(valuesMaxSubFieldName(fullPath()));
+            NumericDocValues sumValues = leafReader.getNumericDocValues(valuesSumSubFieldName(fullPath()));
             if (docValues == null) {
                 // No values in this leaf
                 binaryValue = null;
                 return null;
             }
             return docId -> {
                 if (docValues.advanceExact(docId)) {
+                    // we assume the summary sub-
+                    if (minValues != null) {
+                        minValues.advanceExact(docId);
+                        min = NumericUtils.sortableLongToDouble(minValues.longValue());
+                    } else {
+                        min = Double.NaN;
+                    }
+
+                    if (maxValues != null) {
+                        maxValues.advanceExact(docId);
+                        max = NumericUtils.sortableLongToDouble(maxValues.longValue());
+                    } else {
+                        max = Double.NaN;
+                    }
+
+                    sumValues.advanceExact(docId);
+                    sum = NumericUtils.sortableLongToDouble(sumValues.longValue());
+
                     binaryValue = docValues.binaryValue();
                     return true;
                 }
@@ -536,9 +560,17 @@ public void write(XContentBuilder b) throws IOException {
                 return;
             }
             value.reset(binaryValue);
-
             b.startObject();
+
             // TODO: Load the summary values out of the sub-fields, if they exist
+            if (Double.isNaN(min) == false) {
+                b.field("min", min);
+            }
+            if (Double.isNaN(max) == false) {
+                b.field("max", max);
+            }
+            b.field("sum", sum);
+
             b.startArray(CENTROIDS_NAME);
             while (value.next()) {
                 b.value(value.value());

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/TDigestParser.java

@@ -21,13 +21,11 @@
 import static org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper.MAX_FIELD_NAME;
 import static org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper.MIN_FIELD_NAME;
 import static org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper.SUM_FIELD_NAME;
-import static org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper.TOTAL_COUNT_FIELD_NAME;
 
 public class TDigestParser {
 
     private static final ParseField COUNTS_FIELD = new ParseField(COUNTS_NAME);
     private static final ParseField CENTROIDS_FIELD = new ParseField(CENTROIDS_NAME);
-    private static final ParseField TOTAL_COUNT_FIELD = new ParseField(TOTAL_COUNT_FIELD_NAME);
     private static final ParseField SUM_FIELD = new ParseField(SUM_FIELD_NAME);
     private static final ParseField MAX_FIELD = new ParseField(MAX_FIELD_NAME);
     private static final ParseField MIN_FIELD = new ParseField(MIN_FIELD_NAME);
@@ -37,7 +35,7 @@ public class TDigestParser {
      * @param centroids the centroids, guaranteed to be distinct and in increasing order
      * @param counts the counts, guaranteed to be non-negative and of the same length as the centroids array
      */
-    public record ParsedTDigest(List<Double> centroids, List<Long> counts, Long count, Double sum, Double min, Double max) {
+    public record ParsedTDigest(List<Double> centroids, List<Long> counts, Double sum, Double min, Double max) {
         @Override
         public Double max() {
             if (max != null) {
@@ -75,11 +73,7 @@ public Double sum() {
             return Double.NaN;
         }
 
-        @Override
         public Long count() {
-            if (count != null) {
-                return count;
-            }
             if (counts != null && counts.isEmpty() == false) {
                 long observedCount = 0;
                 for (Long count : counts) {
@@ -102,7 +96,6 @@ public Long count() {
     public static ParsedTDigest parse(String mappedFieldName, XContentParser parser) throws IOException {
         ArrayList<Double> centroids = null;
         ArrayList<Long> counts = null;
-        Long count = null;
         Double sum = null;
         Double min = null;
         Double max = null;
@@ -127,10 +120,6 @@ public static ParsedTDigest parse(String mappedFieldName, XContentParser parser)
                 token = parser.nextToken();
                 ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
                 max = parser.doubleValue();
-            } else if (fieldName.equals(TOTAL_COUNT_FIELD.getPreferredName())) {
-                token = parser.nextToken();
-                ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
-                count = parser.longValue();
             } else {
                 throw new DocumentParsingException(
                     parser.getTokenLocation(),
@@ -169,12 +158,11 @@ public static ParsedTDigest parse(String mappedFieldName, XContentParser parser)
             );
         }
         if (centroids.isEmpty()) {
-            count = 0L;
             sum = 0.0;
             min = null;
             max = null;
         }
-        return new ParsedTDigest(centroids, counts, count, sum, min, max);
+        return new ParsedTDigest(centroids, counts, sum, min, max);
     }
 
     private static ArrayList<Long> getCounts(String mappedFieldName, XContentParser parser) throws IOException {

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestFieldMapperTests.java

@@ -434,6 +434,9 @@ public void testArrayValueSyntheticSource() throws Exception {
         {
             expected.startArray("field");
             expected.startObject();
+            expected.field("min", 1.0d);
+            expected.field("max", 3.0d);
+            expected.field("sum", 14.0d);
             expected.field("centroids", new double[] { 1, 2, 3 });
             expected.field("counts", new int[] { 1, 2, 3 });
             expected.endObject();
@@ -458,10 +461,17 @@ private static Map<String, Object> generateRandomFieldValues(int maxVals) {
         }
         List<Double> centroids = new ArrayList<>();
         List<Long> counts = new ArrayList<>();
+        double sum = 0.0;
         for (Centroid c : digest.centroids()) {
             centroids.add(c.mean());
             counts.add(c.count());
+            sum += c.mean() * c.count();
         }
+        double min = digest.getMin();
+        double max = digest.getMax();
+        value.put("min", min);
+        value.put("max", max);
+        value.put("sum", sum);
         value.put("centroids", centroids);
         value.put("counts", counts);
 

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/t_digest_fieldtype.yml

@@ -19,7 +19,7 @@ setup:
           - '{"index": {"_id": 2}}'
           - '{"latency": {"centroids" : [0, 0.1, 0.2, 0.3, 0.4, 0.5], "counts" : [3, 2, 5, 10, 1, 8]}}'
           - '{"index": {"_id": 3}}'
-          - '{"latency": {"sum": 8.6, "count": 29, "min": 0, "max": 0.5, "centroids" : [0, 0.1, 0.2, 0.3, 0.4, 0.5], "counts" : [3, 2, 5, 10, 1, 8]}}'
+          - '{"latency": {"sum": 8.6, "min": 0, "max": 0.5, "centroids" : [0, 0.1, 0.2, 0.3, 0.4, 0.5], "counts" : [3, 2, 5, 10, 1, 8]}}'
 ---
 "TDigest requires values in increasing order":
   - do:
@@ -47,11 +47,9 @@ TDigest get:
   - match:
       _source:
         latency:
-          # Here the summary fields were in the source, so we get them back
           min: 0
           max: 0.5
           sum: 8.6
-          count: 29
           centroids: [ 0, 0.1, 0.2, 0.3, 0.4, 0.5 ]
           counts: [ 3, 2, 5, 10, 1, 8 ]
 
@@ -89,6 +87,9 @@ TDigest with synthetic source:
   - match:
       _source:
         latency:
+          min: 0.1
+          max: 0.5
+          sum: 16.4
           centroids: [ 0.1, 0.2, 0.3, 0.4, 0.5 ]
           counts: [ 3, 7, 23, 12, 6 ]
 
@@ -99,6 +100,10 @@ TDigest with synthetic source:
   - match:
       _source:
         latency:
+          min: 0.0
+          max: 0.5
+          # $%#@! floating points...
+          sum: 8.600000000000001
           centroids: [ 0.0, 0.1, 0.2, 0.3, 0.4, 0.5 ]
           counts: [ 3, 2, 5, 10, 1, 8 ]
 ---
@@ -124,7 +129,7 @@ TDigest with synthetic source and explicit summary fields:
         refresh: true
         body:
           - '{"index": {"_id": 1}}'
-          - '{"latency": {"sum": 8.6, "count": 29, "min": 0, "max": 0.5, "centroids" : [0, 0.1, 0.2, 0.3, 0.4, 0.5], "counts" : [3, 2, 5, 10, 1, 8]}}'
+          - '{"latency": {"sum": 8.6, "min": 0, "max": 0.5, "centroids" : [0, 0.1, 0.2, 0.3, 0.4, 0.5], "counts" : [3, 2, 5, 10, 1, 8]}}'
 
   - do:
       get:
@@ -133,6 +138,10 @@ TDigest with synthetic source and explicit summary fields:
   - match:
       _source:
         latency:
+          # Note that unlike the stored source case, we get back a float here
+          min: 0.0
+          max: 0.5
+          sum: 8.6
           centroids: [ 0.0, 0.1, 0.2, 0.3, 0.4, 0.5 ]
           counts: [ 3, 2, 5, 10, 1, 8 ]
 
@@ -168,6 +177,10 @@ TDigest with synthetic source and zero counts:
   - match:
       _source:
         latency:
+          # Note that we're storing 0.1 as the min, even though it's count is 0.
+          min: 0.1
+          max: 0.5
+          sum: 3.8000000000000007
           centroids: [ 0.2, 0.4 ]
           counts: [ 7, 6 ]
 
@@ -223,6 +236,9 @@ histogram with synthetic source and ignore_malformed:
   - match:
       _source:
         latency: [ {
+          min: 2.0,
+          max: 2.0,
+          sum: 4.0,
           "centroids": [ 2.0 ],
           "counts": [ 2 ]
         },
@@ -260,6 +276,7 @@ TDigest with synthetic source and empty digest:
   - match:
       _source:
         latency:
+          sum: 0.0
           centroids: [ ]
           counts: [ ]