Make nested alias type support referring to outer context (opensearch-project#4673)

Author: qianheng-aws

integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4559.yml

@@ -0,0 +1,68 @@
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              aws:
+                properties:
+                  cloudtrail:
+                    properties:
+                      event_name:
+                        type: alias
+                        path: api.operation
+                  user_identity:
+                    type: text
+              api:
+                properties:
+                  operation:
+                    type: keyword
+
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled : true
+
+---
+teardown:
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled : false
+
+---
+"Handle nested alias field referring to the outer context":
+  - skip:
+      features:
+        - headers
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{
+               "aws": {
+                 "cloudtrail": {
+                   "user_identity": "test-user"
+                 }
+               },
+               "api": {
+                 "operation": "CreateBucket"
+               }
+             }'
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'source=test | fields aws.cloudtrail.event_name'
+  - match: {"total": 1}
+  - match: { "schema": [ { "name": "aws.cloudtrail.event_name", "type": "string" } ] }
+  - match: { "datarows": [ [ "CreateBucket" ] ] }

opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java

@@ -42,7 +42,8 @@ public enum MappingType {
     HalfFloat("half_float", ExprCoreType.FLOAT),
     ScaledFloat("scaled_float", ExprCoreType.DOUBLE),
     Double("double", ExprCoreType.DOUBLE),
-    Boolean("boolean", ExprCoreType.BOOLEAN);
+    Boolean("boolean", ExprCoreType.BOOLEAN),
+    Alias("alias", ExprCoreType.UNKNOWN);
     // TODO: ranges, geo shape, point, shape
 
     private final String name;
@@ -117,12 +118,7 @@ public static Map<String, OpenSearchDataType> parseMapping(Map<String, Object> i
           // by default, the type is treated as an Object if "type" is not provided
           var type = ((String) innerMap.getOrDefault("type", "object")).replace("_", "");
           if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) {
-            // unknown type, e.g. `alias`
-            // Record fields of the alias type and resolve them later in case their references have
-            // not been resolved.
-            if (OpenSearchAliasType.typeName.equals(type)) {
-              aliasMapping.put(k, (String) innerMap.get(OpenSearchAliasType.pathPropertyName));
-            }
+            // unknown type, skip it.
             return;
           }
           // create OpenSearchDataType
@@ -133,21 +129,6 @@ public static Map<String, OpenSearchDataType> parseMapping(Map<String, Object> i
                   innerMap));
         });
 
-    // Begin to parse alias type fields
-    if (!aliasMapping.isEmpty()) {
-      // The path of alias type may point to a nested field, so we need to flatten the result.
-      Map<String, OpenSearchDataType> flattenResult = traverseAndFlatten(result);
-      aliasMapping.forEach(
-          (k, v) -> {
-            if (flattenResult.containsKey(v)) {
-              result.put(k, new OpenSearchAliasType(v, flattenResult.get(v)));
-            } else {
-              throw new IllegalStateException(
-                  String.format("Cannot find the path [%s] for alias type field [%s]", v, k));
-            }
-          });
-    }
-
     return result;
   }
 
@@ -188,6 +169,10 @@ public static OpenSearchDataType of(MappingType mappingType, Map<String, Object>
         // Default date formatter is used when "" is passed as the second parameter
         String format = (String) innerMap.getOrDefault("format", "");
         return OpenSearchDateType.of(format);
+      case Alias:
+        return new OpenSearchAliasType(
+            (String) innerMap.get(OpenSearchAliasType.pathPropertyName),
+            OpenSearchDateType.of(MappingType.Invalid));
       default:
         return res;
     }
@@ -302,9 +287,20 @@ public void accept(Map<String, OpenSearchDataType> subtree, String prefix) {
           }
         };
     visitLevel.accept(tree, "");
+    validateAliasType(result);
     return result;
   }
 
+  private static void validateAliasType(Map<String, OpenSearchDataType> result) {
+    result.forEach(
+        (key, value) -> {
+          if (value instanceof OpenSearchAliasType && value.getOriginalPath().isPresent()) {
+            String originalPath = value.getOriginalPath().get();
+            result.put(key, new OpenSearchAliasType(originalPath, result.get(originalPath)));
+          }
+        });
+  }
+
   /**
    * Resolve type of identified from parsed mapping tree.
    *

opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java

@@ -165,7 +165,7 @@ public Map<String, String> getAliasMapping() {
     }
     if (aliasMapping == null) {
       aliasMapping =
-          cachedFieldOpenSearchTypes.entrySet().stream()
+          OpenSearchDataType.traverseAndFlatten(cachedFieldOpenSearchTypes).entrySet().stream()
               .filter(entry -> entry.getValue().getOriginalPath().isPresent())
               .collect(
                   Collectors.toUnmodifiableMap(

opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java

@@ -187,7 +187,7 @@ void get_index_mappings() throws IOException {
         () -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("employer")),
         // `employer` is a `text` with `fields`
         () -> assertTrue(((OpenSearchTextType) parsedTypes.get("employer")).getFields().size() > 0),
-        () -> assertEquals("TEXT", mapping.get("employer_alias").legacyTypeName()),
+        () -> assertEquals("TEXT", parsedTypes.get("employer_alias").legacyTypeName()),
         () ->
             assertEquals(
                 new OpenSearchAliasType("employer", OpenSearchTextType.of(MappingType.Text)),

opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java

@@ -191,7 +191,7 @@ void get_index_mappings() throws IOException {
         () -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("employer")),
         // `employer` is a `text` with `fields`
         () -> assertTrue(((OpenSearchTextType) parsedTypes.get("employer")).getFields().size() > 0),
-        () -> assertEquals("TEXT", mapping.get("employer_alias").legacyTypeName()),
+        () -> assertEquals("TEXT", parsedTypes.get("employer_alias").legacyTypeName()),
         () ->
             assertEquals(
                 new OpenSearchAliasType("employer", OpenSearchTextType.of(MappingType.Text)),

opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java

@@ -505,9 +505,14 @@ public void test_parseMapping_on_AliasType() {
             "col0", Map.of("type", "alias", "path", "col1"),
             "col1", Map.of("type", "text"),
             "col2", Map.of("type", "alias", "path", "col3"));
-    IllegalStateException exception =
-        assertThrows(
-            IllegalStateException.class, () -> OpenSearchDataType.parseMapping(indexMapping2));
-    assertEquals("Cannot find the path [col3] for alias type field [col2]", exception.getMessage());
+    assertEquals(
+        Map.of(
+            "col0",
+            new OpenSearchAliasType("col1", textType),
+            "col1",
+            textType,
+            "col2",
+            new OpenSearchAliasType("col1", OpenSearchDataType.of(MappingType.Invalid))),
+        OpenSearchDataType.parseMapping(indexMapping2));
   }
 }