Skip to content

Commit 38cd4d0

Browse files
chakru-rchakru-r
authored
fix(schematron): handle custom props for non-primitive types (#14326)
1 parent dd755d6 commit 38cd4d0

File tree

6 files changed

+70
-34
lines changed

6 files changed

+70
-34
lines changed

‎metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,9 @@ private void processArrayField(
393393
elementSchema.getDoc() != null ? elementSchema.getDoc() : field.doc(),
394394
null // TODO: What is the default value for an array element?
395395
);
396+
if (field.hasProps()) {
397+
field.getObjectProps().forEach(elementField::addProp);
398+
}
396399
processField(
397400
elementField,
398401
fieldPath,
@@ -445,6 +448,9 @@ private void processMapField(
445448
valueSchema.getDoc() != null ? valueSchema.getDoc() : field.doc(),
446449
null // TODO: What is the default value for a map value?
447450
); // Nullability for map values follows the nullability of the map itself
451+
if (field.hasProps()) {
452+
field.getObjectProps().forEach(valueField::addProp);
453+
}
448454
FieldPath valueFieldPath =
449455
fieldPath
450456
.popLast()
@@ -494,13 +500,14 @@ private void processUnionField(
494500
.findFirst()
495501
.orElseThrow(NoSuchElementException::new);
496502

503+
Schema.Field nonNullField = new Schema.Field(field.name(), nonNullSchema, field.doc());
504+
if (field.hasProps()) {
505+
// retain any props in the source field when constructing the non-null field.
506+
Map<String, Object> props = field.getObjectProps();
507+
props.forEach(nonNullField::addProp);
508+
}
497509
processField(
498-
new Schema.Field(field.name(), nonNullSchema, field.doc()),
499-
fieldPath.popLast(),
500-
defaultNullable,
501-
fields,
502-
true,
503-
visitedRecords);
510+
nonNullField, fieldPath.popLast(), defaultNullable, fields, true, visitedRecords);
504511
return;
505512
}
506513

@@ -549,6 +556,9 @@ private void processUnionField(
549556
unionSchema,
550557
unionSchema.getDoc() != null ? unionSchema.getDoc() : unionDescription,
551558
null);
559+
if (field.hasProps()) {
560+
field.getObjectProps().forEach(unionFieldInner::addProp);
561+
}
552562
log.debug(
553563
"TypeIndex: {}, Union Field path : {}, Doc: {}",
554564
typeIndex,

‎metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverterTest.java

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -168,14 +168,16 @@ void testComplexMaps() throws IOException {
168168
new SchemaFieldDataType()
169169
.setType(
170170
SchemaFieldDataType.Type.create(
171-
new MapType().setKeyType("string").setValueType("ComplexType"))));
171+
new MapType().setKeyType("string").setValueType("ComplexType"))),
172+
"{\"custom-prop\":\"custom-value\"}");
172173
assertSchemaField(
173174
schema.getFields().get(2),
174175
"[version=2.0].[type=MapType].[type=map].[type=ComplexType].mapOfComplexType.[type=string].field1",
175176
"string",
176177
false,
177178
false,
178-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
179+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())),
180+
"{\"custom-prop-1\":\"custom-value-1\"}");
179181
assertSchemaField(
180182
schema.getFields().get(3),
181183
"[version=2.0].[type=MapType].[type=map].[type=ComplexType].mapOfComplexType.[type=int].field2",
@@ -209,21 +211,24 @@ void testComplexMaps() throws IOException {
209211
new SchemaFieldDataType()
210212
.setType(
211213
SchemaFieldDataType.Type.create(
212-
new MapType().setKeyType("string").setValueType("union"))));
214+
new MapType().setKeyType("string").setValueType("union"))),
215+
"{\"custom-prop\":\"custom-value\"}");
213216
assertSchemaField(
214217
schema.getFields().get(7),
215218
"[version=2.0].[type=MapType].[type=map].[type=union].[type=ComplexTypeNullable].mapOfNullableComplexType",
216219
"ComplexTypeNullable",
217220
false,
218221
false,
219-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
222+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())),
223+
"{\"custom-prop\":\"custom-value\"}");
220224
assertSchemaField(
221225
schema.getFields().get(8),
222226
"[version=2.0].[type=MapType].[type=map].[type=union].[type=ComplexTypeNullable].mapOfNullableComplexType.[type=string].field1",
223227
"string",
224228
false,
225229
false,
226-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
230+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())),
231+
"{\"custom-prop-1\":\"custom-value-1\"}");
227232
assertSchemaField(
228233
schema.getFields().get(9),
229234
"[version=2.0].[type=MapType].[type=map].[type=union].[type=ComplexTypeNullable].mapOfNullableComplexType.[type=int].field2",
@@ -296,7 +301,8 @@ void testComplexArrays() throws IOException {
296301
new SchemaFieldDataType()
297302
.setType(
298303
SchemaFieldDataType.Type.create(
299-
new ArrayType().setNestedType(new StringArray("string")))));
304+
new ArrayType().setNestedType(new StringArray("string")))),
305+
"{\"custom-prop\":\"custom-value\"}");
300306
assertSchemaField(
301307
schema.getFields().get(1),
302308
"[version=2.0].[type=ArrayType].[type=array].[type=map].arrayOfMap",
@@ -306,7 +312,8 @@ void testComplexArrays() throws IOException {
306312
new SchemaFieldDataType()
307313
.setType(
308314
SchemaFieldDataType.Type.create(
309-
new MapType().setKeyType("string").setValueType("string"))));
315+
new MapType().setKeyType("string").setValueType("string"))),
316+
"{\"custom-prop\":\"custom-value\"}");
310317
assertSchemaField(
311318
schema.getFields().get(2),
312319
"[version=2.0].[type=ArrayType].[type=array].[type=ComplexType].arrayOfRecord",
@@ -316,21 +323,24 @@ void testComplexArrays() throws IOException {
316323
new SchemaFieldDataType()
317324
.setType(
318325
SchemaFieldDataType.Type.create(
319-
new ArrayType().setNestedType(new StringArray("ComplexType")))));
326+
new ArrayType().setNestedType(new StringArray("ComplexType")))),
327+
"{\"custom-prop\":\"custom-value\"}");
320328
assertSchemaField(
321329
schema.getFields().get(3),
322330
"[version=2.0].[type=ArrayType].[type=array].[type=ComplexType].arrayOfRecord.[type=string].field1",
323331
"string",
324332
false,
325333
false,
326-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
334+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())),
335+
"{\"custom-prop-1\":\"custom-value-1\"}");
327336
assertSchemaField(
328337
schema.getFields().get(4),
329338
"[version=2.0].[type=ArrayType].[type=array].[type=ComplexType].arrayOfRecord.[type=int].field2",
330339
"int",
331340
false,
332341
false,
333-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
342+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())),
343+
"{\"custom-prop-2\":\"custom-value-2\"}");
334344
assertSchemaField(
335345
schema.getFields().get(5),
336346
"[version=2.0].[type=ArrayType].[type=array].[type=array].arrayOfArray",
@@ -445,7 +455,8 @@ void testComplexStructs() throws IOException {
445455
"string",
446456
false,
447457
false,
448-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
458+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())),
459+
"{\"custom-prop\":\"custom-value\"}");
449460
assertSchemaField(
450461
schema.getFields().get(2),
451462
"[version=2.0].[type=StructType].[type=ComplexStruct].structField.[type=int].fieldInt",
@@ -556,28 +567,32 @@ void testComplexUnions() throws IOException {
556567
new SchemaFieldDataType()
557568
.setType(
558569
SchemaFieldDataType.Type.create(
559-
new UnionType().setNestedTypes(new StringArray("union")))));
570+
new UnionType().setNestedTypes(new StringArray("union")))),
571+
"{\"custom-prop\":\"custom-value\"}");
560572
assertSchemaField(
561573
schema.getFields().get(1),
562574
"[version=2.0].[type=UnionType].[type=union].[type=string].fieldUnionNullablePrimitives",
563575
"string",
564576
false,
565577
false,
566-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
578+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())),
579+
"{\"custom-prop\":\"custom-value\"}");
567580
assertSchemaField(
568581
schema.getFields().get(2),
569582
"[version=2.0].[type=UnionType].[type=union].[type=int].fieldUnionNullablePrimitives",
570583
"int",
571584
false,
572585
false,
573-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
586+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())),
587+
"{\"custom-prop\":\"custom-value\"}");
574588
assertSchemaField(
575589
schema.getFields().get(3),
576590
"[version=2.0].[type=UnionType].[type=union].[type=boolean].fieldUnionNullablePrimitives",
577591
"boolean",
578592
false,
579593
false,
580-
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new BooleanType())));
594+
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new BooleanType())),
595+
"{\"custom-prop\":\"custom-value\"}");
581596
assertSchemaField(
582597
schema.getFields().get(4),
583598
"[version=2.0].[type=UnionType].[type=union].fieldUnionComplexTypes",
@@ -955,9 +970,9 @@ private void assertSchemaField(
955970
assertEquals(field.isNullable(), expectedNullable);
956971
assertEquals(field.isIsPartOfKey(), expectedIsPartOfKey);
957972
assertEquals(field.getType(), expectedType);
958-
if (expectedJsonProps != null) {
959-
assertEquals(field.getJsonProps(), expectedJsonProps);
960-
}
973+
assertEquals(
974+
field.getJsonProps(),
975+
expectedJsonProps); // ensure this is null if there are no custom properties
961976
}
962977

963978
private Schema readAvroSchema(String schemaFileName) throws IOException {

‎metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_arrays.avsc

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
"name": "arrayOfString",
77
"type": {
88
"type": "array",
9-
"items": "string"
9+
"items": "string",
10+
"custom-prop": "custom-value"
1011
}
1112
},
1213
{
@@ -15,7 +16,8 @@
1516
"type": "array",
1617
"items": {
1718
"type": "map",
18-
"values": "string"
19+
"values": "string",
20+
"custom-prop": "custom-value"
1921
}
2022
}
2123
},
@@ -26,14 +28,17 @@
2628
"items": {
2729
"type": "record",
2830
"name": "ComplexType",
31+
"custom-prop": "custom-value",
2932
"fields": [
3033
{
3134
"name": "field1",
32-
"type": "string"
35+
"type": "string",
36+
"custom-prop-1": "custom-value-1"
3337
},
3438
{
3539
"name": "field2",
36-
"type": "int"
40+
"type": "int",
41+
"custom-prop-2": "custom-value-2"
3742
}
3843
]
3944
}

‎metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_maps.avsc

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,17 @@
1919
"fields": [
2020
{
2121
"name": "field1",
22-
"type": "string"
22+
"type": "string",
23+
"custom-prop-1": "custom-value-1"
2324
},
2425
{
2526
"name": "field2",
2627
"type": "int"
2728
}
2829
]
2930
}
30-
}
31+
},
32+
"custom-prop": "custom-value"
3133
},
3234
{
3335
"name": "mapOfNullableString",
@@ -46,15 +48,17 @@
4648
"fields": [
4749
{
4850
"name": "field1",
49-
"type": "string"
51+
"type": "string",
52+
"custom-prop-1": "custom-value-1"
5053
},
5154
{
5255
"name": "field2",
5356
"type": "int"
5457
}
5558
]
5659
}]
57-
}
60+
},
61+
"custom-prop": "custom-value"
5862
},
5963
{
6064
"name": "mapOfArray",

‎metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_structs.avsc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
"fields": [
1111
{
1212
"name": "fieldString",
13-
"type": "string"
13+
"type": "string",
14+
"custom-prop": "custom-value"
1415
},
1516
{
1617
"name": "fieldInt",

‎metadata-integration/java/datahub-schematron/lib/src/test/resources/complex_unions.avsc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
"string",
1010
"int",
1111
"boolean"
12-
]
12+
],
13+
"custom-prop": "custom-value"
1314
},
1415
{
1516
"name": "fieldUnionComplexTypes",

0 commit comments

Comments
 (0)