Skip to content

Commit 8430c2a

Browse files
aws-sdk-dotnet-automationaws-sdk-dotnet-automation
committed
AWS Glue Zero ETL now supports On-demand snapshot load
1 parent 963dd0b commit 8430c2a

File tree

11 files changed

+129
-10
lines changed

11 files changed

+129
-10
lines changed

‎generator/ServiceModels/glue/glue-2017-03-31.api.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6300,6 +6300,7 @@
63006300
"max":20,
63016301
"min":1
63026302
},
6303+
"ContinuousSync":{"type":"boolean"},
63036304
"Crawl":{
63046305
"type":"structure",
63056306
"members":{
@@ -11238,7 +11239,8 @@
1123811239
"type":"structure",
1123911240
"members":{
1124011241
"RefreshInterval":{"shape":"String128"},
11241-
"SourceProperties":{"shape":"IntegrationSourcePropertiesMap"}
11242+
"SourceProperties":{"shape":"IntegrationSourcePropertiesMap"},
11243+
"ContinuousSync":{"shape":"ContinuousSync"}
1124211244
}
1124311245
},
1124411246
"IntegrationConflictOperationFault":{
@@ -12617,6 +12619,7 @@
1261712619
"IntegrationIdentifier":{"shape":"String128"},
1261812620
"Description":{"shape":"IntegrationDescription"},
1261912621
"DataFilter":{"shape":"String2048"},
12622+
"IntegrationConfig":{"shape":"IntegrationConfig"},
1262012623
"IntegrationName":{"shape":"String128"}
1262112624
}
1262212625
},
@@ -12642,7 +12645,8 @@
1264212645
"Status":{"shape":"IntegrationStatus"},
1264312646
"CreateTime":{"shape":"IntegrationTimestamp"},
1264412647
"Errors":{"shape":"IntegrationErrorList"},
12645-
"DataFilter":{"shape":"String2048"}
12648+
"DataFilter":{"shape":"String2048"},
12649+
"IntegrationConfig":{"shape":"IntegrationConfig"}
1264612650
}
1264712651
},
1264812652
"MongoDBTarget":{

‎generator/ServiceModels/glue/glue-2017-03-31.docs.json‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2068,6 +2068,12 @@
20682068
"GetCustomEntityTypeResponse$ContextWords": "<p>A list of context words if specified when you created the custom pattern. If none of these context words are found within the vicinity of the regular expression the data will not be detected as sensitive data.</p>"
20692069
}
20702070
},
2071+
"ContinuousSync": {
2072+
"base": null,
2073+
"refs": {
2074+
"IntegrationConfig$ContinuousSync": "<p>Enables continuous synchronization for on-demand data extractions from SaaS applications to Amazon Web Services data services like Amazon Redshift and Amazon S3.</p>"
2075+
}
2076+
},
20712077
"Crawl": {
20722078
"base": "<p>The details of a crawl in the workflow.</p>",
20732079
"refs": {
@@ -5800,7 +5806,9 @@
58005806
"CreateIntegrationRequest$IntegrationConfig": "<p>The configuration settings.</p>",
58015807
"CreateIntegrationResponse$IntegrationConfig": "<p>The configuration settings.</p>",
58025808
"InboundIntegration$IntegrationConfig": "<p>Properties associated with the integration.</p>",
5803-
"Integration$IntegrationConfig": "<p>Properties associated with the integration.</p>"
5809+
"Integration$IntegrationConfig": "<p>Properties associated with the integration.</p>",
5810+
"ModifyIntegrationRequest$IntegrationConfig": null,
5811+
"ModifyIntegrationResponse$IntegrationConfig": null
58045812
}
58055813
},
58065814
"IntegrationConflictOperationFault": {
@@ -9948,7 +9956,7 @@
99489956
"IntegrationFilter$Name": "<p>The name of the filter.</p>",
99499957
"IntegrationFilterValues$member": null,
99509958
"IntegrationPartition$FieldName": "<p>The field name used to partition data on the target. Avoid using columns that have unique values for each row (for example, `LastModifiedTimestamp`, `SystemModTimeStamp`) as the partition column. These columns are not suitable for partitioning because they create a large number of small partitions, which can lead to performance issues.</p>",
9951-
"IntegrationPartition$FunctionSpec": "<p>Specifies the function used to partition data on the target. The only accepted value for this parameter is `'identity'` (string). The `'identity'` function ensures that the data partitioning on the target follows the same scheme as the source. In other words, the partitioning structure of the source data is preserved in the target destination.</p>",
9959+
"IntegrationPartition$FunctionSpec": "<p>Specifies the function used to partition data on the target. The accepted values for this parameter are:</p> <ul> <li> <p> <code>identity</code> - Uses source values directly without transformation</p> </li> <li> <p> <code>year</code> - Extracts the year from timestamp values (e.g., 2023)</p> </li> <li> <p> <code>month</code> - Extracts the month from timestamp values (e.g., 2023-01)</p> </li> <li> <p> <code>day</code> - Extracts the day from timestamp values (e.g., 2023-01-15)</p> </li> <li> <p> <code>hour</code> - Extracts the hour from timestamp values (e.g., 2023-01-15-14)</p> </li> </ul>",
99529960
"IntegrationPartition$ConversionSpec": "<p>Specifies the timestamp format of the source data. Valid values are:</p> <ul> <li> <p> <code>epoch_sec</code> - Unix epoch timestamp in seconds</p> </li> <li> <p> <code>epoch_milli</code> - Unix epoch timestamp in milliseconds</p> </li> <li> <p> <code>iso</code> - ISO 8601 formatted timestamp</p> </li> </ul> <note> <p> Only specify <code>ConversionSpec</code> when using timestamp-based partition functions (year, month, day, or hour). Glue Zero-ETL uses this parameter to correctly transform source data into timestamp format before partitioning. </p> <p> Do not use high-cardinality columns with the <code>identity</code> partition function. High-cardinality columns include: </p> <ul> <li> <p>Primary keys</p> </li> <li> <p>Timestamp fields (such as <code>LastModifiedTimestamp</code>, <code>CreatedDate</code>)</p> </li> <li> <p>System-generated timestamps</p> </li> </ul> <p> Using high-cardinality columns with identity partitioning creates many small partitions, which can significantly degrade ingestion performance. </p> </note>",
99539961
"ModifyIntegrationRequest$IntegrationIdentifier": "<p>The Amazon Resource Name (ARN) for the integration.</p>",
99549962
"ModifyIntegrationRequest$IntegrationName": "<p>A unique name for an integration in Glue.</p>",

‎generator/ServiceModels/glue/glue-2017-03-31.normal.json‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8254,6 +8254,7 @@
82548254
"max":20,
82558255
"min":1
82568256
},
8257+
"ContinuousSync":{"type":"boolean"},
82578258
"Crawl":{
82588259
"type":"structure",
82598260
"members":{
@@ -17643,6 +17644,10 @@
1764317644
"SourceProperties":{
1764417645
"shape":"IntegrationSourcePropertiesMap",
1764517646
"documentation":"<p> A collection of key-value pairs that specify additional properties for the integration source. These properties provide configuration options that can be used to customize the behavior of the ODB source during data integration operations. </p>"
17647+
},
17648+
"ContinuousSync":{
17649+
"shape":"ContinuousSync",
17650+
"documentation":"<p>Enables continuous synchronization for on-demand data extractions from SaaS applications to Amazon Web Services data services like Amazon Redshift and Amazon S3.</p>"
1764617651
}
1764717652
},
1764817653
"documentation":"<p>Properties associated with the integration.</p>"
@@ -17725,7 +17730,7 @@
1772517730
},
1772617731
"FunctionSpec":{
1772717732
"shape":"String128",
17728-
"documentation":"<p>Specifies the function used to partition data on the target. The only accepted value for this parameter is `'identity'` (string). The `'identity'` function ensures that the data partitioning on the target follows the same scheme as the source. In other words, the partitioning structure of the source data is preserved in the target destination.</p>"
17733+
"documentation":"<p>Specifies the function used to partition data on the target. The accepted values for this parameter are:</p> <ul> <li> <p> <code>identity</code> - Uses source values directly without transformation</p> </li> <li> <p> <code>year</code> - Extracts the year from timestamp values (e.g., 2023)</p> </li> <li> <p> <code>month</code> - Extracts the month from timestamp values (e.g., 2023-01)</p> </li> <li> <p> <code>day</code> - Extracts the day from timestamp values (e.g., 2023-01-15)</p> </li> <li> <p> <code>hour</code> - Extracts the hour from timestamp values (e.g., 2023-01-15-14)</p> </li> </ul>"
1772917734
},
1773017735
"ConversionSpec":{
1773117736
"shape":"String128",
@@ -20297,6 +20302,7 @@
2029720302
"shape":"String2048",
2029820303
"documentation":"<p>Selects source tables for the integration using Maxwell filter syntax.</p>"
2029920304
},
20305+
"IntegrationConfig":{"shape":"IntegrationConfig"},
2030020306
"IntegrationName":{
2030120307
"shape":"String128",
2030220308
"documentation":"<p>A unique name for an integration in Glue.</p>"
@@ -20361,7 +20367,8 @@
2036120367
"DataFilter":{
2036220368
"shape":"String2048",
2036320369
"documentation":"<p>Selects source tables for the integration using Maxwell filter syntax.</p>"
20364-
}
20370+
},
20371+
"IntegrationConfig":{"shape":"IntegrationConfig"}
2036520372
}
2036620373
},
2036720374
"MongoDBTarget":{

‎sdk/src/Services/Glue/Generated/Model/IntegrationConfig.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,29 @@ namespace Amazon.Glue.Model
3434
/// </summary>
3535
public partial class IntegrationConfig
3636
{
37+
private bool? _continuousSync;
3738
private string _refreshInterval;
3839
private Dictionary<string, string> _sourceProperties = AWSConfigs.InitializeCollections ? new Dictionary<string, string>() : null;
3940

41+
/// <summary>
42+
/// Gets and sets the property ContinuousSync.
43+
/// <para>
44+
/// Enables continuous synchronization for on-demand data extractions from SaaS applications
45+
/// to Amazon Web Services data services like Amazon Redshift and Amazon S3.
46+
/// </para>
47+
/// </summary>
48+
public bool? ContinuousSync
49+
{
50+
get { return this._continuousSync; }
51+
set { this._continuousSync = value; }
52+
}
53+
54+
// Check to see if ContinuousSync property is set
55+
internal bool IsSetContinuousSync()
56+
{
57+
return this._continuousSync.HasValue;
58+
}
59+
4060
/// <summary>
4161
/// Gets and sets the property RefreshInterval.
4262
/// <para>

‎sdk/src/Services/Glue/Generated/Model/IntegrationPartition.cs‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,11 +123,30 @@ internal bool IsSetFieldName()
123123
/// <summary>
124124
/// Gets and sets the property FunctionSpec.
125125
/// <para>
126-
/// Specifies the function used to partition data on the target. The only accepted value
127-
/// for this parameter is `'identity'` (string). The `'identity'` function ensures that
128-
/// the data partitioning on the target follows the same scheme as the source. In other
129-
/// words, the partitioning structure of the source data is preserved in the target destination.
126+
/// Specifies the function used to partition data on the target. The accepted values for
127+
/// this parameter are:
130128
/// </para>
129+
/// <ul> <li>
130+
/// <para>
131+
/// <c>identity</c> - Uses source values directly without transformation
132+
/// </para>
133+
/// </li> <li>
134+
/// <para>
135+
/// <c>year</c> - Extracts the year from timestamp values (e.g., 2023)
136+
/// </para>
137+
/// </li> <li>
138+
/// <para>
139+
/// <c>month</c> - Extracts the month from timestamp values (e.g., 2023-01)
140+
/// </para>
141+
/// </li> <li>
142+
/// <para>
143+
/// <c>day</c> - Extracts the day from timestamp values (e.g., 2023-01-15)
144+
/// </para>
145+
/// </li> <li>
146+
/// <para>
147+
/// <c>hour</c> - Extracts the hour from timestamp values (e.g., 2023-01-15-14)
148+
/// </para>
149+
/// </li> </ul>
131150
/// </summary>
132151
[AWSProperty(Min=1, Max=128)]
133152
public string FunctionSpec

‎sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/IntegrationConfigMarshaller.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ public void Marshall(IntegrationConfig requestObject, JsonMarshallerContext cont
4646
{
4747
if(requestObject == null)
4848
return;
49+
if(requestObject.IsSetContinuousSync())
50+
{
51+
context.Writer.WritePropertyName("ContinuousSync");
52+
context.Writer.WriteBooleanValue(requestObject.ContinuousSync.Value);
53+
}
54+
4955
if(requestObject.IsSetRefreshInterval())
5056
{
5157
context.Writer.WritePropertyName("RefreshInterval");

‎sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/IntegrationConfigUnmarshaller.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ public IntegrationConfig Unmarshall(JsonUnmarshallerContext context, ref Streami
5656
int targetDepth = context.CurrentDepth;
5757
while (context.ReadAtDepth(targetDepth, ref reader))
5858
{
59+
if (context.TestExpression("ContinuousSync", targetDepth))
60+
{
61+
var unmarshaller = NullableBoolUnmarshaller.Instance;
62+
unmarshalledObject.ContinuousSync = unmarshaller.Unmarshall(context, ref reader);
63+
continue;
64+
}
5965
if (context.TestExpression("RefreshInterval", targetDepth))
6066
{
6167
var unmarshaller = StringUnmarshaller.Instance;

‎sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/ModifyIntegrationRequestMarshaller.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,17 @@ public IRequest Marshall(ModifyIntegrationRequest publicRequest)
8787
context.Writer.WriteStringValue(publicRequest.Description);
8888
}
8989

90+
if(publicRequest.IsSetIntegrationConfig())
91+
{
92+
context.Writer.WritePropertyName("IntegrationConfig");
93+
context.Writer.WriteStartObject();
94+
95+
var marshaller = IntegrationConfigMarshaller.Instance;
96+
marshaller.Marshall(publicRequest.IntegrationConfig, context);
97+
98+
context.Writer.WriteEndObject();
99+
}
100+
90101
if(publicRequest.IsSetIntegrationIdentifier())
91102
{
92103
context.Writer.WritePropertyName("IntegrationIdentifier");

‎sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/ModifyIntegrationResponseUnmarshaller.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext cont
8888
response.IntegrationArn = unmarshaller.Unmarshall(context, ref reader);
8989
continue;
9090
}
91+
if (context.TestExpression("IntegrationConfig", targetDepth))
92+
{
93+
var unmarshaller = IntegrationConfigUnmarshaller.Instance;
94+
response.IntegrationConfig = unmarshaller.Unmarshall(context, ref reader);
95+
continue;
96+
}
9197
if (context.TestExpression("IntegrationName", targetDepth))
9298
{
9399
var unmarshaller = StringUnmarshaller.Instance;

‎sdk/src/Services/Glue/Generated/Model/ModifyIntegrationRequest.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public partial class ModifyIntegrationRequest : AmazonGlueRequest
3737
{
3838
private string _dataFilter;
3939
private string _description;
40+
private IntegrationConfig _integrationConfig;
4041
private string _integrationIdentifier;
4142
private string _integrationName;
4243

@@ -78,6 +79,21 @@ internal bool IsSetDescription()
7879
return this._description != null;
7980
}
8081

82+
/// <summary>
83+
/// Gets and sets the property IntegrationConfig.
84+
/// </summary>
85+
public IntegrationConfig IntegrationConfig
86+
{
87+
get { return this._integrationConfig; }
88+
set { this._integrationConfig = value; }
89+
}
90+
91+
// Check to see if IntegrationConfig property is set
92+
internal bool IsSetIntegrationConfig()
93+
{
94+
return this._integrationConfig != null;
95+
}
96+
8197
/// <summary>
8298
/// Gets and sets the property IntegrationIdentifier.
8399
/// <para>

0 commit comments

Comments
 (0)