diff --git a/docs/HTTP-batchsink.md b/docs/HTTP-batchsink.md index 7f3081bf..5d5caf61 100644 --- a/docs/HTTP-batchsink.md +++ b/docs/HTTP-batchsink.md @@ -87,6 +87,36 @@ Skip on error - Ignores erroneous records. **Wait Time Between Request:** Time in milliseconds to wait between HTTP requests. Defaults to 0. (Macro enabled) +### Authentication + +* **OAuth2** + * **Grant Type:** Which OAuth2 grant type flow is used. It can be Refresh Token or Client Credentials Flow. + * **Client Authentication:** Send OAuth2 Credentials in the Request Body or as Query Parameter or as Basic Auth + Header. + * **Auth URL:** Endpoint for the authorization server used to retrieve the authorization code. + * **Token URL:** Endpoint for the resource server, which exchanges the authorization code for an access token. + * **Client ID:** Client identifier obtained during the Application registration process. + * **Client Secret:** Client secret obtained during the Application registration process. + * **Scopes:** Scope of the access request, which might have multiple space-separated values. + * **Refresh Token:** Token used to receive accessToken, which is end product of OAuth2. +* **Service Account** - service account key used for authorization + * **File Path**: Path on the local file system of the service account key used for + authorization. Can be set to 'auto-detect' when running on a Dataproc cluster. + When running on other clusters, the file must be present on every node in the cluster. + * **JSON**: Contents of the service account JSON file. + * **Scope**: The additional Google credential scopes required to access entered url, cloud-platform is included by + default, visit https://developers.google.com/identity/protocols/oauth2/scopes for more information. + * Scope example: + +``` +https://www.googleapis.com/auth/bigquery +https://www.googleapis.com/auth/cloud-platform +``` + +* **Basic Authentication** + * **Username:** Username for basic authentication. + * **Password:** Password for basic authentication. + ### HTTP Proxy **Proxy URL:** Proxy URL. Must contain a protocol, address and port. diff --git a/docs/HTTP-batchsource.md b/docs/HTTP-batchsource.md index 45b85127..f3eb3a45 100644 --- a/docs/HTTP-batchsource.md +++ b/docs/HTTP-batchsource.md @@ -214,6 +214,9 @@ The newline delimiter cannot be within quotes. ### Authentication * **OAuth2** + * **Grant Type:** Which OAuth2 grant type flow is used. It can be Refresh Token or Client Credentials Flow. + * **Client Authentication:** Send OAuth2 Credentials in the Request Body or as Query Parameter or as Basic Auth + Header. * **Auth URL:** Endpoint for the authorization server used to retrieve the authorization code. * **Token URL:** Endpoint for the resource server, which exchanges the authorization code for an access token. * **Client ID:** Client identifier obtained during the Application registration process. diff --git a/docs/HTTP-streamingsource.md b/docs/HTTP-streamingsource.md index 9c9ce140..c8153d83 100644 --- a/docs/HTTP-streamingsource.md +++ b/docs/HTTP-streamingsource.md @@ -209,6 +209,9 @@ can be omitted as long as the field is present in schema. ### Authentication * **OAuth2** + * **Grant Type:** Which OAuth2 grant type flow is used. It can be Refresh Token or Client Credentials Flow. + * **Client Authentication:** Send OAuth2 Credentials in the Request Body or as Query Parameter or as Basic Auth + Header. * **Auth URL:** Endpoint for the authorization server used to retrieve the authorization code. * **Token URL:** Endpoint for the resource server, which exchanges the authorization code for an access token. * **Client ID:** Client identifier obtained during the Application registration process. diff --git a/src/main/java/io/cdap/plugin/http/common/OAuth2ClientAuthentication.java b/src/main/java/io/cdap/plugin/http/common/OAuth2ClientAuthentication.java index 09f13116..e256032d 100644 --- a/src/main/java/io/cdap/plugin/http/common/OAuth2ClientAuthentication.java +++ b/src/main/java/io/cdap/plugin/http/common/OAuth2ClientAuthentication.java @@ -23,7 +23,8 @@ */ public enum OAuth2ClientAuthentication implements EnumWithValue { BODY("body", "Body"), - REQUEST_PARAMETER("request_parameter", "Request Parameter"); + REQUEST_PARAMETER("request_parameter", "Request Parameter"), + BASIC_AUTH_HEADER("basic_auth_header", "Basic Auth Header"); private final String value; private final String label; @@ -37,21 +38,22 @@ public enum OAuth2ClientAuthentication implements EnumWithValue { * Determines the OAuth2 client authentication method based on the provided input. * *
This method checks if the given client authentication type matches the predefined - * BODY authentication type. If it matches, the method returns the BODY authentication. Otherwise, - * it defaults to REQUEST_PARAMETER authentication.
+ * authentication type. If it matches, the method returns the same authentication. Otherwise, + * it defaults to BASIC_AUTH_HEADER authentication. * * @param clientAuthentication The client authentication type as a {@link String}. It can be - * either the value or the label of the BODY authentication method. - * @return {@link OAuth2ClientAuthentication} The corresponding authentication type. Returns - * {@code BODY} if the input matches its value or label; otherwise, returns - * {@code REQUEST_PARAMETER}. + * either the value or the label of the authentication method. + * @return {@link OAuth2ClientAuthentication} The corresponding authentication type. */ public static OAuth2ClientAuthentication getClientAuthentication(String clientAuthentication) { if (Objects.equals(clientAuthentication, BODY.getValue()) || Objects.equals( clientAuthentication, BODY.getLabel())) { return BODY; - } else { + } else if (Objects.equals(clientAuthentication, REQUEST_PARAMETER.getValue()) || Objects.equals( + clientAuthentication, REQUEST_PARAMETER.getLabel())) { return REQUEST_PARAMETER; + } else { + return BASIC_AUTH_HEADER; } } diff --git a/src/main/java/io/cdap/plugin/http/common/http/OAuthUtil.java b/src/main/java/io/cdap/plugin/http/common/http/OAuthUtil.java index e867fc05..02799f5b 100644 --- a/src/main/java/io/cdap/plugin/http/common/http/OAuthUtil.java +++ b/src/main/java/io/cdap/plugin/http/common/http/OAuthUtil.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableSet; import com.google.gson.JsonElement; import io.cdap.plugin.http.common.BaseHttpConfig; -import io.cdap.plugin.http.common.OAuth2ClientAuthentication; import io.cdap.plugin.http.common.OAuth2GrantType; import io.cdap.plugin.http.common.pagination.page.JSONUtil; import io.cdap.plugin.http.source.common.BaseHttpSourceConfig; @@ -31,6 +30,7 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; @@ -44,9 +44,9 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Base64; import java.util.Date; import java.util.List; -import java.util.Objects; import javax.annotation.Nullable; /** @@ -54,6 +54,12 @@ */ public class OAuthUtil { + private static final String PARAM_GRANT_TYPE = "grant_type"; + private static final String PARAM_CLIENT_ID = "client_id"; + private static final String PARAM_CLIENT_SECRET = "client_secret"; + private static final String PARAM_REFRESH_TOKEN = "refresh_token"; + private static final String PARAM_SCOPE = "scope"; + /** * Get Authorization header based on the config parameters provided * @@ -116,8 +122,8 @@ public static AccessToken getAccessToken(CloseableHttpClient httpclient, BaseHtt * Retrieves an OAuth2 access token using the Client Credentials grant type. * *This method constructs an HTTP POST request to fetch an access token from the authorization - * server. The client authentication method (either "BODY" or "REQUEST") determines whether client - * credentials are sent in the request body or as query parameters in the URL.
+ * server. The client authentication method (either "BODY" or "REQUEST" or "BASIC_AUTH_HEADER") determines whether + * client credentials are sent in the request body or as query parameters or as basic auth header. * *Steps:
* 1. If client authentication is set to "BODY": - Constructs a URI using the token URL. - Adds
@@ -127,7 +133,11 @@ public static AccessToken getAccessToken(CloseableHttpClient httpclient, BaseHtt
* 2. If client authentication is set to "REQUEST": - Constructs a URI with client credentials as
* query parameters. - Creates an HTTP POST request with the URI.
* 404. That’s an error.\n";
+
+ Mockito.when(EntityUtils.toString(entity, "UTF-8")).thenReturn(response);
+ PowerMockito.mockStatic(PaginationIteratorFactory.class);
+ BaseHttpPaginationIterator baseHttpPaginationIterator = Mockito.mock(
+ BaseHttpPaginationIterator.class);
+ PowerMockito.when(PaginationIteratorFactory.createInstance(Mockito.any(), Mockito.any()))
+ .thenReturn(baseHttpPaginationIterator);
+ PowerMockito.when(baseHttpPaginationIterator.supportsSkippingPages()).thenReturn(true);
+ PowerMockito.mockStatic(HttpClients.class);
+ HttpClientBuilder httpClientBuilder = Mockito.mock(HttpClientBuilder.class);
+ Mockito.when(HttpClients.custom()).thenReturn(httpClientBuilder);
+ Mockito.when(httpClientBuilder.build()).thenReturn(httpClientMock);
+ config.validate(collector);
+ Assert.assertEquals(1, collector.getValidationFailures().size());
+ }
}
diff --git a/widgets/HTTP-batchsink.json b/widgets/HTTP-batchsink.json
index a205cd92..6fd3fa4e 100644
--- a/widgets/HTTP-batchsink.json
+++ b/widgets/HTTP-batchsink.json
@@ -294,7 +294,8 @@
"widget-attributes": {
"values": [
"Body",
- "Request Parameter"
+ "Request Parameter",
+ "Basic Auth Header"
],
"default": "Body"
}
diff --git a/widgets/HTTP-batchsource.json b/widgets/HTTP-batchsource.json
index 44475524..3b4153cc 100644
--- a/widgets/HTTP-batchsource.json
+++ b/widgets/HTTP-batchsource.json
@@ -184,7 +184,8 @@
"widget-attributes": {
"values": [
"Body",
- "Request Parameter"
+ "Request Parameter",
+ "Basic Auth Header"
],
"default": "Body"
}
diff --git a/widgets/HTTP-streamingsource.json b/widgets/HTTP-streamingsource.json
index 70e17544..88e6833e 100644
--- a/widgets/HTTP-streamingsource.json
+++ b/widgets/HTTP-streamingsource.json
@@ -152,7 +152,8 @@
"widget-attributes": {
"values": [
"Body",
- "Request Parameter"
+ "Request Parameter",
+ "Basic Auth Header"
],
"default": "Body"
}
- * 3. Calls `fetchAccessToken(httpclient,httppost)` to execute the request and retrieve the
+ * 3. If client authentication is set to "BASIC_AUTH_HEADER": - Constructs a URI with client credentials first
+ * concatenated and encoded to Base64 and passed a Basic Authorization Header and
+ * grant type and scope as part of body.
+ *
+ * 4. Calls `fetchAccessToken(httpclient,httppost)` to execute the request and retrieve the
* token.
*
* @param httpclient The HTTP client to execute the request.
@@ -139,29 +149,51 @@ public static AccessToken getAccessTokenByClientCredentials(CloseableHttpClient
BaseHttpConfig config) throws IOException {
URI uri;
HttpPost httppost;
+
try {
- if (Objects.equals(config.getOauth2ClientAuthentication().getValue(),
- OAuth2ClientAuthentication.BODY.getValue())) {
- uri = new URIBuilder(config.getTokenUrl()).build();
- List