Merge branch 'main' into dev-jl-version2

Author: thejoeker12

README.md

@@ -2,11 +2,11 @@
 
 This Go module offers a sophisticated HTTP client designed for seamless API interactions, with a strong emphasis on concurrency management, robust error handling, extensive logging, and adaptive rate limiting. It's particularly suitable for applications requiring high-throughput API interactions with complex authentication and operational resilience.
 
-This client leverages API-specific SDKs to provide a comprehensive and consistent interface for interacting with various APIs, including Microsoft Graph, Jamf Pro, and others. It is designed to be easily extensible to support additional APIs and to be highly configurable to meet specific API requirements. It achieves this through using a modular design, with a core HTTP client and API-specific handlers that encapsulate the unique requirements of each API supported.
+This client is designed to be used with targetted SDK's and terraform providers only. As such the http client cannot be used without a supporting SDK and associated api integration plugin [go-api-http-client-integrations](https://github.com/deploymenttheory/go-api-http-client-integrations).
 
-This HTTP client is intended to be used with targetted SDK's and terraform providers only. As such the http client cannot be used without a supporting SDK.
+The plugin is required to provide the necessary API-specific handlers and configuration to the HTTP client. The plugin approach is designed to provide a consistent interface for interacting with various APIs, including Microsoft Graph, Jamf Pro, and others. It is easily extensible to support additional APIs and highly configurable to meet specific API requirements. It achieves this through using a modular design, with a core HTTP client and API-specific handlers that encapsulate the unique requirements of each API supported. Conseqently the client provides core common HTTP client functionality, such as rate limiting, logging, concurrency and responses while the plugin provides the API-specific logic, such as encoding and decoding requests, managing authentication endpoints, and handling API-specific logic.
 
-## Features
+## HTTP Client Features
 
 - **Comprehensive Authentication Support**: Robust support for various authentication schemes, including OAuth and Bearer Token, with built-in token management and validation.
 - **Advanced Concurrency Management**: An intelligent Concurrency Manager dynamically adjusts concurrent request limits to optimize throughput and adhere to API rate limits.
@@ -20,8 +20,19 @@ This HTTP client is intended to be used with targetted SDK's and terraform provi
 - **API Handler Interface**: Provides a flexible and extensible way to interact with different APIs, including encoding and decoding requests and responses, managing authentication endpoints, and handling API-specific logic.
 - **Configuration via JSON or Environment Variables**: The Go API HTTP Client supports configuration via JSON files or environment variables, providing flexibility in defining authentication credentials, API endpoints, logging settings, and other parameters.
 
-TBC
+## Getting Started
 
+This SDK requires an existing Go environment to build and run. If you do not have Go installed, you can download and install it from the [official Go website](https://golang.org/doc/install).
+
+
+
+## SDK Support
+
+This http client is used in conjuction with the following SDKs:
+
+- [go-api-sdk-m365](https://github.com/deploymenttheory/go-api-sdk-m365)
+
+- [go-api-sdk-jamfpro](https://github.com/deploymenttheory/go-api-sdk-jamfpro)
 
 ## Reporting Issues and Feedback
 
@@ -41,10 +52,6 @@ If you would like to become an active contributor to this repository or project,
 
 ## Learn More
 
-* [GitHub Documentation][GitHubDocs]
-* [Azure DevOps Documentation][AzureDevOpsDocs]
-* [Microsoft Azure Documentation][MicrosoftAzureDocs]
-
 <!-- References -->
 
 <!-- Local -->
@@ -53,13 +60,4 @@ If you would like to become an active contributor to this repository or project,
 [GitHubDocs]: <https://docs.github.com/>
 [AzureDevOpsDocs]: <https://docs.microsoft.com/en-us/azure/devops/?view=azure-devops>
 [GitHubIssues]: <https://github.com/segraef/Template/issues>
-[Contributing]: CONTRIBUTING.md
-
-<!-- External -->
-[Az]: <https://img.shields.io/powershellgallery/v/Az.svg?style=flat-square&label=Az>
-[AzGallery]: <https://www.powershellgallery.com/packages/Az/>
-[PowerShellCore]: <https://github.com/PowerShell/PowerShell/releases/latest>
-
-<!-- Docs -->
-[MicrosoftAzureDocs]: <https://docs.microsoft.com/en-us/azure/>
-[PowerShellDocs]: <https://docs.microsoft.com/en-us/powershell/>
+[Contributing]: CONTRIBUTING.md

httpclient/config_validation.go

@@ -3,37 +3,97 @@
 package httpclient
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
 	"time"
 )
 
 const (
-	DefaultLogLevelString            = "LogLevelInfo"
-	DefaultLogOutputFormatString     = "pretty"
-	DefaultLogConsoleSeparator       = "	"
-	DefaultLogExportPath             = "/defaultlogs"
-	DefaultMaxRetryAttempts          = 3
-	DefaultMaxConcurrentRequests     = 1
-	DefaultExportLogs                = false
-	DefaultHideSensitiveData         = false
-	DefaultEnableDynamicRateLimiting = false
-	DefaultCustomTimeout             = 5 * time.Second
-	DefaultTokenRefreshBufferPeriod  = 2 * time.Minute
-	DefaultTotalRetryDuration        = 5 * time.Minute
-	DefaultFollowRedirects           = false
-	DefaultMaxRedirects              = 5
+	DefaultLogLevelString              = "LogLevelInfo"
+	DefaultLogOutputFormatString       = "pretty"
+	DefaultLogConsoleSeparator         = "	"
+	DefaultLogExportPath               = "/defaultlogs"
+	DefaultMaxRetryAttempts            = 3
+	DefaultMaxConcurrentRequests       = 1
+	DefaultExportLogs                  = false
+	DefaultHideSensitiveData           = false
+	DefaultEnableDynamicRateLimiting   = false
+	DefaultCustomTimeout               = 5 * time.Second
+	DefaultTokenRefreshBufferPeriod    = 2 * time.Minute
+	DefaultTotalRetryDuration          = 5 * time.Minute
+	DefaultFollowRedirects             = false
+	DefaultMaxRedirects                = 5
+	DefaultEnableConcurrencyManagement = false
 )
 
-// TODO migrate all the loose strings
-
-// TODO LoadConfigFromFile Func
+// LoadConfigFromFile loads http client configuration settings from a JSON file.
 func LoadConfigFromFile(filepath string) (*ClientConfig, error) {
-	return nil, nil
+	absPath, err := validateFilePath(filepath)
+	if err != nil {
+		return nil, fmt.Errorf("invalid file path: %v", err)
+	}
+
+	file, err := os.Open(absPath)
+	if err != nil {
+		return nil, fmt.Errorf("could not open file: %v", err)
+	}
+	defer file.Close()
+
+	byteValue, err := io.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("could not read file: %v", err)
+	}
+
+	var config ClientConfig
+	err = json.Unmarshal(byteValue, &config)
+	if err != nil {
+		return nil, fmt.Errorf("could not unmarshal JSON: %v", err)
+	}
+
+	// Set default values for missing fields.
+	SetDefaultValuesClientConfig(&config)
+
+	return &config, nil
 }
 
-// TODO LoadConfigFromEnv Func
+// LoadConfigFromEnv loads HTTP client configuration settings from environment variables.
+// If any environment variables are not set, the default values defined in the constants are used instead.
 func LoadConfigFromEnv() (*ClientConfig, error) {
-	return nil, nil
+	config := &ClientConfig{
+		HideSensitiveData:           getEnvAsBool("HIDE_SENSITIVE_DATA", DefaultHideSensitiveData),
+		MaxRetryAttempts:            getEnvAsInt("MAX_RETRY_ATTEMPTS", DefaultMaxRetryAttempts),
+		MaxConcurrentRequests:       getEnvAsInt("MAX_CONCURRENT_REQUESTS", DefaultMaxConcurrentRequests),
+		EnableDynamicRateLimiting:   getEnvAsBool("ENABLE_DYNAMIC_RATE_LIMITING", DefaultEnableDynamicRateLimiting),
+		CustomTimeout:               getEnvAsDuration("CUSTOM_TIMEOUT", DefaultCustomTimeout),
+		TokenRefreshBufferPeriod:    getEnvAsDuration("TOKEN_REFRESH_BUFFER_PERIOD", DefaultTokenRefreshBufferPeriod),
+		TotalRetryDuration:          getEnvAsDuration("TOTAL_RETRY_DURATION", DefaultTotalRetryDuration),
+		FollowRedirects:             getEnvAsBool("FOLLOW_REDIRECTS", DefaultFollowRedirects),
+		MaxRedirects:                getEnvAsInt("MAX_REDIRECTS", DefaultMaxRedirects),
+		EnableConcurrencyManagement: getEnvAsBool("ENABLE_CONCURRENCY_MANAGEMENT", DefaultEnableConcurrencyManagement),
+	}
+
+	// Load custom cookies from environment variables.
+	customCookies := getEnvAsString("CUSTOM_COOKIES", "")
+	if customCookies != "" {
+		cookies := []*http.Cookie{}
+		for _, cookie := range strings.Split(customCookies, ";") {
+			parts := strings.SplitN(cookie, "=", 2)
+			if len(parts) == 2 {
+				cookies = append(cookies, &http.Cookie{
+					Name:  parts[0],
+					Value: parts[1],
+				})
+			}
+		}
+		config.CustomCookies = cookies
+	}
+
+	return config, nil
 }
 
 // TODO Review validateClientConfig
@@ -45,7 +105,7 @@ func validateClientConfig(config ClientConfig, populateDefaults bool) error {
 
 	// TODO adjust these strings to have links to documentation & centralise them
 	if config.Integration == nil {
-		return errors.New("no api integration supplied, please see documentation")
+		return errors.New("no http client api integration supplied, please see repo documentation for this client and go-api-http-client-integration and provide an implementation")
 	}
 
 	if config.MaxRetryAttempts < 0 {
@@ -77,42 +137,16 @@ func validateClientConfig(config ClientConfig, populateDefaults bool) error {
 	return nil
 }
 
+// SetDefaultValuesClientConfig sets default values for the client configuration. Ensuring that all fields have a valid or minimum value.
 func SetDefaultValuesClientConfig(config *ClientConfig) {
-
-	if !config.HideSensitiveData {
-		config.HideSensitiveData = DefaultHideSensitiveData
-	}
-
-	if config.MaxRetryAttempts == 0 {
-		config.MaxRetryAttempts = DefaultMaxRetryAttempts
-	}
-
-	if config.MaxConcurrentRequests == 0 {
-		config.MaxRetryAttempts = DefaultMaxConcurrentRequests
-	}
-
-	if !config.EnableDynamicRateLimiting {
-		config.EnableDynamicRateLimiting = DefaultEnableDynamicRateLimiting
-	}
-
-	if config.CustomTimeout == 0 {
-		config.CustomTimeout = DefaultCustomTimeout
-	}
-
-	if config.TokenRefreshBufferPeriod == 0 {
-		config.TokenRefreshBufferPeriod = DefaultTokenRefreshBufferPeriod
-	}
-
-	if config.TotalRetryDuration == 0 {
-		config.TotalRetryDuration = DefaultTotalRetryDuration
-	}
-
-	if !config.FollowRedirects {
-		config.FollowRedirects = DefaultFollowRedirects
-	}
-
-	if config.MaxRedirects == 0 {
-		config.MaxRedirects = DefaultMaxRedirects
-	}
-
+	setDefaultBool(&config.HideSensitiveData, DefaultHideSensitiveData)
+	setDefaultInt(&config.MaxRetryAttempts, DefaultMaxRetryAttempts, 1)
+	setDefaultInt(&config.MaxConcurrentRequests, DefaultMaxConcurrentRequests, 1)
+	setDefaultBool(&config.EnableDynamicRateLimiting, DefaultEnableDynamicRateLimiting)
+	setDefaultDuration(&config.CustomTimeout, DefaultCustomTimeout)
+	setDefaultDuration(&config.TokenRefreshBufferPeriod, DefaultTokenRefreshBufferPeriod)
+	setDefaultDuration(&config.TotalRetryDuration, DefaultTotalRetryDuration)
+	setDefaultBool(&config.FollowRedirects, DefaultFollowRedirects)
+	setDefaultInt(&config.MaxRedirects, DefaultMaxRedirects, 0)
+	setDefaultBool(&config.EnableConcurrencyManagement, DefaultEnableConcurrencyManagement)
 }

httpclient/integration.go

@@ -1,11 +1,12 @@
-// apiintegrations/apihandler/apihandler.go
+// httpclient/integration.go
 package httpclient
 
 import (
 	"net/http"
 )
 
-// TODO comment
+// APIIntegration is an interface that defines the methods required for an API integration. These are obtained from go-api-http-client-integrations.
+// The methods defined in this interface are used by the HTTP client to authenticate and prepare requests for the API.
 type APIIntegration interface {
 	GetFQDN() string
 	ConstructURL(endpoint string) string

httpclient/utility.go

@@ -1,17 +1,22 @@
+// httpclient/utility.go
 package httpclient
 
 import (
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
+	"time"
 )
 
 // TODO all func comments in here
 
 const ConfigFileExtension = ".json"
 
+// validateFilePath checks if a file path is valid.
 func validateFilePath(path string) (string, error) {
 	cleanPath := filepath.Clean(path)
 
@@ -32,6 +37,7 @@ func validateFilePath(path string) (string, error) {
 
 }
 
+// validateClientID checks if a client ID is a valid UUID.
 func validateValidClientID(clientID string) error {
 	uuidRegex := `^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`
 	if regexp.MustCompile(uuidRegex).MatchString(clientID) {
@@ -60,6 +66,7 @@ func validateClientSecret(clientSecret string) error {
 	return nil
 }
 
+// validateUsername checks if a username meets the minimum requirements.
 func validateUsername(username string) error {
 	usernameRegex := `^[a-zA-Z0-9!@#$%^&*()_\-\+=\[\]{\}\\|;:'",<.>/?]+$`
 	if !regexp.MustCompile(usernameRegex).MatchString(username) {
@@ -68,9 +75,80 @@ func validateUsername(username string) error {
 	return nil
 }
 
+// validatePassword checks if a password meets the minimum requirements.
 func validatePassword(password string) error {
 	if len(password) < 8 {
 		return errors.New("password not long enough")
 	}
 	return nil
 }
+
+// environment variable mapping helpers
+
+// getEnvAsString reads an environment variable as a string, with a fallback default value.
+func getEnvAsString(name string, defaultVal string) string {
+	if value, exists := os.LookupEnv(name); exists {
+		return value
+	}
+	return defaultVal
+}
+
+// getEnvAsBool reads an environment variable as a boolean, with a fallback default value.
+func getEnvAsBool(name string, defaultVal bool) bool {
+	if value, exists := os.LookupEnv(name); exists {
+		boolValue, err := strconv.ParseBool(value)
+		if err == nil {
+			return boolValue
+		}
+	}
+	return defaultVal
+}
+
+// getEnvAsInt reads an environment variable as an integer, with a fallback default value.
+func getEnvAsInt(name string, defaultVal int) int {
+	if value, exists := os.LookupEnv(name); exists {
+		intValue, err := strconv.Atoi(value)
+		if err == nil {
+			return intValue
+		}
+	}
+	return defaultVal
+}
+
+// getEnvAsDuration reads an environment variable as a duration, with a fallback default value.
+func getEnvAsDuration(name string, defaultVal time.Duration) time.Duration {
+	if value, exists := os.LookupEnv(name); exists {
+		durationValue, err := time.ParseDuration(value)
+		if err == nil {
+			return durationValue
+		}
+	}
+	return defaultVal
+}
+
+// http field validation functions
+
+// setDefaultBool sets a boolean field to a default value if it is not already set during http client config field validation.
+func setDefaultBool(field *bool, defaultValue bool) {
+	if !*field {
+		*field = defaultValue
+	}
+}
+
+// setDefaultInt sets an integer field to a default value if it is not already set during http client config field validation.
+func setDefaultInt(field *int, defaultValue, minValue int) {
+	if *field == 0 {
+		*field = defaultValue
+	} else if *field < minValue {
+		*field = minValue
+	}
+}
+
+// setDefaultDuration sets a duration field to a default value if it is not already set during http client config field validation.
+func setDefaultDuration(field *time.Duration, defaultValue time.Duration) {
+	if *field == 0 {
+		*field = defaultValue
+	} else if *field < 0 {
+		*field = defaultValue
+	}
+}