diff --git a/pcf/.gitignore b/pcf/.gitignore new file mode 100644 index 0000000..bc8e3d5 --- /dev/null +++ b/pcf/.gitignore @@ -0,0 +1,2 @@ +bin/ +myapp diff --git a/pcf/README.md b/pcf/README.md new file mode 100644 index 0000000..9c0722b --- /dev/null +++ b/pcf/README.md @@ -0,0 +1,91 @@ +# Injecting Latency with Failure Flags Sidecar by Proxy on Pivotal Cloud Foundry (PCF): A Step-by-Step Guide + +Follow these steps to run a sample application that loads its dependency www.example.com and injects latency into that dependency using a Failure Flag. No changes to your application code are required to introduce latency. + +## Prerequisites + +- A Pivotal Cloud Foundry (PCF) environment with access to deploy applications +- CF CLI installed and configured + +## Download the failure flags sidecar + +amd64: +``` +wget https://assets.gremlin.com/packages/failure-flags-sidecar/latest/x86_64/failure-flags-sidecar-linux.tar.gz +tar -xzf failure-flags-sidecar-linux.tar.gz +rm failure-flags-sidecar-linux.tar.gz +``` + +arm64: +``` +wget https://assets.gremlin.com/packages/failure-flags-sidecar/latest/arm64/failure-flags-sidecar-linux.tar.gz +tar -xzf failure-flags-sidecar-linux.tar.gz +rm failure-flags-sidecar-linux.tar.gz +``` + +## Build the sample application (myapp) + +amd64: +``` +GOOS=linux GOARCH=amd64 go build -o myapp +``` + +arm64: +``` +GOOS=linux GOARCH=arm64 go build -o myapp +``` + +## Update the Gremlin configuration file + +Update [config.yaml](config.yaml) making sure to replace `team_id`, `team_certificate`, and `team_private_key` with your Gremlin team information. +You can find these at [Settings](https://app.gremlin.com/settings) > Team. + +## Configure the Manifest to deploy the sample application and sidecar to PCF + +Use [manifest.yaml](manifest.yaml) as-is for `amd64`. +For `arm64`, update it to use the `arm64` sidecar. + +## Deploy the sample application along with the Gremlin sidecar + +```bash +cf push myapp -f manifest.yaml +``` + +## Verify the installation + +Navigate to https://app.gremlin.com/failure-flags/list, you should see: +- your service `myapp` +- a dependency named `dependency-www.example.com` + +## Inject Latency with a Failure Flag + +Create a [Gremlin experiment](https://app.gremlin.com/failure-flags/new) with the following: +- Experiment Name: my-latency-experiment +- Failure Flag Selector: dependency-www.example.com +- Service Selector: myapp +- Effects: latency with 1000ms delay +- Impact Probability: 100% + +Click `Save and Run` + +## Verify the latency injection + + +Before the experiment, www.example.com takes ~5ms to load +```bash +cf logs myapp --recent + 2025-08-01T09:57:05.84-0600 [APP/PROC/WEB/0] OUT Request to www.example.com - Status: 200 OK | Duration: 5.393351ms +``` + +During the experiment, www.example.com takes 2s to load +```bash +cf logs myapp --recent + 2025-08-01T09:53:41.11-0600 [APP/PROC/WEB/0] OUT Request to www.example.com - Status: 200 OK | Duration: 2.006431396s +``` + +## Cleanup + +To remove your application and sidecar, run: +```bash +cf delete myapp +``` diff --git a/pcf/config.yaml b/pcf/config.yaml new file mode 100644 index 0000000..c877c12 --- /dev/null +++ b/pcf/config.yaml @@ -0,0 +1,192 @@ +## Gremlin Team Id - you can find this value at https://app.gremlin.com/settings/teams +## Override this using GREMLIN_TEAM_ID +team_id: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +## Failure Flags Service Labels - Add labels to identify unique deployments. +## Override these using environment variables prefixed with GREMLIN_LABEL_ +labels: + project: failure-flags-examples + example: failure-flags-by-proxy + +## Debug, set to true for enhanced debug logging to STDOUT +## Uncomment to enable debugging +## Override with GREMLIN_DEBUG=true +debug: true + +## Trace, set to true for enhanced network tracing to STDOUT +## Uncomment to enable network tracing +## Override with GREMLIN_TRACE=true +trace: false + +############################################## +## Team Certificate - Use One of the Following +############################################## + +## Gremlin Team Certificate - Paste certificate content here. +## Override this with GREMLIN_TEAM_CERTIFICATE +team_certificate: | + -----BEGIN CERTIFICATE----- + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxx= + -----END CERTIFICATE----- + +## Gremlin Team Certificate File Path +## Override this with GREMLIN_TEAM_CERTIFICATE_FILE +# team_certificate_file: "/opt/secrets/gremlin/team-certificate.pem + +## Gremlin Team Certificate ARN +## Supports ssm and secretsmanager ARNs +## Override this with GREMLIN_TEAM_CERTIFICATE_ARN +# team_certificate_arn: "" + +############################################## +## Team Private Key - Use One of the Following +############################################## + +## Gremlin Team Private Key - Paste certificate content here. +## Override this with GREMLIN_TEAM_PRIVATE_KEY +team_private_key: | + -----BEGIN EC PRIVATE KEY----- + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx== + -----END EC PRIVATE KEY----- + +## Gremlin Team Private Key File Path +## Override this with GREMLIN_TEAM_PRIVATE_KEY_FILE +# team_private_key_file: "/opt/secrets/gremlin/team-privatekey.pem" + +## Gremlin Team Private Key ARN +## Supports ssm and secretsmanager ARNs +## Override this with GREMLIN_TEAM_PRIVATE_KEY_ARN +# team_private_key_file: "/opt/secrets/gremlin/team-privatekey.pem" + +############################################## +## Corporate Proxy Configuration +############################################## + +## HTTPS Proxy, set this when routing outbound Gremlin HTTPS traffic through a proxy +## Override this with HTTPS_PROXY +#https_proxy: https://corp.proxy.internal:3128 + +## Custom CA Certificate, set this when using a https proxy with a self-signed certificate. +## Override this with GREMLIN_CUSTOM_ROOT_CERTIFICATE +## Paste certificate content here. +#ssl_cert: | +# -----BEGIN CERTIFICATE----- +# ExampleXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXX +# -----END CERTIFICATE----- + +## Override this with GREMLIN_CUSTOM_ROOT_CERTIFICATE_FILE +#ca_cert_file: "/api/secrets/custom-certificate.crt" + +## Override this with GREMLIN_CUSTOM_ROOT_CERTIFICATE_FILE +#ca_cert_arn: "arn:aws:secretsmanager:::secret:" + +## CA Certificate Bundles +## If you have pre-build certificate bundles you need to use you can provide them +## via file or ARN. TLS configuration provided via bundle will override the +## ssl_cert parameter. Only ssm and secretsmanager ARNs are supported. + +## Override this with GREMLIN_CUSTOM_ROOT_CERTIFICATE_BUNDLE_FILE +# ssl_trust_cert_bundle_file: "/opt/secrets/custom-certificates.crt' + +## Override this with GREMLIN_CUSTOM_ROOT_CERTIFICATE_BUNDLE_ARN +# ssl_trust_cert_bundle_arn: "arn:aws:secretsmanager:::secret:" + +############################################## +## Enabling Failure Flags by Proxy (modes) +############################################## + +## AWS Lambda API Proxy Mode +## ------------------------------------------- + +## Enables a special proxy for the AWS Lambda Runtime API. This proxy allows +## for simple experimentation on AWS Lambda functions withouut code changes. +## Override with GREMLIN_LAMBDA_PROXY_ENABLED. Value must be `true` or `1`. +#lambda_proxy_enabled: true + +## This port must match the AWS_LAMBDA_RUNTIME_API value set in your function. +## Make sure to use the included boostratp script. Defaults to localhost:5033. +## Override with GREMLIN_LAMBDA_PROXY_PORT. +#lambda_proxy_port: localhost:5033 + +## Dependency Proxy Mode +## ------------------------------------------- +## This is an HTTP and HTTPS proxy for intercepting calls to your service +## dependencies. To use this, enable the below and then set the HTTP_PROXY and +## HTTPS_PROXY environment variables in your service to match the connect proxy +## port setting below. + +## This enables the HTTP CONNECT proxy for automatically intercepting outbound +## HTTP requests to your service dependencies. Override this value with +## GREMLIN_DEPENDENCY_PROXY_ENABLED and set it to true, yes, or 1 +## This defaults to false. +dependency_proxy_enabled: true + +## This sets the bind address of the HTTP CONNECT proxy. The proxy will only +## bind on localhost and this value must specify localhost (or 127.0.0.1 or ::1) +## and the port. This defaults to localhost:5034. +## Override this value with GREMLIN_DEPENDENCY_PROXY_ENABLED. +dependency_proxy_port: localhost:5034 + +## Ingress Proxy Mode +## ------------------------------------------- +## This mode routes inbound traffic to your service. + +## Uncomment this property to enable the reverse proxy and begin experimenting +## on inbound requests and responses to those requests without any additional +## changes to your application code. Enabling this requires that the next two +## properties are specified. +## Override this value with GREMLIN_INGRESS_PROXY_ENABLED=true +ingress_proxy_enabled: true + +## This sets the host:port where this reverse proxy should bind. This must match +## the destination where your existing loadbalancer or callers expect the service +## to be running. This value defaults to :5033 +## Override this value with GREMLIN_INGRESS_PROXY_PORT=:9081 +ingress_proxy_port: :9081 + +## If the reverse proxy is enabled this property must be explicitly set to the +## local endpoint URL of the upstream service. This URL should be on localhost +## to avoid making additional network hops. This value has no default. +## Override this value with GREMLIN_INGRESS_PROXIED_ENDPOINT +ingress_proxied_endpoint: http://localhost:9080 + +## Common Proxy Settings +## ------------------------------------------- + +## Uncomment this property to set a custom timeout for idle connections. +## This value is specified using Go duration formats, e.g. 1m, 60s. +## The default value is 2m. +## Override this value with GREMLIN_PROXY_IDLE_TIMEOUT. +#proxy_idle_connection_timeout: "1m" + +## Uncomment this property to set a custom read timeout. +## This value is specified using Go duration formats, e.g. 1m, 60s. +## The default value is 2m. +## Override this value with GREMLIN_PROXY_READ_TIMEOUT. +#proxy_read_timeout: "1m" + +## Uncomment this property to set a custom write timeout. +## This value is specified using Go duration formats, e.g. 1m, 60s. +## The default value is 2m. +## Override this value with GREMLIN_PROXY_WRITE_TIMEOUT. +#proxy_write_timeout: "1m" diff --git a/pcf/go.mod b/pcf/go.mod new file mode 100644 index 0000000..90b5189 --- /dev/null +++ b/pcf/go.mod @@ -0,0 +1,3 @@ +module myapp + +go 1.23.7 diff --git a/pcf/manifest.yaml b/pcf/manifest.yaml new file mode 100644 index 0000000..db578a9 --- /dev/null +++ b/pcf/manifest.yaml @@ -0,0 +1,28 @@ +--- +applications: + - name: myapp # App name in CF + memory: 1G # Memory for the container + disk_quota: 1G # Disk space for the container + instances: 1 # Number of instances + health-check-type: none # Disable health checks + + buildpacks: + - binary_buildpack + + path: . # Push contents of current directory + + command: ./myapp # App that repeatedly loads www.example.com + + env: + SERVICE_NAME: "myapp" + GREMLIN_DEBUG: "true" # Enable debug logging + GREMLIN_SIDECAR_ENABLED: "true" + GREMLIN_CONFIG_FILE: "config.yaml" # Path to Gremlin config file + HTTP_PROXY: "http://localhost:5034" # Needed for detecting dependency failure flags (should match "dependency_proxy_port" in config.yaml) + + sidecars: + - name: gremlin-sidecar + process_types: ["web"] # Attach sidecar to 'web' process + memory: 256M + disk_quota: 256M + command: "./bin/failure-flags-sidecar-amd64-linux" # Run Gremlin sidecar diff --git a/pcf/myapp.go b/pcf/myapp.go new file mode 100644 index 0000000..c872886 --- /dev/null +++ b/pcf/myapp.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "net/http" + "time" +) + +func main() { + for { + start := time.Now() + + resp, err := http.Get("http://www.example.com") + duration := time.Since(start) + + if err != nil { + fmt.Printf("Request failed: %v (after %v)\n", err, duration) + } else { + fmt.Printf("Request to www.example.com - Status: %s | Duration: %v\n", resp.Status, duration) + resp.Body.Close() + } + + time.Sleep(5 * time.Second) + } +}