Updated how local environment file is generated. (#32)

Author: mkananizadeh

.gitignore

@@ -106,6 +106,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
+.accounts
 .env
 .venv
 env/
@@ -115,6 +116,7 @@ env.bak/
 venv.bak/
 requirements.json
 
+
 # Spyder project settings
 .spyderproject
 .spyproject

README.md

@@ -79,43 +79,21 @@ This demo uses Terraform and bash scripting to create and teardown infrastructur
    cd demo-change-data-capture
    ```
 
-1. Create a file to manage all the values you'll need through the setup.
+1. Create an `accounts` by running the following command.
 
    ```bash
-    CONFLUENT_CLOUD_EMAIL=<replace>
-    CONFLUENT_CLOUD_PASSWORD=<replace>
-
-    CCLOUD_API_KEY=api-key
-    CCLOUD_API_SECRET=api-secret
-    CCLOUD_BOOTSTRAP_ENDPOINT=kafka-cluster-endpoint
-
-    ORACLE_USERNAME=admin
-    ORACLE_PASSWORD=demo-cdc-c0nflu3nt!
-    ORACLE_ENDPOINT=oracle-endpoint
-    ORACLE_PORT=1521
-
-    POSTGRES_PRODUCTS_ENDPOINT=postgres-products
-    REDSHIFT_ADDRESS=redshift-address
-
-    SF_PVT_KEY=snowflake-private-key
-
-    export TF_VAR_confluent_cloud_api_key="<replace>"
-    export TF_VAR_confluent_cloud_api_secret="<replace>"
-
-    export SNOWFLAKE_USER="tf-snow"
-    export SNOWFLAKE_PRIVATE_KEY_PATH="../snowflake/snowflake_tf_snow_key.p8"
-    export SNOWFLAKE_ACCOUNT="YOUR_ACCOUNT_LOCATOR"
+   echo "CONFLUENT_CLOUD_EMAIL=add_your_email\nCONFLUENT_CLOUD_PASSWORD=add_your_password\nexport TF_VAR_confluent_cloud_api_key=\"add_your_api_key\"\nexport TF_VAR_confluent_cloud_api_secret=\"add_your_api_secret\"\nexport SNOWFLAKE_ACCOUNT=\"add_your_account_locator\"" > .accounts
    ```
 
-   > **Note:** Run `source .env` at any time to update these values in your terminal session. Do NOT commit this file to a GitHub repo.
+   > **Note:** This repo ignores `.accounts` file
 
 ### Confluent Cloud
 
 1. Create Confluent Cloud API keys by following [this](https://registry.terraform.io/providers/confluentinc/confluent/latest/docs/guides/sample-project#summary) guide.
 
    > **Note:** This is different than Kafka cluster API keys.
 
-1. Update the `.env` file for the following variables with your credentials.
+1. Update the `.accounts` file for the following variables with your credentials.
 
    ```bash
     CONFLUENT_CLOUD_EMAIL=<replace>
@@ -162,11 +140,9 @@ This demo uses Terraform and bash scripting to create and teardown infrastructur
 
    > **Note:** If your Snowflake account isn't in AWS-US-West-2 refer to [doc](https://docs.snowflake.com/en/user-guide/admin-account-identifier#snowflake-region-ids) to identify your account locator.
 
-1. Update your `.env` file and add the newly created credentials for the following variables
+1. Update your `.accounts` file and add the newly created credentials for the following variable
 
    ```bash
-   export SNOWFLAKE_USER="tf-snow"
-   export SNOWFLAKE_PRIVATE_KEY_PATH="../snowflake/snowflake_tf_snow_key.p8"
    export SNOWFLAKE_ACCOUNT="YOUR_ACCOUNT_LOCATOR"
    ```
 
@@ -179,21 +155,31 @@ This demo uses Terraform and bash scripting to create and teardown infrastructur
 
    > **Note:** For troubleshooting or more information review the [doc](https://quickstarts.snowflake.com/guide/terraforming_snowflake/index.html?index=..%2F..index#2).
 
-1. Source the `.env` file.
+### Create a local environment file
 
-   ```
+1. Navigate to the home directory of the project and run `create_env.sh` script. This bash script copies the content of `.accounts` file into a new file called `.env` and append additional variables to it.
+
+   ```bash
    cd demo-change-data-capture
+   ./create_env.sh
+   ```
+
+1. Source `.env` file.
+
+   ```bash
    source .env
    ```
 
+   > **Note:**: if you don't source `.env` file you'll be prompted to manually provide the values through command line when running Terraform commands.
+
 ### Build your cloud infrastructure
 
 1. Log into your AWS account through command line.
 
 1. Navigate to the repo's terraform directory.
 
    ```bash
-   cd terraform
+   cd demo-change-data-capture/terraform
    ```
 
 1. Initialize Terraform within the directory.
@@ -218,18 +204,18 @@ This demo uses Terraform and bash scripting to create and teardown infrastructur
    terraform apply -var sg_package="ADVANCED"
    ```
 
-1. Write the output of `terraform` to a JSON file. The `env.sh` script will parse the JSON file to update the `.env` file.
+1. Write the output of `terraform` to a JSON file. The `setup.sh` script will parse the JSON file to update the `.env` file.
 
    ```bash
    terraform output -json > ../resources.json
    ```
 
    > **Note:** _Verify that the `resources.json` is created at root level of demo-change-data-capture directory._
 
-1. Run the `env.sh` script.
+1. Run the `setup.sh` script.
    ```bash
    cd demo-change-data-capture
-   ./env.sh
+   ./setup.sh
    ```
 1. This script achieves the following:
 

create_env.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+accounts_file=".accounts"
+env_file=".env"
+
+# Check if .accounts file exists
+if [ ! -f "$accounts_file" ]; then
+    echo "$accounts_file not found."
+    exit 1
+fi
+
+# Define the environment variable content
+env_content=$(cat <<EOF
+CCLOUD_API_KEY=api-key
+CCLOUD_API_SECRET=api-secret
+CCLOUD_BOOTSTRAP_ENDPOINT=kafka-cluster-endpoint
+
+ORACLE_USERNAME=admin
+ORACLE_PASSWORD=demo-cdc-c0nflu3nt!
+ORACLE_ENDPOINT=oracle-endpoint
+ORACLE_PORT=1521
+
+POSTGRES_PRODUCTS_ENDPOINT=postgres-products
+REDSHIFT_ADDRESS=redshift-address
+
+SF_PVT_KEY=snowflake-private-key
+
+export SNOWFLAKE_USER="tf-snow"
+export SNOWFLAKE_PRIVATE_KEY_PATH="../snowflake/snowflake_tf_snow_key.p8"
+EOF
+)
+
+# Combine the environment variable content with .accounts and write to .env
+echo "$env_content" | cat - "$accounts_file" > "$env_file"
+
+echo "Created an environment file named: $env_file"

example.env

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+sleep_time=2
+env_file=".env"
+
+# Use confluent environment
+confluent login --save
+export CCLOUD_ENV_ID=$(confluent environment list -o json \
+    | jq -r '.[] | select(.name | contains('\"${CCLOUD_ENV_NAME:-Demo_Change_Data_Capture}\"')) | .id')
+
+confluent env use $CCLOUD_ENV_ID
+
+# Use kafka cluster
+export CCLOUD_CLUSTER_ID=$(confluent kafka cluster list -o json \
+    | jq -r '.[] | select(.name | contains('\"${CCLOUD_CLUSTER_NAME:-demo_kafka_cluster}\"')) | .id')
+
+confluent kafka cluster use $CCLOUD_CLUSTER_ID
+
+# Get cluster bootstrap endpoint
+export CCLOUD_BOOTSTRAP_ENDPOINT=$(confluent kafka cluster describe -o json | jq -r .endpoint)
+STRIPPED_CCLOUD_BOOTSTRAP_ENDPOINT=$(echo $CCLOUD_BOOTSTRAP_ENDPOINT | sed 's/SASL_SSL:\/\///')
+
+# use sed to replace kafka-cluster-endpoint with the replacement string
+sed -i .bak "s/kafka-cluster-endpoint/$STRIPPED_CCLOUD_BOOTSTRAP_ENDPOINT/g" $env_file
+echo "Added Kafka cluster endpoint to $env_file"
+sleep $sleep_time
+
+# Create an API key pair to use for connectors
+echo "Creating Kafka cluster API key"
+CREDENTIALS=$(confluent api-key create --resource $CCLOUD_CLUSTER_ID --description "demo-change-data-capture" -o json)
+kafka_api_key=$(echo $CREDENTIALS | jq -r '.api_key')
+kafka_api_secret=$(echo $CREDENTIALS | jq -r '.api_secret')
+sleep $sleep_time
+
+# use sed to replace all instances of $kafka_api_key with the replacement string
+sed -i .bak "s^api-key^\"$kafka_api_key\"^g" $env_file 
+sed -i .bak "s^api-secret^\"$kafka_api_secret\"^g" $env_file 
+echo "Added Kafka API key and secret to $env_file"
+
+sleep $sleep_time
+
+# Read values from resources.json and update the $env_file file.
+# These resources are created by Terraform
+json=$(cat resources.json)
+
+oracle_endpoint=$(echo "$json" | jq -r '.oracle_endpoint.value.address')
+postgres_products=$(echo "$json" | jq -r '.postgres_instance_products_public_endpoint.value')
+
+raw_snowflake_svc_private_key=$(echo "$json" | jq -r '.snowflake_svc_private_key.value')
+snowflake_svc_private_key=$(echo "$raw_snowflake_svc_private_key" | sed '/-----BEGIN RSA PRIVATE KEY-----/d; /-----END RSA PRIVATE KEY-----/d' | tr -d '\n')
+
+redshift_endpoint=$(echo "$json" | jq -r '.redshift_endpoint.value')
+redshift_address=$(echo $redshift_endpoint | sed 's/:5439//')
+
+# Updating the $env_file file with sed command
+sed -i .bak "s^oracle-endpoint^$oracle_endpoint^g" $env_file 
+sed -i .bak "s^postgres-products^$postgres_products^g" $env_file 
+sed -i .bak "s^snowflake-private-key^\"$snowflake_svc_private_key\"^g" $env_file 
+sed -i .bak "s^redshift-address^$redshift_address^g" $env_file 
+
+echo "Added Oracle endpoint to $env_file"
+echo "Added PostgreSQL endpoint to $env_file"
+echo "Added Snowflake private key to $env_file"
+echo "Added Amazon Redshift address to $env_file"
+
+# sleep $sleep_time