add cloudwatch agent for custom metrics (#1670)

Author: rakshit-souvik

Vagrantfile

@@ -31,6 +31,8 @@ num_workers = 0 # Generic workers that get the code, but don't start any service
 ram_megabytes = 1280
 base_box = "ubuntu/trusty64"
 
+semaphore_job_id = ENV['SEMAPHORE_JOB_ID']
+
 # EC2
 ec2_access_key = ENV['AWS_ACCESS_KEY']
 ec2_secret_key = ENV['AWS_SECRET_KEY']
@@ -50,7 +52,7 @@ ec2_subnet_id = nil
 # Only override this by setting it to false if you're running in a VPC and you
 # are running Vagrant from within that VPC as well.
 ec2_associate_public_ip = nil
-ec2_iam_instance_profile_name = nil
+ec2_iam_instance_profile_name = "semaphore-access"
 
 ebs_volume_type = 'gp3'
 
@@ -212,6 +214,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       name_node(worker, name, ec2_instance_name_prefix)
       ip_address = "192.168.50." + (100 + i).to_s
       assign_local_ip(worker, ip_address)
+      worker.vm.provision "file", source: "vagrant/cloudwatch-agent-configuration.json", destination: "/tmp/cloudwatch-agent-configuration.json"
+      worker.vm.provision "shell", path: "vagrant/cloudwatch-agent-setup.sh", env: {"SEMAPHORE_JOB_ID" => semaphore_job_id}
       worker.vm.provision "shell", path: "vagrant/base.sh", env: {"JDK_MAJOR" => jdk_major, "JDK_FULL" => jdk_full}
     end
   }

vagrant/cloudwatch-agent-configuration.json

@@ -0,0 +1,96 @@
+{
+  "agent": {
+    "metrics_collection_interval": 60,
+    "run_as_user": "root"
+  },
+  "metrics": {
+    "namespace": "system-test-ccs-kafka",
+    "aggregation_dimensions": [
+      [
+        "InstanceId"
+      ]
+    ],
+    "append_dimensions": {
+      "AutoScalingGroupName": "${aws:AutoScalingGroupName}",
+      "ImageId": "${aws:ImageId}",
+      "InstanceId": "${aws:InstanceId}",
+      "InstanceType": "${aws:InstanceType}"
+    },
+    "metrics_collected": {
+      "cpu": {
+        "append_dimensions": {
+          "SemaphoreJobId": "${SEMAPHORE_JOB_ID}"
+        },
+        "measurement": [
+          "usage_idle",
+          "usage_iowait",
+          "usage_user"
+        ],
+        "metrics_collection_interval": 60,
+        "totalcpu": true
+      },
+      "disk": {
+        "append_dimensions": {
+          "SemaphoreJobId": "${SEMAPHORE_JOB_ID}"
+        },
+        "measurement": [
+          "used_percent",
+          "inodes_free"
+        ],
+        "metrics_collection_interval": 60,
+        "resources": [
+          "*"
+        ]
+      },
+      "diskio": {
+        "append_dimensions": {
+          "SemaphoreJobId": "${SEMAPHORE_JOB_ID}"
+        },
+        "measurement": [
+          "io_time",
+          "write_bytes",
+          "read_bytes",
+          "writes",
+          "reads"
+        ],
+        "metrics_collection_interval": 60,
+        "resources": [
+          "*"
+        ]
+      },
+      "mem": {
+        "append_dimensions": {
+          "SemaphoreJobId": "${SEMAPHORE_JOB_ID}"
+        },
+        "measurement": [
+          "mem_used_percent"
+        ],
+        "metrics_collection_interval": 60
+      },
+      "netstat": {
+        "append_dimensions": {
+          "SemaphoreJobId": "${SEMAPHORE_JOB_ID}"
+        },
+        "measurement": [
+          "tcp_established",
+          "tcp_time_wait"
+        ],
+        "metrics_collection_interval": 60
+      },
+      "statsd": {
+        "metrics_aggregation_interval": 60,
+        "metrics_collection_interval": 60,
+        "service_address": ":8125"
+      },
+      "swap": {
+        "append_dimensions": {
+          "SemaphoreJobId": "${SEMAPHORE_JOB_ID}"
+        },
+        "measurement": [
+          "swap_used_percent"
+        ],
+        "metrics_collection_interval": 60
+      }
+    }
+  }
+}

vagrant/cloudwatch-agent-setup.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# This shell script installs Amazon CloudWatch Agent on all EC2 instances that
+# are provisioned and spinned-off as worker-nodes for system-test runs.
+#
+# This helps in collecting and monitoring useful system-level metrics from these EC2
+# instances aka worker-nodes to identify performance patterns and anomalies,
+# and devise methods to address them.
+#
+# Example : Disk Space Utilization and Memory Utilization Metrics.
+
+set -ex
+
+architecture=arm64
+arch=$(uname -m)
+
+if [ "$arch" == "x86_64" ]; then
+    architecture=amd64
+fi
+
+downloadUrl=https://amazoncloudwatch-agent.s3.amazonaws.com/ubuntu/$architecture/latest/amazon-cloudwatch-agent.deb
+
+echo "Installing amazon-cloudwatch-agent...."
+wget -nv $downloadUrl
+
+
+dpkg -i -E ./amazon-cloudwatch-agent.deb
+apt-get -y update && apt-get -y install collectd
+
+
+pushd /tmp
+
+JQ_URL="https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64"
+
+# Download the binary
+wget "$JQ_URL" -O jq
+
+# Make it executable
+chmod +x jq
+
+# Copy it to a directory in your PATH (e.g., /usr/local/bin)
+sudo cp jq /usr/local/bin/
+
+jq --version
+
+popd
+
+CONFIG_FILE="/tmp/cloudwatch-agent-configuration.json"
+TEMP_FILE="cloudwatch-agent-configuration.json.tmp"
+
+# traverse the JSON file and update the SemaphoreJobId value
+jq \
+  --arg job_id "$SEMAPHORE_JOB_ID" \
+  '
+  walk(
+    if type == "object" and has("append_dimensions") and (.append_dimensions | has("SemaphoreJobId")) then
+      .append_dimensions.SemaphoreJobId = $job_id
+    else
+      # Otherwise, return the element unchanged
+      .
+    end
+  )
+  ' \
+   "$CONFIG_FILE" > "$TEMP_FILE"
+
+
+# Check if the jq command was successful
+if [ $? -eq 0 ]; then
+  cp "$TEMP_FILE" /opt/aws/amazon-cloudwatch-agent/bin/config.json
+  cat /opt/aws/amazon-cloudwatch-agent/bin/config.json
+else
+  echo "Error: Failed to update '$CONFIG_FILE' using jq."
+  rm -f "$TEMP_FILE"
+  exit 1
+fi
+
+/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json
+/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -m ec2 -a status
+echo "Installation completed for amazon-cloudwatch-agent !!!"