Merge pull request #267 from dippynark/264-cluster-autoscaler-documentation

264 upgrade cluster autoscaler

Author: simonswine

docs/spelling_wordlist.txt

@@ -65,3 +65,12 @@ Jenkins
 prepended
 username
 loopback
+addons
+autoscaler
+prometheus
+ubuntu
+offline
+admin
+plugin
+checklist
+localhost

docs/user-guide.rst

@@ -150,6 +150,32 @@ The configuration file can be found at ``$HOME/.tarmak/tarmak.yaml`` (default).
 The Pod Security Policy manifests can be found within the tarmak directory at
 ``puppet/modules/kubernetes/templates/pod-security-policy.yaml.erb``
 
+Cluster Autoscaler
+~~~~~~~~~~~~~~~~~~
+
+Tarmak supports deploying `Cluster Autoscaler
+<https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler>`_ when
+spinning up a Kubernetes cluster. The following `tarmak.yaml` snippet shows how
+you would enable Cluster Autoscaler. 
+
+.. code-block:: yaml
+
+    kubernetes:
+      clusterAutoscaler:
+        enabled: true
+    ...
+
+The above configuration would deploy Cluster Autoscaler with an image of
+`gcr.io/google_containers/cluster-autoscaler` using the recommend version based
+on the version of your Kubernetes cluster. The configuration block accepts two
+optional fields of `image` and `version` allowing you to change these defaults.
+Note that the final image tag used when deploying Cluster Autoscaler will be the
+configured version prepended with the letter `v`.
+
+The current implementation will configure the first instance pool of type worker
+in your cluster configuration to scale between `minCount` and `maxCount`. We
+plan to add support for an arbitrary number of worker instance pools.
+
 Logging
 ~~~~~~~
 
@@ -336,7 +362,6 @@ certificate is valid for ``jenkins.<environment>.<zone>``.
       type: ssd
   ...
 
-
 Tiller
 ~~~~~~
 
@@ -362,7 +387,6 @@ allows to override the deployed version:
    consider Helm's `security best practices
    <https://github.com/kubernetes/helm/blob/master/docs/securing_installation.md>`_.
 
-
 Prometheus
 ~~~~~~~~~~
 

pkg/puppet/puppet.go

@@ -306,11 +306,32 @@ func (p *Puppet) writeHieraData(puppetPath string, cluster interfaces.Cluster) e
 		return fmt.Errorf("error writing global hiera config: %s", err)
 	}
 
+	// retrieve details for first worker instance pool
+	workerMinCount := 0
+	workerMaxCount := 0
+	workerInstancePoolName := ""
+	if cluster.Config().Kubernetes.ClusterAutoscaler != nil && cluster.Config().Kubernetes.ClusterAutoscaler.Enabled {
+		for _, instancePool := range cluster.InstancePools() {
+			if instancePool.Role().Name() == clusterv1alpha1.KubernetesWorkerRoleName {
+				workerMinCount = instancePool.MinCount()
+				workerMaxCount = instancePool.MaxCount()
+				workerInstancePoolName = instancePool.Name()
+				break
+			}
+		}
+	}
+
 	// loop through instance pools
 	for _, instancePool := range cluster.InstancePools() {
 
 		classes, variables := contentInstancePoolConfig(cluster.Config(), instancePool.Config(), instancePool.Role().Name())
 
+		if instancePool.Role().Name() == clusterv1alpha1.KubernetesMasterRoleName && cluster.Config().Kubernetes.ClusterAutoscaler != nil && cluster.Config().Kubernetes.ClusterAutoscaler.Enabled {
+			variables = append(variables, fmt.Sprintf(`kubernetes_addons::cluster_autoscaler::min_instances: %d`, workerMinCount))
+			variables = append(variables, fmt.Sprintf(`kubernetes_addons::cluster_autoscaler::max_instances: %d`, workerMaxCount))
+			variables = append(variables, fmt.Sprintf(`kubernetes_addons::cluster_autoscaler::instance_pool_name: "%s"`, workerInstancePoolName))
+		}
+
 		//  classes
 		err = p.writeLines(
 			filepath.Join(hieraPath, "instance_pools", fmt.Sprintf("%s_classes.yaml", instancePool.Name())), classes,

pkg/tarmak/cluster/cluster.go

@@ -392,7 +392,8 @@ func (c *Cluster) Variables() map[string]interface{} {
 		if ok {
 			output[fmt.Sprintf("%s_ami", instancePool.TFName())] = ids
 		}
-		output[fmt.Sprintf("%s_instance_count", instancePool.TFName())] = instancePool.Config().MinCount
+		output[fmt.Sprintf("%s_min_instance_count", instancePool.TFName())] = instancePool.Config().MinCount
+		output[fmt.Sprintf("%s_max_instance_count", instancePool.TFName())] = instancePool.Config().MaxCount
 	}
 
 	// set network cidr

pkg/tarmak/instance_pool/instance_pool.go

@@ -148,8 +148,11 @@ func (n *InstancePool) RootVolume() interfaces.Volume {
 	return n.rootVolume
 }
 
-func (n *InstancePool) Count() int {
-	// TODO: this needs to be replaced by Max/Min
+func (n *InstancePool) MinCount() int {
+	return n.conf.MinCount
+}
+
+func (n *InstancePool) MaxCount() int {
 	return n.conf.MaxCount
 }
 

pkg/tarmak/interfaces/interfaces.go

@@ -248,6 +248,8 @@ type InstancePool interface {
 	Role() *role.Role
 	Volumes() []Volume
 	Zones() []string
+	MinCount() int
+	MaxCount() int
 }
 
 type Volume interface {

puppet/hieradata/common.yaml

@@ -15,10 +15,6 @@ tarmak::kubernetes_api_url: "https://api.%{::tarmak_cluster}.%{::tarmak_dns_root
 # TODO: This should come from terraform
 tarmak::etcd_instances: 3
 
-# cluster scaler config
-kubernetes_addons::cluster_autoscaler::min_instances: 3
-kubernetes_addons::cluster_autoscaler::max_instances: 10
-
 # point heapster to influxdb
 kubernetes_addons::heapster::sink: influxdb:http://monitoring-influxdb.kube-system:8086
 

puppet/modules/kubernetes_addons/manifests/cluster_autoscaler.pp

@@ -5,6 +5,7 @@
   String $limit_mem='500Mi',
   String $request_cpu='100m',
   String $request_mem='300Mi',
+  String $instance_pool_name='worker',
   Integer $min_instances=3,
   Integer $max_instances=6,
   $ca_mounts=$::kubernetes_addons::params::ca_mounts,
@@ -20,14 +21,20 @@
     $rbac_enabled = false
   }
 
-  if defined('$kubernetes::cluster_name') {
-    $asg_name="kubernetes-${::kubernetes::cluster_name}-worker"
+  if defined('$kubernetes::cluster_name') and $instance_pool_name != '' {
+    $asg_name="${::kubernetes::cluster_name}-kubernetes-${instance_pool_name}"
   } else {
-    $asg_name=undef
+    fail('asg name must be set')
   }
 
   if $version == '' {
-    if versioncmp($::kubernetes::version, '1.7.0') >= 0 {
+    if versioncmp($::kubernetes::version, '1.10.0') >= 0 {
+      $_version = '1.2.0'
+    } elsif versioncmp($::kubernetes::version, '1.9.0') >= 0 {
+      $_version = '1.1.0'
+    } elsif versioncmp($::kubernetes::version, '1.8.0') >= 0 {
+      $_version = '1.0.0'
+    } elsif versioncmp($::kubernetes::version, '1.7.0') >= 0 {
       $_version = '0.6.0'
     } elsif versioncmp($::kubernetes::version, '1.6.0') >= 0 {
       $_version = '0.5.4'
@@ -40,6 +47,10 @@
     $_version = $version
   }
 
+  if versioncmp($_version, '0.6.0') >= 0 {
+    $balance_similar_node_groups = true
+  }
+
   if versioncmp($::kubernetes::version, '1.6.0') >= 0 {
     $version_before_1_6 = false
   } else {
@@ -50,6 +61,7 @@
   kubernetes::apply{'cluster-autoscaler':
     manifests => [
       template('kubernetes_addons/cluster-autoscaler-deployment.yaml.erb'),
+      template('kubernetes_addons/cluster-autoscaler-rbac.yaml.erb'),
     ],
   }
 }

puppet/modules/kubernetes_addons/spec/classes/cluster_autoscaler_spec.rb

@@ -49,25 +49,37 @@ class kubernetes{
     end
 
     it 'has asg configured' do
-      expect(manifests[0]).to match(%{3:6:kubernetes-cluster1-worker})
+      expect(manifests[0]).to match(%{3:6:cluster1-kubernetes-worker})
     end
 
     it 'has cloud_provider configured' do
       expect(manifests[0]).to match(%{cloud-provider=aws})
     end
 
-    it 'has cert path set' do
-      expect(manifests[0]).to match(%{path: /etc/ssl/certs})
-    end
-
     it 'has AWS_REGION set' do
       expect(manifests[0]).to match(%r{value: eu-west-1$})
     end
+
+    it 'has host network set' do
+      expect(manifests[0]).to match(%r{hostNetwork: true$})
+    end
+
+    it 'has master toleration set' do
+      expect(manifests[0]).to match(%r{tolerations:\s+- key: "node-role\.kubernetes\.io\/master"\s+operator: "Exists"\s+effect: "NoSchedule"})
+    end
+
+    it 'has critical addon toleration set' do
+      expect(manifests[0]).to match(%r{- key: "CriticalAddonsOnly"\s+operator: "Exists"})
+    end
+
+    it 'has master node affinity set' do
+      expect(manifests[0]).to match(%r{nodeAffinity:\s+requiredDuringSchedulingIgnoredDuringExecution:\s+nodeSelectorTerms:\s+- matchExpressions:\s+- key: "node-role\.kubernetes\.io\/master"\s+operator: "Exists"})
+    end
   end
 
   context 'with kubernetes 1.5' do
     let(:kubernetes_version) do
-      '1.5.6'
+      '1.5.0'
     end
     it 'uses correct image version' do
       expect(manifests[0]).to match(%r{gcr.io/google_containers/cluster-autoscaler:v0.4.0})
@@ -76,7 +88,7 @@ class kubernetes{
 
   context 'with kubernetes 1.6' do
     let(:kubernetes_version) do
-      '1.6.6'
+      '1.6.0'
     end
     it 'uses correct image version' do
       expect(manifests[0]).to match(%r{gcr.io/google_containers/cluster-autoscaler:v0.5.4})
@@ -85,10 +97,37 @@ class kubernetes{
 
   context 'with kubernetes 1.7' do
     let(:kubernetes_version) do
-      '1.7.1'
+      '1.7.0'
     end
     it 'uses correct image version' do
       expect(manifests[0]).to match(%r{gcr.io/google_containers/cluster-autoscaler:v0.6.0})
     end
   end
+
+  context 'with kubernetes 1.8' do
+    let(:kubernetes_version) do
+      '1.8.0'
+    end
+    it 'uses correct image version' do
+      expect(manifests[0]).to match(%r{gcr.io/google_containers/cluster-autoscaler:v1.0.0})
+    end
+  end
+
+  context 'with kubernetes 1.9' do
+    let(:kubernetes_version) do
+      '1.9.0'
+    end
+    it 'uses correct image version' do
+      expect(manifests[0]).to match(%r{gcr.io/google_containers/cluster-autoscaler:v1.1.0})
+    end
+  end
+
+  context 'with kubernetes 1.10' do
+    let(:kubernetes_version) do
+      '1.10.0'
+    end
+    it 'uses correct image version' do
+      expect(manifests[0]).to match(%r{gcr.io/google_containers/cluster-autoscaler:v1.2.0})
+    end
+  end
 end

puppet/modules/kubernetes_addons/templates/cluster-autoscaler-deployment.yaml.erb

@@ -7,6 +7,7 @@ metadata:
   labels:
     app: cluster-autoscaler
     kubernetes.io/cluster-service: "true"
+    version: <%= @_version %>
 spec:
   replicas: 1
   selector:
@@ -17,10 +18,32 @@ spec:
       labels:
         app: cluster-autoscaler
       annotations:
+        prometheus.io/port: "8085"
+        prometheus.io/scrape: "true"
         scheduler.alpha.kubernetes.io/critical-pod: ''
+<%- if @version_before_1_6 -%>
         scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+<%- end -%>
     spec:
+      tolerations:
+      - key: "node-role.kubernetes.io/master"
+        operator: "Exists"
+        effect: "NoSchedule"
+<%- unless @version_before_1_6 -%>
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"      
+<%- end -%>
+<%- if @rbac_enabled -%>
+      serviceAccountName: cluster-autoscaler
+<%- end -%>
       hostNetwork: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: "node-role.kubernetes.io/master"
+                operator: "Exists"
       containers:
       - image: "<%= @image %>:v<%= @_version %>"
         name: cluster-autoscaler
@@ -31,27 +54,23 @@ spec:
           requests:
             cpu: <%= @request_cpu %>
             memory: <%= @request_mem %>
+        livenessProbe:
+          httpGet:
+            path: "/health-check"
+            port: 8085
         command:
         - ./cluster-autoscaler
         - --v=4
         - --cloud-provider=<%= @cloud_provider %>
         - --skip-nodes-with-local-storage=false
         - --nodes=<%= @min_instances %>:<%= @max_instances %>:<%= @asg_name %>
+        - --skip-nodes-with-system-pods=false
+<% if @balance_similar_node_groups -%>
+        - --balance-similar-node-groups
+<% end -%>
 <% if @aws_region -%>
         env:
         - name: AWS_REGION
           value: <%= @aws_region %>
 <% end -%>
-        imagePullPolicy: Always
-        volumeMounts:
-<% @ca_mounts.each do |ca| -%>
-        - name: <%= ca['name'] %>
-          readOnly: <%= ca['readOnly'] %>
-          mountPath: <%= ca['mountPath'] %>
-<% end -%>
-      volumes:
-<% @ca_mounts.each do |ca| -%>
-      - name: <%= ca['name'] %>
-        hostPath:
-          path: <%= ca['mountPath'] %>
-<% end -%>
+        imagePullPolicy: Always