handing single node and standard mode

Author: tomarv2

README.md

@@ -43,28 +43,57 @@
 ##### [Create Cluster:](examples/cluster)
 
 - This is where you would normally start with if you just deployed your databricks workspace.
- Two options are available:
-  - Minimum configuration required to bring up a cluster.
-  - Bring up cluster with most of the available options.
-Note: Some option may be missing.
+
+Two Cluster Mode are supported by the module:
+
+- `Single Node` mode: To deploy cluster in Single Node mode, update `fixed_value` to `0`:
+```
+fixed_value         = 0
+```
+
+- `Standard` mode: To deploy in Standard mode, two options are available:
+
+```
+fixed_value         = 1
+```
+OR
+```
+auto_scaling         = [1,3]
+```
+
+**Note:** If you need to configure `Instance Pool`:
+
+```
+deploy_instance_pool                  = true
+min_idle_instances                    = 1
+max_capacity                          = 5
+idle_instance_autotermination_minutes = 30
+```
+
+> ❗️ **Important**
+>
+> If `deploy_instance_pool` is set to `true` and `auto_scaling` is enabled.
+> Ensure `max_capacity` is more than `auto_scaling`
+ max value.
 
 ##### [Deploy Job on new or existing cluster:](examples/job)
 
 - Deploy Job to an existing cluster.
 - Deploy Cluster and deploy Job.
-Note: Job name and Notebook name is same.
+
+Note: `Job name` and `Notebook name` is same.
 
 ##### [Deploy Notebook:](examples/notebook)
 
 - Once you have Notebook ready put them in the notebooks folder and specify the job as below:
 
 ```
 notebook_info = {
-  default994 = {
+  demo1 = {
     language        = "PYTHON"
     local_path      = "notebooks/demo_notebook_1.py"
   }
-  default140 = {
+  demo2 = {
     language        = "PYTHON"
     local_path      = "notebooks/demo_notebook_2.py"
   }

cluster.tf

@@ -1,11 +1,12 @@
 resource "databricks_cluster" "cluster" {
-  count = var.deploy_cluster ? 1 : 0
+  count = var.deploy_cluster == true && var.fixed_value != 0 ? 1 : 0
 
-  cluster_name            = "${var.teamid}-${var.prjid} (${data.databricks_current_user.me.alphanumeric})"
-  spark_version           = data.databricks_spark_version.latest.id
-  autotermination_minutes = var.cluster_autotermination_minutes
+  cluster_name  = "${var.teamid}-${var.prjid} (${data.databricks_current_user.me.alphanumeric})"
+  spark_version = data.databricks_spark_version.latest.id
+
+  driver_node_type_id = var.driver_node_type_id
+  node_type_id        = var.deploy_instance_pool != true ? join("", data.databricks_node_type.cluster_node_type.*.id) : null
 
-  node_type_id     = var.deploy_instance_pool != true ? join("", data.databricks_node_type.cluster_node_type.*.id) : null
   instance_pool_id = var.deploy_instance_pool == true ? join("", databricks_instance_pool.instance_nodes.*.id) : null
 
   num_workers = var.fixed_value != null ? var.fixed_value : null
@@ -17,18 +18,26 @@ resource "databricks_cluster" "cluster" {
       max_workers = autoscale.value[1]
     }
   }
-  autoscale {
-    min_workers = var.cluster_min_workers
-    max_workers = var.cluster_max_workers
-  }
+  autotermination_minutes = var.cluster_autotermination_minutes
+  custom_tags             = merge(local.shared_tags)
+}
 
-  custom_tags = merge(local.shared_tags)
+resource "databricks_cluster" "single_node_cluster" {
+  count = var.deploy_cluster == true && var.fixed_value == 0 && var.auto_scaling == null ? 1 : 0
 
-  /*
-   spark_conf = {
-      # Single-node
-      "spark.databricks.cluster.profile" : "singleNode"
-      "spark.master" : "local[*]"
-    }
-  */
+  cluster_name            = "${var.teamid}-${var.prjid} (${data.databricks_current_user.me.alphanumeric})"
+  spark_version           = data.databricks_spark_version.latest.id
+  autotermination_minutes = var.cluster_autotermination_minutes
+  node_type_id            = var.deploy_instance_pool != true ? join("", data.databricks_node_type.cluster_node_type.*.id) : null
+  num_workers             = 0
+
+  custom_tags = {
+    "ResourceClass" = "SingleNode"
+  }
+
+  spark_conf = {
+    # Single-node
+    "spark.databricks.cluster.profile" : "singleNode"
+    "spark.master" : "local[*]"
+  }
 }

cluster_permissions.tf

@@ -1,7 +1,7 @@
 resource "databricks_permissions" "cluster" {
   count = var.deploy_cluster ? 1 : 0
 
-  cluster_id = join("", databricks_cluster.cluster.*.id)
+  cluster_id = var.fixed_value != 0 ? join("", databricks_cluster.cluster.*.id) : join("", databricks_cluster.single_node_cluster.*.id)
   access_control {
     user_name        = join("", databricks_user.users.*.user_name)
     permission_level = "CAN_RESTART"

examples/all/sample-maximum_config/main.tf

@@ -19,8 +19,8 @@ module "databricks_workspace_management" {
   deploy_cluster = true
   # cluster_id                      = "0507-210128-assay460"
   cluster_autotermination_minutes = 30
-  cluster_min_workers             = 1
-  cluster_max_workers             = 2
+  fixed_value                     = 1
+  auto_scaling                    = [2, 3]
   # ------------------------------------------------
   # Cluster Policy
   # ------------------------------------------------
@@ -29,6 +29,7 @@ module "databricks_workspace_management" {
   # ------------------------------------------------
   # Cluster Instance Pool
   # ------------------------------------------------
+  deploy_instance_pool                  = false
   min_idle_instances                    = 1
   max_capacity                          = 2
   idle_instance_autotermination_minutes = 30

examples/cluster/sample-maximum_config/main.tf

@@ -1,7 +1,7 @@
 module "databricks_workspace_management" {
   source = "git::git@github.com:tomarv2/terraform-databricks-workspace-management.git"
 
-  workspace_url = "https://<workspace_url>.cloud.databricks.com"
+  workspace_url = "https://https://<workspace_url>.cloud.databricks.com"
   dapi_token    = "dapi123456789012"
   # ------------------------------------------------
   # Admin Console
@@ -16,20 +16,21 @@ module "databricks_workspace_management" {
   # ------------------------------------------------
   deploy_cluster                  = true
   cluster_autotermination_minutes = 30
-  cluster_min_workers             = 1
-  cluster_max_workers             = 2
-  # ------------------------------------------------
-  # Cluster Policy
-  # ------------------------------------------------
-  cluster_policy_max_dbus_per_hour       = 5
-  cluster_policy_autotermination_minutes = 5
+  fixed_value                     = 1
+  auto_scaling                    = [2, 3]
   # ------------------------------------------------
   # Cluster Instance Pool
   # ------------------------------------------------
+  deploy_instance_pool                  = false
   min_idle_instances                    = 1
-  max_capacity                          = 2
+  max_capacity                          = 5
   idle_instance_autotermination_minutes = 30
   # ------------------------------------------------
+  # Cluster Policy
+  # ------------------------------------------------
+  cluster_policy_max_dbus_per_hour       = 5
+  cluster_policy_autotermination_minutes = 5
+  # ------------------------------------------------
   # Cluster Worker Type
   # ------------------------------------------------
   local_disk    = 0

examples/job/sample/output.tf

@@ -49,11 +49,6 @@ output "databricks_group_member" {
   value       = module.databricks_workspace_management.databricks_group_member
 }
 
-output "databricks_permissions_notebook" {
-  description = "databricks notebook permissions"
-  value       = module.databricks_workspace_management.databricks_permissions_notebook
-}
-
 /*
 output "databricks_permissions_job" {
   value = module.databricks_workspace_management.databricks_permissions_job

examples/notebook/sample/output.tf

@@ -49,17 +49,6 @@ output "databricks_group_member" {
   value       = module.databricks_workspace_management.databricks_group_member
 }
 
-output "databricks_permissions_notebook" {
-  description = "databricks notebook permissions"
-  value       = module.databricks_workspace_management.databricks_permissions_notebook
-}
-
-/*
-output "databricks_permissions_job" {
-  value = module.databricks_workspace_management.databricks_permissions_job
-}
-*/
-
 output "databricks_permissions_cluster" {
   description = "databricks cluster permissions"
   value       = module.databricks_workspace_management.databricks_permissions_cluster

output.tf

@@ -49,12 +49,8 @@ output "databricks_group_member" {
   description = "databricks group members"
   value       = join("", databricks_group_member.group_members.*.group_id)
 }
-/*
-output "databricks_permissions_notebook" {
-  description = "databricks notebook permissions"
-  value       = join("", databricks_permissions.notebook.*.notebook_path)
-}
 
+/*
 output "databricks_permissions_job" {
   value = databricks_permissions.
 }

variables.tf

@@ -49,13 +49,13 @@ variable "databricks_secret_key" {
 variable "min_idle_instances" {
   description = "instance pool minimum idle instances"
   type        = number
-  default     = 0
+  default     = 1
 }
 
 variable "max_capacity" {
   description = "instance pool maximum capacity"
   type        = number
-  default     = 30
+  default     = 3
 }
 
 variable "idle_instance_autotermination_minutes" {
@@ -79,46 +79,28 @@ variable "cluster_autotermination_minutes" {
   default     = 20
 }
 
-variable "cluster_min_workers" {
-  description = "cluster minimum workers"
-  type        = number
-  default     = 1
-}
-
-variable "cluster_max_workers" {
-  description = "cluster maximum workers"
-  type        = number
-  default     = 10
-}
-
 variable "cluster_policy_max_dbus_per_hour" {
   description = "cluster maximum dbus per hour"
   type        = number
   default     = 10
 }
 
 variable "cluster_policy_autotermination_minutes" {
-  description = "cluster policy auto termination minutes"
+  description = "cluster policy auto termination minutes."
   type        = number
   default     = 20
 }
 
-variable "cluster_type" {
-  description = "Type of cluster(autoscale or fixed)"
-  type        = string
-  default     = "fixed"
-}
-
 variable "auto_scaling" {
-  description = "Type of cluster(autoscale or fixed)"
+  description = "Number of min and max workers in auto scale."
   type        = list(any)
-  default     = [1, 1]
+  default     = null
 }
 
 variable "fixed_value" {
-  description = "Type of cluster(autoscale or fixed)"
+  description = "Number of nodes in the cluster."
   type        = number
-  default     = null
+  default     = 0
 }
 # ------------------------------------------------
 # Job
@@ -226,15 +208,15 @@ variable "cluster_id" {
 }
 
 variable "note_type_id" {
-  description = "Existing cluster id"
+  description = "Type of node"
   type        = string
   default     = null
 }
 
 variable "deploy_instance_pool" {
   description = "Deploy instance pool"
   type        = bool
-  default     = true
+  default     = false
 }
 # ------------------------------------------------
 # Spark version
@@ -276,3 +258,9 @@ variable "create_group" {
   type        = bool
   default     = true
 }
+#
+variable "driver_node_type_id" {
+  description = "The node type of the Spark driver. This field is optional; if unset, API will set the driver node type to the same value as node_type_id"
+  type        = string
+  default     = null
+}