Merge pull request #62 from tomarv2/develop

tomarv2 · web-flow · commit f3274f6bd478 · 2022-06-14T14:47:27.000-07:00
adding file for azure and example
diff --git a/examples/all/sample-maximum_config/main.tf b/examples/all/sample-maximum_config/main.tf
@@ -16,8 +16,6 @@ terraform {
 module "databricks_workspace_management" {
   source = "../../../"
 
-  workspace_url = "https://<workspace_url>.cloud.databricks.com"
-  dapi_token    = "dapi1234567890"
   # ------------------------------------------------
   # CLUSTER
   # ------------------------------------------------
diff --git a/examples/all/sample/main.tf b/examples/all/sample/main.tf
@@ -16,9 +16,6 @@ terraform {
 module "databricks_workspace_management" {
   source = "../../../"
 
-  workspace_url = "https://<workspace_url>.cloud.databricks.com"
-  dapi_token    = "dapi1234567890"
-  # ------------------------------------------------
   deploy_cluster = true
   deploy_jobs    = true
   local_notebooks = [
diff --git a/examples/jobs/1_job_on_new_cluster_with_new_and_existing_notebooks/main.tf b/examples/jobs/1_job_on_new_cluster_with_new_and_existing_notebooks/main.tf
@@ -15,9 +15,6 @@ terraform {
 
 module "databricks_workspace_management" {
   source = "../../../"
-
-  workspace_url = var.workspace_url
-  dapi_token    = var.dapi_token
   # ------------------------------------------------
   # JOB
   # ------------------------------------------------
diff --git a/examples/jobs/1_job_on_new_cluster_with_new_and_existing_notebooks/outputs.tf b/examples/jobs/1_job_on_new_cluster_with_new_and_existing_notebooks/outputs.tf
@@ -7,42 +7,17 @@ output "notebook_url" {
   description = "databricks notebook url"
   value       = module.databricks_workspace_management.notebook_url
 }
-output "new_cluster_new_job_new_notebooks_job" {
+output "job_url" {
   description = "databricks job url"
   value       = module.databricks_workspace_management.new_cluster_new_job_new_notebooks_job
 }
 
-output "new_cluster_new_job_new_notebooks_id" {
+output "job_id" {
   description = "databricks job id"
   value       = module.databricks_workspace_management.new_cluster_new_job_new_notebooks_id
 }
 
-output "existing_cluster_new_job_new_notebooks_job" {
-  description = "databricks new cluster job url"
-  value       = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_job
-}
-
-output "existing_cluster_new_job_new_notebooks_id" {
-  description = "databricks new cluster job id"
-  value       = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_id
-}
-
-output "new_cluster_new_job_existing_notebooks_job" {
-  description = "databricks job url"
-  value       = module.databricks_workspace_management.new_cluster_new_job_existing_notebooks_job
-}
-
-output "new_cluster_new_job_existing_notebooks_id" {
-  description = "databricks job id"
+output "notebooks_id" {
+  description = "databricks notebook id"
   value       = module.databricks_workspace_management.new_cluster_new_job_existing_notebooks_id
 }
-
-output "existing_cluster_new_job_existing_notebooks_job" {
-  description = "databricks new cluster job url"
-  value       = module.databricks_workspace_management.existing_cluster_new_job_existing_notebooks_job
-}
-
-output "existing_cluster_new_job_existing_notebooks_id" {
-  description = "databricks new cluster job id"
-  value       = module.databricks_workspace_management.existing_cluster_new_job_existing_notebooks_id
-}
diff --git a/examples/jobs/2_job_on_existing_cluster_with_new_notebooks/outputs.tf b/examples/jobs/2_job_on_existing_cluster_with_new_notebooks/outputs.tf
diff --git a/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/main.tf b/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/main.tf
@@ -15,14 +15,11 @@ terraform {
 
 module "databricks_workspace_management" {
   source = "../../../"
-
-  workspace_url = var.workspace_url
-  dapi_token    = var.dapi_token
   # ------------------------------------------------
   # JOB
   # ------------------------------------------------
   deploy_jobs               = true
-  cluster_id                = "0907-052446-bike152"
+  deploy_cluster            = true
   fixed_value               = 1
   retry_on_timeout          = false
   max_retries               = 3
@@ -45,15 +42,6 @@ module "databricks_workspace_management" {
     on_success                = ["demo@demo.com"]
   }
   # ------------------------------------------------
-  # JOB ACCESS CONTROL
-  # ------------------------------------------------
-  jobs_access_control = [
-    {
-      group_name       = "demo"
-      permission_level = "CAN_MANAGE_RUN"
-    }
-  ]
-  # ------------------------------------------------
   # NOTEBOOK
   # ------------------------------------------------
   local_notebooks = [
@@ -68,15 +56,12 @@ module "databricks_workspace_management" {
       local_path = "notebooks/sample2.py"
     }
   ]
-  # ------------------------------------------------
-  # NOTEBOOK ACCESS CONTROL
-  # ------------------------------------------------
-  notebooks_access_control = [
-    {
-      group_name       = "demo"
-      permission_level = "CAN_READ"
-    }
-  ]
+#  remote_notebooks = [
+#    {
+#      job_name = "remote_demo_job"
+#      path     = "/Shared/demo/sample1.py"
+#    }
+#  ]
   # ------------------------------------------------
   # Do not change the teamid, prjid once set.
   teamid = var.teamid
diff --git a/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/notebooks/sample1.py b/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/notebooks/sample1.py
@@ -0,0 +1,3 @@
+token = dbutils.secrets.get('${databricks_secret_scope.this.name}', '${databricks_secret.token.key}')
+print(f'This should be redacted: {token}')
+print(f'Hello world - today is holiday')
diff --git a/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/notebooks/sample2.py b/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/notebooks/sample2.py
@@ -0,0 +1,3 @@
+token = dbutils.secrets.get('${databricks_secret_scope.this.name}', '${databricks_secret.token.key}')
+print(f'This should be recacted: {token}')
+print(f'Hello world')
diff --git a/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/outputs.tf b/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/outputs.tf
@@ -0,0 +1,23 @@
+output "cluster_id" {
+  description = "databricks cluster id"
+  value       = module.databricks_workspace_management.cluster_id
+}
+
+output "notebook_url" {
+  description = "databricks notebook url"
+  value       = module.databricks_workspace_management.notebook_url
+}
+output "job_url" {
+  description = "databricks job url"
+  value       = module.databricks_workspace_management.new_cluster_new_job_new_notebooks_job
+}
+
+output "job_id" {
+  description = "databricks job id"
+  value       = module.databricks_workspace_management.new_cluster_new_job_new_notebooks_id
+}
+
+output "notebooks_id" {
+  description = "databricks notebook id"
+  value       = module.databricks_workspace_management.new_cluster_new_job_existing_notebooks_id
+}
diff --git a/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/variables.tf b/examples/jobs/2_job_on_new_cluster_with_new_and_existing_notebooks_without_acl/variables.tf
diff --git a/examples/jobs/3_job_on_existing_cluster_with_existing_notebooks/main.tf b/examples/jobs/3_job_on_existing_cluster_with_existing_notebooks/main.tf
@@ -14,23 +14,11 @@ terraform {
 
 module "databricks_workspace_management" {
   source = "../../../"
-
-  workspace_url = var.workspace_url
-  dapi_token    = var.dapi_token
   # ------------------------------------------------
   # JOB
   # ------------------------------------------------
   deploy_jobs = true
-  cluster_id  = "0907-052446-bike152"
-  # ------------------------------------------------
-  # JOB ACCESS CONTROL
-  # ------------------------------------------------
-  jobs_access_control = [
-    {
-      group_name       = "demo"
-      permission_level = "CAN_MANAGE_RUN"
-    }
-  ]
+  cluster_id  = "0609-190359-dbzlwtsp"
   # ------------------------------------------------
   # NOTEBOOK
   # ------------------------------------------------
diff --git a/examples/jobs/3_job_on_existing_cluster_with_existing_notebooks/outputs.tf b/examples/jobs/3_job_on_existing_cluster_with_existing_notebooks/outputs.tf
@@ -3,42 +3,12 @@ output "notebook_url" {
   value       = module.databricks_workspace_management.notebook_url
 }
 
-output "new_cluster_new_job_new_notebooks_job" {
-  description = "databricks job url"
-  value       = module.databricks_workspace_management.new_cluster_new_job_new_notebooks_job
-}
-
-output "new_cluster_new_job_new_notebooks_id" {
-  description = "databricks job id"
-  value       = module.databricks_workspace_management.new_cluster_new_job_new_notebooks_id
-}
-
-output "existing_cluster_new_job_new_notebooks_job" {
-  description = "databricks new cluster job url"
-  value       = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_job
-}
-
-output "existing_cluster_new_job_new_notebooks_id" {
-  description = "databricks new cluster job id"
-  value       = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_id
-}
-
-output "new_cluster_new_job_existing_notebooks_job" {
-  description = "databricks job url"
-  value       = module.databricks_workspace_management.new_cluster_new_job_existing_notebooks_job
-}
-
-output "new_cluster_new_job_existing_notebooks_id" {
-  description = "databricks job id"
-  value       = module.databricks_workspace_management.new_cluster_new_job_existing_notebooks_id
-}
-
-output "existing_cluster_new_job_existing_notebooks_job" {
+output "job_url" {
   description = "databricks new cluster job url"
   value       = module.databricks_workspace_management.existing_cluster_new_job_existing_notebooks_job
 }
 
-output "existing_cluster_new_job_existing_notebooks_id" {
-  description = "databricks new cluster job id"
+output "job_id" {
+  description = "job id"
   value       = module.databricks_workspace_management.existing_cluster_new_job_existing_notebooks_id
 }
diff --git a/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/main.tf b/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/main.tf
@@ -0,0 +1,63 @@
+provider "databricks" {
+  host  = var.workspace_url
+  token = var.dapi_token
+}
+
+terraform {
+  required_providers {
+    databricks = {
+      source  = "databrickslabs/databricks"
+      version = "~> 0.5.7"
+    }
+  }
+}
+
+
+module "databricks_workspace_management" {
+  source = "../../../"
+  # ------------------------------------------------
+  # JOB
+  # ------------------------------------------------
+  deploy_jobs               = true
+  cluster_id                = "0606-193951-30y13dr5"
+  fixed_value               = 1
+  retry_on_timeout          = false
+  max_retries               = 3
+  timeout                   = 30
+  min_retry_interval_millis = 10
+  max_concurrent_runs       = 1
+  task_parameters = {
+    "hello" = "world",
+    "ping"  = "pong"
+  }
+  schedule = {
+    cron_expression = "1 0 7 * * ?",
+    timezone_id     = "America/Los_Angeles",
+    pause_status    = "UNPAUSED"
+  }
+  email_notifications = {
+    on_failure                = ["demo@demo.com"],
+    no_alert_for_skipped_runs = true,
+    on_start                  = ["demo@demo.com"],
+    on_success                = ["demo@demo.com"]
+  }
+  # ------------------------------------------------
+  # NOTEBOOK
+  # ------------------------------------------------
+  local_notebooks = [
+    {
+      job_name   = "local_demo_job1"
+      language   = "PYTHON"
+      local_path = "notebooks/sample1.py"
+      path       = "/Shared/demo/sample1.py"
+    },
+    {
+      job_name   = "local_demo_job2"
+      local_path = "notebooks/sample2.py"
+    }
+  ]
+  # ------------------------------------------------
+  # Do not change the teamid, prjid once set.
+  teamid = var.teamid
+  prjid  = var.prjid
+}
diff --git a/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/notebooks/sample1.py b/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/notebooks/sample1.py
diff --git a/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/notebooks/sample2.py b/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/notebooks/sample2.py
diff --git a/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/outputs.tf b/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/outputs.tf
@@ -0,0 +1,15 @@
+output "notebook_url" {
+  description = "databricks notebook url"
+  value       = module.databricks_workspace_management.notebook_url
+}
+
+output "job_url" {
+  description = "databricks job url"
+  value       = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_job
+}
+
+output "job_id" {
+  description = "databricks job id"
+  value       = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_id
+}
+
diff --git a/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/variables.tf b/examples/jobs/4_job_on_existing_cluster_with_new_notebooks/variables.tf
@@ -0,0 +1,19 @@
+variable "teamid" {
+  description = "Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
+  type        = string
+}
+
+variable "prjid" {
+  description = "Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
+  type        = string
+}
+
+variable "workspace_url" {
+  description = "databricks workspace url"
+  type        = string
+}
+
+variable "dapi_token" {
+  description = "databricks dapi token"
+  type        = string
+}
diff --git a/examples/notebooks/1_deploy_notebooks/main.tf b/examples/notebooks/1_deploy_notebooks/main.tf
@@ -15,9 +15,6 @@ terraform {
 
 module "databricks_workspace_management" {
   source = "../../../"
-
-  workspace_url = var.workspace_url
-  dapi_token    = var.dapi_token
   # ------------------------------------------------
   # NOTEBOOK
   # ------------------------------------------------
diff --git a/job.tf b/job.tf
diff --git a/main.tf b/main.tf
diff --git a/outputs.tf b/outputs.tf

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+token = dbutils.secrets.get('${databricks_secret_scope.this.name}', '${databricks_secret.token.key}')`
	`2`	`+print(f'This should be redacted: {token}')`
	`3`	`+print(f'Hello world - today is holiday')`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+token = dbutils.secrets.get('${databricks_secret_scope.this.name}', '${databricks_secret.token.key}')`
	`2`	`+print(f'This should be recacted: {token}')`
	`3`	`+print(f'Hello world')`