I would like to create a single terraform .tf that creates my databricks environment from scratch. I have followed an example that creates an databricks workspace in azure and it works fine.
https://learn.microsoft.com/en-us/azure/databricks/dev-tools/terraform/azure-workspace
Seems like this also creates components needed for unity catalog like Managed identity, access connector and storage account. But unity catalog is not enabled. In the code example, this is the part that actually makes the resource:
resource "azurerm_databricks_workspace" "this" {
name = "${local.prefix}-workspace"
resource_group_name = azurerm_resource_group.this.name
location = azurerm_resource_group.this.location
sku = "premium"
managed_resource_group_name = "${local.prefix}-workspace-rg"
tags = local.tags
}
What am I missing to be able to get the Unity catalog enabled through terraform?
What am I missing to be able to get the Unity catalog enabled through terraform?
Here is the updated terraform code to create a meta store
and enabled Unity catalog.
terraform {
required_providers {
azurerm = {
source = "hashicorp/azurerm"
}
databricks = {
source = "databricks/databricks"
}
}
}
provider "azurerm" {
subscription_id = ""
features {}
}
resource "azurerm_resource_group" "rgname" {
name = "databricks-temp-1"
location = "centralus"
}
resource "azurerm_databricks_workspace" "example" {
name = "databricks-test"
resource_group_name = azurerm_resource_group.rgname.name
location = azurerm_resource_group.rgname.location
sku = "premium"
}
provider "databricks" {
host = azurerm_databricks_workspace.example.workspace_url
}
resource "azurerm_databricks_access_connector" "unity" {
name = "venkatdatabricksmi1"
resource_group_name = azurerm_resource_group.rgname.name
location = azurerm_resource_group.rgname.location
identity {
type = "SystemAssigned"
}
}
resource "azurerm_storage_account" "unity_catalog" {
name = "thejadatabricksdemo2"
resource_group_name = azurerm_resource_group.rgname.name
location = azurerm_resource_group.rgname.location
account_tier = "Standard"
account_replication_type = "GRS"
is_hns_enabled = true
}
resource "azurerm_storage_container" "unity_catalog" {
name = "venkat-container2"
storage_account_name = azurerm_storage_account.unity_catalog.name
container_access_type = "private"
}
resource "azurerm_role_assignment" "example" {
scope = azurerm_storage_account.unity_catalog.id
role_definition_name = "Storage Blob Data Contributor"
principal_id = azurerm_databricks_access_connector.unity.identity[0].principal_id
}
resource "databricks_metastore" "this" {
name = "demometastoretest"
storage_root = format("abfss://%s@%s.dfs.core.windows.net/",
azurerm_storage_container.unity_catalog.name,
azurerm_storage_account.unity_catalog.name)
force_destroy = true
region = "centralus"
}
resource "databricks_metastore_assignment" "this" {
provider = databricks
workspace_id = azurerm_databricks_workspace.example.workspace_id
metastore_id = databricks_metastore.this.id
default_catalog_name = "hive_metastore"
}
resource "databricks_metastore_data_access" "this" {
metastore_id = databricks_metastore.this.id
name = "mi_dac"
azure_managed_identity {
access_connector_id = azurerm_databricks_access_connector.unity.id
}
is_default = true
depends_on = [ databricks_metastore_assignment.this ]
}
Terraform apply:
After running the code, the Unity catalog
has been enabled.
The Databricks workspace
was also successfully added to the metastore.
Reference: databricks_metastore Resource