Error when programmatically deleting Recovery Service Vault using Terraform

I have the following Terraform code for:

• Creating:

  1. Storage account
  2. 1 Azure File Share
  3. 1 Azure Blob Container
  4. 1 Recovery Service Vault

• Setting:

  1. Backup policy for the Azure File Share

• Enabling backup for the Azure File Share and Azure Blob Container

resource "azurerm_storage_account" "stg" {
  name                     = local.storage_name
  resource_group_name      = azurerm_resource_group.ws.name
  location                 = azurerm_resource_group.ws.location
  account_tier             = "Standard"
  account_replication_type = "GRS"
  is_hns_enabled           = true
  tags                     = local.tre_workspace_tags

  lifecycle { ignore_changes = [tags] }
}

resource "azurerm_storage_share" "shared_storage" {
  name                 = local.shared_storage_name
  storage_account_name = azurerm_storage_account.stg.name
  quota                = var.shared_storage_quota
}

resource "azurerm_storage_container" "stgcontainer" {
  name                  = local.blob_container_name
  storage_account_name  = azurerm_storage_account.stg.name
  container_access_type = "private"
}

resource "azurerm_recovery_services_vault" "recovery_services_vault" {
  name                = local.recovery_services_vault_name
  location            = azurerm_storage_account.stg.location
  resource_group_name = azurerm_storage_account.stg.resource_group_name
  sku                 = "Standard"
  tags                = local.tre_workspace_tags

  lifecycle { ignore_changes = [tags] }

  depends_on = [azurerm_storage_account.stg]
}

resource "azurerm_backup_policy_file_share" "vault_policy" {
  name                = "rsv-policy-${local.workspace_resource_name_suffix}"
  resource_group_name = azurerm_recovery_services_vault.recovery_services_vault.resource_group_name
  recovery_vault_name = azurerm_recovery_services_vault.recovery_services_vault.name

  backup {
    frequency = "Daily"
    time      = "23:00"
  }

  retention_daily {
    count = 30
  }

  retention_weekly {
    count    = 4
    weekdays = ["Saturday"]
  }

  retention_monthly {
    count    = 2
    weekdays = ["Sunday"]
    weeks    = ["Last"]
  }

  depends_on = [azurerm_recovery_services_vault.recovery_services_vault]
}

resource "azurerm_backup_container_storage_account" "shared_storage_register" {
  resource_group_name = azurerm_backup_policy_file_share.vault_policy.resource_group_name
  recovery_vault_name = azurerm_backup_policy_file_share.vault_policy.recovery_vault_name
  storage_account_id  = azurerm_storage_account.stg.id

  depends_on = [
    azurerm_backup_policy_file_share.vault_policy,
    azurerm_storage_account.stg
  ]
}

resource "azurerm_backup_protected_file_share" "shared_storage_backup" {
  resource_group_name       = azurerm_backup_container_storage_account.shared_storage_register.resource_group_name
  recovery_vault_name       = azurerm_backup_container_storage_account.shared_storage_register.recovery_vault_name
  source_storage_account_id = azurerm_backup_container_storage_account.shared_storage_register.storage_account_id
  source_file_share_name    = azurerm_storage_share.shared_storage.name
  backup_policy_id          = azurerm_backup_policy_file_share.vault_policy.id

  depends_on = [
    azurerm_backup_container_storage_account.shared_storage_register,
    azurerm_backup_policy_file_share.vault_policy
  ]
}

# This code is executed on destroy-time before removing the Recovery Service Vault.
resource "terraform_data" "disable_rsv_configurations" {
  input = {
    subscription_id     = data.azurerm_client_config.current.subscription_id
    resource_group_name = azurerm_resource_group.ws.name
    rsv_name            = local.recovery_services_vault_name
    container_name      = local.storage_name
    item_name           = local.shared_storage_name
  }
  provisioner "local-exec" {
    when       = destroy
    on_failure = continue
    command    = <<EOT
      az login --identity &&
      az backup vault backup-properties set \
        --subscription ${self.input.subscription_id} \
        --resource-group ${self.input.resource_group_name} \
        --name ${self.input.rsv_name} \
        --soft-delete-feature-state Disable \
        --hybrid-backup-security-features Disable &&
      az backup protection disable \
        --subscription ${self.input.subscription_id} \
        --resource-group ${self.input.resource_group_name} \
        --vault-name ${self.input.rsv_name} \
        --container-name ${self.input.container_name} \
        --backup-management-type AzureStorage \
        --item-name ${self.input.item_name} \
        --delete-backup-data true \
        --yes &&
      az backup container unregister \
        --resource-group ${self.input.resource_group_name} \
        --vault-name ${self.input.rsv_name} \
        --container-name ${self.input.container_name} \
        --backup-management-type AzureStorage \
        --yes
    EOT
  }

  depends_on = [azurerm_backup_container_storage_account.shared_storage_register]
}

All the resources are creating with no problem. Including the terraform_data resource.

However, problems appear when I have to delete the resources. I have been using depends_on so that I could force an specific deletion order. According to Azure documentation, before deleting the Recovery Service Vault I have to:

1. Disable soft delete
2. Remove backup items (the Azure File Share is the only backup item)
3. Unregister Storage Accounts (the Storage Account created is the only one available)

I can execute the steps above through Azure Portal with no problem, and then successfully destroy everything. I researched, found and tested in the console the commands defined in the terraform_data. It means that if I run then manually, and destroy the resources, everything goes fine. On the other hand, if I try to destroy the resources automatically, it fails. The error message is:

Error: deleting Azure File Share backups item StorageContainer;storage;<RESOURCE_GROUP_NAME>;<STORAGE_ACCOUNT_NAME> (Vault <RECOVERY_SERVICE_VAULT_NAME>): Location header missing or empty

Any suggestions where is this error coming from?

EDIT 1

@vinay-b answer given below helped me to understand a little bit better what was happening. He suggests using null_resource instead of terraform_data. however, given that essentially everywhere terraform_data is said to be the natural evolution of null_resource, I couldn't believe that it would be the real issue.

It turned out that disabling backup protection and unregistering container, and letting Terraform form remove the resources azurerm_backup_protected_file_share.shared_storage_backup and azurerm_backup_container_storage_account.shared_storage_register, made such resources being destroyed twice.

Besides, such operations take a while and must be serialized, so I added a few time_sleep and used depends_on. The following code is working as expected:

resource "azurerm_storage_account" "stg" {
  name                     = local.storage_name
  resource_group_name      = azurerm_resource_group.ws.name
  location                 = azurerm_resource_group.ws.location
  account_tier             = "Standard"
  account_replication_type = "GRS"
  is_hns_enabled           = true
  tags                     = local.tre_workspace_tags

  lifecycle { ignore_changes = [tags] }
}

resource "azurerm_storage_share" "shared_storage" {
  name                 = local.shared_storage_name
  storage_account_name = azurerm_storage_account.stg.name
  quota                = var.shared_storage_quota
}

resource "azurerm_storage_container" "stgcontainer" {
  name                  = local.blob_container_name
  storage_account_name  = azurerm_storage_account.stg.name
  container_access_type = "private"
}

resource "azurerm_recovery_services_vault" "recovery_services_vault" {
  name                = local.recovery_services_vault_name
  location            = azurerm_storage_account.stg.location
  resource_group_name = azurerm_storage_account.stg.resource_group_name
  sku                 = "Standard"
  tags                = local.tre_workspace_tags

  lifecycle { ignore_changes = [tags] }

  depends_on = [azurerm_storage_account.stg]
}

resource "time_sleep" "wait_1_minute_point_d" {
  create_duration  = "30s"
  destroy_duration = "120s"

  depends_on = [azurerm_recovery_services_vault.recovery_services_vault]
}

resource "azurerm_backup_policy_file_share" "vault_policy" {
  name                = "rsv-policy-${local.workspace_resource_name_suffix}"
  resource_group_name = azurerm_recovery_services_vault.recovery_services_vault.resource_group_name
  recovery_vault_name = azurerm_recovery_services_vault.recovery_services_vault.name

  backup {
    frequency = "Daily"
    time      = "23:00"
  }

  retention_daily {
    count = 30
  }

  retention_weekly {
    count    = 4
    weekdays = ["Saturday"]
  }

  retention_monthly {
    count    = 2
    weekdays = ["Sunday"]
    weeks    = ["Last"]
  }

  depends_on = [time_sleep.wait_1_minute_point_d]
}

resource "time_sleep" "wait_1_minute_point_c" {
  create_duration  = "30s"
  destroy_duration = "120s"

  depends_on = [azurerm_backup_policy_file_share.vault_policy]
}

resource "azurerm_backup_container_storage_account" "shared_storage_register" {
  resource_group_name = azurerm_backup_policy_file_share.vault_policy.resource_group_name
  recovery_vault_name = azurerm_backup_policy_file_share.vault_policy.recovery_vault_name
  storage_account_id  = azurerm_storage_account.stg.id

  depends_on = [time_sleep.wait_1_minute_point_c]
}

resource "time_sleep" "wait_1_minute_point_b" {
  create_duration  = "30s"
  destroy_duration = "120s"

  depends_on = [azurerm_backup_container_storage_account.shared_storage_register]
}

resource "azurerm_backup_protected_file_share" "shared_storage_backup" {
  resource_group_name       = azurerm_backup_container_storage_account.shared_storage_register.resource_group_name
  recovery_vault_name       = azurerm_backup_container_storage_account.shared_storage_register.recovery_vault_name
  source_storage_account_id = azurerm_backup_container_storage_account.shared_storage_register.storage_account_id
  source_file_share_name    = azurerm_storage_share.shared_storage.name
  backup_policy_id          = azurerm_backup_policy_file_share.vault_policy.id

  depends_on = [
    azurerm_backup_policy_file_share.vault_policy,
    time_sleep.wait_1_minute_point_b
  ]
}

resource "time_sleep" "wait_1_minute_point_a" {
  create_duration  = "30s"
  destroy_duration = "120s"

  depends_on = [azurerm_backup_protected_file_share.shared_storage_backup]
}

# This code is executed on destroy-time before removing the Recovery Service Vault.
resource "terraform_data" "disable_rsv_configurations" {
  input = {
    client_id           = data.azurerm_client_config.current.client_id
    subscription_id     = data.azurerm_client_config.current.subscription_id
    resource_group_name = azurerm_resource_group.ws.name
    rsv_name            = local.recovery_services_vault_name
  }

  provisioner "local-exec" {
    when       = destroy
    on_failure = continue
    command    = <<EOT
      az login --identity  -u ${self.input.client_id}
      sleep 30
      echo "Disable Soft Delete"
      az backup vault backup-properties set \
        --subscription ${self.input.subscription_id} \
        --resource-group ${self.input.resource_group_name} \
        --name ${self.input.rsv_name} \
        --soft-delete-feature-state Disable \
        --hybrid-backup-security-features Disable
      sleep 30
    EOT
  }

  depends_on = [time_sleep.wait_1_minute_point_a]
}

Deleting the dependencies before deleting Recovery Service Vault using Terraform

Here in this approach the terraform-data is used for fetching or querying existing information but not executing actions i.e., Terraform data sources cannot execute actions like az backup protection disable or az backup container unregister.

Even when I tried using this I faced the blocker as mentioned below

I tried an updated configuration using null resource local exec where we can achieve all the requirements as mentioned like disable soft delete, remove backup items and unregister storage accounts etc dependences before destroying the recovery vault.

My initial setup in the portal

Configuration:

resource "azurerm_recovery_services_vault" "recovery_services_vault" {
  name                = "evksb-recovery-vault"
  location            = azurerm_storage_account.stg.location
  resource_group_name = azurerm_resource_group.ws.name
  sku                 = "Standard"
}

resource "azurerm_backup_policy_file_share" "vault_policy" {
  name                = "rsv-policy-example"
  resource_group_name = azurerm_recovery_services_vault.recovery_services_vault.resource_group_name
  recovery_vault_name = azurerm_recovery_services_vault.recovery_services_vault.name

  backup {
    frequency = "Daily"
    time      = "23:00"
  }

  retention_daily {
    count = 30
  }

  retention_weekly {
    count    = 4
    weekdays = ["Saturday"]
  }

  retention_monthly {
    count    = 2
    weekdays = ["Sunday"]
    weeks    = ["Last"]
  }
}

resource "azurerm_backup_container_storage_account" "shared_storage_register" {
  resource_group_name = azurerm_backup_policy_file_share.vault_policy.resource_group_name
  recovery_vault_name = azurerm_backup_policy_file_share.vault_policy.recovery_vault_name
  storage_account_id  = azurerm_storage_account.stg.id
}

resource "azurerm_backup_protected_file_share" "shared_storage_backup" {
  resource_group_name       = azurerm_backup_container_storage_account.shared_storage_register.resource_group_name
  recovery_vault_name       = azurerm_backup_container_storage_account.shared_storage_register.recovery_vault_name
  source_storage_account_id = azurerm_backup_container_storage_account.shared_storage_register.storage_account_id
  source_file_share_name    = azurerm_storage_share.shared_storage.name
  backup_policy_id          = azurerm_backup_policy_file_share.vault_policy.id
}

resource "null_resource" "disable_rsv_configurations" {
  provisioner "local-exec" {
    interpreter = ["pwsh", "-Command"]
    when        = destroy
    command     = <<EOT
     
      $env:AZURE_CLIENT_ID = "yourclientid";
      $env:AZURE_CLIENT_SECRET = "yourclientsecret";
      $env:AZURE_TENANT_ID = "yourtenentid";
      az login --service-principal --username $env:AZURE_CLIENT_ID --password $env:AZURE_CLIENT_SECRET --tenant $env:AZURE_TENANT_ID

     
      az backup protection disable `
        --resource-group ${self.triggers.resource_group} `
        --vault-name ${self.triggers.recovery_vault_name} `
        --container-name ${self.triggers.storage_account} `
        --backup-management-type AzureStorage `
        --item-name ${self.triggers.file_share_name} `
        --delete-backup-data true `
        --yes
      
   
      az backup container unregister `
        --resource-group ${self.triggers.resource_group} `
        --vault-name ${self.triggers.recovery_vault_name} `
        --container-name ${self.triggers.storage_account} `
        --backup-management-type AzureStorage `
        --yes
      
      Start-Sleep -Seconds 10

      
      az backup vault backup-properties set `
        --resource-group ${self.triggers.resource_group} `
        --name ${self.triggers.recovery_vault_name} `
        --soft-delete-feature-state Disable

      Start-Sleep -Seconds 10
      
      # Delete the Recovery Services Vault after backups are cleaned up
      az resource delete `
        --resource-group ${self.triggers.resource_group} `
        --name ${self.triggers.recovery_vault_name} `
        --resource-type "Microsoft.RecoveryServices/vaults"
    EOT
  }

  triggers = {
    resource_group      = azurerm_resource_group.ws.name
    recovery_vault_name = azurerm_recovery_services_vault.recovery_services_vault.name
    storage_account     = azurerm_storage_account.stg.name
    file_share_name     = azurerm_storage_share.shared_storage.name
  }

  depends_on = [
    azurerm_backup_protected_file_share.shared_storage_backup
  ]
}

deployment:

Refer:

https://learn.microsoft.com/en-us/azure/backup/

https://learn.microsoft.com/en-us/cli/azure/backup?view=azure-cli-latest

https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/backup_protected_file_share