Private endpoint support for container apps

app/backend/prepdocs.py

@@ -4,6 +4,7 @@
 import os
 from typing import Optional, Union
 
+import aiohttp
 from azure.core.credentials import AzureKeyCredential
 from azure.core.credentials_async import AsyncTokenCredential
 from azure.identity.aio import AzureDeveloperCliCredential, get_bearer_token_provider
@@ -45,6 +46,19 @@ def clean_key_if_exists(key: Union[str, None]) -> Union[str, None]:
     return None
 
 
+async def check_search_service_connectivity(search_service: str) -> bool:
+    """Check if the search service is accessible by hitting the /ping endpoint."""
+    ping_url = f"https://{search_service}.search.windows.net/ping"
+
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(ping_url, timeout=aiohttp.ClientTimeout(total=10)) as response:
+                return response.status == 200
+    except Exception as e:
+        logger.debug(f"Search service ping failed: {e}")
+        return False
+
+
 async def setup_search_info(
     search_service: str,
     index_name: str,
@@ -323,7 +337,10 @@ async def main(strategy: Strategy, setup_index: bool = True):
 
     load_azd_env()
 
-    if os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled":
+    if (
+        os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled"
+        and os.getenv("AZURE_USE_VPN_GATEWAY", "").lower() != "true"
+    ):
         logger.error("AZURE_PUBLIC_NETWORK_ACCESS is set to Disabled. Exiting.")
         exit(0)
 
@@ -372,6 +389,23 @@ async def main(strategy: Strategy, setup_index: bool = True):
             search_key=clean_key_if_exists(args.searchkey),
         )
     )
+
+    # Check search service connectivity
+    search_service = os.environ["AZURE_SEARCH_SERVICE"]
+    is_connected = loop.run_until_complete(check_search_service_connectivity(search_service))
+
+    if not is_connected:
+        if os.getenv("AZURE_USE_PRIVATE_ENDPOINT"):
+            logger.error(
+                "Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration. You have AZURE_USE_PRIVATE_ENDPOINT enabled. Perhaps you're not yet connected to the VPN? Download the VPN configuration from the Azure portal here: %s",
+                os.getenv("AZURE_VPN_CONFIG_DOWNLOAD_LINK"),
+            )
+        else:
+            logger.error(
+                "Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration."
+            )
+        exit(1)
+
     blob_manager = setup_blob_manager(
         azure_credential=azd_credential,
         storage_account=os.environ["AZURE_STORAGE_ACCOUNT"],

docs/deploy_private.md

@@ -31,9 +31,75 @@ If you want to disable public access when deploying the Chat App, you can do so
 
 Deploying with public access disabled adds additional cost to your deployment. Please see pricing for the following products:
 
-1. [Private Endpoints](https://azure.microsoft.com/pricing/details/private-link/)
-    1. The exact number of private endpoints created depends on the [optional features](./deploy_features.md) used.
-1. [Private DNS Zones](https://azure.microsoft.com/pricing/details/dns/)
+* [Azure Container Registry](https://azure.microsoft.com/pricing/details/container-registry/): Premium tier is used when virtual network is added (required for private links), which incurs additional costs.
+* [Azure Container Apps](https://azure.microsoft.com/pricing/details/container-apps/): Workload profiles environment is used when virtual network is added (required for private links), which incurs additional costs. Additionally, min replica count is set to 1, so you will be charged for at least one instance.
+* [VPN Gateway](https://azure.microsoft.com/pricing/details/vpn-gateway/): VpnGw2 SKU. Pricing includes a base monthly cost plus an hourly cost based on the number of connections.
+* [Virtual Network](https://azure.microsoft.com/pricing/details/virtual-network/): Pay-as-you-go tier. Costs based on data processed.
+
+The pricing for the following features depends on the [optional features](./deploy_features.md) used. Most deployments will have at least 5 private endpoints (Azure OpenAI, Azure Cognitive Services, Azure AI Search, Azure Blob Storage, and either Azure App Service or Azure Container Apps).
+
+* [Azure Private Endpoints](https://azure.microsoft.com/pricing/details/private-link/): Pricing is per hour per endpoint.
+* [Private DNS Zones](https://azure.microsoft.com/pricing/details/dns/): Pricing is per month and zones.
+* [Azure Private DNS Resolver](https://azure.microsoft.com/pricing/details/dns/): Pricing is per month and zones.
+
+⚠️ To avoid unnecessary costs, remember to take down your app if it's no longer in use,
+either by deleting the resource group in the Portal or running `azd down`.
+You might also decide to delete the VPN Gateway when not in use.
+
+## Recommended deployment strategy for private access
+
+1. Configure the azd environment variables to use private endpoints and a VPN gateway, with public network access disabled. This will allow you to connect to the chat app from inside the virtual network, but not from the public Internet.
+
+    ```shell
+    azd env set AZURE_USE_PRIVATE_ENDPOINT true
+    azd env set AZURE_USE_VPN_GATEWAY true
+    azd env set AZURE_PUBLIC_NETWORK_ACCESS Disabled
+    azd up
+    ```
+
+2. Provision all the Azure resources:
+
+    ```bash
+    azd provision
+    ```
+
+3. Once provisioning is complete, you will see an error when it tries to run the data ingestion script, because you are not yet connected to the VPN. That message should provide a URL for the VPN configuration file download. If you don't see that URL, run this command:
+
+    ```bash
+    azd env get-value AZURE_VPN_CONFIG_DOWNLOAD_LINK
+    ```
+
+    Open that link in your browser. Select "Download VPN client" to download a ZIP file containing the VPN configuration.
+
+4. Open `AzureVPN/azurevpnconfig.xml`, and replace the `<clientconfig>` empty tag with the following:
+
+    ```xml
+      <clientconfig>
+        <dnsservers>
+          <dnsserver>10.0.11.4</dnsserver>
+        </dnsservers>
+      </clientconfig>
+    ```
+
+    > **Note:** We use the IP address `10.0.11.4` since it is the first available IP in the `dns-resolver-subnet`(10.0.11.0/28) from the provisioned virtual network, as Azure reserves the first four IP addresses in each subnet. Adding this DNS server allows your VPN client to resolve private DNS names for Azure services accessed through private endpoints. See the network configuration in [network-isolation.bicep](../infra/network-isolation.bicep) for details.
+
+5. Install the [Azure VPN Client](https://learn.microsoft.com/azure/vpn-gateway/azure-vpn-client-versions).
+
+6. Open the Azure VPN Client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified.
+
+7. Select "Connect" and the new VPN connection. You will be prompted to select your Microsoft account and login.
+
+8. Once you're successfully connected to VPN, you can run the data ingestion script:
+
+    ```bash
+    azd hooks run postprovision
+    ```
+
+9. Finally, you can deploy the app:
+
+    ```bash
+    azd deploy
+    ```
 
 ## Environment variables controlling private access
 
@@ -43,23 +109,8 @@ Deploying with public access disabled adds additional cost to your deployment. P
 1. `AZURE_USE_PRIVATE_ENDPOINT`: Controls deployment of [private endpoints](https://learn.microsoft.com/azure/private-link/private-endpoint-overview) which connect Azure resources to the virtual network.
     1. When set to 'true', ensures private endpoints are deployed for connectivity even when `AZURE_PUBLIC_NETWORK_ACCESS` is 'Disabled'.
     1. Note that private endpoints do not make the chat app accessible from the internet. Connections must be initiated from inside the virtual network.
+1. `AZURE_USE_VPN_GATEWAY`: Controls deployment of a VPN gateway for the virtual network. If you do not use this and public access is disabled, you will need a different way to connect to the virtual network.
 
-## Recommended deployment strategy for private access
-
-1. Deploy the app with private endpoints enabled and public access enabled.
-
-  ```shell
-  azd env set AZURE_USE_PRIVATE_ENDPOINT true
-  azd env set AZURE_PUBLIC_NETWORK_ACCESS Enabled
-  azd up
-  ```
-
-1. Validate that you can connect to the chat app and it's working as expected from the internet.
-1. Re-provision the app with public access disabled.
-
-  ```shell
-  azd env set AZURE_PUBLIC_NETWORK_ACCESS Disabled
-  azd provision
-  ```
+## Compatibility with other features
 
-1. Log into your network using a tool like [Azure VPN Gateway](https://azure.microsoft.com/services/vpn-gateway/) and validate that you can connect to the chat app from inside the network.
+* **GitHub Actions / Azure DevOps**: The private access deployment is not compatible with the built-in CI/CD pipelines, as it requires a VPN connection to deploy the app. You could modify the pipeline to only do provisioning, and set up a different deployment strategy for the app.

infra/abbreviations.json

@@ -110,6 +110,7 @@
   "privateEndpoint": "pe-",
   "privateLink": "pl-",
   "purviewAccounts": "pview-",
+  "privateDnsResolver": "pdr-",
   "recoveryServicesVaults": "rsv-",
   "resourcesResourceGroups": "rg-",
   "searchSearchServices": "srch-",

infra/core/host/container-app-upsert.bicep

@@ -78,9 +78,6 @@ param serviceBinds array = []
 @description('The target port for the container')
 param targetPort int = 80
 
-@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100'])
-param workloadProfile string = 'Consumption'
-
 param allowedOrigins array = []
 
 resource existingApp 'Microsoft.App/containerApps@2023-05-02-preview' existing = if (exists) {
@@ -98,7 +95,6 @@ module app 'container-app.bicep' = {
   name: '${deployment().name}-update'
   params: {
     name: name
-    workloadProfile: workloadProfile
     location: location
     tags: tags
     identityType: identityType

infra/core/host/container-app.bicep

@@ -79,8 +79,6 @@ param serviceType string = ''
 @description('The target port for the container')
 param targetPort int = 80
 
-param workloadProfile string = 'Consumption'
-
 resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!empty(identityName)) {
   name: identityName
 }
@@ -125,7 +123,6 @@ resource app 'Microsoft.App/containerApps@2023-05-02-preview' = {
   }
   properties: {
     managedEnvironmentId: containerAppsEnvironment.id
-    workloadProfileName: workloadProfile
     configuration: {
       activeRevisionsMode: revisionMode
       ingress: ingressEnabled ? {

infra/core/host/container-apps-environment.bicep

@@ -0,0 +1,73 @@
+param name string
+param location string = resourceGroup().location
+param tags object = {}
+
+param daprEnabled bool = false
+param logAnalyticsWorkspaceName string = ''
+param applicationInsightsName string = ''
+
+param subnetResourceId string
+
+param usePrivateIngress bool = true
+
+@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100'])
+param workloadProfile string
+
+// Make sure that we are using a non-consumption workload profile for private endpoints
+var finalWorkloadProfile = (usePrivateIngress && workloadProfile == 'Consumption') ? 'D4' : workloadProfile
+
+var minimumCount = usePrivateIngress ? 1 : 0
+var maximumCount = usePrivateIngress ? 3 : 2
+
+resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2025-02-02-preview' = {
+  name: name
+  location: location
+  tags: tags
+  properties: {
+    // We can't use a conditional here due to an issue with the Container Apps ARM parsing
+    appLogsConfiguration: {
+      destination: 'log-analytics'
+      logAnalyticsConfiguration: {
+        customerId: logAnalyticsWorkspace.properties.customerId
+        sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey
+      }
+    }
+    daprAIInstrumentationKey: daprEnabled && !empty(applicationInsightsName) ? applicationInsights.properties.InstrumentationKey : ''
+    publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled'
+    vnetConfiguration: usePrivateIngress ? {
+      infrastructureSubnetId: subnetResourceId
+      internal: true
+    } : null
+    workloadProfiles: usePrivateIngress
+    ? [
+      {
+        name: 'Consumption'
+        workloadProfileType: 'Consumption'
+      }
+      {
+        name: 'Warm'
+        workloadProfileType: finalWorkloadProfile
+        minimumCount: minimumCount
+        maximumCount: maximumCount
+      }
+    ]
+    : [
+      {
+        name: 'Consumption'
+        workloadProfileType: 'Consumption'
+      }
+    ]
+  }
+}
+
+resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = if (!empty(logAnalyticsWorkspaceName)) {
+  name: logAnalyticsWorkspaceName
+}
+
+resource applicationInsights 'Microsoft.Insights/components@2020-02-02' existing = if (daprEnabled && !empty(applicationInsightsName)){
+  name: applicationInsightsName
+}
+
+output defaultDomain string = containerAppsEnvironment.properties.defaultDomain
+output name string = containerAppsEnvironment.name
+output resourceId string = containerAppsEnvironment.id

infra/core/host/container-apps.bicep

@@ -1,78 +1,48 @@
-metadata description = 'Creates an Azure Container Registry and an Azure Container Apps environment.'
 param name string
 param location string = resourceGroup().location
 param tags object = {}
 
 param containerAppsEnvironmentName string
 param containerRegistryName string
-param containerRegistryResourceGroupName string = ''
-param containerRegistryAdminUserEnabled bool = false
-param logAnalyticsWorkspaceResourceId string
-param applicationInsightsName string = '' // Not used here, was used for DAPR
-param virtualNetworkSubnetId string = ''
+param logAnalyticsWorkspaceName string = ''
+param applicationInsightsName string = ''
+
+@description('Virtual network name for container apps environment.')
+param vnetName string = ''
+
 @allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100'])
 param workloadProfile string
 
-var workloadProfiles = workloadProfile == 'Consumption'
-  ? [
-      {
-        name: 'Consumption'
-        workloadProfileType: 'Consumption'
-      }
-    ]
-  : [
-      {
-        name: 'Consumption'
-        workloadProfileType: 'Consumption'
-      }
-      {
-        minimumCount: 0
-        maximumCount: 2
-        name: workloadProfile
-        workloadProfileType: workloadProfile
-      }
-    ]
+param subnetResourceId string = ''
 
-@description('Optional user assigned identity IDs to assign to the resource')
-param userAssignedIdentityResourceIds array = []
+param usePrivateIngress bool = true
 
-module containerAppsEnvironment 'br/public:avm/res/app/managed-environment:0.8.0' = {
+module containerAppsEnvironment 'container-apps-environment.bicep' = {
   name: '${name}-container-apps-environment'
   params: {
-    // Required parameters
-    logAnalyticsWorkspaceResourceId: logAnalyticsWorkspaceResourceId
-
-    managedIdentities: empty(userAssignedIdentityResourceIds) ? {
-      systemAssigned: true
-    } : {
-      userAssignedResourceIds: userAssignedIdentityResourceIds
-    }
-
     name: containerAppsEnvironmentName
-    // Non-required parameters
-    infrastructureResourceGroupName: containerRegistryResourceGroupName
-    infrastructureSubnetId: virtualNetworkSubnetId
     location: location
     tags: tags
-    zoneRedundant: false
-    workloadProfiles: workloadProfiles
+    logAnalyticsWorkspaceName: logAnalyticsWorkspaceName
+    applicationInsightsName: applicationInsightsName
+    usePrivateIngress: usePrivateIngress
+    subnetResourceId: subnetResourceId
+    workloadProfile: workloadProfile
   }
 }
 
-module containerRegistry 'br/public:avm/res/container-registry/registry:0.5.1' = {
+module containerRegistry 'container-registry.bicep' = {
   name: '${name}-container-registry'
-  scope: resourceGroup(!empty(containerRegistryResourceGroupName) ? containerRegistryResourceGroupName : resourceGroup().name)
   params: {
     name: containerRegistryName
     location: location
-    acrAdminUserEnabled: containerRegistryAdminUserEnabled
     tags: tags
+    useVnet: !empty(vnetName)
   }
 }
 
 output defaultDomain string = containerAppsEnvironment.outputs.defaultDomain
 output environmentName string = containerAppsEnvironment.outputs.name
 output environmentId string = containerAppsEnvironment.outputs.resourceId
-
 output registryLoginServer string = containerRegistry.outputs.loginServer
 output registryName string = containerRegistry.outputs.name