diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index f03baac0dc..db2131a94a 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -4,6 +4,7 @@ import os from typing import Optional, Union +import aiohttp from azure.core.credentials import AzureKeyCredential from azure.core.credentials_async import AsyncTokenCredential from azure.identity.aio import AzureDeveloperCliCredential, get_bearer_token_provider @@ -45,6 +46,19 @@ def clean_key_if_exists(key: Union[str, None]) -> Union[str, None]: return None +async def check_search_service_connectivity(search_service: str) -> bool: + """Check if the search service is accessible by hitting the /ping endpoint.""" + ping_url = f"https://{search_service}.search.windows.net/ping" + + try: + async with aiohttp.ClientSession() as session: + async with session.get(ping_url, timeout=aiohttp.ClientTimeout(total=10)) as response: + return response.status == 200 + except Exception as e: + logger.debug(f"Search service ping failed: {e}") + return False + + async def setup_search_info( search_service: str, index_name: str, @@ -323,7 +337,10 @@ async def main(strategy: Strategy, setup_index: bool = True): load_azd_env() - if os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled": + if ( + os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled" + and os.getenv("AZURE_USE_VPN_GATEWAY", "").lower() != "true" + ): logger.error("AZURE_PUBLIC_NETWORK_ACCESS is set to Disabled. Exiting.") exit(0) @@ -372,6 +389,23 @@ async def main(strategy: Strategy, setup_index: bool = True): search_key=clean_key_if_exists(args.searchkey), ) ) + + # Check search service connectivity + search_service = os.environ["AZURE_SEARCH_SERVICE"] + is_connected = loop.run_until_complete(check_search_service_connectivity(search_service)) + + if not is_connected: + if os.getenv("AZURE_USE_PRIVATE_ENDPOINT"): + logger.error( + "Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration. You have AZURE_USE_PRIVATE_ENDPOINT enabled. Perhaps you're not yet connected to the VPN? Download the VPN configuration from the Azure portal here: %s", + os.getenv("AZURE_VPN_CONFIG_DOWNLOAD_LINK"), + ) + else: + logger.error( + "Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration." + ) + exit(1) + blob_manager = setup_blob_manager( azure_credential=azd_credential, storage_account=os.environ["AZURE_STORAGE_ACCOUNT"], diff --git a/docs/deploy_private.md b/docs/deploy_private.md index 8e2321c9e7..bfd9b2331f 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -31,9 +31,75 @@ If you want to disable public access when deploying the Chat App, you can do so Deploying with public access disabled adds additional cost to your deployment. Please see pricing for the following products: -1. [Private Endpoints](https://azure.microsoft.com/pricing/details/private-link/) - 1. The exact number of private endpoints created depends on the [optional features](./deploy_features.md) used. -1. [Private DNS Zones](https://azure.microsoft.com/pricing/details/dns/) +* [Azure Container Registry](https://azure.microsoft.com/pricing/details/container-registry/): Premium tier is used when virtual network is added (required for private links), which incurs additional costs. +* [Azure Container Apps](https://azure.microsoft.com/pricing/details/container-apps/): Workload profiles environment is used when virtual network is added (required for private links), which incurs additional costs. Additionally, min replica count is set to 1, so you will be charged for at least one instance. +* [VPN Gateway](https://azure.microsoft.com/pricing/details/vpn-gateway/): VpnGw2 SKU. Pricing includes a base monthly cost plus an hourly cost based on the number of connections. +* [Virtual Network](https://azure.microsoft.com/pricing/details/virtual-network/): Pay-as-you-go tier. Costs based on data processed. + +The pricing for the following features depends on the [optional features](./deploy_features.md) used. Most deployments will have at least 5 private endpoints (Azure OpenAI, Azure Cognitive Services, Azure AI Search, Azure Blob Storage, and either Azure App Service or Azure Container Apps). + +* [Azure Private Endpoints](https://azure.microsoft.com/pricing/details/private-link/): Pricing is per hour per endpoint. +* [Private DNS Zones](https://azure.microsoft.com/pricing/details/dns/): Pricing is per month and zones. +* [Azure Private DNS Resolver](https://azure.microsoft.com/pricing/details/dns/): Pricing is per month and zones. + +⚠️ To avoid unnecessary costs, remember to take down your app if it's no longer in use, +either by deleting the resource group in the Portal or running `azd down`. +You might also decide to delete the VPN Gateway when not in use. + +## Recommended deployment strategy for private access + +1. Configure the azd environment variables to use private endpoints and a VPN gateway, with public network access disabled. This will allow you to connect to the chat app from inside the virtual network, but not from the public Internet. + + ```shell + azd env set AZURE_USE_PRIVATE_ENDPOINT true + azd env set AZURE_USE_VPN_GATEWAY true + azd env set AZURE_PUBLIC_NETWORK_ACCESS Disabled + azd up + ``` + +2. Provision all the Azure resources: + + ```bash + azd provision + ``` + +3. Once provisioning is complete, you will see an error when it tries to run the data ingestion script, because you are not yet connected to the VPN. That message should provide a URL for the VPN configuration file download. If you don't see that URL, run this command: + + ```bash + azd env get-value AZURE_VPN_CONFIG_DOWNLOAD_LINK + ``` + + Open that link in your browser. Select "Download VPN client" to download a ZIP file containing the VPN configuration. + +4. Open `AzureVPN/azurevpnconfig.xml`, and replace the `` empty tag with the following: + + ```xml + + + 10.0.11.4 + + + ``` + + > **Note:** We use the IP address `10.0.11.4` since it is the first available IP in the `dns-resolver-subnet`(10.0.11.0/28) from the provisioned virtual network, as Azure reserves the first four IP addresses in each subnet. Adding this DNS server allows your VPN client to resolve private DNS names for Azure services accessed through private endpoints. See the network configuration in [network-isolation.bicep](../infra/network-isolation.bicep) for details. + +5. Install the [Azure VPN Client](https://learn.microsoft.com/azure/vpn-gateway/azure-vpn-client-versions). + +6. Open the Azure VPN Client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified. + +7. Select "Connect" and the new VPN connection. You will be prompted to select your Microsoft account and login. + +8. Once you're successfully connected to VPN, you can run the data ingestion script: + + ```bash + azd hooks run postprovision + ``` + +9. Finally, you can deploy the app: + + ```bash + azd deploy + ``` ## Environment variables controlling private access @@ -43,23 +109,8 @@ Deploying with public access disabled adds additional cost to your deployment. P 1. `AZURE_USE_PRIVATE_ENDPOINT`: Controls deployment of [private endpoints](https://learn.microsoft.com/azure/private-link/private-endpoint-overview) which connect Azure resources to the virtual network. 1. When set to 'true', ensures private endpoints are deployed for connectivity even when `AZURE_PUBLIC_NETWORK_ACCESS` is 'Disabled'. 1. Note that private endpoints do not make the chat app accessible from the internet. Connections must be initiated from inside the virtual network. +1. `AZURE_USE_VPN_GATEWAY`: Controls deployment of a VPN gateway for the virtual network. If you do not use this and public access is disabled, you will need a different way to connect to the virtual network. -## Recommended deployment strategy for private access - -1. Deploy the app with private endpoints enabled and public access enabled. - - ```shell - azd env set AZURE_USE_PRIVATE_ENDPOINT true - azd env set AZURE_PUBLIC_NETWORK_ACCESS Enabled - azd up - ``` - -1. Validate that you can connect to the chat app and it's working as expected from the internet. -1. Re-provision the app with public access disabled. - - ```shell - azd env set AZURE_PUBLIC_NETWORK_ACCESS Disabled - azd provision - ``` +## Compatibility with other features -1. Log into your network using a tool like [Azure VPN Gateway](https://azure.microsoft.com/services/vpn-gateway/) and validate that you can connect to the chat app from inside the network. +* **GitHub Actions / Azure DevOps**: The private access deployment is not compatible with the built-in CI/CD pipelines, as it requires a VPN connection to deploy the app. You could modify the pipeline to only do provisioning, and set up a different deployment strategy for the app. diff --git a/infra/abbreviations.json b/infra/abbreviations.json index 3673672a7e..6477bca0aa 100644 --- a/infra/abbreviations.json +++ b/infra/abbreviations.json @@ -110,6 +110,7 @@ "privateEndpoint": "pe-", "privateLink": "pl-", "purviewAccounts": "pview-", + "privateDnsResolver": "pdr-", "recoveryServicesVaults": "rsv-", "resourcesResourceGroups": "rg-", "searchSearchServices": "srch-", diff --git a/infra/core/host/container-app-upsert.bicep b/infra/core/host/container-app-upsert.bicep index 8b3918df2a..c8fc7d5bcd 100644 --- a/infra/core/host/container-app-upsert.bicep +++ b/infra/core/host/container-app-upsert.bicep @@ -78,9 +78,6 @@ param serviceBinds array = [] @description('The target port for the container') param targetPort int = 80 -@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) -param workloadProfile string = 'Consumption' - param allowedOrigins array = [] resource existingApp 'Microsoft.App/containerApps@2023-05-02-preview' existing = if (exists) { @@ -98,7 +95,6 @@ module app 'container-app.bicep' = { name: '${deployment().name}-update' params: { name: name - workloadProfile: workloadProfile location: location tags: tags identityType: identityType diff --git a/infra/core/host/container-app.bicep b/infra/core/host/container-app.bicep index 6fcff1dfc4..dc103e7ead 100644 --- a/infra/core/host/container-app.bicep +++ b/infra/core/host/container-app.bicep @@ -79,8 +79,6 @@ param serviceType string = '' @description('The target port for the container') param targetPort int = 80 -param workloadProfile string = 'Consumption' - resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!empty(identityName)) { name: identityName } @@ -125,7 +123,6 @@ resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { } properties: { managedEnvironmentId: containerAppsEnvironment.id - workloadProfileName: workloadProfile configuration: { activeRevisionsMode: revisionMode ingress: ingressEnabled ? { diff --git a/infra/core/host/container-apps-environment.bicep b/infra/core/host/container-apps-environment.bicep new file mode 100644 index 0000000000..2a35daceae --- /dev/null +++ b/infra/core/host/container-apps-environment.bicep @@ -0,0 +1,73 @@ +param name string +param location string = resourceGroup().location +param tags object = {} + +param daprEnabled bool = false +param logAnalyticsWorkspaceName string = '' +param applicationInsightsName string = '' + +param subnetResourceId string + +param usePrivateIngress bool = true + +@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) +param workloadProfile string + +// Make sure that we are using a non-consumption workload profile for private endpoints +var finalWorkloadProfile = (usePrivateIngress && workloadProfile == 'Consumption') ? 'D4' : workloadProfile + +var minimumCount = usePrivateIngress ? 1 : 0 +var maximumCount = usePrivateIngress ? 3 : 2 + +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2025-02-02-preview' = { + name: name + location: location + tags: tags + properties: { + // We can't use a conditional here due to an issue with the Container Apps ARM parsing + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logAnalyticsWorkspace.properties.customerId + sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey + } + } + daprAIInstrumentationKey: daprEnabled && !empty(applicationInsightsName) ? applicationInsights.properties.InstrumentationKey : '' + publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled' + vnetConfiguration: usePrivateIngress ? { + infrastructureSubnetId: subnetResourceId + internal: true + } : null + workloadProfiles: usePrivateIngress + ? [ + { + name: 'Consumption' + workloadProfileType: 'Consumption' + } + { + name: 'Warm' + workloadProfileType: finalWorkloadProfile + minimumCount: minimumCount + maximumCount: maximumCount + } + ] + : [ + { + name: 'Consumption' + workloadProfileType: 'Consumption' + } + ] + } +} + +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = if (!empty(logAnalyticsWorkspaceName)) { + name: logAnalyticsWorkspaceName +} + +resource applicationInsights 'Microsoft.Insights/components@2020-02-02' existing = if (daprEnabled && !empty(applicationInsightsName)){ + name: applicationInsightsName +} + +output defaultDomain string = containerAppsEnvironment.properties.defaultDomain +output name string = containerAppsEnvironment.name +output resourceId string = containerAppsEnvironment.id diff --git a/infra/core/host/container-apps.bicep b/infra/core/host/container-apps.bicep index 0428571215..00941b1cb2 100644 --- a/infra/core/host/container-apps.bicep +++ b/infra/core/host/container-apps.bicep @@ -1,78 +1,48 @@ -metadata description = 'Creates an Azure Container Registry and an Azure Container Apps environment.' param name string param location string = resourceGroup().location param tags object = {} param containerAppsEnvironmentName string param containerRegistryName string -param containerRegistryResourceGroupName string = '' -param containerRegistryAdminUserEnabled bool = false -param logAnalyticsWorkspaceResourceId string -param applicationInsightsName string = '' // Not used here, was used for DAPR -param virtualNetworkSubnetId string = '' +param logAnalyticsWorkspaceName string = '' +param applicationInsightsName string = '' + +@description('Virtual network name for container apps environment.') +param vnetName string = '' + @allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) param workloadProfile string -var workloadProfiles = workloadProfile == 'Consumption' - ? [ - { - name: 'Consumption' - workloadProfileType: 'Consumption' - } - ] - : [ - { - name: 'Consumption' - workloadProfileType: 'Consumption' - } - { - minimumCount: 0 - maximumCount: 2 - name: workloadProfile - workloadProfileType: workloadProfile - } - ] +param subnetResourceId string = '' -@description('Optional user assigned identity IDs to assign to the resource') -param userAssignedIdentityResourceIds array = [] +param usePrivateIngress bool = true -module containerAppsEnvironment 'br/public:avm/res/app/managed-environment:0.8.0' = { +module containerAppsEnvironment 'container-apps-environment.bicep' = { name: '${name}-container-apps-environment' params: { - // Required parameters - logAnalyticsWorkspaceResourceId: logAnalyticsWorkspaceResourceId - - managedIdentities: empty(userAssignedIdentityResourceIds) ? { - systemAssigned: true - } : { - userAssignedResourceIds: userAssignedIdentityResourceIds - } - name: containerAppsEnvironmentName - // Non-required parameters - infrastructureResourceGroupName: containerRegistryResourceGroupName - infrastructureSubnetId: virtualNetworkSubnetId location: location tags: tags - zoneRedundant: false - workloadProfiles: workloadProfiles + logAnalyticsWorkspaceName: logAnalyticsWorkspaceName + applicationInsightsName: applicationInsightsName + usePrivateIngress: usePrivateIngress + subnetResourceId: subnetResourceId + workloadProfile: workloadProfile } } -module containerRegistry 'br/public:avm/res/container-registry/registry:0.5.1' = { +module containerRegistry 'container-registry.bicep' = { name: '${name}-container-registry' - scope: resourceGroup(!empty(containerRegistryResourceGroupName) ? containerRegistryResourceGroupName : resourceGroup().name) params: { name: containerRegistryName location: location - acrAdminUserEnabled: containerRegistryAdminUserEnabled tags: tags + useVnet: !empty(vnetName) } } output defaultDomain string = containerAppsEnvironment.outputs.defaultDomain output environmentName string = containerAppsEnvironment.outputs.name output environmentId string = containerAppsEnvironment.outputs.resourceId - output registryLoginServer string = containerRegistry.outputs.loginServer output registryName string = containerRegistry.outputs.name diff --git a/infra/core/host/container-registry.bicep b/infra/core/host/container-registry.bicep new file mode 100644 index 0000000000..3bee20c870 --- /dev/null +++ b/infra/core/host/container-registry.bicep @@ -0,0 +1,38 @@ +param name string +param location string = resourceGroup().location +param tags object = {} + +param adminUserEnabled bool = false +param anonymousPullEnabled bool = false +param dataEndpointEnabled bool = false +param encryption object = { + status: 'disabled' +} +param networkRuleBypassOptions string = 'AzureServices' +param publicNetworkAccess string = useVnet ? 'Disabled' : 'Enabled' // Public network access is disabled if VNet integration is enabled +param useVnet bool = false // Determines if VNet integration is enabled +param sku object = { + name: useVnet ? 'Premium' : 'Standard' // Use Premium if VNet is required, otherwise Standard +} +param zoneRedundancy string = 'Disabled' + +// 2022-02-01-preview needed for anonymousPullEnabled +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { + name: name + location: location + tags: tags + sku: sku + properties: { + adminUserEnabled: adminUserEnabled + anonymousPullEnabled: anonymousPullEnabled + dataEndpointEnabled: dataEndpointEnabled + encryption: encryption + networkRuleBypassOptions: networkRuleBypassOptions + publicNetworkAccess: publicNetworkAccess + zoneRedundancy: zoneRedundancy + } +} + +output loginServer string = containerRegistry.properties.loginServer +output name string = containerRegistry.name +output resourceId string = containerRegistry.id diff --git a/infra/core/storage/storage-account.bicep b/infra/core/storage/storage-account.bicep index 670199162c..46489ceb7d 100644 --- a/infra/core/storage/storage-account.bicep +++ b/infra/core/storage/storage-account.bicep @@ -59,7 +59,7 @@ resource storage 'Microsoft.Storage/storageAccounts@2024-01-01' = { resource container 'containers' = [for container in containers: { name: container.name properties: { - publicAccess: contains(container, 'publicAccess') ? container.publicAccess : 'None' + publicAccess: container.?publicAccess ?? 'None' } }] } diff --git a/infra/main.bicep b/infra/main.bicep index da31b2b385..d74047f8bc 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -244,6 +244,9 @@ param publicNetworkAccess string = 'Enabled' @description('Add a private endpoints for network connectivity') param usePrivateEndpoint bool = false +@description('Use a P2S VPN Gateway for secure access to the private endpoints') +param useVpnGateway bool = false + @description('Id of the user or app to assign application roles') param principalId string = '' @@ -488,7 +491,7 @@ module backend 'core/host/appservice.bicep' = if (deploymentTarget == 'appservic appCommandLine: 'python3 -m gunicorn main:app' scmDoBuildDuringDeployment: true managedIdentity: true - virtualNetworkSubnetId: isolation.outputs.appSubnetId + virtualNetworkSubnetId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' publicNetworkAccess: publicNetworkAccess allowedOrigins: allowedOrigins clientAppId: clientAppId @@ -525,10 +528,12 @@ module containerApps 'core/host/container-apps.bicep' = if (deploymentTarget == name: 'app' tags: tags location: location - workloadProfile: azureContainerAppsWorkloadProfile containerAppsEnvironmentName: acaManagedEnvironmentName containerRegistryName: '${containerRegistryName}${resourceToken}' - logAnalyticsWorkspaceResourceId: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceId : '' + logAnalyticsWorkspaceName: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceName : '' + subnetResourceId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' + usePrivateIngress: usePrivateEndpoint + workloadProfile: azureContainerAppsWorkloadProfile } } @@ -545,15 +550,13 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget location: location identityName: (deploymentTarget == 'containerapps') ? acaIdentityName : '' exists: webAppExists - workloadProfile: azureContainerAppsWorkloadProfile containerRegistryName: (deploymentTarget == 'containerapps') ? containerApps.outputs.registryName : '' containerAppsEnvironmentName: (deploymentTarget == 'containerapps') ? containerApps.outputs.environmentName : '' - identityType: 'UserAssigned' tags: union(tags, { 'azd-service-name': 'backend' }) targetPort: 8000 containerCpuCoreCount: '1.0' containerMemory: '2Gi' - containerMinReplicas: 0 + containerMinReplicas: usePrivateEndpoint ? 1 : 0 allowedOrigins: allowedOrigins env: union(appEnvVariables, { // For using managed identity to access Azure resources. See https://github.com/microsoft/azure-container-apps/issues/442 @@ -789,7 +792,7 @@ module searchService 'core/search/search-services.bicep' = { publicNetworkAccess: publicNetworkAccess == 'Enabled' ? 'enabled' : (publicNetworkAccess == 'Disabled' ? 'disabled' : null) - sharedPrivateLinkStorageAccounts: usePrivateEndpoint ? [storage.outputs.id] : [] + sharedPrivateLinkStorageAccounts: (usePrivateEndpoint && useIntegratedVectorization) ? [storage.outputs.id] : [] } } @@ -1156,37 +1159,66 @@ module cosmosDbRoleBackend 'core/security/documentdb-sql-role.bicep' = if (useAu } } -module isolation 'network-isolation.bicep' = { +module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { name: 'networks' scope: resourceGroup params: { - deploymentTarget: deploymentTarget location: location tags: tags vnetName: '${abbrs.virtualNetworks}${resourceToken}' - // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 - appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : '' - usePrivateEndpoint: usePrivateEndpoint + deploymentTarget: deploymentTarget + useVpnGateway: useVpnGateway + vpnGatewayName: useVpnGateway ? '${abbrs.networkVpnGateways}${resourceToken}' : '' + dnsResolverName: useVpnGateway ? '${abbrs.privateDnsResolver}${resourceToken}' : '' } } var environmentData = environment() -var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi && deploymentTarget == 'appservice') +var openAiPrivateEndpointConnection = (usePrivateEndpoint && isAzureOpenAiHost && deployAzureOpenAi) ? [ { groupId: 'account' dnsZoneName: 'privatelink.openai.azure.com' + resourceIds: [openAi.outputs.resourceId] + } + ] + : [] + +var cognitiveServicesPrivateEndpointConnection = (usePrivateEndpoint && (!useLocalPdfParser || useGPT4V || useMediaDescriberAzureCU)) + ? [ + { + groupId: 'account' + dnsZoneName: 'privatelink.cognitiveservices.azure.com' resourceIds: concat( - [openAi.outputs.resourceId], + !useLocalPdfParser ? [documentIntelligence.outputs.resourceId] : [], useGPT4V ? [computerVision.outputs.resourceId] : [], - useMediaDescriberAzureCU ? [contentUnderstanding.outputs.resourceId] : [], - !useLocalPdfParser ? [documentIntelligence.outputs.resourceId] : [] + useMediaDescriberAzureCU ? [contentUnderstanding.outputs.resourceId] : [] ) } ] : [] -var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget == 'appservice') + +var containerAppsPrivateEndpointConnection = (usePrivateEndpoint && deploymentTarget == 'containerapps') + ? [ + { + groupId: 'managedEnvironments' + dnsZoneName: 'privatelink.${location}.azurecontainerapps.io' + resourceIds: [containerApps.outputs.environmentId] + } + ] + : [] + +var appServicePrivateEndpointConnection = (usePrivateEndpoint && deploymentTarget == 'appservice') + ? [ + { + groupId: 'sites' + dnsZoneName: 'privatelink.azurewebsites.net' + resourceIds: [backend.outputs.id] + } + ] + : [] +var otherPrivateEndpointConnections = (usePrivateEndpoint) ? [ { groupId: 'blob' @@ -1198,11 +1230,6 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget == dnsZoneName: 'privatelink.search.windows.net' resourceIds: [searchService.outputs.id] } - { - groupId: 'sites' - dnsZoneName: 'privatelink.azurewebsites.net' - resourceIds: [backend.outputs.id] - } { groupId: 'sql' dnsZoneName: 'privatelink.documents.azure.com' @@ -1211,9 +1238,9 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget == ] : [] -var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection) +var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection, cognitiveServicesPrivateEndpointConnection, containerAppsPrivateEndpointConnection, appServicePrivateEndpointConnection) -module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint && deploymentTarget == 'appservice') { +module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { name: 'privateEndpoints' scope: resourceGroup params: { @@ -1224,7 +1251,7 @@ module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint && de applicationInsightsId: useApplicationInsights ? monitoring.outputs.applicationInsightsId : '' logAnalyticsWorkspaceId: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceId : '' vnetName: isolation.outputs.vnetName - vnetPeSubnetName: isolation.outputs.backendSubnetId + vnetPeSubnetId: isolation.outputs.backendSubnetId } } @@ -1352,3 +1379,5 @@ output BACKEND_URI string = deploymentTarget == 'appservice' ? backend.outputs.u output AZURE_CONTAINER_REGISTRY_ENDPOINT string = deploymentTarget == 'containerapps' ? containerApps.outputs.registryLoginServer : '' + +output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource${isolation.outputs.virtualNetworkGatewayId}/pointtositeconfiguration' : '' diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 1c34063020..5202fdcde0 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -314,6 +314,9 @@ "usePrivateEndpoint": { "value": "${AZURE_USE_PRIVATE_ENDPOINT=false}" }, + "useVpnGateway": { + "value": "${AZURE_USE_VPN_GATEWAY=false}" + }, "bypass": { "value": "${AZURE_NETWORK_BYPASS=AzureServices}" }, diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 4dd1e49f86..2454421b79 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -9,69 +9,341 @@ param location string = resourceGroup().location @description('The tags to apply to all resources') param tags object = {} -@description('The name of an existing App Service Plan to connect to the VNet') -param appServicePlanName string - -param usePrivateEndpoint bool = false - @allowed(['appservice', 'containerapps']) param deploymentTarget string -resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (deploymentTarget == 'appservice') { - name: appServicePlanName +param useVpnGateway bool = false + +param vpnGatewayName string = '${vnetName}-vpn-gateway' +param dnsResolverName string = '${vnetName}-dns-resolver' + +// Subnet name constants +var backendSubnetName = 'backend-subnet' +var gatewaySubnetName = 'GatewaySubnet' // Required name for Gateway subnet +var dnsResolverSubnetName = 'dns-resolver-subnet' +var appServiceSubnetName = 'app-service-subnet' +var containerAppsSubnetName = 'container-apps-subnet' + +module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = if (deploymentTarget == 'containerapps') { + name: 'container-apps-nsg' + params: { + name: '${vnetName}-container-apps-nsg' + location: location + tags: tags + securityRules: [ + // Inbound rules for Container Apps (Workload Profiles) + { + name: 'AllowAzureLoadBalancerInbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationPortRange: '30000-32767' + destinationAddressPrefix: '10.0.0.0/21' // Container apps subnet + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + // Outbound rules for Container Apps (Workload Profiles) + { + name: 'AllowMicrosoftContainerRegistryOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'MicrosoftContainerRegistry' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'AllowAzureFrontDoorOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'AzureFrontDoor.FirstParty' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'AllowContainerAppsSubnetOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '*' + destinationAddressPrefix: '10.0.0.0/21' // Container apps subnet + access: 'Allow' + priority: 120 + direction: 'Outbound' + } + } + { + name: 'AllowAzureActiveDirectoryOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'AzureActiveDirectory' + access: 'Allow' + priority: 130 + direction: 'Outbound' + } + } + { + name: 'AllowAzureMonitorOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'AzureMonitor' + access: 'Allow' + priority: 140 + direction: 'Outbound' + } + } + { + name: 'AllowAzureDnsOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '53' + destinationAddressPrefix: '168.63.129.16' + access: 'Allow' + priority: 150 + direction: 'Outbound' + } + } + { + name: 'AllowStorageRegionOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'Storage.${location}' + access: 'Allow' + priority: 160 + direction: 'Outbound' + } + } + ] + } } -module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) { - name: 'vnet' +module privateEndpointsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = { + name: 'private-endpoints-nsg' params: { - name: vnetName + name: '${vnetName}-private-endpoints-nsg' location: location tags: tags - subnets: [ + securityRules: [ { - name: 'backend-subnet' + name: 'AllowVnetInBound' properties: { - addressPrefix: '10.0.1.0/24' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 100 + direction: 'Inbound' } } { - name: 'AzureBastionSubnet' + name: 'AllowAzureLoadBalancerInbound' properties: { - addressPrefix: '10.0.2.0/24' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 110 + direction: 'Inbound' } } { - name: 'app-int-subnet' + name: 'DenyInternetInbound' properties: { - addressPrefix: '10.0.3.0/24' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' - delegations: [ - { - id: appServicePlan.id - name: appServicePlan.name - properties: { - serviceName: 'Microsoft.Web/serverFarms' - } - } - ] + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 4096 + direction: 'Inbound' + } + } + { + name: 'AllowVnetOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'AllowAzureCloudOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '443' + destinationAddressPrefix: 'AzureCloud' + access: 'Allow' + priority: 110 + direction: 'Outbound' } } { - name: 'vm-subnet' + name: 'AllowDnsOutbound' properties: { - addressPrefix: '10.0.4.0/24' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '53' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 120 + direction: 'Outbound' } } + { + name: 'DenyInternetOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: 'Internet' + access: 'Deny' + priority: 4096 + direction: 'Outbound' + } + } + ] + } +} + +module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { + name: 'vnet' + params: { + name: vnetName + location: location + tags: tags + addressPrefixes: [ + '10.0.0.0/16' ] + subnets: union( + [ + { + name: backendSubnetName + addressPrefix: '10.0.8.0/24' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + networkSecurityGroupResourceId: privateEndpointsNSG.outputs.resourceId + } + { + name: gatewaySubnetName // Required name for Gateway subnet + addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet + } + { + name: dnsResolverSubnetName // Dedicated subnet for Azure Private DNS Resolver + addressPrefix: '10.0.11.0/28' + delegation: 'Microsoft.Network/dnsResolvers' + } + ], + deploymentTarget == 'appservice' + ? [ + { + name: appServiceSubnetName + addressPrefix: '10.0.9.0/24' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + delegation: 'Microsoft.Web/serverFarms' + } + ] + : [ + { + name: containerAppsSubnetName + addressPrefix: '10.0.0.0/21' + delegation: 'Microsoft.App/environments' + networkSecurityGroupResourceId: containerAppsNSG!.outputs.resourceId + } + ] + ) } } +// Helper variables to find subnet resource IDs by name instead of hardcoded indices +var dnsResolverSubnetIndex = indexOf(vnet.outputs.subnetNames, dnsResolverSubnetName) +var backendSubnetIndex = indexOf(vnet.outputs.subnetNames, backendSubnetName) +var appSubnetIndex = deploymentTarget == 'appservice' ? indexOf(vnet.outputs.subnetNames, appServiceSubnetName) : indexOf(vnet.outputs.subnetNames, containerAppsSubnetName) + +module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.8.0' = if (useVpnGateway) { + name: 'virtual-network-gateway' + params: { + name: vpnGatewayName + clusterSettings: { + clusterMode: 'activePassiveNoBgp' + } + gatewayType: 'Vpn' + virtualNetworkResourceId: vnet.outputs.resourceId + vpnGatewayGeneration: 'Generation2' + vpnClientAddressPoolPrefix: '172.16.201.0/24' + skuName: 'VpnGw2' + vpnClientAadConfiguration: { + aadAudience: 'c632b3df-fb67-4d84-bdcf-b95ad541b5c8' // Azure VPN client + aadIssuer: 'https://sts.windows.net/${tenant().tenantId}/' + aadTenant: '${environment().authentication.loginEndpoint}${tenant().tenantId}' + vpnAuthenticationTypes: [ + 'AAD' + ] + vpnClientProtocols: [ + 'OpenVPN' + ] + } + } +} + +// Based on https://luke.geek.nz/azure/azure-point-to-site-vpn-and-private-dns-resolver/ +// Manual step required of updating azurevpnconfig.xml to use the correct DNS server IP address +module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.4' = if (useVpnGateway) { + name: 'dns-resolver' + params: { + name: dnsResolverName + location: location + virtualNetworkResourceId: vnet.outputs.resourceId + inboundEndpoints: [ + { + name: 'inboundEndpoint' + subnetResourceId: useVpnGateway ? vnet.outputs.subnetResourceIds[dnsResolverSubnetIndex] : '' + } + ] + } +} -output appSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[2].id : '' -output backendSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[0].id : '' -output vnetName string = usePrivateEndpoint ? vnet.outputs.name : '' +output backendSubnetId string = vnet.outputs.subnetResourceIds[backendSubnetIndex] +output privateDnsResolverSubnetId string = useVpnGateway ? vnet.outputs.subnetResourceIds[dnsResolverSubnetIndex] : '' +output appSubnetId string = vnet.outputs.subnetResourceIds[appSubnetIndex] +output vnetName string = vnet.outputs.name +output vnetId string = vnet.outputs.resourceId +output virtualNetworkGatewayId string = useVpnGateway ? virtualNetworkGateway!.outputs.resourceId : '' diff --git a/infra/private-endpoints.bicep b/infra/private-endpoints.bicep index 58fe14177e..6d6357ab8e 100644 --- a/infra/private-endpoints.bicep +++ b/infra/private-endpoints.bicep @@ -7,7 +7,7 @@ param vnetName string @description('The location to create the private endpoints') param location string = resourceGroup().location -param vnetPeSubnetName string +param vnetPeSubnetId string @description('A formatted array of private endpoint connections containing the dns zone name, group id, and list of resource ids of Private Endpoints to create') param privateEndpointConnections array @@ -32,8 +32,8 @@ param logAnalyticsWorkspaceId string var abbrs = loadJsonContent('abbreviations.json') // DNS Zones -module dnsZones './core/networking/private-dns-zones.bicep' = [for privateEndpointConnection in privateEndpointConnections: { - name: '${privateEndpointConnection.groupId}-dnszone' +module dnsZones './core/networking/private-dns-zones.bicep' = [for (privateEndpointConnection, i) in privateEndpointConnections: { + name: '${privateEndpointConnection.groupId}-${i}-dnszone' params: { dnsZoneName: privateEndpointConnection.dnsZoneName tags: tags @@ -56,7 +56,7 @@ module privateEndpoints './core/networking/private-endpoint.bicep' = [for privat location: location name: '${privateEndpointInfo.name}${abbrs.privateEndpoint}${resourceToken}' tags: tags - subnetId: vnetPeSubnetName + subnetId: vnetPeSubnetId serviceId: privateEndpointInfo.resourceId groupIds: [ privateEndpointInfo.groupId ] dnsZoneId: dnsZones[privateEndpointInfo.dnsZoneIndex].outputs.id @@ -81,7 +81,9 @@ module monitorDnsZones './core/networking/private-dns-zones.bicep' = [for monito } }] // Get blob DNS zone index for monitor private link -var dnsZoneBlobIndex = filter(flatten(privateEndpointInfo), info => info.groupId == 'blob')[0].dnsZoneIndex +var blobEndpointInfo = filter(flatten(privateEndpointInfo), info => info.groupId == 'blob') +// Assert that blob endpoints exist (required for this application) +var dnsZoneBlobIndex = blobEndpointInfo[0].dnsZoneIndex // Azure Monitor Private Link Scope // https://learn.microsoft.com/azure/azure-monitor/logs/private-link-security @@ -120,7 +122,7 @@ module monitorPrivateEndpoint './core/networking/private-endpoint.bicep' = { name: 'monitor${abbrs.privateEndpoint}${resourceToken}' location: location tags: tags - subnetId: vnetPeSubnetName + subnetId: vnetPeSubnetId serviceId: monitorPrivateLinkScope.id groupIds: [ 'azuremonitor' ] // Add multiple DNS zone configs for Azure Monitor