From 6fac970ebc41490ac036834acb51ba465ee8e4a7 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 7 Jan 2025 12:47:05 -0800 Subject: [PATCH 01/24] Configure Azure Developer Pipeline From b16fd31ad48af8644f67c46863056ffbb0520783 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 21 Jan 2025 09:31:54 -0800 Subject: [PATCH 02/24] Private endpoints draft --- infra/main.bicep | 11 +++++------ infra/network-isolation.bicep | 37 ++++++++++------------------------- infra/private-endpoints.bicep | 7 ++++--- 3 files changed, 19 insertions(+), 36 deletions(-) diff --git a/infra/main.bicep b/infra/main.bicep index 5c181cd525..0c86eaf7c0 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -431,7 +431,7 @@ module backend 'core/host/appservice.bicep' = if (deploymentTarget == 'appservic appCommandLine: 'python3 -m gunicorn main:app' scmDoBuildDuringDeployment: true managedIdentity: true - virtualNetworkSubnetId: isolation.outputs.appSubnetId + virtualNetworkSubnetId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' publicNetworkAccess: publicNetworkAccess allowedOrigins: allowedOrigins clientAppId: clientAppId @@ -472,6 +472,7 @@ module containerApps 'core/host/container-apps.bicep' = if (deploymentTarget == containerAppsEnvironmentName: acaManagedEnvironmentName containerRegistryName: '${containerRegistryName}${resourceToken}' logAnalyticsWorkspaceResourceId: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceId : '' + virtualNetworkSubnetId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' } } @@ -1047,17 +1048,15 @@ module cosmosDbRoleBackend 'core/security/documentdb-sql-role.bicep' = if (useAu } } -module isolation 'network-isolation.bicep' = { +module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { name: 'networks' scope: resourceGroup params: { - deploymentTarget: deploymentTarget location: location tags: tags vnetName: '${abbrs.virtualNetworks}${resourceToken}' - // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 - appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : '' usePrivateEndpoint: usePrivateEndpoint + containerAppsEnvName: acaManagedEnvironmentName } } @@ -1103,7 +1102,7 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget == var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection) -module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint && deploymentTarget == 'appservice') { +module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { name: 'privateEndpoints' scope: resourceGroup params: { diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 4dd1e49f86..c623eb7cae 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -9,18 +9,15 @@ param location string = resourceGroup().location @description('The tags to apply to all resources') param tags object = {} -@description('The name of an existing App Service Plan to connect to the VNet') -param appServicePlanName string - param usePrivateEndpoint bool = false -@allowed(['appservice', 'containerapps']) -param deploymentTarget string +param containerAppsEnvName string -resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (deploymentTarget == 'appservice') { - name: appServicePlanName +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = { + name: containerAppsEnvName } + module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) { name: 'vnet' params: { @@ -36,42 +33,28 @@ module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) { privateLinkServiceNetworkPolicies: 'Enabled' } } - { - name: 'AzureBastionSubnet' - properties: { - addressPrefix: '10.0.2.0/24' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' - } - } - { + { // App Service / Container Apps specific subnet name: 'app-int-subnet' properties: { - addressPrefix: '10.0.3.0/24' + addressPrefix: '10.0.4.0/23' privateEndpointNetworkPolicies: 'Enabled' privateLinkServiceNetworkPolicies: 'Enabled' delegations: [ { - id: appServicePlan.id - name: appServicePlan.name + id: containerAppsEnvironment.id + name: containerAppsEnvironment.name properties: { - serviceName: 'Microsoft.Web/serverFarms' + serviceName: 'Microsoft.App/environments' } } ] } } - { - name: 'vm-subnet' - properties: { - addressPrefix: '10.0.4.0/24' - } - } ] } } -output appSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[2].id : '' +output appSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[1].id : '' output backendSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[0].id : '' output vnetName string = usePrivateEndpoint ? vnet.outputs.name : '' diff --git a/infra/private-endpoints.bicep b/infra/private-endpoints.bicep index 58fe14177e..a26ad08947 100644 --- a/infra/private-endpoints.bicep +++ b/infra/private-endpoints.bicep @@ -81,7 +81,8 @@ module monitorDnsZones './core/networking/private-dns-zones.bicep' = [for monito } }] // Get blob DNS zone index for monitor private link -var dnsZoneBlobIndex = filter(flatten(privateEndpointInfo), info => info.groupId == 'blob')[0].dnsZoneIndex +var blobEndpointInfo = filter(flatten(privateEndpointInfo), info => info.groupId == 'blob') +var dnsZoneBlobIndex = empty(blobEndpointInfo) ? 0 : blobEndpointInfo[0].dnsZoneIndex // Azure Monitor Private Link Scope // https://learn.microsoft.com/azure/azure-monitor/logs/private-link-security @@ -150,9 +151,9 @@ module monitorPrivateEndpoint './core/networking/private-endpoint.bicep' = { } } { - name: dnsZones[dnsZoneBlobIndex].name + name: 'blob-dnszone' // dnsZones[dnsZoneBlobIndex].name properties: { - privateDnsZoneId: dnsZones[dnsZoneBlobIndex].outputs.id + privateDnsZoneId: '/subscriptions/77d8a3d0-8b18-47e9-b773-08bee327bb4a/resourceGroups/rg-pf-ragprivate/providers/Microsoft.Network/privateDnsZones/privatelink.blob.core.windows.net' // dnsZones[dnsZoneBlobIndex].outputs.id } } ] From 7ad962a7d5698587610addf7890cf8e1be5b8a10 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 3 Feb 2025 13:59:44 -0800 Subject: [PATCH 03/24] Conditional for SPL --- infra/core/search/search-services.bicep | 1 + infra/main.bicep | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/core/search/search-services.bicep b/infra/core/search/search-services.bicep index 4ee8d6a8fb..fc743758cf 100644 --- a/infra/core/search/search-services.bicep +++ b/infra/core/search/search-services.bicep @@ -55,6 +55,7 @@ resource search 'Microsoft.Search/searchServices@2023-11-01' = { } sku: sku + // https://github.com/Azure/bicep-types-az/issues/2421 resource sharedPrivateLinkResource 'sharedPrivateLinkResources@2023-11-01' = [for (resourceId, i) in sharedPrivateLinkStorageAccounts: { name: 'search-shared-private-link-${i}' properties: { diff --git a/infra/main.bicep b/infra/main.bicep index 0c86eaf7c0..bd6ca4f695 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -701,7 +701,7 @@ module searchService 'core/search/search-services.bicep' = { publicNetworkAccess: publicNetworkAccess == 'Enabled' ? 'enabled' : (publicNetworkAccess == 'Disabled' ? 'disabled' : null) - sharedPrivateLinkStorageAccounts: usePrivateEndpoint ? [storage.outputs.id] : [] + sharedPrivateLinkStorageAccounts: (usePrivateEndpoint && useIntegratedVectorization) ? [storage.outputs.id] : [] } } From 7a82bde8287f12086ced1b0afebe08d479aa8001 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 5 Feb 2025 16:47:21 -0800 Subject: [PATCH 04/24] private endpoint for ACA --- infra/main.bicep | 10 +++++----- infra/network-isolation.bicep | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/infra/main.bicep b/infra/main.bicep index bd6ca4f695..935217b9ec 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1062,7 +1062,7 @@ module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { var environmentData = environment() -var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi && deploymentTarget == 'appservice') +var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi) ? [ { groupId: 'account' @@ -1075,7 +1075,7 @@ var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi && } ] : [] -var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget == 'appservice') +var otherPrivateEndpointConnections = (usePrivateEndpoint) ? [ { groupId: 'blob' @@ -1088,9 +1088,9 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget == resourceIds: [searchService.outputs.id] } { - groupId: 'sites' - dnsZoneName: 'privatelink.azurewebsites.net' - resourceIds: [backend.outputs.id] + groupId: 'managedEnvironments' + dnsZoneName: 'privatelink.${location}.azurecontainerapps.io' + resourceIds: [containerApps.outputs.environmentId] } { groupId: 'cosmosdb' diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index c623eb7cae..90e1396453 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -57,4 +57,6 @@ module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) { output appSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[1].id : '' output backendSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[0].id : '' +output bastionSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[2].id : '' +output vmSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[3].id : '' output vnetName string = usePrivateEndpoint ? vnet.outputs.name : '' From 0f18906896338bf9787fdbe7a8b497adaf607ac6 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 29 May 2025 15:22:27 -0700 Subject: [PATCH 05/24] Add P2S VPN gateway and other improvements --- app/frontend/package-lock.json | 2 +- infra/abbreviations.json | 1 + infra/core/host/container-app-upsert.bicep | 2 +- infra/core/host/container-app.bicep | 2 +- infra/core/host/container-apps.bicep | 14 ++- infra/main.bicep | 62 +++++++++++- infra/network-isolation.bicep | 105 +++++++++++++++------ infra/private-endpoints.bicep | 4 +- 8 files changed, 149 insertions(+), 43 deletions(-) diff --git a/app/frontend/package-lock.json b/app/frontend/package-lock.json index b4ec8fb7a0..6da48b3591 100644 --- a/app/frontend/package-lock.json +++ b/app/frontend/package-lock.json @@ -44,7 +44,7 @@ "vite": "^5.4.18" }, "engines": { - "node": ">=14.0.0" + "node": ">=20.0.0" } }, "node_modules/@ampproject/remapping": { diff --git a/infra/abbreviations.json b/infra/abbreviations.json index 3673672a7e..6477bca0aa 100644 --- a/infra/abbreviations.json +++ b/infra/abbreviations.json @@ -110,6 +110,7 @@ "privateEndpoint": "pe-", "privateLink": "pl-", "purviewAccounts": "pview-", + "privateDnsResolver": "pdr-", "recoveryServicesVaults": "rsv-", "resourcesResourceGroups": "rg-", "searchSearchServices": "srch-", diff --git a/infra/core/host/container-app-upsert.bicep b/infra/core/host/container-app-upsert.bicep index 8b3918df2a..0621dc1889 100644 --- a/infra/core/host/container-app-upsert.bicep +++ b/infra/core/host/container-app-upsert.bicep @@ -15,7 +15,7 @@ param containerMaxReplicas int = 10 param containerMemory string = '1.0Gi' @description('The minimum number of replicas to run. Must be at least 1 for non-consumption workloads.') -param containerMinReplicas int = 1 +param containerMinReplicas int = 0 @description('The name of the container') param containerName string = 'main' diff --git a/infra/core/host/container-app.bicep b/infra/core/host/container-app.bicep index 6fcff1dfc4..9c8c80df91 100644 --- a/infra/core/host/container-app.bicep +++ b/infra/core/host/container-app.bicep @@ -20,7 +20,7 @@ param containerMaxReplicas int = 10 param containerMemory string = '1.0Gi' @description('The minimum number of replicas to run. Must be at least 1.') -param containerMinReplicas int = 1 +param containerMinReplicas int = 0 @description('The name of the container') param containerName string = 'main' diff --git a/infra/core/host/container-apps.bicep b/infra/core/host/container-apps.bicep index f48a6b3610..f29eec4ea2 100644 --- a/infra/core/host/container-apps.bicep +++ b/infra/core/host/container-apps.bicep @@ -8,8 +8,11 @@ param containerRegistryName string param containerRegistryResourceGroupName string = '' param containerRegistryAdminUserEnabled bool = false param logAnalyticsWorkspaceResourceId string -param applicationInsightsName string = '' // Not used here, was used for DAPR -param virtualNetworkSubnetId string = '' + +param subnetResourceId string = '' + +param usePrivateIngress bool = true + @allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) param workloadProfile string @@ -26,7 +29,7 @@ var workloadProfiles = workloadProfile == 'Consumption' workloadProfileType: 'Consumption' } { - minimumCount: 0 + minimumCount: 1 maximumCount: 2 name: workloadProfile workloadProfileType: workloadProfile @@ -51,7 +54,8 @@ module containerAppsEnvironment 'br/public:avm/res/app/managed-environment:0.8.0 name: containerAppsEnvironmentName // Non-required parameters infrastructureResourceGroupName: containerRegistryResourceGroupName - infrastructureSubnetId: virtualNetworkSubnetId + infrastructureSubnetId: subnetResourceId + internal: usePrivateIngress location: location tags: tags zoneRedundant: false @@ -67,7 +71,9 @@ module containerRegistry 'br/public:avm/res/container-registry/registry:0.5.1' = params: { name: containerRegistryName location: location + acrSku: usePrivateIngress ? 'Premium' : 'Standard' acrAdminUserEnabled: containerRegistryAdminUserEnabled + publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled' tags: tags } } diff --git a/infra/main.bicep b/infra/main.bicep index 6ce01f7143..8cf1c6b831 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -244,6 +244,9 @@ param publicNetworkAccess string = 'Enabled' @description('Add a private endpoints for network connectivity') param usePrivateEndpoint bool = false +@description('Use a P2S VPN Gateway for secure access to the private endpoints') +param useVpnGateway bool = false + @description('Id of the user or app to assign application roles') param principalId string = '' @@ -529,7 +532,8 @@ module containerApps 'core/host/container-apps.bicep' = if (deploymentTarget == containerAppsEnvironmentName: acaManagedEnvironmentName containerRegistryName: '${containerRegistryName}${resourceToken}' logAnalyticsWorkspaceResourceId: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceId : '' - virtualNetworkSubnetId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' + subnetResourceId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' + usePrivateIngress: usePrivateEndpoint } } @@ -542,8 +546,8 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget acaIdentity ] params: { - name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesContainerApps}backend-${resourceToken}' - location: location + name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesContainerApps}backend${resourceToken}' + location: 'westus2' identityName: (deploymentTarget == 'containerapps') ? acaIdentityName : '' exists: webAppExists workloadProfile: azureContainerAppsWorkloadProfile @@ -554,7 +558,7 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget targetPort: 8000 containerCpuCoreCount: '1.0' containerMemory: '2Gi' - containerMinReplicas: 0 + containerMinReplicas: 1 allowedOrigins: allowedOrigins env: union(appEnvVariables, { // For using managed identity to access Azure resources. See https://github.com/microsoft/azure-container-apps/issues/442 @@ -1165,7 +1169,10 @@ module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { tags: tags vnetName: '${abbrs.virtualNetworks}${resourceToken}' usePrivateEndpoint: usePrivateEndpoint - containerAppsEnvName: acaManagedEnvironmentName + deploymentTarget: deploymentTarget + // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 + appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : '' + containerAppsEnvName: deploymentTarget == 'containerapps' ? acaManagedEnvironmentName : '' } } @@ -1227,6 +1234,51 @@ module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { } } +// Based on https://luke.geek.nz/azure/azure-point-to-site-vpn-and-private-dns-resolver/ +// Manual step required of updating azurevpnconfig.xml to use the correct DNS server IP address +module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.3' = if (useVpnGateway) { + name: 'dnsResolverDeployment' + scope: resourceGroup + params: { + name: '${abbrs.privateDnsResolver}${resourceToken}' + location: location + virtualNetworkResourceId: isolation.outputs.vnetId + inboundEndpoints: [ + { + name: 'inboundEndpoint' + subnetResourceId: useVpnGateway ? isolation.outputs.privateDnsResolverSubnetId : '' + } + ] + } +} + +module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.6.1' = if (useVpnGateway) { + name: 'virtualNetworkGatewayDeployment' + scope: resourceGroup + params: { + name: '${abbrs.networkVpnGateways}${resourceToken}' + clusterSettings: { + clusterMode: 'activePassiveNoBgp' + } + gatewayType: 'Vpn' + virtualNetworkResourceId: isolation.outputs.vnetId + vpnGatewayGeneration: 'Generation2' + vpnClientAddressPoolPrefix: '172.16.201.0/24' + skuName: 'VpnGw2' + vpnClientAadConfiguration: { + aadAudience: 'c632b3df-fb67-4d84-bdcf-b95ad541b5c8' // Azure VPN client + aadIssuer: 'https://sts.windows.net/${tenant().tenantId}/' + aadTenant: '${environment().authentication.loginEndpoint}${tenant().tenantId}' + vpnAuthenticationTypes: [ + 'AAD' + ] + vpnClientProtocols: [ + 'OpenVPN' + ] + } + } +} + // Used to read index definitions (required when using authentication) // https://learn.microsoft.com/azure/search/search-security-rbac module searchReaderRoleBackend 'core/security/role.bicep' = if (useAuthentication) { diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 90e1396453..e6ae56f8fc 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -11,21 +11,27 @@ param tags object = {} param usePrivateEndpoint bool = false +@allowed(['appservice', 'containerapps']) +param deploymentTarget string + +@description('The name of an existing App Service Plan to connect to the VNet') +param appServicePlanName string + +@description('The name of an existing Container Apps Environment to connect to the VNet') param containerAppsEnvName string -resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = { - name: containerAppsEnvName +param deployVpnGateway bool = false + +resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (deploymentTarget == 'appservice') { + name: appServicePlanName } +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = if (deploymentTarget == 'containerapps') { + name: containerAppsEnvName +} -module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) { - name: 'vnet' - params: { - name: vnetName - location: location - tags: tags - subnets: [ - { +// Always need this one +var backendSubnet = { name: 'backend-subnet' properties: { addressPrefix: '10.0.1.0/24' @@ -33,30 +39,71 @@ module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) { privateLinkServiceNetworkPolicies: 'Enabled' } } - { // App Service / Container Apps specific subnet - name: 'app-int-subnet' - properties: { - addressPrefix: '10.0.4.0/23' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' - delegations: [ - { - id: containerAppsEnvironment.id - name: containerAppsEnvironment.name - properties: { - serviceName: 'Microsoft.App/environments' - } + +var appServiceSubnet = { + name: 'app-int-subnet' + properties: { + addressPrefix: '10.0.3.0/24' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + delegations: [ + { + id: appServicePlan.id + name: appServicePlan.name + properties: { + serviceName: 'Microsoft.Web/serverFarms' } - ] - } + } + ] + } + } + +var containerAppsSubnet = { + name: 'app-int-subnet' + properties: { + addressPrefix: '10.0.4.0/23' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + delegations: [ + { + id: containerAppsEnvironment.id + name: containerAppsEnvironment.name + properties: { + serviceName: 'Microsoft.App/environments' + } + } + ] } - ] - } } +var gatewaySubnet = { + name: 'GatewaySubnet' // Required name for Gateway subnet + addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet + } + +var privateDnsResolverSubnet = { + name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver + addressPrefix: '10.0.11.0/28' // Original value kept as requested + delegation: 'Microsoft.Network/dnsResolvers' + } + +var subnets = union( + [backendSubnet, deploymentTarget == 'appservice' ? appServiceSubnet : containerAppsSubnet], + deployVpnGateway ? [gatewaySubnet, privateDnsResolverSubnet] : []) + +module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) { + name: 'vnet' + params: { + name: vnetName + location: location + tags: tags + subnets: subnets + } +} output appSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[1].id : '' +output appSubnetName string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[1].name : '' output backendSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[0].id : '' -output bastionSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[2].id : '' -output vmSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[3].id : '' +output privateDnsResolverSubnetId string = deployVpnGateway ? vnet.outputs.vnetSubnets[3].id : '' output vnetName string = usePrivateEndpoint ? vnet.outputs.name : '' +output vnetId string = usePrivateEndpoint ? vnet.outputs.id : '' diff --git a/infra/private-endpoints.bicep b/infra/private-endpoints.bicep index a26ad08947..856970169b 100644 --- a/infra/private-endpoints.bicep +++ b/infra/private-endpoints.bicep @@ -151,9 +151,9 @@ module monitorPrivateEndpoint './core/networking/private-endpoint.bicep' = { } } { - name: 'blob-dnszone' // dnsZones[dnsZoneBlobIndex].name + name: dnsZones[dnsZoneBlobIndex].name properties: { - privateDnsZoneId: '/subscriptions/77d8a3d0-8b18-47e9-b773-08bee327bb4a/resourceGroups/rg-pf-ragprivate/providers/Microsoft.Network/privateDnsZones/privatelink.blob.core.windows.net' // dnsZones[dnsZoneBlobIndex].outputs.id + privateDnsZoneId: dnsZones[dnsZoneBlobIndex].outputs.id } } ] From ced598349ff7699e8384937a7745cb2f34ed4f25 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 24 Jul 2025 11:33:24 -0700 Subject: [PATCH 06/24] Private endpoint almost working --- app/backend/prepdocs.py | 2 +- docs/deploy_private.md | 57 +++++-- infra/core/host/container-app-upsert.bicep | 106 ++++--------- infra/core/host/container-app.bicep | 149 ++++++------------ .../host/container-apps-environment.bicep | 65 ++++++++ infra/core/host/container-apps.bicep | 69 ++------ infra/core/host/container-registry.bicep | 69 ++++++++ infra/core/storage/storage-account.bicep | 2 +- infra/main.bicep | 48 +++--- infra/main.parameters.json | 3 + infra/network-isolation.bicep | 17 +- infra/private-endpoints.bicep | 4 +- todo.txt | 1 + 13 files changed, 330 insertions(+), 262 deletions(-) create mode 100644 infra/core/host/container-apps-environment.bicep create mode 100644 infra/core/host/container-registry.bicep create mode 100644 todo.txt diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index f03baac0dc..2c4bd49c26 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -323,7 +323,7 @@ async def main(strategy: Strategy, setup_index: bool = True): load_azd_env() - if os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled": + if os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled" and os.getenv("AZURE_USE_VPN_GATEWAY") != "true": logger.error("AZURE_PUBLIC_NETWORK_ACCESS is set to Disabled. Exiting.") exit(0) diff --git a/docs/deploy_private.md b/docs/deploy_private.md index 8e2321c9e7..a70e6b5087 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -46,20 +46,51 @@ Deploying with public access disabled adds additional cost to your deployment. P ## Recommended deployment strategy for private access -1. Deploy the app with private endpoints enabled and public access enabled. +1. Deploy the app with private endpoints enabled, public network access disabled, and a VPN gateway configured. This will allow you to connect to the chat app from inside the virtual network. - ```shell - azd env set AZURE_USE_PRIVATE_ENDPOINT true - azd env set AZURE_PUBLIC_NETWORK_ACCESS Enabled - azd up - ``` + ```shell + azd env set AZURE_USE_PRIVATE_ENDPOINT true + azd env set AZURE_USE_VPN_GATEWAY true + azd env set AZURE_PUBLIC_NETWORK_ACCESS Enabled + azd up + ``` -1. Validate that you can connect to the chat app and it's working as expected from the internet. -1. Re-provision the app with public access disabled. +2. First provision all the resources: - ```shell - azd env set AZURE_PUBLIC_NETWORK_ACCESS Disabled - azd provision - ``` + ```bash + azd provision + ``` -1. Log into your network using a tool like [Azure VPN Gateway](https://azure.microsoft.com/services/vpn-gateway/) and validate that you can connect to the chat app from inside the network. +3. Once provisioning is complete, run this command to get the VPN configuration download link: + + ```bash + azd env get-value AZURE_VPN_CONFIG_DOWNLOAD_LINK + ``` + + Select "Download VPN client" to download a ZIP file containing the VPN configuration. + +4. Open `AzureVPN/azurevpnconfig.xml`, and replace the `` empty tag with the following: + + ```xml + + + 10.0.11.4 + + + ``` + +5. Open the "Azure VPN" client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified. + +6. Select "Connect" and the new VPN connection. You will be prompted to select your Microsoft account and login. + +7. Once you're successfully connected to VPN, you can run the data ingestion script: + + ```bash + azd hooks run postprovision + ``` + +8. Finally, you can deploy the app: + + ```bash + azd deploy + ``` diff --git a/infra/core/host/container-app-upsert.bicep b/infra/core/host/container-app-upsert.bicep index 0621dc1889..71f6a7d6b7 100644 --- a/infra/core/host/container-app-upsert.bicep +++ b/infra/core/host/container-app-upsert.bicep @@ -1,89 +1,57 @@ -metadata description = 'Creates or updates an existing Azure Container App.' param name string param location string = resourceGroup().location param tags object = {} - -@description('The number of CPU cores allocated to a single container instance, e.g., 0.5') -param containerCpuCoreCount string = '0.5' - -@description('The maximum number of replicas to run. Must be at least 1.') -@minValue(1) -param containerMaxReplicas int = 10 - -@description('The amount of memory allocated to a single container instance, e.g., 1Gi') -param containerMemory string = '1.0Gi' - -@description('The minimum number of replicas to run. Must be at least 1 for non-consumption workloads.') -param containerMinReplicas int = 0 - -@description('The name of the container') +param containerAppsEnvironmentName string param containerName string = 'main' - -@description('The environment name for the container apps') -param containerAppsEnvironmentName string = '${containerName}env' - -@description('The name of the container registry') param containerRegistryName string -@description('Hostname suffix for container registry. Set when deploying to sovereign clouds') -param containerRegistryHostSuffix string = 'azurecr.io' - -@allowed(['http', 'grpc']) -@description('The protocol used by Dapr to connect to the app, e.g., HTTP or gRPC') -param daprAppProtocol string = 'http' - -@description('Enable or disable Dapr for the container app') -param daprEnabled bool = false - -@description('The Dapr app ID') -param daprAppId string = containerName - -@description('Specifies if the resource already exists') -param exists bool = false - -@description('Specifies if Ingress is enabled for the container app') -param ingressEnabled bool = true - -@description('The type of identity for the resource') -@allowed(['None', 'SystemAssigned', 'UserAssigned']) -param identityType string = 'None' - -@description('The name of the user-assigned identity') -param identityName string = '' - -@description('The name of the container image') -param imageName string = '' +@description('Minimum number of replicas to run') +@minValue(1) +param containerMinReplicas int = 1 +@description('Maximum number of replicas to run') +@minValue(1) +param containerMaxReplicas int = 10 @description('The secrets required for the container') @secure() param secrets object = {} -@description('The keyvault identities required for the container') -@secure() -param keyvaultIdentities object = {} - @description('The environment variables for the container in key value pairs') param env object = {} @description('The environment variables with secret references') param envSecrets array = [] -@description('Specifies if the resource ingress is exposed externally') param external bool = true +param targetPort int = 80 +param exists bool -@description('The service binds associated with the container') -param serviceBinds array = [] +@description('User assigned identity name') +param identityName string -@description('The target port for the container') -param targetPort int = 80 +@description('Enabled Ingress for container app') +param ingressEnabled bool = true + +// Dapr Options +@description('Enable Dapr') +param daprEnabled bool = false +@description('Dapr app ID') +param daprAppId string = containerName +@allowed([ 'http', 'grpc' ]) +@description('Protocol used by Dapr to connect to the app, e.g. http or grpc') +param daprAppProtocol string = 'http' -@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) -param workloadProfile string = 'Consumption' +@description('CPU cores allocated to a single container instance, e.g. 0.5') +param containerCpuCoreCount string = '0.5' + +@description('Memory allocated to a single container instance, e.g. 1Gi') +param containerMemory string = '1.0Gi' -param allowedOrigins array = [] +@description('Workload profile name to use for the container app when using private ingress') +param workloadProfileName string = 'Warm' -resource existingApp 'Microsoft.App/containerApps@2023-05-02-preview' existing = if (exists) { +resource existingApp 'Microsoft.App/containerApps@2022-03-01' existing = if (exists) { name: name } @@ -98,16 +66,13 @@ module app 'container-app.bicep' = { name: '${deployment().name}-update' params: { name: name - workloadProfile: workloadProfile location: location tags: tags - identityType: identityType identityName: identityName ingressEnabled: ingressEnabled containerName: containerName containerAppsEnvironmentName: containerAppsEnvironmentName containerRegistryName: containerRegistryName - containerRegistryHostSuffix: containerRegistryHostSuffix containerCpuCoreCount: containerCpuCoreCount containerMemory: containerMemory containerMinReplicas: containerMinReplicas @@ -116,20 +81,19 @@ module app 'container-app.bicep' = { daprAppId: daprAppId daprAppProtocol: daprAppProtocol secrets: secrets - keyvaultIdentities: keyvaultIdentities - allowedOrigins: allowedOrigins external: external env: concat(envAsArray, envSecrets) - imageName: !empty(imageName) ? imageName : exists ? existingApp.properties.template.containers[0].image : '' + imageName: exists ? existingApp.properties.template.containers[0].image : '' targetPort: targetPort - serviceBinds: serviceBinds + // Pass workload profile name parameter + workloadProfileName: workloadProfileName } } output defaultDomain string = app.outputs.defaultDomain output imageName string = app.outputs.imageName output name string = app.outputs.name +output hostName string = app.outputs.hostName output uri string = app.outputs.uri -output id string = app.outputs.id -output identityPrincipalId string = app.outputs.identityPrincipalId output identityResourceId string = app.outputs.identityResourceId +output identityPrincipalId string = app.outputs.identityPrincipalId diff --git a/infra/core/host/container-app.bicep b/infra/core/host/container-app.bicep index 9c8c80df91..ddb65fb7f9 100644 --- a/infra/core/host/container-app.bicep +++ b/infra/core/host/container-app.bicep @@ -1,116 +1,71 @@ -metadata description = 'Creates a container app in an Azure Container App environment.' param name string param location string = resourceGroup().location param tags object = {} -@description('Allowed origins') -param allowedOrigins array = [] - -@description('Name of the environment for container apps') param containerAppsEnvironmentName string +param containerName string = 'main' +param containerRegistryName string -@description('CPU cores allocated to a single container instance, e.g., 0.5') -param containerCpuCoreCount string = '0.5' - -@description('The maximum number of replicas to run. Must be at least 1.') +@description('Minimum number of replicas to run') +@minValue(1) +param containerMinReplicas int = 1 +@description('Maximum number of replicas to run') @minValue(1) param containerMaxReplicas int = 10 -@description('Memory allocated to a single container instance, e.g., 1Gi') -param containerMemory string = '1.0Gi' - -@description('The minimum number of replicas to run. Must be at least 1.') -param containerMinReplicas int = 0 - -@description('The name of the container') -param containerName string = 'main' - -@description('The name of the container registry') -param containerRegistryName string = '' - -@description('Hostname suffix for container registry. Set when deploying to sovereign clouds') -param containerRegistryHostSuffix string = 'azurecr.io' - -@description('The protocol used by Dapr to connect to the app, e.g., http or grpc') -@allowed([ 'http', 'grpc' ]) -param daprAppProtocol string = 'http' - -@description('The Dapr app ID') -param daprAppId string = containerName - -@description('Enable Dapr') -param daprEnabled bool = false +@description('The secrets required for the container') +@secure() +param secrets object = {} @description('The environment variables for the container') param env array = [] -@description('Specifies if the resource ingress is exposed externally') param external bool = true +param imageName string +param targetPort int = 80 -@description('The name of the user-assigned identity') -param identityName string = '' - -@description('The type of identity for the resource') -@allowed([ 'None', 'SystemAssigned', 'UserAssigned' ]) -param identityType string = 'None' - -@description('The name of the container image') -param imageName string = '' +@description('User assigned identity name') +param identityName string -@description('Specifies if Ingress is enabled for the container app') +@description('Enabled Ingress for container app') param ingressEnabled bool = true -param revisionMode string = 'Single' - -@description('The secrets required for the container') -@secure() -param secrets object = {} - -@description('The keyvault identities required for the container') -@secure() -param keyvaultIdentities object = {} - -@description('The service binds associated with the container') -param serviceBinds array = [] - -@description('The name of the container apps add-on to use. e.g. redis') -param serviceType string = '' - -@description('The target port for the container') -param targetPort int = 80 - -param workloadProfile string = 'Consumption' +// Dapr Options +@description('Enable Dapr') +param daprEnabled bool = false +@description('Dapr app ID') +param daprAppId string = containerName +@allowed([ 'http', 'grpc' ]) +@description('Protocol used by Dapr to connect to the app, e.g. http or grpc') +param daprAppProtocol string = 'http' -resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!empty(identityName)) { - name: identityName -} +@description('CPU cores allocated to a single container instance, e.g. 0.5') +param containerCpuCoreCount string = '0.5' -// Private registry support requires both an ACR name and a User Assigned managed identity -var usePrivateRegistry = !empty(identityName) && !empty(containerRegistryName) +@description('Memory allocated to a single container instance, e.g. 1Gi') +param containerMemory string = '1.0Gi' -// Automatically set to `UserAssigned` when an `identityName` has been set -var normalizedIdentityType = !empty(identityName) ? 'UserAssigned' : identityType +@description('Workload profile name to use for the container app when using private ingress') +param workloadProfileName string = 'Warm' var keyvalueSecrets = [for secret in items(secrets): { name: secret.key value: secret.value }] -var keyvaultIdentitySecrets = [for secret in items(keyvaultIdentities): { - name: secret.key - keyVaultUrl: secret.value.keyVaultUrl - identity: secret.value.identity -}] +resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { + name: identityName +} -module containerRegistryAccess '../security/registry-access.bicep' = if (usePrivateRegistry) { +module containerRegistryAccess '../security/registry-access.bicep' = { name: '${deployment().name}-registry-access' params: { containerRegistryName: containerRegistryName - principalId: usePrivateRegistry ? userIdentity.properties.principalId : '' + principalId: userIdentity.properties.principalId } } -resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { +resource app 'Microsoft.App/containerApps@2025-01-01' = { name: name location: location tags: tags @@ -118,23 +73,19 @@ resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { // otherwise the container app will throw a provision error // This also forces us to use an user assigned managed identity since there would no way to // provide the system assigned identity with the ACR pull access before the app is created - dependsOn: usePrivateRegistry ? [ containerRegistryAccess ] : [] + dependsOn: [ containerRegistryAccess ] identity: { - type: normalizedIdentityType - userAssignedIdentities: !empty(identityName) && normalizedIdentityType == 'UserAssigned' ? { '${userIdentity.id}': {} } : null + type: 'UserAssigned' + userAssignedIdentities: { '${userIdentity.id}': {} } } properties: { managedEnvironmentId: containerAppsEnvironment.id - workloadProfileName: workloadProfile configuration: { - activeRevisionsMode: revisionMode + activeRevisionsMode: 'single' ingress: ingressEnabled ? { external: external targetPort: targetPort transport: 'auto' - corsPolicy: { - allowedOrigins: union([ 'https://portal.azure.com', 'https://ms.portal.azure.com' ], allowedOrigins) - } } : null dapr: daprEnabled ? { enabled: true @@ -142,17 +93,15 @@ resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { appProtocol: daprAppProtocol appPort: ingressEnabled ? targetPort : 0 } : { enabled: false } - secrets: concat(keyvalueSecrets, keyvaultIdentitySecrets) - service: !empty(serviceType) ? { type: serviceType } : null - registries: usePrivateRegistry ? [ + secrets: keyvalueSecrets + registries: [ { - server: '${containerRegistryName}.${containerRegistryHostSuffix}' + server: '${containerRegistry.name}.azurecr.io' identity: userIdentity.id } - ] : [] + ] } template: { - serviceBinds: !empty(serviceBinds) ? serviceBinds : null containers: [ { image: !empty(imageName) ? imageName : 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' @@ -172,15 +121,19 @@ resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { } } -resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = { +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2022-03-01' existing = { name: containerAppsEnvironmentName } +// 2022-02-01-preview needed for anonymousPullEnabled +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' existing = { + name: containerRegistryName +} + output defaultDomain string = containerAppsEnvironment.properties.defaultDomain -output identityPrincipalId string = normalizedIdentityType == 'None' ? '' : (empty(identityName) ? app.identity.principalId : userIdentity.properties.principalId) -output identityResourceId string = normalizedIdentityType == 'UserAssigned' ? resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', userIdentity.name) : '' output imageName string = imageName output name string = app.name -output serviceBind object = !empty(serviceType) ? { serviceId: app.id, name: name } : {} +output hostName string = app.properties.configuration.ingress.fqdn output uri string = ingressEnabled ? 'https://${app.properties.configuration.ingress.fqdn}' : '' -output id string = app.id +output identityResourceId string = resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', userIdentity.name) +output identityPrincipalId string = userIdentity.properties.principalId diff --git a/infra/core/host/container-apps-environment.bicep b/infra/core/host/container-apps-environment.bicep new file mode 100644 index 0000000000..22ee019c19 --- /dev/null +++ b/infra/core/host/container-apps-environment.bicep @@ -0,0 +1,65 @@ +param name string +param location string = resourceGroup().location +param tags object = {} + +param daprEnabled bool = false +param logAnalyticsWorkspaceName string = '' +param applicationInsightsName string = '' + +@description('Virtual network name for container apps environment.') +param vnetName string = '' +@description('Subnet name for container apps environment integration.') +param subnetName string = '' +param subnetResourceId string + +param usePrivateIngress bool = true + +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2025-02-02-preview' = { + name: name + location: location + tags: tags + properties: { + // We can't use a conditional here due to an issue with the Container Apps ARM parsing + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logAnalyticsWorkspace.properties.customerId + sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey + } + } + daprAIInstrumentationKey: daprEnabled && !empty(applicationInsightsName) ? applicationInsights.properties.InstrumentationKey : '' + publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled' + vnetConfiguration: (!empty(vnetName) && !empty(subnetName)) ? { + // Use proper subnet resource ID format + infrastructureSubnetId: subnetResourceId + internal: usePrivateIngress + } : null + // Configure workload profile for dedicated environment (not consumption) + workloadProfiles: usePrivateIngress + ? [ + { + name: 'Consumption' + workloadProfileType: 'Consumption' + } + { + name: 'Warm' + workloadProfileType: 'D4' + minimumCount: 1 + maximumCount: 3 + } + ] + : [] + } +} + +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = if (!empty(logAnalyticsWorkspaceName)) { + name: logAnalyticsWorkspaceName +} + +resource applicationInsights 'Microsoft.Insights/components@2020-02-02' existing = if (daprEnabled && !empty(applicationInsightsName)){ + name: applicationInsightsName +} + +output defaultDomain string = containerAppsEnvironment.properties.defaultDomain +output name string = containerAppsEnvironment.name +output resourceId string = containerAppsEnvironment.id diff --git a/infra/core/host/container-apps.bicep b/infra/core/host/container-apps.bicep index 4598a5b901..53af2d56d1 100644 --- a/infra/core/host/container-apps.bicep +++ b/infra/core/host/container-apps.bicep @@ -1,84 +1,49 @@ -metadata description = 'Creates an Azure Container Registry and an Azure Container Apps environment.' param name string param location string = resourceGroup().location param tags object = {} param containerAppsEnvironmentName string param containerRegistryName string -param containerRegistryResourceGroupName string = '' -param containerRegistryAdminUserEnabled bool = false -param logAnalyticsWorkspaceResourceId string +param logAnalyticsWorkspaceName string = '' +param applicationInsightsName string = '' + +@description('Virtual network name for container apps environment.') +param vnetName string = '' +@description('Subnet name for container apps environment integration.') +param subnetName string = '' param subnetResourceId string = '' param usePrivateIngress bool = true -@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) -param workloadProfile string - -var workloadProfiles = workloadProfile == 'Consumption' - ? [ - { - name: 'Consumption' - workloadProfileType: 'Consumption' - } - ] - : [ - { - name: 'Consumption' - workloadProfileType: 'Consumption' - } - { - minimumCount: 1 - maximumCount: 2 - name: workloadProfile - workloadProfileType: workloadProfile - } - ] - -@description('Optional user assigned identity IDs to assign to the resource') -param userAssignedIdentityResourceIds array = [] - -module containerAppsEnvironment 'br/public:avm/res/app/managed-environment:0.8.0' = { +module containerAppsEnvironment 'container-apps-environment.bicep' = { name: '${name}-container-apps-environment' params: { - // Required parameters - logAnalyticsWorkspaceResourceId: logAnalyticsWorkspaceResourceId - - managedIdentities: empty(userAssignedIdentityResourceIds) ? { - systemAssigned: true - } : { - userAssignedResourceIds: userAssignedIdentityResourceIds - } - name: containerAppsEnvironmentName - // Non-required parameters - infrastructureResourceGroupName: containerRegistryResourceGroupName - infrastructureSubnetId: subnetResourceId - internal: usePrivateIngress location: location tags: tags - zoneRedundant: false - workloadProfiles: workloadProfiles + logAnalyticsWorkspaceName: logAnalyticsWorkspaceName + applicationInsightsName: applicationInsightsName + vnetName: vnetName + subnetName: subnetName + usePrivateIngress: usePrivateIngress + subnetResourceId: subnetResourceId } } -module containerRegistry 'br/public:avm/res/container-registry/registry:0.5.1' = { +module containerRegistry 'container-registry.bicep' = { name: '${name}-container-registry' - scope: resourceGroup(!empty(containerRegistryResourceGroupName) ? containerRegistryResourceGroupName : resourceGroup().name) params: { name: containerRegistryName location: location - acrSku: usePrivateIngress ? 'Premium' : 'Standard' - acrAdminUserEnabled: containerRegistryAdminUserEnabled - publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled' tags: tags + useVnet: !empty(vnetName) } } output defaultDomain string = containerAppsEnvironment.outputs.defaultDomain output environmentName string = containerAppsEnvironment.outputs.name output environmentId string = containerAppsEnvironment.outputs.resourceId - output registryLoginServer string = containerRegistry.outputs.loginServer output registryName string = containerRegistry.outputs.name +output registryId string = containerRegistry.outputs.resourceId diff --git a/infra/core/host/container-registry.bicep b/infra/core/host/container-registry.bicep new file mode 100644 index 0000000000..8f48611c8b --- /dev/null +++ b/infra/core/host/container-registry.bicep @@ -0,0 +1,69 @@ +param name string +param location string = resourceGroup().location +param tags object = {} + +param adminUserEnabled bool = true +param anonymousPullEnabled bool = false +param dataEndpointEnabled bool = false +param encryption object = { + status: 'disabled' +} +param networkRuleBypassOptions string = 'AzureServices' +param publicNetworkAccess string = useVnet ? 'Disabled' : 'Enabled' // Public network access is disabled if VNet integration is enabled +param useVnet bool = false // Determines if VNet integration is enabled +param sku object = { + name: useVnet ? 'Premium' : 'Standard' // Use Premium if VNet is required, otherwise Standard +} +param zoneRedundancy string = 'Disabled' + +@description('The log analytics workspace id used for logging & monitoring') +param workspaceId string = '' + +// 2022-02-01-preview needed for anonymousPullEnabled +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { + name: name + location: location + tags: tags + sku: sku + properties: { + adminUserEnabled: adminUserEnabled + anonymousPullEnabled: anonymousPullEnabled + dataEndpointEnabled: dataEndpointEnabled + encryption: encryption + networkRuleBypassOptions: networkRuleBypassOptions + publicNetworkAccess: publicNetworkAccess + zoneRedundancy: zoneRedundancy + } +} + +// TODO: Update diagnostics to be its own module +// Blocking issue: https://github.com/Azure/bicep/issues/622 +// Unable to pass in a `resource` scope or unable to use string interpolation in resource types +resource diagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(workspaceId)) { + name: 'registry-diagnostics' + scope: containerRegistry + properties: { + workspaceId: workspaceId + logs: [ + { + category: 'ContainerRegistryRepositoryEvents' + enabled: true + } + { + category: 'ContainerRegistryLoginEvents' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + timeGrain: 'PT1M' + } + ] + } +} + +output loginServer string = containerRegistry.properties.loginServer +output name string = containerRegistry.name +output resourceId string = containerRegistry.id diff --git a/infra/core/storage/storage-account.bicep b/infra/core/storage/storage-account.bicep index 670199162c..46489ceb7d 100644 --- a/infra/core/storage/storage-account.bicep +++ b/infra/core/storage/storage-account.bicep @@ -59,7 +59,7 @@ resource storage 'Microsoft.Storage/storageAccounts@2024-01-01' = { resource container 'containers' = [for container in containers: { name: container.name properties: { - publicAccess: contains(container, 'publicAccess') ? container.publicAccess : 'None' + publicAccess: container.?publicAccess ?? 'None' } }] } diff --git a/infra/main.bicep b/infra/main.bicep index 74fc05d176..a9ebe446ee 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -297,13 +297,10 @@ param runningOnAdo string = '' @description('Used by azd for containerapps deployment') param webAppExists bool -@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) -param azureContainerAppsWorkloadProfile string - @allowed(['appservice', 'containerapps']) param deploymentTarget string = 'appservice' param acaIdentityName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-identity' : '' -param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-env' : '' +param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-env2' : '' param containerRegistryName string = deploymentTarget == 'containerapps' ? '${replace(toLower(environmentName), '-', '')}acr' : '' @@ -528,10 +525,9 @@ module containerApps 'core/host/container-apps.bicep' = if (deploymentTarget == name: 'app' tags: tags location: location - workloadProfile: azureContainerAppsWorkloadProfile containerAppsEnvironmentName: acaManagedEnvironmentName containerRegistryName: '${containerRegistryName}${resourceToken}' - logAnalyticsWorkspaceResourceId: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceId : '' + logAnalyticsWorkspaceName: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceName : '' subnetResourceId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' usePrivateIngress: usePrivateEndpoint } @@ -547,19 +543,17 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget ] params: { name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesContainerApps}backend${resourceToken}' - location: 'westus2' + location: location identityName: (deploymentTarget == 'containerapps') ? acaIdentityName : '' exists: webAppExists - workloadProfile: azureContainerAppsWorkloadProfile containerRegistryName: (deploymentTarget == 'containerapps') ? containerApps.outputs.registryName : '' containerAppsEnvironmentName: (deploymentTarget == 'containerapps') ? containerApps.outputs.environmentName : '' - identityType: 'UserAssigned' tags: union(tags, { 'azd-service-name': 'backend' }) targetPort: 8000 containerCpuCoreCount: '1.0' containerMemory: '2Gi' containerMinReplicas: 1 - allowedOrigins: allowedOrigins + //allowedOrigins: allowedOrigins env: union(appEnvVariables, { // For using managed identity to access Azure resources. See https://github.com/microsoft/azure-container-apps/issues/442 AZURE_CLIENT_ID: (deploymentTarget == 'containerapps') ? acaIdentity.outputs.clientId : '' @@ -705,6 +699,8 @@ module documentIntelligence 'br/public:avm/res/cognitive-services/account:0.7.2' name: !empty(documentIntelligenceServiceName) ? documentIntelligenceServiceName : '${abbrs.cognitiveServicesDocumentIntelligence}${resourceToken}' + location: documentIntelligenceResourceGroupLocation + tags: tags kind: 'FormRecognizer' customSubDomainName: !empty(documentIntelligenceServiceName) ? documentIntelligenceServiceName @@ -713,10 +709,8 @@ module documentIntelligence 'br/public:avm/res/cognitive-services/account:0.7.2' networkAcls: { defaultAction: 'Allow' } - location: documentIntelligenceResourceGroupLocation - disableLocalAuth: true - tags: tags sku: documentIntelligenceSkuName + disableLocalAuth: true } } @@ -1169,6 +1163,7 @@ module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { tags: tags vnetName: '${abbrs.virtualNetworks}${resourceToken}' usePrivateEndpoint: usePrivateEndpoint + deployVpnGateway: useVpnGateway deploymentTarget: deploymentTarget // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : '' @@ -1183,11 +1178,20 @@ var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi) { groupId: 'account' dnsZoneName: 'privatelink.openai.azure.com' + resourceIds: [openAi.outputs.resourceId] + } + ] + : [] + +var cognitiveServicesPrivateEndpointConnection = (usePrivateEndpoint && (!useLocalPdfParser || useGPT4V || useMediaDescriberAzureCU)) + ? [ + { + groupId: 'account' + dnsZoneName: 'privatelink.cognitiveservices.azure.com' resourceIds: concat( - [openAi.outputs.resourceId], + !useLocalPdfParser ? [documentIntelligence.outputs.resourceId] : [], useGPT4V ? [computerVision.outputs.resourceId] : [], - useMediaDescriberAzureCU ? [contentUnderstanding.outputs.resourceId] : [], - !useLocalPdfParser ? [documentIntelligence.outputs.resourceId] : [] + useMediaDescriberAzureCU ? [contentUnderstanding.outputs.resourceId] : [] ) } ] @@ -1217,7 +1221,7 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint) ] : [] -var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection) +var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection, cognitiveServicesPrivateEndpointConnection) module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { name: 'privateEndpoints' @@ -1236,8 +1240,8 @@ module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { // Based on https://luke.geek.nz/azure/azure-point-to-site-vpn-and-private-dns-resolver/ // Manual step required of updating azurevpnconfig.xml to use the correct DNS server IP address -module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.3' = if (useVpnGateway) { - name: 'dnsResolverDeployment' +module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.4' = if (useVpnGateway) { + name: 'dns-resolver' scope: resourceGroup params: { name: '${abbrs.privateDnsResolver}${resourceToken}' @@ -1252,8 +1256,8 @@ module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.3' = if (useVpnGa } } -module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.6.1' = if (useVpnGateway) { - name: 'virtualNetworkGatewayDeployment' +module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.8.0' = if (useVpnGateway) { + name: 'virtual-network-gateway' scope: resourceGroup params: { name: '${abbrs.networkVpnGateways}${resourceToken}' @@ -1403,3 +1407,5 @@ output BACKEND_URI string = deploymentTarget == 'appservice' ? backend.outputs.u output AZURE_CONTAINER_REGISTRY_ENDPOINT string = deploymentTarget == 'containerapps' ? containerApps.outputs.registryLoginServer : '' + + output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource/subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup.name}/providers/Microsoft.Network/virtualNetworkGateways/${virtualNetworkGateway.outputs.name}/pointtositeconfiguration' : '' diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 1c34063020..5202fdcde0 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -314,6 +314,9 @@ "usePrivateEndpoint": { "value": "${AZURE_USE_PRIVATE_ENDPOINT=false}" }, + "useVpnGateway": { + "value": "${AZURE_USE_VPN_GATEWAY=false}" + }, "bypass": { "value": "${AZURE_NETWORK_BYPASS=AzureServices}" }, diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index e6ae56f8fc..d3b2eab687 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -78,13 +78,24 @@ var containerAppsSubnet = { var gatewaySubnet = { name: 'GatewaySubnet' // Required name for Gateway subnet - addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet + properties: { + addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet + } } var privateDnsResolverSubnet = { name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver - addressPrefix: '10.0.11.0/28' // Original value kept as requested - delegation: 'Microsoft.Network/dnsResolvers' + properties: { + addressPrefix: '10.0.11.0/28' // Original value kept as requested + delegations: [ + { + name: 'Microsoft.Network.dnsResolvers' + properties: { + serviceName: 'Microsoft.Network/dnsResolvers' + } + } + ] + } } var subnets = union( diff --git a/infra/private-endpoints.bicep b/infra/private-endpoints.bicep index 856970169b..2a7ca8d75c 100644 --- a/infra/private-endpoints.bicep +++ b/infra/private-endpoints.bicep @@ -32,8 +32,8 @@ param logAnalyticsWorkspaceId string var abbrs = loadJsonContent('abbreviations.json') // DNS Zones -module dnsZones './core/networking/private-dns-zones.bicep' = [for privateEndpointConnection in privateEndpointConnections: { - name: '${privateEndpointConnection.groupId}-dnszone' +module dnsZones './core/networking/private-dns-zones.bicep' = [for (privateEndpointConnection, i) in privateEndpointConnections: { + name: '${privateEndpointConnection.groupId}-${i}-dnszone' params: { dnsZoneName: privateEndpointConnection.dnsZoneName tags: tags diff --git a/todo.txt b/todo.txt new file mode 100644 index 0000000000..bf72b81792 --- /dev/null +++ b/todo.txt @@ -0,0 +1 @@ +* Better error when youre trying to run prepdocs and not connected to VPN. From abb00aa7e26ad47c54b98c1c672866aa529844cf Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 24 Jul 2025 14:38:24 -0700 Subject: [PATCH 07/24] Usving avm for the subnets --- infra/main.bicep | 55 +++++++- infra/network-isolation.bicep | 242 +++++++++++++++++++++++----------- 2 files changed, 217 insertions(+), 80 deletions(-) diff --git a/infra/main.bicep b/infra/main.bicep index a9ebe446ee..ad798aa35f 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1155,6 +1155,8 @@ module cosmosDbRoleBackend 'core/security/documentdb-sql-role.bicep' = if (useAu } } + + module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { name: 'networks' scope: resourceGroup @@ -1162,7 +1164,6 @@ module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { location: location tags: tags vnetName: '${abbrs.virtualNetworks}${resourceToken}' - usePrivateEndpoint: usePrivateEndpoint deployVpnGateway: useVpnGateway deploymentTarget: deploymentTarget // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 @@ -1208,11 +1209,6 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint) dnsZoneName: 'privatelink.search.windows.net' resourceIds: [searchService.outputs.id] } - { - groupId: 'managedEnvironments' - dnsZoneName: 'privatelink.${location}.azurecontainerapps.io' - resourceIds: [containerApps.outputs.environmentId] - } { groupId: 'sql' dnsZoneName: 'privatelink.documents.azure.com' @@ -1256,6 +1252,53 @@ module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.4' = if (useVpnGa } } +// Container Apps Private DNS Zone +module containerAppsPrivateDnsZone 'br/public:avm/res/network/private-dns-zone:0.7.1' = if (usePrivateEndpoint && deploymentTarget == 'containerapps') { + name: 'container-apps-dns-zone' + scope: resourceGroup + params: { + name: 'privatelink.${location}.azurecontainerapps.io' + tags: tags + virtualNetworkLinks: [ + { + registrationEnabled: false + virtualNetworkResourceId: isolation.outputs.vnetId + } + ] + } +} + +// Container Apps Environment Private Endpoint +// https://learn.microsoft.com/azure/container-apps/how-to-use-private-endpoint +module containerAppsEnvironmentPrivateEndpoint 'br/public:avm/res/network/private-endpoint:0.11.0' = if (usePrivateEndpoint && deploymentTarget == 'containerapps') { + name: 'containerAppsEnvironmentPrivateEndpointDeployment' + scope: resourceGroup + params: { + name: 'container-apps-env-pe${resourceToken}' + location: location + tags: tags + subnetResourceId: isolation.outputs.appSubnetId + privateDnsZoneGroup: { + privateDnsZoneGroupConfigs: [ + { + privateDnsZoneResourceId: containerAppsPrivateDnsZone.outputs.resourceId + } + ] + } + privateLinkServiceConnections: [ + { + name: 'containerAppsEnvironmentConnection' + properties: { + groupIds: [ + 'managedEnvironments' + ] + privateLinkServiceId: containerApps.outputs.environmentId + } + } + ] + } +} + module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.8.0' = if (useVpnGateway) { name: 'virtual-network-gateway' scope: resourceGroup diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index d3b2eab687..639d06085e 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -9,8 +9,6 @@ param location string = resourceGroup().location @description('The tags to apply to all resources') param tags object = {} -param usePrivateEndpoint bool = false - @allowed(['appservice', 'containerapps']) param deploymentTarget string @@ -26,95 +24,191 @@ resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (de name: appServicePlanName } -resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = if (deploymentTarget == 'containerapps') { - name: containerAppsEnvName +module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = if (deploymentTarget == 'containerapps') { + name: 'container-apps-nsg' + params: { + name: '${vnetName}-container-apps-nsg' + location: location + tags: tags + securityRules: [ + { + name: 'AllowHttpsInbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRange: '443' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + ] + } } -// Always need this one -var backendSubnet = { - name: 'backend-subnet' +module privateEndpointsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = if (deploymentTarget == 'containerapps') { + name: 'private-endpoints-nsg' + params: { + name: '${vnetName}-private-endpoints-nsg' + location: location + tags: tags + securityRules: [ + { + name: 'AllowVnetInBound' properties: { - addressPrefix: '10.0.1.0/24' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 100 + direction: 'Inbound' } } - -var appServiceSubnet = { - name: 'app-int-subnet' - properties: { - addressPrefix: '10.0.3.0/24' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' - delegations: [ - { - id: appServicePlan.id - name: appServicePlan.name - properties: { - serviceName: 'Microsoft.Web/serverFarms' - } - } - ] + { + name: 'AllowAzureLoadBalancerInbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 110 + direction: 'Inbound' + } } - } - -var containerAppsSubnet = { - name: 'app-int-subnet' - properties: { - addressPrefix: '10.0.4.0/23' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' - delegations: [ - { - id: containerAppsEnvironment.id - name: containerAppsEnvironment.name - properties: { - serviceName: 'Microsoft.App/environments' - } - } - ] + { + name: 'DenyInternetInbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 4096 + direction: 'Inbound' + } } -} - -var gatewaySubnet = { - name: 'GatewaySubnet' // Required name for Gateway subnet - properties: { - addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet - } + { + name: 'AllowVnetOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'AllowAzureCloudOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '443' + destinationAddressPrefix: 'AzureCloud' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'AllowDnsOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '53' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 120 + direction: 'Outbound' + } + } + { + name: 'DenyInternetOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: 'Internet' + access: 'Deny' + priority: 4096 + direction: 'Outbound' + } + } + ] } +} -var privateDnsResolverSubnet = { - name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver - properties: { - addressPrefix: '10.0.11.0/28' // Original value kept as requested - delegations: [ - { - name: 'Microsoft.Network.dnsResolvers' - properties: { - serviceName: 'Microsoft.Network/dnsResolvers' - } +var appServiceSubnet = { + name: 'app-int-subnet' + properties: { + addressPrefix: '10.0.3.0/24' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + delegations: [ + { + id: appServicePlan.id + name: appServicePlan.name + properties: { + serviceName: 'Microsoft.Web/serverFarms' } - ] - } + } + ] } +} -var subnets = union( - [backendSubnet, deploymentTarget == 'appservice' ? appServiceSubnet : containerAppsSubnet], - deployVpnGateway ? [gatewaySubnet, privateDnsResolverSubnet] : []) - -module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) { +module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { name: 'vnet' params: { name: vnetName location: location tags: tags - subnets: subnets + addressPrefixes: [ + '10.0.0.0/16' + ] + subnets: [ + { + name: 'backend-subnet' + addressPrefix: '10.0.1.0/24' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + networkSecurityGroupResourceId: privateEndpointsNSG.outputs.resourceId + } + { + name: 'GatewaySubnet' // Required name for Gateway subnet + addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet + } + { + name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver + addressPrefix: '10.0.11.0/28' // Original value kept as requested + delegation: 'Microsoft.Network/dnsResolvers' + } + { + name: 'app-int-subnet' + addressPrefix: '10.0.4.0/23' + //privateEndpointNetworkPolicies: 'Enabled' + //privateLinkServiceNetworkPolicies: 'Enabled' + networkSecurityGroupResourceId: containerAppsNSG.outputs.resourceId + delegation: 'Microsoft.App/environments' + } + ] } } -output appSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[1].id : '' -output appSubnetName string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[1].name : '' -output backendSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[0].id : '' -output privateDnsResolverSubnetId string = deployVpnGateway ? vnet.outputs.vnetSubnets[3].id : '' -output vnetName string = usePrivateEndpoint ? vnet.outputs.name : '' -output vnetId string = usePrivateEndpoint ? vnet.outputs.id : '' + +output backendSubnetId string = vnet.outputs.subnetResourceIds[0] +output privateDnsResolverSubnetId string = deployVpnGateway ? vnet.outputs.subnetResourceIds[2] : '' +output appSubnetId string = vnet.outputs.subnetResourceIds[3] +output vnetName string = vnet.outputs.name +output vnetId string = vnet.outputs.resourceId From 4b3eb2af90029c7e827c19853c841763f242b883 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 24 Jul 2025 23:04:07 -0700 Subject: [PATCH 08/24] Connected app to vnet --- app/backend/app.py | 2 +- .../host/container-apps-environment.bicep | 2 +- infra/main.bicep | 8 +-- infra/network-isolation.bicep | 55 ++++++++++++++++--- 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 1b4563bb98..5e8e65c153 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -821,7 +821,7 @@ def create_app(): # Log levels should be one of https://docs.python.org/3/library/logging.html#logging-levels # Set root level to WARNING to avoid seeing overly verbose logs from SDKS - logging.basicConfig(level=logging.WARNING) + logging.basicConfig(level=logging.DEBUG) # Set our own logger levels to INFO by default app_level = os.getenv("APP_LOG_LEVEL", "INFO") app.logger.setLevel(os.getenv("APP_LOG_LEVEL", app_level)) diff --git a/infra/core/host/container-apps-environment.bicep b/infra/core/host/container-apps-environment.bicep index 22ee019c19..3f3c2be36c 100644 --- a/infra/core/host/container-apps-environment.bicep +++ b/infra/core/host/container-apps-environment.bicep @@ -29,7 +29,7 @@ resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2025-02-02- } daprAIInstrumentationKey: daprEnabled && !empty(applicationInsightsName) ? applicationInsights.properties.InstrumentationKey : '' publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled' - vnetConfiguration: (!empty(vnetName) && !empty(subnetName)) ? { + vnetConfiguration: usePrivateIngress ? { // Use proper subnet resource ID format infrastructureSubnetId: subnetResourceId internal: usePrivateIngress diff --git a/infra/main.bicep b/infra/main.bicep index ad798aa35f..23ed64cb84 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -300,7 +300,7 @@ param webAppExists bool @allowed(['appservice', 'containerapps']) param deploymentTarget string = 'appservice' param acaIdentityName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-identity' : '' -param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-env2' : '' +param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-envnet' : '' param containerRegistryName string = deploymentTarget == 'containerapps' ? '${replace(toLower(environmentName), '-', '')}acr' : '' @@ -1168,13 +1168,13 @@ module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { deploymentTarget: deploymentTarget // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : '' - containerAppsEnvName: deploymentTarget == 'containerapps' ? acaManagedEnvironmentName : '' + //containerAppsEnvName: deploymentTarget == 'containerapps' ? acaManagedEnvironmentName : '' } } var environmentData = environment() -var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi) +var openAiPrivateEndpointConnection = (usePrivateEndpoint && isAzureOpenAiHost && deployAzureOpenAi) ? [ { groupId: 'account' @@ -1277,7 +1277,7 @@ module containerAppsEnvironmentPrivateEndpoint 'br/public:avm/res/network/privat name: 'container-apps-env-pe${resourceToken}' location: location tags: tags - subnetResourceId: isolation.outputs.appSubnetId + subnetResourceId: isolation.outputs.backendSubnetId privateDnsZoneGroup: { privateDnsZoneGroupConfigs: [ { diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 639d06085e..65efb830d1 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -15,11 +15,9 @@ param deploymentTarget string @description('The name of an existing App Service Plan to connect to the VNet') param appServicePlanName string -@description('The name of an existing Container Apps Environment to connect to the VNet') -param containerAppsEnvName string - param deployVpnGateway bool = false +// TODO: Bring back app service option resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (deploymentTarget == 'appservice') { name: appServicePlanName } @@ -44,6 +42,45 @@ module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' direction: 'Inbound' } } + { // TODO: Were any of these rules really needed?? + name: 'AllowPrivateEndpointsOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' + destinationPortRange: '443' + destinationAddressPrefix: '10.0.8.0/24' + access: 'Allow' + priority: 200 + direction: 'Outbound' + } + } + { + name: 'AllowDnsOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '53' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 210 + direction: 'Outbound' + } + } + { + name: 'AllowVNetOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 220 + direction: 'Outbound' + } + } ] } } @@ -180,7 +217,7 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { subnets: [ { name: 'backend-subnet' - addressPrefix: '10.0.1.0/24' + addressPrefix: '10.0.8.0/24' privateEndpointNetworkPolicies: 'Enabled' privateLinkServiceNetworkPolicies: 'Enabled' networkSecurityGroupResourceId: privateEndpointsNSG.outputs.resourceId @@ -191,16 +228,16 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { } { name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver - addressPrefix: '10.0.11.0/28' // Original value kept as requested + addressPrefix: '10.0.11.0/28' delegation: 'Microsoft.Network/dnsResolvers' } { name: 'app-int-subnet' - addressPrefix: '10.0.4.0/23' - //privateEndpointNetworkPolicies: 'Enabled' - //privateLinkServiceNetworkPolicies: 'Enabled' + addressPrefix: '10.0.0.0/21' + //privateEndpointNetworkPolicies: 'Enabled' // TODO: Needed? + //privateLinkServiceNetworkPolicies: 'Enabled' // TODO: Needed? networkSecurityGroupResourceId: containerAppsNSG.outputs.resourceId - delegation: 'Microsoft.App/environments' + delegation: 'Microsoft.App/environments' // TODO: Needed? } ] } From ba69870b4c203b4535d73777099cb0f413da6e4e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 29 Jul 2025 14:15:03 -0700 Subject: [PATCH 09/24] Feedback from Matt --- app/backend/app.py | 2 +- docs/deploy_private.md | 2 +- infra/network-isolation.bicep | 4 +--- todo.txt | 4 ++++ 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 5e8e65c153..1b4563bb98 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -821,7 +821,7 @@ def create_app(): # Log levels should be one of https://docs.python.org/3/library/logging.html#logging-levels # Set root level to WARNING to avoid seeing overly verbose logs from SDKS - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.WARNING) # Set our own logger levels to INFO by default app_level = os.getenv("APP_LOG_LEVEL", "INFO") app.logger.setLevel(os.getenv("APP_LOG_LEVEL", app_level)) diff --git a/docs/deploy_private.md b/docs/deploy_private.md index a70e6b5087..865400c460 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -51,7 +51,7 @@ Deploying with public access disabled adds additional cost to your deployment. P ```shell azd env set AZURE_USE_PRIVATE_ENDPOINT true azd env set AZURE_USE_VPN_GATEWAY true - azd env set AZURE_PUBLIC_NETWORK_ACCESS Enabled + azd env set AZURE_PUBLIC_NETWORK_ACCESS Disabled azd up ``` diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 65efb830d1..8cbd718edc 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -234,10 +234,8 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { { name: 'app-int-subnet' addressPrefix: '10.0.0.0/21' - //privateEndpointNetworkPolicies: 'Enabled' // TODO: Needed? - //privateLinkServiceNetworkPolicies: 'Enabled' // TODO: Needed? networkSecurityGroupResourceId: containerAppsNSG.outputs.resourceId - delegation: 'Microsoft.App/environments' // TODO: Needed? + delegation: 'Microsoft.App/environments' } ] } diff --git a/todo.txt b/todo.txt index bf72b81792..86d8ce1134 100644 --- a/todo.txt +++ b/todo.txt @@ -1 +1,5 @@ * Better error when youre trying to run prepdocs and not connected to VPN. + * Hit up /ping and check that it returns 200 OK. Anything else = no bueno. + * If they have AZURE_USE_PRIVATE_ENDPOINT and it gets non-200, reminds them to set up VPN. +* Bring back App Service's subnet +* Move vnet-y things into network-isolation.bicep From eac9f9d5f25c911ef26aed8890cb2d52a02bad0b Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 29 Jul 2025 22:28:26 -0700 Subject: [PATCH 10/24] Move resources into modules --- infra/main.bicep | 112 +++++----------------------------- infra/network-isolation.bicep | 50 ++++++++++++++- infra/private-endpoints.bicep | 6 +- 3 files changed, 67 insertions(+), 101 deletions(-) diff --git a/infra/main.bicep b/infra/main.bicep index 23ed64cb84..678a923d80 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1164,11 +1164,13 @@ module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { location: location tags: tags vnetName: '${abbrs.virtualNetworks}${resourceToken}' - deployVpnGateway: useVpnGateway + useVpnGateway: useVpnGateway deploymentTarget: deploymentTarget // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : '' //containerAppsEnvName: deploymentTarget == 'containerapps' ? acaManagedEnvironmentName : '' + vpnGatewayName: useVpnGateway ? '${abbrs.networkVpnGateways}${resourceToken}' : '' + dnsResolverName: useVpnGateway ? '${abbrs.privateDnsResolver}${resourceToken}' : '' } } @@ -1197,6 +1199,16 @@ var cognitiveServicesPrivateEndpointConnection = (usePrivateEndpoint && (!useLoc } ] : [] + +var containerAppsPrivateEndpointConnection = (usePrivateEndpoint && deploymentTarget == 'containerapps') + ? [ + { + groupId: 'managedEnvironments' + dnsZoneName: 'privatelink.${location}.azurecontainerapps.io' + resourceIds: [containerApps.outputs.environmentId] + } + ] + : [] var otherPrivateEndpointConnections = (usePrivateEndpoint) ? [ { @@ -1217,7 +1229,7 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint) ] : [] -var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection, cognitiveServicesPrivateEndpointConnection) +var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection, cognitiveServicesPrivateEndpointConnection, containerAppsPrivateEndpointConnection) module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { name: 'privateEndpoints' @@ -1230,99 +1242,7 @@ module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { applicationInsightsId: useApplicationInsights ? monitoring.outputs.applicationInsightsId : '' logAnalyticsWorkspaceId: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceId : '' vnetName: isolation.outputs.vnetName - vnetPeSubnetName: isolation.outputs.backendSubnetId - } -} - -// Based on https://luke.geek.nz/azure/azure-point-to-site-vpn-and-private-dns-resolver/ -// Manual step required of updating azurevpnconfig.xml to use the correct DNS server IP address -module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.4' = if (useVpnGateway) { - name: 'dns-resolver' - scope: resourceGroup - params: { - name: '${abbrs.privateDnsResolver}${resourceToken}' - location: location - virtualNetworkResourceId: isolation.outputs.vnetId - inboundEndpoints: [ - { - name: 'inboundEndpoint' - subnetResourceId: useVpnGateway ? isolation.outputs.privateDnsResolverSubnetId : '' - } - ] - } -} - -// Container Apps Private DNS Zone -module containerAppsPrivateDnsZone 'br/public:avm/res/network/private-dns-zone:0.7.1' = if (usePrivateEndpoint && deploymentTarget == 'containerapps') { - name: 'container-apps-dns-zone' - scope: resourceGroup - params: { - name: 'privatelink.${location}.azurecontainerapps.io' - tags: tags - virtualNetworkLinks: [ - { - registrationEnabled: false - virtualNetworkResourceId: isolation.outputs.vnetId - } - ] - } -} - -// Container Apps Environment Private Endpoint -// https://learn.microsoft.com/azure/container-apps/how-to-use-private-endpoint -module containerAppsEnvironmentPrivateEndpoint 'br/public:avm/res/network/private-endpoint:0.11.0' = if (usePrivateEndpoint && deploymentTarget == 'containerapps') { - name: 'containerAppsEnvironmentPrivateEndpointDeployment' - scope: resourceGroup - params: { - name: 'container-apps-env-pe${resourceToken}' - location: location - tags: tags - subnetResourceId: isolation.outputs.backendSubnetId - privateDnsZoneGroup: { - privateDnsZoneGroupConfigs: [ - { - privateDnsZoneResourceId: containerAppsPrivateDnsZone.outputs.resourceId - } - ] - } - privateLinkServiceConnections: [ - { - name: 'containerAppsEnvironmentConnection' - properties: { - groupIds: [ - 'managedEnvironments' - ] - privateLinkServiceId: containerApps.outputs.environmentId - } - } - ] - } -} - -module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.8.0' = if (useVpnGateway) { - name: 'virtual-network-gateway' - scope: resourceGroup - params: { - name: '${abbrs.networkVpnGateways}${resourceToken}' - clusterSettings: { - clusterMode: 'activePassiveNoBgp' - } - gatewayType: 'Vpn' - virtualNetworkResourceId: isolation.outputs.vnetId - vpnGatewayGeneration: 'Generation2' - vpnClientAddressPoolPrefix: '172.16.201.0/24' - skuName: 'VpnGw2' - vpnClientAadConfiguration: { - aadAudience: 'c632b3df-fb67-4d84-bdcf-b95ad541b5c8' // Azure VPN client - aadIssuer: 'https://sts.windows.net/${tenant().tenantId}/' - aadTenant: '${environment().authentication.loginEndpoint}${tenant().tenantId}' - vpnAuthenticationTypes: [ - 'AAD' - ] - vpnClientProtocols: [ - 'OpenVPN' - ] - } + vnetPeSubnetId: isolation.outputs.backendSubnetId } } @@ -1451,4 +1371,4 @@ output AZURE_CONTAINER_REGISTRY_ENDPOINT string = deploymentTarget == 'container ? containerApps.outputs.registryLoginServer : '' - output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource/subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup.name}/providers/Microsoft.Network/virtualNetworkGateways/${virtualNetworkGateway.outputs.name}/pointtositeconfiguration' : '' + output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource/subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup.name}/providers/Microsoft.Network/virtualNetworkGateways/${isolation.outputs.virtualNetworkGatewayName}/pointtositeconfiguration' : '' diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 8cbd718edc..35dcd761d3 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -15,7 +15,10 @@ param deploymentTarget string @description('The name of an existing App Service Plan to connect to the VNet') param appServicePlanName string -param deployVpnGateway bool = false +param useVpnGateway bool = false + +param vpnGatewayName string = '${vnetName}-vpn-gateway' +param dnsResolverName string = '${vnetName}-dns-resolver' // TODO: Bring back app service option resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (deploymentTarget == 'appservice') { @@ -241,9 +244,52 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { } } +module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.8.0' = if (useVpnGateway) { + name: 'virtual-network-gateway' + params: { + name: vpnGatewayName + clusterSettings: { + clusterMode: 'activePassiveNoBgp' + } + gatewayType: 'Vpn' + virtualNetworkResourceId: vnet.outputs.resourceId + vpnGatewayGeneration: 'Generation2' + vpnClientAddressPoolPrefix: '172.16.201.0/24' + skuName: 'VpnGw2' + vpnClientAadConfiguration: { + aadAudience: 'c632b3df-fb67-4d84-bdcf-b95ad541b5c8' // Azure VPN client + aadIssuer: 'https://sts.windows.net/${tenant().tenantId}/' + aadTenant: '${environment().authentication.loginEndpoint}${tenant().tenantId}' + vpnAuthenticationTypes: [ + 'AAD' + ] + vpnClientProtocols: [ + 'OpenVPN' + ] + } + } +} + +// Based on https://luke.geek.nz/azure/azure-point-to-site-vpn-and-private-dns-resolver/ +// Manual step required of updating azurevpnconfig.xml to use the correct DNS server IP address +module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.4' = if (useVpnGateway) { + name: 'dns-resolver' + params: { + name: dnsResolverName + location: location + virtualNetworkResourceId: vnet.outputs.resourceId + inboundEndpoints: [ + { + name: 'inboundEndpoint' + subnetResourceId: useVpnGateway ? vnet.outputs.subnetResourceIds[2] : '' + } + ] + } +} output backendSubnetId string = vnet.outputs.subnetResourceIds[0] -output privateDnsResolverSubnetId string = deployVpnGateway ? vnet.outputs.subnetResourceIds[2] : '' +output privateDnsResolverSubnetId string = useVpnGateway ? vnet.outputs.subnetResourceIds[2] : '' output appSubnetId string = vnet.outputs.subnetResourceIds[3] output vnetName string = vnet.outputs.name output vnetId string = vnet.outputs.resourceId +output virtualNetworkGatewayName string = useVpnGateway ? virtualNetworkGateway.outputs.name : '' diff --git a/infra/private-endpoints.bicep b/infra/private-endpoints.bicep index 2a7ca8d75c..241aa07a9b 100644 --- a/infra/private-endpoints.bicep +++ b/infra/private-endpoints.bicep @@ -7,7 +7,7 @@ param vnetName string @description('The location to create the private endpoints') param location string = resourceGroup().location -param vnetPeSubnetName string +param vnetPeSubnetId string @description('A formatted array of private endpoint connections containing the dns zone name, group id, and list of resource ids of Private Endpoints to create') param privateEndpointConnections array @@ -56,7 +56,7 @@ module privateEndpoints './core/networking/private-endpoint.bicep' = [for privat location: location name: '${privateEndpointInfo.name}${abbrs.privateEndpoint}${resourceToken}' tags: tags - subnetId: vnetPeSubnetName + subnetId: vnetPeSubnetId serviceId: privateEndpointInfo.resourceId groupIds: [ privateEndpointInfo.groupId ] dnsZoneId: dnsZones[privateEndpointInfo.dnsZoneIndex].outputs.id @@ -121,7 +121,7 @@ module monitorPrivateEndpoint './core/networking/private-endpoint.bicep' = { name: 'monitor${abbrs.privateEndpoint}${resourceToken}' location: location tags: tags - subnetId: vnetPeSubnetName + subnetId: vnetPeSubnetId serviceId: monitorPrivateLinkScope.id groupIds: [ 'azuremonitor' ] // Add multiple DNS zone configs for Azure Monitor From 4b3cd8c46ca38e1e4e4d34760e6f8e79c221f2a3 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 29 Jul 2025 23:16:32 -0700 Subject: [PATCH 11/24] Bring back unneeded changes --- infra/core/host/container-app-upsert.bicep | 6 +- infra/core/host/container-app.bicep | 76 +++++--- infra/core/host/container-apps.bicep | 1 - infra/core/search/search-services.bicep | 1 - infra/main.bicep | 11 +- infra/main.parameters.json | 3 - infra/network-isolation.bicep | 194 ++++++++++----------- todo.txt | 3 +- 8 files changed, 150 insertions(+), 145 deletions(-) diff --git a/infra/core/host/container-app-upsert.bicep b/infra/core/host/container-app-upsert.bicep index 71f6a7d6b7..aba512cd24 100644 --- a/infra/core/host/container-app-upsert.bicep +++ b/infra/core/host/container-app-upsert.bicep @@ -51,6 +51,8 @@ param containerMemory string = '1.0Gi' @description('Workload profile name to use for the container app when using private ingress') param workloadProfileName string = 'Warm' +param allowedOrigins array = [] + resource existingApp 'Microsoft.App/containerApps@2022-03-01' existing = if (exists) { name: name } @@ -81,19 +83,17 @@ module app 'container-app.bicep' = { daprAppId: daprAppId daprAppProtocol: daprAppProtocol secrets: secrets + allowedOrigins: allowedOrigins external: external env: concat(envAsArray, envSecrets) imageName: exists ? existingApp.properties.template.containers[0].image : '' targetPort: targetPort - // Pass workload profile name parameter - workloadProfileName: workloadProfileName } } output defaultDomain string = app.outputs.defaultDomain output imageName string = app.outputs.imageName output name string = app.outputs.name -output hostName string = app.outputs.hostName output uri string = app.outputs.uri output identityResourceId string = app.outputs.identityResourceId output identityPrincipalId string = app.outputs.identityPrincipalId diff --git a/infra/core/host/container-app.bicep b/infra/core/host/container-app.bicep index ddb65fb7f9..4fcfeb8e63 100644 --- a/infra/core/host/container-app.bicep +++ b/infra/core/host/container-app.bicep @@ -1,11 +1,23 @@ +metadata description = 'Creates a container app in an Azure Container App environment.' param name string param location string = resourceGroup().location param tags object = {} +@description('Allowed origins') +param allowedOrigins array = [] + +@description('Name of the environment for container apps') param containerAppsEnvironmentName string + +@description('The name of the container') param containerName string = 'main' + +@description('The name of the container registry') param containerRegistryName string +@description('Hostname suffix for container registry. Set when deploying to sovereign clouds') +param containerRegistryHostSuffix string = 'azurecr.io' + @description('Minimum number of replicas to run') @minValue(1) param containerMinReplicas int = 1 @@ -20,16 +32,27 @@ param secrets object = {} @description('The environment variables for the container') param env array = [] +@description('Specifies if the resource ingress is exposed externally') param external bool = true -param imageName string -param targetPort int = 80 @description('User assigned identity name') param identityName string +@description('The type of identity for the resource') +@allowed([ 'None', 'SystemAssigned', 'UserAssigned' ]) +param identityType string = 'None' + +@description('The name of the container image') +param imageName string + @description('Enabled Ingress for container app') param ingressEnabled bool = true +param revisionMode string = 'Single' + +@description('The target port for the container') +param targetPort int = 80 + // Dapr Options @description('Enable Dapr') param daprEnabled bool = false @@ -45,47 +68,53 @@ param containerCpuCoreCount string = '0.5' @description('Memory allocated to a single container instance, e.g. 1Gi') param containerMemory string = '1.0Gi' -@description('Workload profile name to use for the container app when using private ingress') -param workloadProfileName string = 'Warm' - var keyvalueSecrets = [for secret in items(secrets): { name: secret.key value: secret.value }] -resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { +resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!empty(identityName)) { name: identityName } -module containerRegistryAccess '../security/registry-access.bicep' = { +// Private registry support requires both an ACR name and a User Assigned managed identity +var usePrivateRegistry = !empty(identityName) && !empty(containerRegistryName) + +// Automatically set to `UserAssigned` when an `identityName` has been set +var normalizedIdentityType = !empty(identityName) ? 'UserAssigned' : identityType + +module containerRegistryAccess '../security/registry-access.bicep' = if (usePrivateRegistry) { name: '${deployment().name}-registry-access' params: { containerRegistryName: containerRegistryName - principalId: userIdentity.properties.principalId + principalId: usePrivateRegistry ? userIdentity.properties.principalId : '' } } -resource app 'Microsoft.App/containerApps@2025-01-01' = { - name: name +resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { +name: name location: location tags: tags // It is critical that the identity is granted ACR pull access before the app is created // otherwise the container app will throw a provision error // This also forces us to use an user assigned managed identity since there would no way to // provide the system assigned identity with the ACR pull access before the app is created - dependsOn: [ containerRegistryAccess ] + dependsOn: usePrivateRegistry ? [ containerRegistryAccess ] : [] identity: { - type: 'UserAssigned' - userAssignedIdentities: { '${userIdentity.id}': {} } + type: normalizedIdentityType + userAssignedIdentities: !empty(identityName) && normalizedIdentityType == 'UserAssigned' ? { '${userIdentity.id}': {} } : null } properties: { managedEnvironmentId: containerAppsEnvironment.id configuration: { - activeRevisionsMode: 'single' + activeRevisionsMode: revisionMode ingress: ingressEnabled ? { external: external targetPort: targetPort transport: 'auto' + corsPolicy: { + allowedOrigins: union([ 'https://portal.azure.com', 'https://ms.portal.azure.com' ], allowedOrigins) + } } : null dapr: daprEnabled ? { enabled: true @@ -94,12 +123,12 @@ resource app 'Microsoft.App/containerApps@2025-01-01' = { appPort: ingressEnabled ? targetPort : 0 } : { enabled: false } secrets: keyvalueSecrets - registries: [ + registries: usePrivateRegistry ? [ { - server: '${containerRegistry.name}.azurecr.io' + server: '${containerRegistryName}.${containerRegistryHostSuffix}' identity: userIdentity.id } - ] + ] : [] } template: { containers: [ @@ -121,19 +150,14 @@ resource app 'Microsoft.App/containerApps@2025-01-01' = { } } -resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2022-03-01' existing = { +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = { name: containerAppsEnvironmentName } -// 2022-02-01-preview needed for anonymousPullEnabled -resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' existing = { - name: containerRegistryName -} - output defaultDomain string = containerAppsEnvironment.properties.defaultDomain +output identityPrincipalId string = userIdentity.properties.principalId +output identityResourceId string = resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', userIdentity.name) output imageName string = imageName output name string = app.name -output hostName string = app.properties.configuration.ingress.fqdn output uri string = ingressEnabled ? 'https://${app.properties.configuration.ingress.fqdn}' : '' -output identityResourceId string = resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', userIdentity.name) -output identityPrincipalId string = userIdentity.properties.principalId +output id string = app.id diff --git a/infra/core/host/container-apps.bicep b/infra/core/host/container-apps.bicep index 53af2d56d1..ff3b0cb9e9 100644 --- a/infra/core/host/container-apps.bicep +++ b/infra/core/host/container-apps.bicep @@ -46,4 +46,3 @@ output environmentName string = containerAppsEnvironment.outputs.name output environmentId string = containerAppsEnvironment.outputs.resourceId output registryLoginServer string = containerRegistry.outputs.loginServer output registryName string = containerRegistry.outputs.name -output registryId string = containerRegistry.outputs.resourceId diff --git a/infra/core/search/search-services.bicep b/infra/core/search/search-services.bicep index fc743758cf..4ee8d6a8fb 100644 --- a/infra/core/search/search-services.bicep +++ b/infra/core/search/search-services.bicep @@ -55,7 +55,6 @@ resource search 'Microsoft.Search/searchServices@2023-11-01' = { } sku: sku - // https://github.com/Azure/bicep-types-az/issues/2421 resource sharedPrivateLinkResource 'sharedPrivateLinkResources@2023-11-01' = [for (resourceId, i) in sharedPrivateLinkStorageAccounts: { name: 'search-shared-private-link-${i}' properties: { diff --git a/infra/main.bicep b/infra/main.bicep index 678a923d80..e4cfe5ec6b 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -553,7 +553,7 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget containerCpuCoreCount: '1.0' containerMemory: '2Gi' containerMinReplicas: 1 - //allowedOrigins: allowedOrigins + allowedOrigins: allowedOrigins env: union(appEnvVariables, { // For using managed identity to access Azure resources. See https://github.com/microsoft/azure-container-apps/issues/442 AZURE_CLIENT_ID: (deploymentTarget == 'containerapps') ? acaIdentity.outputs.clientId : '' @@ -699,8 +699,6 @@ module documentIntelligence 'br/public:avm/res/cognitive-services/account:0.7.2' name: !empty(documentIntelligenceServiceName) ? documentIntelligenceServiceName : '${abbrs.cognitiveServicesDocumentIntelligence}${resourceToken}' - location: documentIntelligenceResourceGroupLocation - tags: tags kind: 'FormRecognizer' customSubDomainName: !empty(documentIntelligenceServiceName) ? documentIntelligenceServiceName @@ -709,8 +707,10 @@ module documentIntelligence 'br/public:avm/res/cognitive-services/account:0.7.2' networkAcls: { defaultAction: 'Allow' } - sku: documentIntelligenceSkuName + location: documentIntelligenceResourceGroupLocation disableLocalAuth: true + tags: tags + sku: documentIntelligenceSkuName } } @@ -1166,9 +1166,6 @@ module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { vnetName: '${abbrs.virtualNetworks}${resourceToken}' useVpnGateway: useVpnGateway deploymentTarget: deploymentTarget - // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 - appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : '' - //containerAppsEnvName: deploymentTarget == 'containerapps' ? acaManagedEnvironmentName : '' vpnGatewayName: useVpnGateway ? '${abbrs.networkVpnGateways}${resourceToken}' : '' dnsResolverName: useVpnGateway ? '${abbrs.privateDnsResolver}${resourceToken}' : '' } diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 5202fdcde0..6767191b6e 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -344,9 +344,6 @@ "webAppExists": { "value": "${SERVICE_WEB_RESOURCE_EXISTS=false}" }, - "azureContainerAppsWorkloadProfile": { - "value": "${AZURE_CONTAINER_APPS_WORKLOAD_PROFILE=Consumption}" - }, "useMediaDescriberAzureCU": { "value": "${USE_MEDIA_DESCRIBER_AZURE_CU=false}" }, diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 35dcd761d3..7c9bdaed2c 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -12,19 +12,11 @@ param tags object = {} @allowed(['appservice', 'containerapps']) param deploymentTarget string -@description('The name of an existing App Service Plan to connect to the VNet') -param appServicePlanName string - param useVpnGateway bool = false param vpnGatewayName string = '${vnetName}-vpn-gateway' param dnsResolverName string = '${vnetName}-dns-resolver' -// TODO: Bring back app service option -resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (deploymentTarget == 'appservice') { - name: appServicePlanName -} - module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = if (deploymentTarget == 'containerapps') { name: 'container-apps-nsg' params: { @@ -32,58 +24,59 @@ module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' location: location tags: tags securityRules: [ - { - name: 'AllowHttpsInbound' - properties: { - protocol: 'Tcp' - sourcePortRange: '*' - sourceAddressPrefix: 'Internet' - destinationPortRange: '443' - destinationAddressPrefix: '*' - access: 'Allow' - priority: 100 - direction: 'Inbound' - } - } - { // TODO: Were any of these rules really needed?? - name: 'AllowPrivateEndpointsOutbound' - properties: { - protocol: 'Tcp' - sourcePortRange: '*' - sourceAddressPrefix: '10.0.0.0/21' - destinationPortRange: '443' - destinationAddressPrefix: '10.0.8.0/24' - access: 'Allow' - priority: 200 - direction: 'Outbound' - } - } - { - name: 'AllowDnsOutbound' - properties: { - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: '*' - destinationPortRange: '53' - destinationAddressPrefix: '*' - access: 'Allow' - priority: 210 - direction: 'Outbound' - } - } - { - name: 'AllowVNetOutbound' - properties: { - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: '*' - destinationPortRange: '*' - destinationAddressPrefix: 'VirtualNetwork' - access: 'Allow' - priority: 220 - direction: 'Outbound' - } - } + { + name: 'AllowHttpsInbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRange: '443' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + { + // TODO: Were any of these rules really needed?? + name: 'AllowPrivateEndpointsOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' + destinationPortRange: '443' + destinationAddressPrefix: '10.0.8.0/24' + access: 'Allow' + priority: 200 + direction: 'Outbound' + } + } + { + name: 'AllowDnsOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '53' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 210 + direction: 'Outbound' + } + } + { + name: 'AllowVNetOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 220 + direction: 'Outbound' + } + } ] } } @@ -190,24 +183,6 @@ module privateEndpointsNSG 'br/public:avm/res/network/network-security-group:0.5 } } -var appServiceSubnet = { - name: 'app-int-subnet' - properties: { - addressPrefix: '10.0.3.0/24' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' - delegations: [ - { - id: appServicePlan.id - name: appServicePlan.name - properties: { - serviceName: 'Microsoft.Web/serverFarms' - } - } - ] - } -} - module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { name: 'vnet' params: { @@ -217,30 +192,45 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { addressPrefixes: [ '10.0.0.0/16' ] - subnets: [ - { - name: 'backend-subnet' - addressPrefix: '10.0.8.0/24' - privateEndpointNetworkPolicies: 'Enabled' - privateLinkServiceNetworkPolicies: 'Enabled' - networkSecurityGroupResourceId: privateEndpointsNSG.outputs.resourceId - } - { - name: 'GatewaySubnet' // Required name for Gateway subnet - addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet - } - { - name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver - addressPrefix: '10.0.11.0/28' - delegation: 'Microsoft.Network/dnsResolvers' - } - { - name: 'app-int-subnet' - addressPrefix: '10.0.0.0/21' - networkSecurityGroupResourceId: containerAppsNSG.outputs.resourceId - delegation: 'Microsoft.App/environments' - } - ] + subnets: union( + [ + { + name: 'backend-subnet' + addressPrefix: '10.0.8.0/24' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + networkSecurityGroupResourceId: privateEndpointsNSG.outputs.resourceId + } + { + name: 'GatewaySubnet' // Required name for Gateway subnet + addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet + } + { + name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver + addressPrefix: '10.0.11.0/28' + delegation: 'Microsoft.Network/dnsResolvers' + } + ], + deploymentTarget == 'appservice' + ? [ + { + name: 'app-int-subnet' + addressPrefix: '10.0.3.0/24' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + // TODO: Are we sure we don't need App Service Plan/ID? Test this. + delegation: 'Microsoft.Web/serverFarms' + } + ] + : [ + { + name: 'app-int-subnet' + addressPrefix: '10.0.0.0/21' + networkSecurityGroupResourceId: containerAppsNSG.outputs.resourceId + delegation: 'Microsoft.App/environments' + } + ] + ) } } diff --git a/todo.txt b/todo.txt index 86d8ce1134..d5c82d8b88 100644 --- a/todo.txt +++ b/todo.txt @@ -1,5 +1,4 @@ * Better error when youre trying to run prepdocs and not connected to VPN. * Hit up /ping and check that it returns 200 OK. Anything else = no bueno. * If they have AZURE_USE_PRIVATE_ENDPOINT and it gets non-200, reminds them to set up VPN. -* Bring back App Service's subnet -* Move vnet-y things into network-isolation.bicep +* Manually test App Service From 7c0147ae33192ee7dc1a06e40146cdc127bad11c Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 29 Jul 2025 23:29:06 -0700 Subject: [PATCH 12/24] Update prepdocs with ping and update docs --- app/backend/prepdocs.py | 31 +++++++++++++++++++++++++++++++ docs/deploy_private.md | 18 ++++++++++-------- todo.txt | 4 +--- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index 2c4bd49c26..3c37056796 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -4,6 +4,7 @@ import os from typing import Optional, Union +import aiohttp from azure.core.credentials import AzureKeyCredential from azure.core.credentials_async import AsyncTokenCredential from azure.identity.aio import AzureDeveloperCliCredential, get_bearer_token_provider @@ -45,6 +46,19 @@ def clean_key_if_exists(key: Union[str, None]) -> Union[str, None]: return None +async def check_search_service_connectivity(search_service: str) -> bool: + """Check if the search service is accessible by hitting the /ping endpoint.""" + ping_url = f"https://{search_service}.search.windows.net/ping" + + try: + async with aiohttp.ClientSession() as session: + async with session.get(ping_url, timeout=aiohttp.ClientTimeout(total=10)) as response: + return response.status == 200 + except Exception as e: + logger.debug(f"Search service ping failed: {e}") + return False + + async def setup_search_info( search_service: str, index_name: str, @@ -372,6 +386,23 @@ async def main(strategy: Strategy, setup_index: bool = True): search_key=clean_key_if_exists(args.searchkey), ) ) + + # Check search service connectivity + search_service = os.environ["AZURE_SEARCH_SERVICE"] + is_connected = loop.run_until_complete(check_search_service_connectivity(search_service)) + + if not is_connected: + if os.getenv("AZURE_USE_PRIVATE_ENDPOINT"): + logger.error( + "Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration. You have AZURE_USE_PRIVATE_ENDPOINT enabled. Perhaps you're not yet connected to the VPN? Download the VPN configuration from the Azure portal here: %s", + os.getenv("AZURE_VPN_CONFIG_DOWNLOAD_LINK"), + ) + else: + logger.error( + "Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration." + ) + exit(1) + blob_manager = setup_blob_manager( azure_credential=azd_credential, storage_account=os.environ["AZURE_STORAGE_ACCOUNT"], diff --git a/docs/deploy_private.md b/docs/deploy_private.md index 865400c460..a453a8ce49 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -46,7 +46,7 @@ Deploying with public access disabled adds additional cost to your deployment. P ## Recommended deployment strategy for private access -1. Deploy the app with private endpoints enabled, public network access disabled, and a VPN gateway configured. This will allow you to connect to the chat app from inside the virtual network. +1. Configure the azd environment variables to use private endpoints and a VPN gateway, with public network access disabled. This will allow you to connect to the chat app from inside the virtual network, but not from the public Internet. ```shell azd env set AZURE_USE_PRIVATE_ENDPOINT true @@ -55,19 +55,19 @@ Deploying with public access disabled adds additional cost to your deployment. P azd up ``` -2. First provision all the resources: +2. Provision all the Azure resources: ```bash azd provision ``` -3. Once provisioning is complete, run this command to get the VPN configuration download link: +3. Once provisioning is complete, you will see an error when it tries to run the data ingestion script, because you are not yet connected to the VPN. That message should provide a URL for the VPN configuration file download. If you don't see that URL, run this command: ```bash azd env get-value AZURE_VPN_CONFIG_DOWNLOAD_LINK ``` - Select "Download VPN client" to download a ZIP file containing the VPN configuration. + Open that link in your browser. Select "Download VPN client" to download a ZIP file containing the VPN configuration. 4. Open `AzureVPN/azurevpnconfig.xml`, and replace the `` empty tag with the following: @@ -79,17 +79,19 @@ Deploying with public access disabled adds additional cost to your deployment. P ``` -5. Open the "Azure VPN" client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified. +5. Install the [Azure VPN Client](https://learn.microsoft.com/azure/vpn-gateway/vpn-gateway-howto-vpn-client-install). -6. Select "Connect" and the new VPN connection. You will be prompted to select your Microsoft account and login. +6. Open the "Azure VPN" client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified. -7. Once you're successfully connected to VPN, you can run the data ingestion script: +7. Select "Connect" and the new VPN connection. You will be prompted to select your Microsoft account and login. + +8. Once you're successfully connected to VPN, you can run the data ingestion script: ```bash azd hooks run postprovision ``` -8. Finally, you can deploy the app: +9. Finally, you can deploy the app: ```bash azd deploy diff --git a/todo.txt b/todo.txt index d5c82d8b88..f9d2ffb0ca 100644 --- a/todo.txt +++ b/todo.txt @@ -1,4 +1,2 @@ -* Better error when youre trying to run prepdocs and not connected to VPN. - * Hit up /ping and check that it returns 200 OK. Anything else = no bueno. - * If they have AZURE_USE_PRIVATE_ENDPOINT and it gets non-200, reminds them to set up VPN. + * Manually test App Service From d1f1de8a584531e95448de0d0492c0b4255cf793 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 29 Jul 2025 23:32:47 -0700 Subject: [PATCH 13/24] Fix VPN client link --- docs/deploy_private.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deploy_private.md b/docs/deploy_private.md index a453a8ce49..77ab81cd66 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -79,9 +79,9 @@ Deploying with public access disabled adds additional cost to your deployment. P ``` -5. Install the [Azure VPN Client](https://learn.microsoft.com/azure/vpn-gateway/vpn-gateway-howto-vpn-client-install). +5. Install the [Azure VPN Client](https://learn.microsoft.com/azure/vpn-gateway/azure-vpn-client-versions). -6. Open the "Azure VPN" client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified. +6. Open the Azure VPN Client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified. 7. Select "Connect" and the new VPN connection. You will be prompted to select your Microsoft account and login. From 5a92db1c1a1d3bc7c09f39a670371a3ab5ff5a1e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 30 Jul 2025 11:22:48 -0700 Subject: [PATCH 14/24] Add App Service private endpoint for deployment --- infra/main.bicep | 12 +++++++++++- infra/network-isolation.bicep | 7 +++---- todo.txt | 2 -- 3 files changed, 14 insertions(+), 7 deletions(-) delete mode 100644 todo.txt diff --git a/infra/main.bicep b/infra/main.bicep index e4cfe5ec6b..5d560ee0e9 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1206,6 +1206,16 @@ var containerAppsPrivateEndpointConnection = (usePrivateEndpoint && deploymentTa } ] : [] + +var appServicePrivateEndpointConnection = (usePrivateEndpoint && deploymentTarget == 'appservice') + ? [ + { + groupId: 'sites' + dnsZoneName: 'privatelink.azurewebsites.net' + resourceIds: [backend.outputs.id] + } + ] + : [] var otherPrivateEndpointConnections = (usePrivateEndpoint) ? [ { @@ -1226,7 +1236,7 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint) ] : [] -var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection, cognitiveServicesPrivateEndpointConnection, containerAppsPrivateEndpointConnection) +var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection, cognitiveServicesPrivateEndpointConnection, containerAppsPrivateEndpointConnection, appServicePrivateEndpointConnection) module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { name: 'privateEndpoints' diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 7c9bdaed2c..7f3363773e 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -214,17 +214,16 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { deploymentTarget == 'appservice' ? [ { - name: 'app-int-subnet' - addressPrefix: '10.0.3.0/24' + name: 'app-service-subnet' + addressPrefix: '10.0.9.0/24' privateEndpointNetworkPolicies: 'Enabled' privateLinkServiceNetworkPolicies: 'Enabled' - // TODO: Are we sure we don't need App Service Plan/ID? Test this. delegation: 'Microsoft.Web/serverFarms' } ] : [ { - name: 'app-int-subnet' + name: 'container-apps-subnet' addressPrefix: '10.0.0.0/21' networkSecurityGroupResourceId: containerAppsNSG.outputs.resourceId delegation: 'Microsoft.App/environments' diff --git a/todo.txt b/todo.txt deleted file mode 100644 index f9d2ffb0ca..0000000000 --- a/todo.txt +++ /dev/null @@ -1,2 +0,0 @@ - -* Manually test App Service From d7f070bc361b33b65e5cd36ee7ed3ab7882d4f1f Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 30 Jul 2025 13:06:41 -0700 Subject: [PATCH 15/24] Revert unneeded bicep changes --- infra/core/host/container-app-upsert.bicep | 100 ++++++++++++++------- infra/core/host/container-app.bicep | 92 +++++++++++-------- 2 files changed, 122 insertions(+), 70 deletions(-) diff --git a/infra/core/host/container-app-upsert.bicep b/infra/core/host/container-app-upsert.bicep index aba512cd24..64e7d412bf 100644 --- a/infra/core/host/container-app-upsert.bicep +++ b/infra/core/host/container-app-upsert.bicep @@ -1,59 +1,86 @@ +metadata description = 'Creates or updates an existing Azure Container App.' param name string param location string = resourceGroup().location param tags object = {} -param containerAppsEnvironmentName string -param containerName string = 'main' -param containerRegistryName string +@description('The number of CPU cores allocated to a single container instance, e.g., 0.5') +param containerCpuCoreCount string = '0.5' -@description('Minimum number of replicas to run') -@minValue(1) -param containerMinReplicas int = 1 -@description('Maximum number of replicas to run') +@description('The maximum number of replicas to run. Must be at least 1.') @minValue(1) param containerMaxReplicas int = 10 +@description('The amount of memory allocated to a single container instance, e.g., 1Gi') +param containerMemory string = '1.0Gi' + +@description('The minimum number of replicas to run. Must be at least 1 for non-consumption workloads.') +param containerMinReplicas int = 1 + +@description('The name of the container') +param containerName string = 'main' + +@description('The environment name for the container apps') +param containerAppsEnvironmentName string = '${containerName}env' + +@description('The name of the container registry') +param containerRegistryName string + +@description('Hostname suffix for container registry. Set when deploying to sovereign clouds') +param containerRegistryHostSuffix string = 'azurecr.io' + + +@allowed(['http', 'grpc']) +@description('The protocol used by Dapr to connect to the app, e.g., HTTP or gRPC') +param daprAppProtocol string = 'http' + +@description('Enable or disable Dapr for the container app') +param daprEnabled bool = false + +@description('The Dapr app ID') +param daprAppId string = containerName + +@description('Specifies if the resource already exists') +param exists bool = false + +@description('Specifies if Ingress is enabled for the container app') +param ingressEnabled bool = true + +@description('The type of identity for the resource') +@allowed(['None', 'SystemAssigned', 'UserAssigned']) +param identityType string = 'None' + +@description('The name of the user-assigned identity') +param identityName string = '' + +@description('The name of the container image') +param imageName string = '' + @description('The secrets required for the container') @secure() param secrets object = {} +@description('The keyvault identities required for the container') +@secure() +param keyvaultIdentities object = {} + @description('The environment variables for the container in key value pairs') param env object = {} @description('The environment variables with secret references') param envSecrets array = [] +@description('Specifies if the resource ingress is exposed externally') param external bool = true -param targetPort int = 80 -param exists bool - -@description('User assigned identity name') -param identityName string - -@description('Enabled Ingress for container app') -param ingressEnabled bool = true - -// Dapr Options -@description('Enable Dapr') -param daprEnabled bool = false -@description('Dapr app ID') -param daprAppId string = containerName -@allowed([ 'http', 'grpc' ]) -@description('Protocol used by Dapr to connect to the app, e.g. http or grpc') -param daprAppProtocol string = 'http' -@description('CPU cores allocated to a single container instance, e.g. 0.5') -param containerCpuCoreCount string = '0.5' +@description('The service binds associated with the container') +param serviceBinds array = [] -@description('Memory allocated to a single container instance, e.g. 1Gi') -param containerMemory string = '1.0Gi' - -@description('Workload profile name to use for the container app when using private ingress') -param workloadProfileName string = 'Warm' +@description('The target port for the container') +param targetPort int = 80 param allowedOrigins array = [] -resource existingApp 'Microsoft.App/containerApps@2022-03-01' existing = if (exists) { +resource existingApp 'Microsoft.App/containerApps@2023-05-02-preview' existing = if (exists) { name: name } @@ -70,11 +97,13 @@ module app 'container-app.bicep' = { name: name location: location tags: tags + identityType: identityType identityName: identityName ingressEnabled: ingressEnabled containerName: containerName containerAppsEnvironmentName: containerAppsEnvironmentName containerRegistryName: containerRegistryName + containerRegistryHostSuffix: containerRegistryHostSuffix containerCpuCoreCount: containerCpuCoreCount containerMemory: containerMemory containerMinReplicas: containerMinReplicas @@ -83,11 +112,13 @@ module app 'container-app.bicep' = { daprAppId: daprAppId daprAppProtocol: daprAppProtocol secrets: secrets + keyvaultIdentities: keyvaultIdentities allowedOrigins: allowedOrigins external: external env: concat(envAsArray, envSecrets) - imageName: exists ? existingApp.properties.template.containers[0].image : '' + imageName: !empty(imageName) ? imageName : exists ? existingApp.properties.template.containers[0].image : '' targetPort: targetPort + serviceBinds: serviceBinds } } @@ -95,5 +126,6 @@ output defaultDomain string = app.outputs.defaultDomain output imageName string = app.outputs.imageName output name string = app.outputs.name output uri string = app.outputs.uri -output identityResourceId string = app.outputs.identityResourceId +output id string = app.outputs.id output identityPrincipalId string = app.outputs.identityPrincipalId +output identityResourceId string = app.outputs.identityResourceId diff --git a/infra/core/host/container-app.bicep b/infra/core/host/container-app.bicep index 4fcfeb8e63..dc103e7ead 100644 --- a/infra/core/host/container-app.bicep +++ b/infra/core/host/container-app.bicep @@ -9,25 +9,37 @@ param allowedOrigins array = [] @description('Name of the environment for container apps') param containerAppsEnvironmentName string +@description('CPU cores allocated to a single container instance, e.g., 0.5') +param containerCpuCoreCount string = '0.5' + +@description('The maximum number of replicas to run. Must be at least 1.') +@minValue(1) +param containerMaxReplicas int = 10 + +@description('Memory allocated to a single container instance, e.g., 1Gi') +param containerMemory string = '1.0Gi' + +@description('The minimum number of replicas to run. Must be at least 1.') +param containerMinReplicas int = 1 + @description('The name of the container') param containerName string = 'main' @description('The name of the container registry') -param containerRegistryName string +param containerRegistryName string = '' @description('Hostname suffix for container registry. Set when deploying to sovereign clouds') param containerRegistryHostSuffix string = 'azurecr.io' -@description('Minimum number of replicas to run') -@minValue(1) -param containerMinReplicas int = 1 -@description('Maximum number of replicas to run') -@minValue(1) -param containerMaxReplicas int = 10 +@description('The protocol used by Dapr to connect to the app, e.g., http or grpc') +@allowed([ 'http', 'grpc' ]) +param daprAppProtocol string = 'http' -@description('The secrets required for the container') -@secure() -param secrets object = {} +@description('The Dapr app ID') +param daprAppId string = containerName + +@description('Enable Dapr') +param daprEnabled bool = false @description('The environment variables for the container') param env array = [] @@ -35,43 +47,37 @@ param env array = [] @description('Specifies if the resource ingress is exposed externally') param external bool = true -@description('User assigned identity name') -param identityName string +@description('The name of the user-assigned identity') +param identityName string = '' @description('The type of identity for the resource') @allowed([ 'None', 'SystemAssigned', 'UserAssigned' ]) param identityType string = 'None' @description('The name of the container image') -param imageName string +param imageName string = '' -@description('Enabled Ingress for container app') +@description('Specifies if Ingress is enabled for the container app') param ingressEnabled bool = true param revisionMode string = 'Single' -@description('The target port for the container') -param targetPort int = 80 +@description('The secrets required for the container') +@secure() +param secrets object = {} -// Dapr Options -@description('Enable Dapr') -param daprEnabled bool = false -@description('Dapr app ID') -param daprAppId string = containerName -@allowed([ 'http', 'grpc' ]) -@description('Protocol used by Dapr to connect to the app, e.g. http or grpc') -param daprAppProtocol string = 'http' +@description('The keyvault identities required for the container') +@secure() +param keyvaultIdentities object = {} -@description('CPU cores allocated to a single container instance, e.g. 0.5') -param containerCpuCoreCount string = '0.5' +@description('The service binds associated with the container') +param serviceBinds array = [] -@description('Memory allocated to a single container instance, e.g. 1Gi') -param containerMemory string = '1.0Gi' +@description('The name of the container apps add-on to use. e.g. redis') +param serviceType string = '' -var keyvalueSecrets = [for secret in items(secrets): { - name: secret.key - value: secret.value -}] +@description('The target port for the container') +param targetPort int = 80 resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!empty(identityName)) { name: identityName @@ -83,6 +89,17 @@ var usePrivateRegistry = !empty(identityName) && !empty(containerRegistryName) // Automatically set to `UserAssigned` when an `identityName` has been set var normalizedIdentityType = !empty(identityName) ? 'UserAssigned' : identityType +var keyvalueSecrets = [for secret in items(secrets): { + name: secret.key + value: secret.value +}] + +var keyvaultIdentitySecrets = [for secret in items(keyvaultIdentities): { + name: secret.key + keyVaultUrl: secret.value.keyVaultUrl + identity: secret.value.identity +}] + module containerRegistryAccess '../security/registry-access.bicep' = if (usePrivateRegistry) { name: '${deployment().name}-registry-access' params: { @@ -92,7 +109,7 @@ module containerRegistryAccess '../security/registry-access.bicep' = if (usePriv } resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { -name: name + name: name location: location tags: tags // It is critical that the identity is granted ACR pull access before the app is created @@ -122,7 +139,8 @@ name: name appProtocol: daprAppProtocol appPort: ingressEnabled ? targetPort : 0 } : { enabled: false } - secrets: keyvalueSecrets + secrets: concat(keyvalueSecrets, keyvaultIdentitySecrets) + service: !empty(serviceType) ? { type: serviceType } : null registries: usePrivateRegistry ? [ { server: '${containerRegistryName}.${containerRegistryHostSuffix}' @@ -131,6 +149,7 @@ name: name ] : [] } template: { + serviceBinds: !empty(serviceBinds) ? serviceBinds : null containers: [ { image: !empty(imageName) ? imageName : 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' @@ -155,9 +174,10 @@ resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' } output defaultDomain string = containerAppsEnvironment.properties.defaultDomain -output identityPrincipalId string = userIdentity.properties.principalId -output identityResourceId string = resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', userIdentity.name) +output identityPrincipalId string = normalizedIdentityType == 'None' ? '' : (empty(identityName) ? app.identity.principalId : userIdentity.properties.principalId) +output identityResourceId string = normalizedIdentityType == 'UserAssigned' ? resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', userIdentity.name) : '' output imageName string = imageName output name string = app.name +output serviceBind object = !empty(serviceType) ? { serviceId: app.id, name: name } : {} output uri string = ingressEnabled ? 'https://${app.properties.configuration.ingress.fqdn}' : '' output id string = app.id From 0d0abb01320d45ce30fc1b05357355f7bbec99e4 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 30 Jul 2025 16:44:23 -0700 Subject: [PATCH 16/24] Revert unneeded changes --- docs/deploy_private.md | 4 ++++ infra/core/host/container-app-upsert.bicep | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/deploy_private.md b/docs/deploy_private.md index 77ab81cd66..919044be06 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -96,3 +96,7 @@ Deploying with public access disabled adds additional cost to your deployment. P ```bash azd deploy ``` + +## Compatibility with other features + +* **GitHub Actions / Azure DevOps**: The private access deployment is not compatible with the built-in CI/CD pipelines, as it requires a VPN connection to deploy the app. You could modify the pipeline to only do provisioning, and set up a different deployment strategy for the app. diff --git a/infra/core/host/container-app-upsert.bicep b/infra/core/host/container-app-upsert.bicep index 64e7d412bf..c8fc7d5bcd 100644 --- a/infra/core/host/container-app-upsert.bicep +++ b/infra/core/host/container-app-upsert.bicep @@ -3,6 +3,7 @@ param name string param location string = resourceGroup().location param tags object = {} + @description('The number of CPU cores allocated to a single container instance, e.g., 0.5') param containerCpuCoreCount string = '0.5' @@ -28,7 +29,6 @@ param containerRegistryName string @description('Hostname suffix for container registry. Set when deploying to sovereign clouds') param containerRegistryHostSuffix string = 'azurecr.io' - @allowed(['http', 'grpc']) @description('The protocol used by Dapr to connect to the app, e.g., HTTP or gRPC') param daprAppProtocol string = 'http' From 8aaf0c0fc710f1bf4d3e698c59359801a671dbab Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 30 Jul 2025 17:02:05 -0700 Subject: [PATCH 17/24] Remove unneeded NSG rules --- infra/network-isolation.bicep | 40 ----------------------------------- 1 file changed, 40 deletions(-) diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 7f3363773e..2a29ff1e4a 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -37,46 +37,6 @@ module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' direction: 'Inbound' } } - { - // TODO: Were any of these rules really needed?? - name: 'AllowPrivateEndpointsOutbound' - properties: { - protocol: 'Tcp' - sourcePortRange: '*' - sourceAddressPrefix: '10.0.0.0/21' - destinationPortRange: '443' - destinationAddressPrefix: '10.0.8.0/24' - access: 'Allow' - priority: 200 - direction: 'Outbound' - } - } - { - name: 'AllowDnsOutbound' - properties: { - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: '*' - destinationPortRange: '53' - destinationAddressPrefix: '*' - access: 'Allow' - priority: 210 - direction: 'Outbound' - } - } - { - name: 'AllowVNetOutbound' - properties: { - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: '*' - destinationPortRange: '*' - destinationAddressPrefix: 'VirtualNetwork' - access: 'Allow' - priority: 220 - direction: 'Outbound' - } - } ] } } From fc628e4c757a5baefad16c30df8d350f688383a6 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 30 Jul 2025 17:28:02 -0700 Subject: [PATCH 18/24] Address feedback from Copilot --- infra/core/host/container-registry.bicep | 31 -------------------- infra/network-isolation.bicep | 36 ++++++++++++++++-------- infra/private-endpoints.bicep | 3 +- 3 files changed, 26 insertions(+), 44 deletions(-) diff --git a/infra/core/host/container-registry.bicep b/infra/core/host/container-registry.bicep index 8f48611c8b..9736beee48 100644 --- a/infra/core/host/container-registry.bicep +++ b/infra/core/host/container-registry.bicep @@ -16,9 +16,6 @@ param sku object = { } param zoneRedundancy string = 'Disabled' -@description('The log analytics workspace id used for logging & monitoring') -param workspaceId string = '' - // 2022-02-01-preview needed for anonymousPullEnabled resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { name: name @@ -36,34 +33,6 @@ resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-pr } } -// TODO: Update diagnostics to be its own module -// Blocking issue: https://github.com/Azure/bicep/issues/622 -// Unable to pass in a `resource` scope or unable to use string interpolation in resource types -resource diagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(workspaceId)) { - name: 'registry-diagnostics' - scope: containerRegistry - properties: { - workspaceId: workspaceId - logs: [ - { - category: 'ContainerRegistryRepositoryEvents' - enabled: true - } - { - category: 'ContainerRegistryLoginEvents' - enabled: true - } - ] - metrics: [ - { - category: 'AllMetrics' - enabled: true - timeGrain: 'PT1M' - } - ] - } -} - output loginServer string = containerRegistry.properties.loginServer output name string = containerRegistry.name output resourceId string = containerRegistry.id diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 2a29ff1e4a..8f6ecb4fa2 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -17,6 +17,13 @@ param useVpnGateway bool = false param vpnGatewayName string = '${vnetName}-vpn-gateway' param dnsResolverName string = '${vnetName}-dns-resolver' +// Subnet name constants +var backendSubnetName = 'backend-subnet' +var gatewaySubnetName = 'GatewaySubnet' // Required name for Gateway subnet +var dnsResolverSubnetName = 'dns-resolver-subnet' +var appServiceSubnetName = 'app-service-subnet' +var containerAppsSubnetName = 'container-apps-subnet' + module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = if (deploymentTarget == 'containerapps') { name: 'container-apps-nsg' params: { @@ -41,7 +48,7 @@ module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' } } -module privateEndpointsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = if (deploymentTarget == 'containerapps') { +module privateEndpointsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = { name: 'private-endpoints-nsg' params: { name: '${vnetName}-private-endpoints-nsg' @@ -155,18 +162,18 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { subnets: union( [ { - name: 'backend-subnet' + name: backendSubnetName addressPrefix: '10.0.8.0/24' privateEndpointNetworkPolicies: 'Enabled' privateLinkServiceNetworkPolicies: 'Enabled' networkSecurityGroupResourceId: privateEndpointsNSG.outputs.resourceId } { - name: 'GatewaySubnet' // Required name for Gateway subnet + name: gatewaySubnetName // Required name for Gateway subnet addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet } { - name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver + name: dnsResolverSubnetName // Dedicated subnet for Azure Private DNS Resolver addressPrefix: '10.0.11.0/28' delegation: 'Microsoft.Network/dnsResolvers' } @@ -174,7 +181,7 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { deploymentTarget == 'appservice' ? [ { - name: 'app-service-subnet' + name: appServiceSubnetName addressPrefix: '10.0.9.0/24' privateEndpointNetworkPolicies: 'Enabled' privateLinkServiceNetworkPolicies: 'Enabled' @@ -183,16 +190,21 @@ module vnet 'br/public:avm/res/network/virtual-network:0.6.1' = { ] : [ { - name: 'container-apps-subnet' + name: containerAppsSubnetName addressPrefix: '10.0.0.0/21' - networkSecurityGroupResourceId: containerAppsNSG.outputs.resourceId delegation: 'Microsoft.App/environments' + networkSecurityGroupResourceId: containerAppsNSG!.outputs.resourceId } ] ) } } +// Helper variables to find subnet resource IDs by name instead of hardcoded indices +var dnsResolverSubnetIndex = indexOf(vnet.outputs.subnetNames, dnsResolverSubnetName) +var backendSubnetIndex = indexOf(vnet.outputs.subnetNames, backendSubnetName) +var appSubnetIndex = deploymentTarget == 'appservice' ? indexOf(vnet.outputs.subnetNames, appServiceSubnetName) : indexOf(vnet.outputs.subnetNames, containerAppsSubnetName) + module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.8.0' = if (useVpnGateway) { name: 'virtual-network-gateway' params: { @@ -230,15 +242,15 @@ module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.4' = if (useVpnGa inboundEndpoints: [ { name: 'inboundEndpoint' - subnetResourceId: useVpnGateway ? vnet.outputs.subnetResourceIds[2] : '' + subnetResourceId: useVpnGateway ? vnet.outputs.subnetResourceIds[dnsResolverSubnetIndex] : '' } ] } } -output backendSubnetId string = vnet.outputs.subnetResourceIds[0] -output privateDnsResolverSubnetId string = useVpnGateway ? vnet.outputs.subnetResourceIds[2] : '' -output appSubnetId string = vnet.outputs.subnetResourceIds[3] +output backendSubnetId string = vnet.outputs.subnetResourceIds[backendSubnetIndex] +output privateDnsResolverSubnetId string = useVpnGateway ? vnet.outputs.subnetResourceIds[dnsResolverSubnetIndex] : '' +output appSubnetId string = vnet.outputs.subnetResourceIds[appSubnetIndex] output vnetName string = vnet.outputs.name output vnetId string = vnet.outputs.resourceId -output virtualNetworkGatewayName string = useVpnGateway ? virtualNetworkGateway.outputs.name : '' +output virtualNetworkGatewayName string = useVpnGateway ? virtualNetworkGateway!.outputs.name : '' diff --git a/infra/private-endpoints.bicep b/infra/private-endpoints.bicep index 241aa07a9b..6d6357ab8e 100644 --- a/infra/private-endpoints.bicep +++ b/infra/private-endpoints.bicep @@ -82,7 +82,8 @@ module monitorDnsZones './core/networking/private-dns-zones.bicep' = [for monito }] // Get blob DNS zone index for monitor private link var blobEndpointInfo = filter(flatten(privateEndpointInfo), info => info.groupId == 'blob') -var dnsZoneBlobIndex = empty(blobEndpointInfo) ? 0 : blobEndpointInfo[0].dnsZoneIndex +// Assert that blob endpoints exist (required for this application) +var dnsZoneBlobIndex = blobEndpointInfo[0].dnsZoneIndex // Azure Monitor Private Link Scope // https://learn.microsoft.com/azure/azure-monitor/logs/private-link-security From a426a3a25b649b7e8b7aed11c4d18d947e22ac55 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 30 Jul 2025 22:07:57 -0700 Subject: [PATCH 19/24] Address feedback from Copilot --- app/backend/prepdocs.py | 5 ++++- infra/core/host/container-apps-environment.bicep | 8 +------- infra/core/host/container-apps.bicep | 2 -- infra/main.bicep | 10 ++++------ 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index 3c37056796..db2131a94a 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -337,7 +337,10 @@ async def main(strategy: Strategy, setup_index: bool = True): load_azd_env() - if os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled" and os.getenv("AZURE_USE_VPN_GATEWAY") != "true": + if ( + os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled" + and os.getenv("AZURE_USE_VPN_GATEWAY", "").lower() != "true" + ): logger.error("AZURE_PUBLIC_NETWORK_ACCESS is set to Disabled. Exiting.") exit(0) diff --git a/infra/core/host/container-apps-environment.bicep b/infra/core/host/container-apps-environment.bicep index 3f3c2be36c..019410acc2 100644 --- a/infra/core/host/container-apps-environment.bicep +++ b/infra/core/host/container-apps-environment.bicep @@ -6,10 +6,6 @@ param daprEnabled bool = false param logAnalyticsWorkspaceName string = '' param applicationInsightsName string = '' -@description('Virtual network name for container apps environment.') -param vnetName string = '' -@description('Subnet name for container apps environment integration.') -param subnetName string = '' param subnetResourceId string param usePrivateIngress bool = true @@ -30,11 +26,9 @@ resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2025-02-02- daprAIInstrumentationKey: daprEnabled && !empty(applicationInsightsName) ? applicationInsights.properties.InstrumentationKey : '' publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled' vnetConfiguration: usePrivateIngress ? { - // Use proper subnet resource ID format infrastructureSubnetId: subnetResourceId - internal: usePrivateIngress + internal: true } : null - // Configure workload profile for dedicated environment (not consumption) workloadProfiles: usePrivateIngress ? [ { diff --git a/infra/core/host/container-apps.bicep b/infra/core/host/container-apps.bicep index ff3b0cb9e9..8597f49288 100644 --- a/infra/core/host/container-apps.bicep +++ b/infra/core/host/container-apps.bicep @@ -24,8 +24,6 @@ module containerAppsEnvironment 'container-apps-environment.bicep' = { tags: tags logAnalyticsWorkspaceName: logAnalyticsWorkspaceName applicationInsightsName: applicationInsightsName - vnetName: vnetName - subnetName: subnetName usePrivateIngress: usePrivateIngress subnetResourceId: subnetResourceId } diff --git a/infra/main.bicep b/infra/main.bicep index 5d560ee0e9..807f2d89d6 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -300,7 +300,7 @@ param webAppExists bool @allowed(['appservice', 'containerapps']) param deploymentTarget string = 'appservice' param acaIdentityName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-identity' : '' -param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-envnet' : '' +param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-env' : '' param containerRegistryName string = deploymentTarget == 'containerapps' ? '${replace(toLower(environmentName), '-', '')}acr' : '' @@ -542,7 +542,7 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget acaIdentity ] params: { - name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesContainerApps}backend${resourceToken}' + name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesContainerApps}backend-${resourceToken}' location: location identityName: (deploymentTarget == 'containerapps') ? acaIdentityName : '' exists: webAppExists @@ -552,7 +552,7 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget targetPort: 8000 containerCpuCoreCount: '1.0' containerMemory: '2Gi' - containerMinReplicas: 1 + containerMinReplicas: usePrivateEndpoint ? 1 : 0 allowedOrigins: allowedOrigins env: union(appEnvVariables, { // For using managed identity to access Azure resources. See https://github.com/microsoft/azure-container-apps/issues/442 @@ -1155,8 +1155,6 @@ module cosmosDbRoleBackend 'core/security/documentdb-sql-role.bicep' = if (useAu } } - - module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { name: 'networks' scope: resourceGroup @@ -1164,8 +1162,8 @@ module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) { location: location tags: tags vnetName: '${abbrs.virtualNetworks}${resourceToken}' - useVpnGateway: useVpnGateway deploymentTarget: deploymentTarget + useVpnGateway: useVpnGateway vpnGatewayName: useVpnGateway ? '${abbrs.networkVpnGateways}${resourceToken}' : '' dnsResolverName: useVpnGateway ? '${abbrs.privateDnsResolver}${resourceToken}' : '' } From a46875be8bc8f5ebc246717430bd0ba5951c270a Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 30 Jul 2025 23:26:14 -0700 Subject: [PATCH 20/24] Address Copilot feedback --- docs/deploy_private.md | 2 ++ infra/main.bicep | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/deploy_private.md b/docs/deploy_private.md index 919044be06..a4db747da7 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -79,6 +79,8 @@ Deploying with public access disabled adds additional cost to your deployment. P ``` + > **Note:** The IP address `10.0.11.4` is the first available IP in the `dns-resolver-subnet`(10.0.11.0/28), as Azure reserves the first four IP addresses in each subnet. Adding this DNS server allows your VPN client to resolve private DNS names for Azure services accessed through private endpoints. See the network configuration in [network-isolation.bicep](../infra/network-isolation.bicep) for details. + 5. Install the [Azure VPN Client](https://learn.microsoft.com/azure/vpn-gateway/azure-vpn-client-versions). 6. Open the Azure VPN Client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified. diff --git a/infra/main.bicep b/infra/main.bicep index 807f2d89d6..baf68e0089 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1376,4 +1376,4 @@ output AZURE_CONTAINER_REGISTRY_ENDPOINT string = deploymentTarget == 'container ? containerApps.outputs.registryLoginServer : '' - output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource/subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup.name}/providers/Microsoft.Network/virtualNetworkGateways/${isolation.outputs.virtualNetworkGatewayName}/pointtositeconfiguration' : '' +output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource/subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup.name}/providers/Microsoft.Network/virtualNetworkGateways/${isolation.outputs.virtualNetworkGatewayName}/pointtositeconfiguration' : '' From 287aa0ec7bfb98d9e53cae860be5023659ee035b Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 1 Aug 2025 09:29:32 -0700 Subject: [PATCH 21/24] Update NSG, container registry --- infra/core/host/container-registry.bicep | 2 +- infra/main.bicep | 5 +- infra/network-isolation.bicep | 105 +++++++++++++++++++++-- 3 files changed, 103 insertions(+), 9 deletions(-) diff --git a/infra/core/host/container-registry.bicep b/infra/core/host/container-registry.bicep index 9736beee48..3bee20c870 100644 --- a/infra/core/host/container-registry.bicep +++ b/infra/core/host/container-registry.bicep @@ -2,7 +2,7 @@ param name string param location string = resourceGroup().location param tags object = {} -param adminUserEnabled bool = true +param adminUserEnabled bool = false param anonymousPullEnabled bool = false param dataEndpointEnabled bool = false param encryption object = { diff --git a/infra/main.bicep b/infra/main.bicep index baf68e0089..8877a55510 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -300,7 +300,7 @@ param webAppExists bool @allowed(['appservice', 'containerapps']) param deploymentTarget string = 'appservice' param acaIdentityName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-identity' : '' -param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-env' : '' +param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-envnet' : '' param containerRegistryName string = deploymentTarget == 'containerapps' ? '${replace(toLower(environmentName), '-', '')}acr' : '' @@ -1376,4 +1376,5 @@ output AZURE_CONTAINER_REGISTRY_ENDPOINT string = deploymentTarget == 'container ? containerApps.outputs.registryLoginServer : '' -output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource/subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup.name}/providers/Microsoft.Network/virtualNetworkGateways/${isolation.outputs.virtualNetworkGatewayName}/pointtositeconfiguration' : '' +// TODO: test this +output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource/${isolation.outputs.virtualNetworkGatewayId}/pointtositeconfiguration' : '' diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 8f6ecb4fa2..323b1d9c16 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -22,7 +22,7 @@ var backendSubnetName = 'backend-subnet' var gatewaySubnetName = 'GatewaySubnet' // Required name for Gateway subnet var dnsResolverSubnetName = 'dns-resolver-subnet' var appServiceSubnetName = 'app-service-subnet' -var containerAppsSubnetName = 'container-apps-subnet' +var containerAppsSubnetName = 'app-int-subnet' module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = if (deploymentTarget == 'containerapps') { name: 'container-apps-nsg' @@ -31,19 +31,112 @@ module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' location: location tags: tags securityRules: [ + // Inbound rules for Container Apps (Workload Profiles) { - name: 'AllowHttpsInbound' + name: 'AllowAzureLoadBalancerInbound' properties: { protocol: 'Tcp' sourcePortRange: '*' - sourceAddressPrefix: 'Internet' - destinationPortRange: '443' - destinationAddressPrefix: '*' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationPortRange: '30000-32767' + destinationAddressPrefix: '10.0.0.0/21' // Container apps subnet access: 'Allow' priority: 100 direction: 'Inbound' } } + // Outbound rules for Container Apps (Workload Profiles) + { + name: 'AllowMicrosoftContainerRegistryOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'MicrosoftContainerRegistry' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'AllowAzureFrontDoorOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'AzureFrontDoor.FirstParty' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'AllowContainerAppsSubnetOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '*' + destinationAddressPrefix: '10.0.0.0/21' // Container apps subnet + access: 'Allow' + priority: 120 + direction: 'Outbound' + } + } + { + name: 'AllowAzureActiveDirectoryOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'AzureActiveDirectory' + access: 'Allow' + priority: 130 + direction: 'Outbound' + } + } + { + name: 'AllowAzureMonitorOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'AzureMonitor' + access: 'Allow' + priority: 140 + direction: 'Outbound' + } + } + { + name: 'AllowAzureDnsOutbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '53' + destinationAddressPrefix: '168.63.129.16' + access: 'Allow' + priority: 150 + direction: 'Outbound' + } + } + { + name: 'AllowStorageRegionOutbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '10.0.0.0/21' // Container apps subnet + destinationPortRange: '443' + destinationAddressPrefix: 'Storage.${location}' + access: 'Allow' + priority: 160 + direction: 'Outbound' + } + } ] } } @@ -253,4 +346,4 @@ output privateDnsResolverSubnetId string = useVpnGateway ? vnet.outputs.subnetRe output appSubnetId string = vnet.outputs.subnetResourceIds[appSubnetIndex] output vnetName string = vnet.outputs.name output vnetId string = vnet.outputs.resourceId -output virtualNetworkGatewayName string = useVpnGateway ? virtualNetworkGateway!.outputs.name : '' +output virtualNetworkGatewayId string = useVpnGateway ? virtualNetworkGateway!.outputs.resourceId : '' From c2509d01e30170cdcc00ea89a0e56c3f4a96ad1a Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 1 Aug 2025 09:29:59 -0700 Subject: [PATCH 22/24] Update NSG, container registry --- infra/main.bicep | 2 +- infra/network-isolation.bicep | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/main.bicep b/infra/main.bicep index 8877a55510..8824c27806 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -300,7 +300,7 @@ param webAppExists bool @allowed(['appservice', 'containerapps']) param deploymentTarget string = 'appservice' param acaIdentityName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-identity' : '' -param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-envnet' : '' +param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-env' : '' param containerRegistryName string = deploymentTarget == 'containerapps' ? '${replace(toLower(environmentName), '-', '')}acr' : '' diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index 323b1d9c16..2454421b79 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -22,7 +22,7 @@ var backendSubnetName = 'backend-subnet' var gatewaySubnetName = 'GatewaySubnet' // Required name for Gateway subnet var dnsResolverSubnetName = 'dns-resolver-subnet' var appServiceSubnetName = 'app-service-subnet' -var containerAppsSubnetName = 'app-int-subnet' +var containerAppsSubnetName = 'container-apps-subnet' module containerAppsNSG 'br/public:avm/res/network/network-security-group:0.5.1' = if (deploymentTarget == 'containerapps') { name: 'container-apps-nsg' From ee8a447b9d4c6dfc1fa0ddd359ccd5632a355673 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 1 Aug 2025 11:21:52 -0700 Subject: [PATCH 23/24] Address feedback --- docs/deploy_private.md | 34 +++++++++++++++++++++++----------- infra/main.bicep | 3 +-- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/deploy_private.md b/docs/deploy_private.md index a4db747da7..8418797f3d 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -31,18 +31,20 @@ If you want to disable public access when deploying the Chat App, you can do so Deploying with public access disabled adds additional cost to your deployment. Please see pricing for the following products: -1. [Private Endpoints](https://azure.microsoft.com/pricing/details/private-link/) - 1. The exact number of private endpoints created depends on the [optional features](./deploy_features.md) used. -1. [Private DNS Zones](https://azure.microsoft.com/pricing/details/dns/) +* [Azure Container Registry](https://azure.microsoft.com/pricing/details/container-registry/): Premium tier is used when virtual network is added (required for private links), which incurs additional costs. +* [Azure Container Apps](https://azure.microsoft.com/pricing/details/container-apps/): Workload profiles environment is used when virtual network is added (required for private links), which incurs additional costs. Additionally, min replica count is set to 1, so you will be charged for at least one instance. If you need to customize the environment configuration further, edit the container-apps-environment.bicep file. +* [VPN Gateway](https://azure.microsoft.com/pricing/details/vpn-gateway/): VpnGw2 SKU. Pricing includes a base monthly cost plus an hourly cost based on the number of connections. +* [Virtual Network](https://azure.microsoft.com/pricing/details/virtual-network/): Pay-as-you-go tier. Costs based on data processed. -## Environment variables controlling private access +The pricing for the following features depends on the [optional features](./deploy_features.md) used. Most deployments will have at least 5 private endpoints (Azure OpenAI, Azure Cognitive Services, Azure AI Search, Azure Blob Storage, and either Azure App Service or Azure Container Apps). -1. `AZURE_PUBLIC_NETWORK_ACCESS`: Controls the value of public network access on supported Azure resources. Valid values are 'Enabled' or 'Disabled'. - 1. When public network access is 'Enabled', Azure resources are open to the internet. - 1. When public network access is 'Disabled', Azure resources are only accessible over a virtual network. -1. `AZURE_USE_PRIVATE_ENDPOINT`: Controls deployment of [private endpoints](https://learn.microsoft.com/azure/private-link/private-endpoint-overview) which connect Azure resources to the virtual network. - 1. When set to 'true', ensures private endpoints are deployed for connectivity even when `AZURE_PUBLIC_NETWORK_ACCESS` is 'Disabled'. - 1. Note that private endpoints do not make the chat app accessible from the internet. Connections must be initiated from inside the virtual network. +* [Azure Private Endpoints](https://azure.microsoft.com/pricing/details/private-link/): Pricing is per hour per endpoint. +* [Private DNS Zones](https://azure.microsoft.com/pricing/details/dns/): Pricing is per month and zones. +* [Azure Private DNS Resolver](https://azure.microsoft.com/pricing/details/dns/): Pricing is per month and zones. + +⚠️ To avoid unnecessary costs, remember to take down your app if it's no longer in use, +either by deleting the resource group in the Portal or running `azd down`. +You might also decide to delete the VPN Gateway when not in use. ## Recommended deployment strategy for private access @@ -79,7 +81,7 @@ Deploying with public access disabled adds additional cost to your deployment. P ``` - > **Note:** The IP address `10.0.11.4` is the first available IP in the `dns-resolver-subnet`(10.0.11.0/28), as Azure reserves the first four IP addresses in each subnet. Adding this DNS server allows your VPN client to resolve private DNS names for Azure services accessed through private endpoints. See the network configuration in [network-isolation.bicep](../infra/network-isolation.bicep) for details. + > **Note:** We use the IP address `10.0.11.4` since it is the first available IP in the `dns-resolver-subnet`(10.0.11.0/28) from the provisioned virtual network, as Azure reserves the first four IP addresses in each subnet. Adding this DNS server allows your VPN client to resolve private DNS names for Azure services accessed through private endpoints. See the network configuration in [network-isolation.bicep](../infra/network-isolation.bicep) for details. 5. Install the [Azure VPN Client](https://learn.microsoft.com/azure/vpn-gateway/azure-vpn-client-versions). @@ -99,6 +101,16 @@ Deploying with public access disabled adds additional cost to your deployment. P azd deploy ``` +## Environment variables controlling private access + +1. `AZURE_PUBLIC_NETWORK_ACCESS`: Controls the value of public network access on supported Azure resources. Valid values are 'Enabled' or 'Disabled'. + 1. When public network access is 'Enabled', Azure resources are open to the internet. + 1. When public network access is 'Disabled', Azure resources are only accessible over a virtual network. +1. `AZURE_USE_PRIVATE_ENDPOINT`: Controls deployment of [private endpoints](https://learn.microsoft.com/azure/private-link/private-endpoint-overview) which connect Azure resources to the virtual network. + 1. When set to 'true', ensures private endpoints are deployed for connectivity even when `AZURE_PUBLIC_NETWORK_ACCESS` is 'Disabled'. + 1. Note that private endpoints do not make the chat app accessible from the internet. Connections must be initiated from inside the virtual network. +1. `AZURE_USE_VPN_GATEWAY`: Controls deployment of a VPN gateway for the virtual network. If you do not use this and public access is disabled, you will need a different way to connect to the virtual network. + ## Compatibility with other features * **GitHub Actions / Azure DevOps**: The private access deployment is not compatible with the built-in CI/CD pipelines, as it requires a VPN connection to deploy the app. You could modify the pipeline to only do provisioning, and set up a different deployment strategy for the app. diff --git a/infra/main.bicep b/infra/main.bicep index 8824c27806..75cf98aff2 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1376,5 +1376,4 @@ output AZURE_CONTAINER_REGISTRY_ENDPOINT string = deploymentTarget == 'container ? containerApps.outputs.registryLoginServer : '' -// TODO: test this -output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource/${isolation.outputs.virtualNetworkGatewayId}/pointtositeconfiguration' : '' +output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource${isolation.outputs.virtualNetworkGatewayId}/pointtositeconfiguration' : '' From 6ec8a2f1890a78c28370dc22f5ac005326a45906 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 1 Aug 2025 12:06:01 -0700 Subject: [PATCH 24/24] Bring back workload profile name --- docs/deploy_private.md | 2 +- .../host/container-apps-environment.bicep | 22 +++++++++++++++---- infra/core/host/container-apps.bicep | 6 +++-- infra/main.bicep | 4 ++++ infra/main.parameters.json | 3 +++ 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/deploy_private.md b/docs/deploy_private.md index 8418797f3d..bfd9b2331f 100644 --- a/docs/deploy_private.md +++ b/docs/deploy_private.md @@ -32,7 +32,7 @@ If you want to disable public access when deploying the Chat App, you can do so Deploying with public access disabled adds additional cost to your deployment. Please see pricing for the following products: * [Azure Container Registry](https://azure.microsoft.com/pricing/details/container-registry/): Premium tier is used when virtual network is added (required for private links), which incurs additional costs. -* [Azure Container Apps](https://azure.microsoft.com/pricing/details/container-apps/): Workload profiles environment is used when virtual network is added (required for private links), which incurs additional costs. Additionally, min replica count is set to 1, so you will be charged for at least one instance. If you need to customize the environment configuration further, edit the container-apps-environment.bicep file. +* [Azure Container Apps](https://azure.microsoft.com/pricing/details/container-apps/): Workload profiles environment is used when virtual network is added (required for private links), which incurs additional costs. Additionally, min replica count is set to 1, so you will be charged for at least one instance. * [VPN Gateway](https://azure.microsoft.com/pricing/details/vpn-gateway/): VpnGw2 SKU. Pricing includes a base monthly cost plus an hourly cost based on the number of connections. * [Virtual Network](https://azure.microsoft.com/pricing/details/virtual-network/): Pay-as-you-go tier. Costs based on data processed. diff --git a/infra/core/host/container-apps-environment.bicep b/infra/core/host/container-apps-environment.bicep index 019410acc2..2a35daceae 100644 --- a/infra/core/host/container-apps-environment.bicep +++ b/infra/core/host/container-apps-environment.bicep @@ -10,6 +10,15 @@ param subnetResourceId string param usePrivateIngress bool = true +@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) +param workloadProfile string + +// Make sure that we are using a non-consumption workload profile for private endpoints +var finalWorkloadProfile = (usePrivateIngress && workloadProfile == 'Consumption') ? 'D4' : workloadProfile + +var minimumCount = usePrivateIngress ? 1 : 0 +var maximumCount = usePrivateIngress ? 3 : 2 + resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2025-02-02-preview' = { name: name location: location @@ -37,12 +46,17 @@ resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2025-02-02- } { name: 'Warm' - workloadProfileType: 'D4' - minimumCount: 1 - maximumCount: 3 + workloadProfileType: finalWorkloadProfile + minimumCount: minimumCount + maximumCount: maximumCount + } + ] + : [ + { + name: 'Consumption' + workloadProfileType: 'Consumption' } ] - : [] } } diff --git a/infra/core/host/container-apps.bicep b/infra/core/host/container-apps.bicep index 8597f49288..00941b1cb2 100644 --- a/infra/core/host/container-apps.bicep +++ b/infra/core/host/container-apps.bicep @@ -9,8 +9,9 @@ param applicationInsightsName string = '' @description('Virtual network name for container apps environment.') param vnetName string = '' -@description('Subnet name for container apps environment integration.') -param subnetName string = '' + +@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) +param workloadProfile string param subnetResourceId string = '' @@ -26,6 +27,7 @@ module containerAppsEnvironment 'container-apps-environment.bicep' = { applicationInsightsName: applicationInsightsName usePrivateIngress: usePrivateIngress subnetResourceId: subnetResourceId + workloadProfile: workloadProfile } } diff --git a/infra/main.bicep b/infra/main.bicep index 75cf98aff2..d74047f8bc 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -297,6 +297,9 @@ param runningOnAdo string = '' @description('Used by azd for containerapps deployment') param webAppExists bool +@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) +param azureContainerAppsWorkloadProfile string + @allowed(['appservice', 'containerapps']) param deploymentTarget string = 'appservice' param acaIdentityName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-identity' : '' @@ -530,6 +533,7 @@ module containerApps 'core/host/container-apps.bicep' = if (deploymentTarget == logAnalyticsWorkspaceName: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceName : '' subnetResourceId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' usePrivateIngress: usePrivateEndpoint + workloadProfile: azureContainerAppsWorkloadProfile } } diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 6767191b6e..5202fdcde0 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -344,6 +344,9 @@ "webAppExists": { "value": "${SERVICE_WEB_RESOURCE_EXISTS=false}" }, + "azureContainerAppsWorkloadProfile": { + "value": "${AZURE_CONTAINER_APPS_WORKLOAD_PROFILE=Consumption}" + }, "useMediaDescriberAzureCU": { "value": "${USE_MEDIA_DESCRIBER_AZURE_CU=false}" },