diff --git a/README.md b/README.md index 530ecc1b..66b0d767 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ Optionally, the module supports advanced security group management for the worke - Ensure that you have an up-to-date version of the [IBM Cloud CLI](https://cloud.ibm.com/docs/cli?topic=cli-getting-started). - Ensure that you have an up-to-date version of the [IBM Cloud Kubernetes service CLI](https://cloud.ibm.com/docs/containers?topic=containers-kubernetes-service-cli). - Ensure that you have an up-to-date version of the [IBM Cloud VPC Infrastructure service CLI](https://cloud.ibm.com/docs/vpc?topic=vpc-vpc-reference). Only required if providing additional security groups with the `var.additional_lb_security_group_ids`. -- Ensure that you have an up-to-date version of the [jq](https://jqlang.github.io/jq) -- Ensure that you have an up-to-date version of the [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) +- Ensure that you have an up-to-date version of the [jq](https://jqlang.github.io/jq). +- Ensure that you have an up-to-date version of the [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl). @@ -229,6 +229,14 @@ If you prefer not to use moved blocks, you can manually use the terraform state ### Troubleshooting +#### The specified API key could not be found + +During cluster provisioning a containers apikey is created if one does not already exist for the given resource group and region ([learn more](https://cloud.ibm.com/docs/containers?topic=containers-access-creds)). Occasionally replication of the newly created apikey can be delayed causing the cluster creation to fail with an error like this: + +`Error: Request failed with status code: 404, ServerErrorResponse: {"incidentID":"c5caf83e-5f08-48c9-9778-6f3eb0ce1d16,c5caf83e-5f08-48c9-9778-6f3eb0ce1d16","code":"E06f9","description":"The specified API key could not be found.","type":""}` + +To workaround the issue simply attempt a re-apply of the terraform and it should pass on second attempt. If you still face issues, an IBM Cloud support case should be created with the `Kubernetes service` and include the `incidentID` from the error. + #### New kube_version message - When you run a `terraform plan` command, you might get a message about a new version of Kubernetes, as in the following example: @@ -304,7 +312,6 @@ Optionally, you need the following permissions to attach Access Management tags | Name | Type | |------|------| | [ibm_container_addons.addons](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_addons) | resource | -| [ibm_container_api_key_reset.reset_api_key](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_api_key_reset) | resource | | [ibm_container_ingress_instance.instance](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_ingress_instance) | resource | | [ibm_container_vpc_cluster.autoscaling_cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource | | [ibm_container_vpc_cluster.autoscaling_cluster_with_upgrade](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource | @@ -318,7 +325,6 @@ Optionally, you need the following permissions to attach Access Management tags | [null_resource.confirm_network_healthy](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.ocp_console_management](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [time_sleep.wait_for_auth_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | -| [time_sleep.wait_for_reset_api_key](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [ibm_container_addons.existing_addons](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_addons) | data source | | [ibm_container_cluster_config.cluster_config](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster_config) | data source | | [ibm_container_cluster_versions.cluster_versions](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster_versions) | data source | @@ -363,7 +369,6 @@ Optionally, you need the following permissions to attach Access Management tags | [resource\_group\_id](#input\_resource\_group\_id) | The ID of an existing IBM Cloud resource group where the cluster is grouped. | `string` | n/a | yes | | [secrets\_manager\_secret\_group\_id](#input\_secrets\_manager\_secret\_group\_id) | Secret group ID where Ingress secrets are stored in the Secrets Manager instance. | `string` | `null` | no | | [service\_subnet\_cidr](#input\_service\_subnet\_cidr) | Specify a custom subnet CIDR to provide private IP addresses for services. The subnet must be at least `/24` or larger. Default value is `172.21.0.0/16` when the variable is set to `null`. | `string` | `null` | no | -| [skip\_cluster\_apikey\_creation](#input\_skip\_cluster\_apikey\_creation) | Set to true to skip explicit creation of the `containers-kubernetes-key` for the given region and resource group. You can set this to false if you plan to manually create this key, or if you want to allow the cluster creation process to create it. Please be aware that it may take multiple apply attempts when allowing the cluster creation process to create it it before it will be successful. | `bool` | `false` | no | | [skip\_ocp\_secrets\_manager\_iam\_auth\_policy](#input\_skip\_ocp\_secrets\_manager\_iam\_auth\_policy) | To skip creating auth policy that allows OCP cluster 'Manager' role access in the existing Secrets Manager instance for managing ingress certificates. | `bool` | `false` | no | | [tags](#input\_tags) | Metadata labels describing this cluster deployment, i.e. test | `list(string)` | `[]` | no | | [use\_existing\_cos](#input\_use\_existing\_cos) | Flag indicating whether or not to use an existing COS instance for OpenShift internal registry storage. Only applicable if 'enable\_registry\_storage' is true | `bool` | `false` | no | diff --git a/ibm_catalog.json b/ibm_catalog.json index 5855f18f..f4316fba 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -923,9 +923,6 @@ "key": "skip_ocp_secrets_manager_iam_auth_policy", "hidden": true }, - { - "key": "skip_cluster_apikey_creation" - }, { "key": "subnets", "default_value": "{\n zone-1 = [\n {\n name = \"subnet-a\"\n cidr = \"10.10.10.0/24\"\n public_gateway = true\n acl_name = \"vpc-acl\"\n no_addr_prefix = false\n }\n ],\n zone-2 = [\n {\n name = \"subnet-b\"\n cidr = \"10.20.10.0/24\"\n public_gateway = true\n acl_name = \"vpc-acl\"\n no_addr_prefix = false\n }\n ],\n zone-3 = [\n {\n name = \"subnet-c\"\n cidr = \"10.30.10.0/24\"\n public_gateway = true\n acl_name = \"vpc-acl\"\n no_addr_prefix = false\n }\n ]\n }", @@ -1521,10 +1518,6 @@ "key": "address_prefix", "hidden": true }, - { - "key": "skip_cluster_apikey_creation", - "hidden": true - }, { "key": "ocp_entitlement" }, diff --git a/main.tf b/main.tf index 41d483d0..49fbad8b 100644 --- a/main.tf +++ b/main.tf @@ -135,7 +135,6 @@ resource "ibm_resource_tag" "cos_access_tag" { ############################################################################## resource "ibm_container_vpc_cluster" "cluster" { - depends_on = [time_sleep.wait_for_reset_api_key] count = var.enable_openshift_version_upgrade ? 0 : (var.ignore_worker_pool_size_changes ? 0 : 1) name = var.cluster_name vpc_id = var.vpc_id @@ -206,7 +205,6 @@ resource "ibm_container_vpc_cluster" "cluster" { # copy of the cluster resource above which allows major openshift version upgrade resource "ibm_container_vpc_cluster" "cluster_with_upgrade" { - depends_on = [time_sleep.wait_for_reset_api_key] count = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? 0 : 1) : 0 name = var.cluster_name vpc_id = var.vpc_id @@ -276,7 +274,6 @@ resource "ibm_container_vpc_cluster" "cluster_with_upgrade" { # copy of the cluster resource above which ignores changes to the worker pool for use in autoscaling scenarios resource "ibm_container_vpc_cluster" "autoscaling_cluster" { - depends_on = [time_sleep.wait_for_reset_api_key] count = var.enable_openshift_version_upgrade ? 0 : (var.ignore_worker_pool_size_changes ? 1 : 0) name = var.cluster_name vpc_id = var.vpc_id @@ -347,7 +344,6 @@ resource "ibm_container_vpc_cluster" "autoscaling_cluster" { # copy of the cluster resource above which allows major openshift version upgrade resource "ibm_container_vpc_cluster" "autoscaling_cluster_with_upgrade" { - depends_on = [time_sleep.wait_for_reset_api_key] count = var.enable_openshift_version_upgrade ? (var.ignore_worker_pool_size_changes ? 1 : 0) : 0 name = var.cluster_name vpc_id = var.vpc_id @@ -429,27 +425,6 @@ resource "ibm_resource_tag" "cluster_access_tag" { tag_type = "access" } -# Cluster provisioning will automatically create an IAM API key called "containers-kubernetes-key" if one does not exist -# for the given region and resource group. The API key is used to access several services, such as the IBM Cloud classic -# infrastructure portfolio, and is required to manage the cluster. Immediately after the IAM API key is created and -# added to the new resource group, it is replicated across IAM Cloudant instances. There is a small period of time from -# when the IAM API key is initially created and when it is fully replicated across Cloudant instances where the API key -# does not work because it is not fully replicated, so commands that require the API key may fail with 404. -# -# Enhancement Request: Add support to skip API key reset if a valid key already exists (https://github.com/IBM-Cloud/terraform-provider-ibm/issues/6468). - -resource "ibm_container_api_key_reset" "reset_api_key" { - count = var.skip_cluster_apikey_creation ? 0 : 1 - region = var.region - resource_group_id = var.resource_group_id -} - -resource "time_sleep" "wait_for_reset_api_key" { - count = var.skip_cluster_apikey_creation ? 0 : 1 - depends_on = [ibm_container_api_key_reset.reset_api_key] - create_duration = "10s" -} - ############################################################################## # Access cluster to kick off RBAC synchronisation ############################################################################## diff --git a/modules/fscloud/README.md b/modules/fscloud/README.md index 8f403827..5a61899c 100644 --- a/modules/fscloud/README.md +++ b/modules/fscloud/README.md @@ -135,7 +135,6 @@ No resources. | [region](#input\_region) | The IBM Cloud region where the cluster will be provisioned. | `string` | n/a | yes | | [resource\_group\_id](#input\_resource\_group\_id) | The Id of an existing IBM Cloud resource group where the cluster will be grouped. | `string` | n/a | yes | | [service\_subnet\_cidr](#input\_service\_subnet\_cidr) | Specify a custom subnet CIDR to provide private IP addresses for services. The subnet must be at least `/24` or larger. Default value is `172.21.0.0/16` when the variable is set to `null`. | `string` | `null` | no | -| [skip\_cluster\_apikey\_creation](#input\_skip\_cluster\_apikey\_creation) | Set to true to skip explicit creation of the `containers-kubernetes-key` for the given region and resource group. You can set this to false if you plan to manually create this key, or if you want to allow the cluster creation process to create it. Please be aware that it may take multiple apply attempts when allowing the cluster creation process to create it it before it will be successful. | `bool` | `false` | no | | [tags](#input\_tags) | Metadata labels describing this cluster deployment | `list(string)` | `[]` | no | | [verify\_worker\_network\_readiness](#input\_verify\_worker\_network\_readiness) | By setting this to true, a script will run kubectl commands to verify that all worker nodes can communicate successfully with the master. If the runtime does not have access to the kube cluster to run kubectl commands, this should be set to false. | `bool` | `true` | no | | [vpc\_id](#input\_vpc\_id) | ID of the VPC instance where this cluster will be provisioned | `string` | n/a | yes | diff --git a/modules/fscloud/main.tf b/modules/fscloud/main.tf index 2755b033..39337005 100644 --- a/modules/fscloud/main.tf +++ b/modules/fscloud/main.tf @@ -34,5 +34,4 @@ module "fscloud" { additional_vpe_security_group_ids = var.additional_vpe_security_group_ids cbr_rules = var.cbr_rules enable_ocp_console = var.enable_ocp_console - skip_cluster_apikey_creation = var.skip_cluster_apikey_creation } diff --git a/modules/fscloud/variables.tf b/modules/fscloud/variables.tf index 8224bde4..7874a651 100644 --- a/modules/fscloud/variables.tf +++ b/modules/fscloud/variables.tf @@ -287,9 +287,3 @@ variable "enable_ocp_console" { type = bool default = true } - -variable "skip_cluster_apikey_creation" { - type = bool - description = "Set to true to skip explicit creation of the `containers-kubernetes-key` for the given region and resource group. You can set this to false if you plan to manually create this key, or if you want to allow the cluster creation process to create it. Please be aware that it may take multiple apply attempts when allowing the cluster creation process to create it it before it will be successful." - default = false -} diff --git a/solutions/fully-configurable/main.tf b/solutions/fully-configurable/main.tf index 121ff93d..6e65b9d2 100644 --- a/solutions/fully-configurable/main.tf +++ b/solutions/fully-configurable/main.tf @@ -239,7 +239,6 @@ module "ocp_base" { existing_secrets_manager_instance_crn = var.existing_secrets_manager_instance_crn secrets_manager_secret_group_id = var.secrets_manager_secret_group_id != null ? var.secrets_manager_secret_group_id : (var.enable_secrets_manager_integration ? module.secret_group[0].secret_group_id : null) skip_ocp_secrets_manager_iam_auth_policy = var.skip_ocp_secrets_manager_iam_auth_policy - skip_cluster_apikey_creation = var.skip_cluster_apikey_creation } module "existing_secrets_manager_instance_parser" { diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf index 3633a88c..c5710c9c 100644 --- a/solutions/fully-configurable/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -601,9 +601,3 @@ variable "audit_webhook_listener_image_tag_digest" { description = "The tag or digest for the audit webhook listener image to deploy. If changing the value, ensure it is compatible with `audit_webhook_listener_image`." default = "deaabcb8225e800385413ba420cf3f819d3b0671@sha256:acf123f4dba63534cbc104c6886abedff9d25a22a34ab7b549ede988ed6e7144" } - -variable "skip_cluster_apikey_creation" { - type = bool - description = "To skip resetting the `containers-kubernetes-key` for the given region and resource group." - default = false -} diff --git a/solutions/quickstart/main.tf b/solutions/quickstart/main.tf index f47dbf15..a3dcbea2 100644 --- a/solutions/quickstart/main.tf +++ b/solutions/quickstart/main.tf @@ -144,5 +144,4 @@ module "ocp_base" { access_tags = var.access_tags disable_public_endpoint = !var.allow_public_access_to_cluster_management cluster_config_endpoint_type = "default" - skip_cluster_apikey_creation = var.skip_cluster_apikey_creation } diff --git a/solutions/quickstart/variables.tf b/solutions/quickstart/variables.tf index 3d2a03ac..cc4dfd24 100644 --- a/solutions/quickstart/variables.tf +++ b/solutions/quickstart/variables.tf @@ -102,9 +102,3 @@ variable "allow_outbound_traffic" { description = "Set to true to allow public outbound access from the cluster workers." default = true } - -variable "skip_cluster_apikey_creation" { - type = bool - description = "Set to true to skip explicit creation of the `containers-kubernetes-key` for the given region and resource group. You can set this to false if you plan to manually create this key, or if you want to allow the cluster creation process to create it. Please be aware that it may take multiple apply attempts when allowing the cluster creation process to create it it before it will be successful." - default = false -} diff --git a/variables.tf b/variables.tf index f652401f..4827b952 100644 --- a/variables.tf +++ b/variables.tf @@ -472,9 +472,3 @@ variable "skip_ocp_secrets_manager_iam_auth_policy" { description = "To skip creating auth policy that allows OCP cluster 'Manager' role access in the existing Secrets Manager instance for managing ingress certificates." default = false } - -variable "skip_cluster_apikey_creation" { - type = bool - description = "Set to true to skip explicit creation of the `containers-kubernetes-key` for the given region and resource group. You can set this to false if you plan to manually create this key, or if you want to allow the cluster creation process to create it. Please be aware that it may take multiple apply attempts when allowing the cluster creation process to create it it before it will be successful." - default = false -}