diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 227ab5b4da8..88c8daed79a 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -13,6 +13,14 @@ set -euoE pipefail ## -E option will cause functions to inherit trap mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests} +# Restart the local registry service if it's running to pick up new TLS certificates +# This is needed for agent-based installs where the registry.service may already be +# running with localhost-only certs and needs to switch to api-int certs +if systemctl is-active --quiet registry.service; then + echo "Restarting registry.service to pick up new TLS certificates..." + systemctl restart registry.service +fi + {{- if .BootstrapInPlace }} BOOTSTRAP_INPLACE=true {{ else }} diff --git a/pkg/asset/agent/image/unconfigured_ignition.go b/pkg/asset/agent/image/unconfigured_ignition.go index 3f66f0cdad1..51fc02c1b35 100644 --- a/pkg/asset/agent/image/unconfigured_ignition.go +++ b/pkg/asset/agent/image/unconfigured_ignition.go @@ -22,6 +22,7 @@ import ( "github.com/openshift/installer/pkg/asset/agent/workflow" "github.com/openshift/installer/pkg/asset/ignition" "github.com/openshift/installer/pkg/asset/ignition/bootstrap" + "github.com/openshift/installer/pkg/asset/tls" "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/version" ) @@ -58,6 +59,8 @@ func GetConfigImageFiles() []string { "/opt/agent/tls/kube-apiserver-localhost-signer.crt", "/opt/agent/tls/kube-apiserver-service-network-signer.key", "/opt/agent/tls/kube-apiserver-service-network-signer.crt", + "/opt/agent/tls/internal-release-registry-signer.key", + "/opt/agent/tls/internal-release-registry-signer.crt", rendezvousHostEnvPath, // This file must be last in the list } } @@ -87,6 +90,8 @@ func (a *UnconfiguredIgnition) Dependencies() []asset.Asset { &mirror.RegistriesConf{}, &mirror.CaBundle{}, &common.InfraEnvID{}, + &tls.InternalReleaseRegistrySignerCertKey{}, + &tls.InternalReleaseRegistryLocalhostCertKey{}, } } @@ -205,6 +210,35 @@ func (a *UnconfiguredIgnition) Generate(_ context.Context, dependencies asset.Pa // Required by assisted-service. a.ignAddFolders(&config, "/opt/agent/tls") + // Add internal release registry certificates + registryCA := &tls.InternalReleaseRegistrySignerCertKey{} + registryLocalhostCert := &tls.InternalReleaseRegistryLocalhostCertKey{} + dependencies.Get(registryCA, registryLocalhostCert) + + // Add CA cert+key to /opt/agent/tls for assisted-service to load + for _, f := range registryCA.Files() { + certFile := ignition.FileFromBytes(path.Join("/opt/agent", f.Filename), "root", 0600, f.Data) + config.Storage.Files = append(config.Storage.Files, certFile) + } + + // Add CA cert to system trust store + config.Storage.Files = append(config.Storage.Files, + ignition.FileFromBytes("/etc/pki/ca-trust/source/anchors/internal-release-registry-ca.crt", + "root", 0644, registryCA.Cert())) + + // Create /opt/registry/tls directory and add localhost server cert+key + a.ignAddFolders(&config, "/opt/registry/tls") + for _, f := range registryLocalhostCert.Files() { + var filename string + if strings.HasSuffix(f.Filename, ".key") { + filename = "tls.key" + } else { + filename = "tls.crt" + } + certFile := ignition.FileFromBytes(path.Join("/opt/registry/tls", filename), "root", 0600, f.Data) + config.Storage.Files = append(config.Storage.Files, certFile) + } + // Configure static networking if required. if len(nmStateConfigs.StaticNetworkConfig) > 0 { err = addStaticNetworkConfig(&config, nmStateConfigs.StaticNetworkConfig) diff --git a/pkg/asset/ignition/bootstrap/common.go b/pkg/asset/ignition/bootstrap/common.go index b2abdd29275..435badf7578 100644 --- a/pkg/asset/ignition/bootstrap/common.go +++ b/pkg/asset/ignition/bootstrap/common.go @@ -171,6 +171,8 @@ func (a *Common) Dependencies() []asset.Asset { &tls.RootCA{}, &tls.ServiceAccountKeyPair{}, &tls.IronicTLSCert{}, + &tls.InternalReleaseRegistrySignerCertKey{}, + &tls.InternalReleaseRegistryCertKey{}, &releaseimage.Image{}, new(rhcos.Image), } @@ -684,6 +686,23 @@ func (a *Common) addParentFiles(dependencies asset.Parents) { rootCA := &tls.RootCA{} dependencies.Get(rootCA) a.Config.Storage.Files = replaceOrAppend(a.Config.Storage.Files, ignition.FileFromBytes(path.Join(rootDir, rootCA.CertFile().Filename), "root", 0644, rootCA.Cert())) + + // Add internal release registry server certificate if the CA was loaded from disk (agent-based installs) + registryCA := &tls.InternalReleaseRegistrySignerCertKey{} + registryServerCert := &tls.InternalReleaseRegistryCertKey{} + dependencies.Get(registryCA, registryServerCert) + if registryCA.LoadedFromDisk { + for _, f := range registryServerCert.Files() { + var filename string + if strings.HasSuffix(f.Filename, ".key") { + filename = "tls.key" + } else { + filename = "tls.crt" + } + a.Config.Storage.Files = replaceOrAppend(a.Config.Storage.Files, + ignition.FileFromBytes(path.Join("/opt/registry/tls", filename), "root", 0600, f.Data)) + } + } } func replaceOrAppend(files []igntypes.File, file igntypes.File) []igntypes.File { diff --git a/pkg/asset/ignition/machine/master.go b/pkg/asset/ignition/machine/master.go index 2e26809716c..db7304b4206 100644 --- a/pkg/asset/ignition/machine/master.go +++ b/pkg/asset/ignition/machine/master.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "os" + "path" + "strings" igntypes "github.com/coreos/ignition/v2/config/v3_2/types" "github.com/pkg/errors" @@ -33,6 +35,8 @@ func (a *Master) Dependencies() []asset.Asset { return []asset.Asset{ &installconfig.InstallConfig{}, &tls.RootCA{}, + &tls.InternalReleaseRegistrySignerCertKey{}, + &tls.InternalReleaseRegistryCertKey{}, } } @@ -40,10 +44,26 @@ func (a *Master) Dependencies() []asset.Asset { func (a *Master) Generate(_ context.Context, dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} rootCA := &tls.RootCA{} - dependencies.Get(installConfig, rootCA) + registryCA := &tls.InternalReleaseRegistrySignerCertKey{} + registryServerCert := &tls.InternalReleaseRegistryCertKey{} + dependencies.Get(installConfig, rootCA, registryCA, registryServerCert) a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "master") + // Add internal release registry server certificate if the CA was loaded from disk (agent-based installs) + if registryCA.LoadedFromDisk { + for _, f := range registryServerCert.Files() { + var filename string + if strings.HasSuffix(f.Filename, ".key") { + filename = "tls.key" + } else { + filename = "tls.crt" + } + a.Config.Storage.Files = append(a.Config.Storage.Files, + ignition.FileFromBytes(path.Join("/opt/registry/tls", filename), "root", 0600, f.Data)) + } + } + if installConfig.Config.Platform.Name() == azure.Name { logrus.Debugf("Adding /var partition to skip CoreOS growfs step") // See https://issues.redhat.com/browse/OCPBUGS-43625 diff --git a/pkg/asset/manifests/additionaltrustbundleconfig.go b/pkg/asset/manifests/additionaltrustbundleconfig.go index d9876cdf659..ddc9844facb 100644 --- a/pkg/asset/manifests/additionaltrustbundleconfig.go +++ b/pkg/asset/manifests/additionaltrustbundleconfig.go @@ -17,6 +17,7 @@ import ( "github.com/openshift/api/annotations" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/tls" ) var ( @@ -46,19 +47,36 @@ func (*AdditionalTrustBundleConfig) Name() string { func (*AdditionalTrustBundleConfig) Dependencies() []asset.Asset { return []asset.Asset{ &installconfig.InstallConfig{}, + &tls.InternalReleaseRegistrySignerCertKey{}, } } // Generate generates the CloudProviderConfig. func (atbc *AdditionalTrustBundleConfig) Generate(_ context.Context, dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} - dependencies.Get(installConfig) + registryCA := &tls.InternalReleaseRegistrySignerCertKey{} + dependencies.Get(installConfig, registryCA) - if installConfig.Config.AdditionalTrustBundle == "" { + // Combine user-provided trust bundle with registry CA if it was loaded from disk + var combinedBundle strings.Builder + if installConfig.Config.AdditionalTrustBundle != "" { + combinedBundle.WriteString(installConfig.Config.AdditionalTrustBundle) + } + + // Add registry CA if it was loaded from disk (for agent-based installs) + if registryCA.LoadedFromDisk { + if combinedBundle.Len() > 0 { + combinedBundle.WriteString("\n") + } + combinedBundle.Write(registryCA.Cert()) + } + + // If no trust bundle at all, return early + if combinedBundle.Len() == 0 { return nil } - data, err := ParseCertificates(installConfig.Config.AdditionalTrustBundle) + data, err := ParseCertificates(combinedBundle.String()) if err != nil { return err } diff --git a/pkg/asset/tls/registry.go b/pkg/asset/tls/registry.go new file mode 100644 index 00000000000..85cbb527742 --- /dev/null +++ b/pkg/asset/tls/registry.go @@ -0,0 +1,132 @@ +package tls + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "net" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/installconfig" +) + +// InternalReleaseRegistrySignerCertKey is a key/cert pair that signs the internal release registry server certs. +type InternalReleaseRegistrySignerCertKey struct { + SelfSignedCertKey + LoadedFromDisk bool +} + +var _ asset.WritableAsset = (*InternalReleaseRegistrySignerCertKey)(nil) + +// Dependencies returns the dependency of the root-ca, which is empty. +func (c *InternalReleaseRegistrySignerCertKey) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +// Generate generates the root-ca key and cert pair. +func (c *InternalReleaseRegistrySignerCertKey) Generate(ctx context.Context, parents asset.Parents) error { + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "internal-release-registry-signer", OrganizationalUnit: []string{"openshift"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + Validity: ValidityTenYears(), + IsCA: true, + } + + return c.SelfSignedCertKey.Generate(ctx, cfg, "internal-release-registry-signer") +} + +// Load reads the asset files from disk. +func (c *InternalReleaseRegistrySignerCertKey) Load(f asset.FileFetcher) (bool, error) { + loaded, err := c.loadCertKey(f, "internal-release-registry-signer") + if err != nil { + return false, err + } + c.LoadedFromDisk = loaded + return loaded, nil +} + +// Name returns the human-friendly name of the asset. +func (c *InternalReleaseRegistrySignerCertKey) Name() string { + return "Certificate (internal-release-registry-signer)" +} + +// InternalReleaseRegistryCertKey is the asset that generates the internal release registry +// serving key/cert pair for both localhost and api-int. +type InternalReleaseRegistryCertKey struct { + SignedCertKey +} + +var _ asset.Asset = (*InternalReleaseRegistryCertKey)(nil) + +// Dependencies returns the dependency of the cert/key pair. +func (a *InternalReleaseRegistryCertKey) Dependencies() []asset.Asset { + return []asset.Asset{ + &InternalReleaseRegistrySignerCertKey{}, + &installconfig.InstallConfig{}, + } +} + +// Generate generates the cert/key pair based on its dependencies. +func (a *InternalReleaseRegistryCertKey) Generate(ctx context.Context, dependencies asset.Parents) error { + ca := &InternalReleaseRegistrySignerCertKey{} + installConfig := &installconfig.InstallConfig{} + dependencies.Get(ca, installConfig) + + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "internal-release-registry", Organization: []string{"openshift"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + Validity: ValidityTenYears(), + DNSNames: []string{ + "localhost", + internalAPIAddress(installConfig.Config), + }, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + } + + return a.SignedCertKey.Generate(ctx, cfg, ca, "internal-release-registry", AppendParent) +} + +// Name returns the human-friendly name of the asset. +func (a *InternalReleaseRegistryCertKey) Name() string { + return "Certificate (internal-release-registry)" +} + +// InternalReleaseRegistryLocalhostCertKey is the asset that generates the internal release registry +// serving key/cert pair for localhost only (used in unconfigured-ignition before cluster name is known). +type InternalReleaseRegistryLocalhostCertKey struct { + SignedCertKey +} + +var _ asset.Asset = (*InternalReleaseRegistryLocalhostCertKey)(nil) + +// Dependencies returns the dependency of the cert/key pair. +func (a *InternalReleaseRegistryLocalhostCertKey) Dependencies() []asset.Asset { + return []asset.Asset{ + &InternalReleaseRegistrySignerCertKey{}, + } +} + +// Generate generates the cert/key pair based on its dependencies. +func (a *InternalReleaseRegistryLocalhostCertKey) Generate(ctx context.Context, dependencies asset.Parents) error { + ca := &InternalReleaseRegistrySignerCertKey{} + dependencies.Get(ca) + + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "internal-release-registry-localhost", Organization: []string{"openshift"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + Validity: ValidityTenYears(), + DNSNames: []string{ + "localhost", + }, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + } + + return a.SignedCertKey.Generate(ctx, cfg, ca, "internal-release-registry-localhost", AppendParent) +} + +// Name returns the human-friendly name of the asset. +func (a *InternalReleaseRegistryLocalhostCertKey) Name() string { + return "Certificate (internal-release-registry-localhost)" +} diff --git a/pkg/asset/tls/registry_test.go b/pkg/asset/tls/registry_test.go new file mode 100644 index 00000000000..f30fd8b5e89 --- /dev/null +++ b/pkg/asset/tls/registry_test.go @@ -0,0 +1,202 @@ +package tls + +import ( + "context" + "crypto/x509" + "net" + "testing" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/ipnet" + "github.com/openshift/installer/pkg/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestInternalReleaseRegistrySignerCertKeyGenerate(t *testing.T) { + ca := &InternalReleaseRegistrySignerCertKey{} + parents := asset.Parents{} + + if err := ca.Generate(context.TODO(), parents); err != nil { + t.Fatalf("failed to generate internal release registry signer cert key: %v", err) + } + + if len(ca.Cert()) == 0 { + t.Error("expected certificate data") + } + if len(ca.Key()) == 0 { + t.Error("expected key data") + } + + cert, err := PemToCertificate(ca.Cert()) + if err != nil { + t.Fatalf("failed to parse certificate: %v", err) + } + + if !cert.IsCA { + t.Error("expected certificate to be a CA") + } + if cert.Subject.CommonName != "internal-release-registry-signer" { + t.Errorf("expected CommonName to be 'internal-release-registry-signer', got %s", cert.Subject.CommonName) + } +} + +func TestInternalReleaseRegistryCertKeyGenerate(t *testing.T) { + ca := &InternalReleaseRegistrySignerCertKey{} + if err := ca.Generate(context.TODO(), asset.Parents{}); err != nil { + t.Fatalf("failed to generate CA: %v", err) + } + + installConfig := &installconfig.InstallConfig{} + installConfig.Config = &types.InstallConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + BaseDomain: "test.example.com", + Networking: &types.Networking{ + MachineNetwork: []types.MachineNetworkEntry{{ + CIDR: *ipnet.MustParseCIDR("10.0.0.0/16"), + }}, + ServiceNetwork: []ipnet.IPNet{*ipnet.MustParseCIDR("172.30.0.0/16")}, + }, + } + + parents := asset.Parents{} + parents.Add(ca, installConfig) + + serverCert := &InternalReleaseRegistryCertKey{} + if err := serverCert.Generate(context.TODO(), parents); err != nil { + t.Fatalf("failed to generate server certificate: %v", err) + } + + if len(serverCert.Cert()) == 0 { + t.Error("expected certificate data") + } + if len(serverCert.Key()) == 0 { + t.Error("expected key data") + } + + cert, err := PemToCertificate(serverCert.Cert()) + if err != nil { + t.Fatalf("failed to parse certificate: %v", err) + } + + // Check SANs include localhost and api-int + expectedDNSNames := map[string]bool{ + "localhost": true, + "api-int.test-cluster.test.example.com": true, + } + + for _, dnsName := range cert.DNSNames { + if !expectedDNSNames[dnsName] { + t.Errorf("unexpected DNS name in certificate: %s", dnsName) + } + delete(expectedDNSNames, dnsName) + } + + if len(expectedDNSNames) > 0 { + t.Errorf("missing expected DNS names: %v", expectedDNSNames) + } + + // Check IP addresses + expectedIPs := map[string]bool{ + "127.0.0.1": true, + "::1": true, + } + + for _, ipAddr := range cert.IPAddresses { + ipStr := ipAddr.String() + if !expectedIPs[ipStr] { + t.Errorf("unexpected IP address in certificate: %s", ipStr) + } + delete(expectedIPs, ipStr) + } + + if len(expectedIPs) > 0 { + t.Errorf("missing expected IP addresses: %v", expectedIPs) + } + + // Verify ExtKeyUsage includes ServerAuth + hasServerAuth := false + for _, usage := range cert.ExtKeyUsage { + if usage == x509.ExtKeyUsageServerAuth { + hasServerAuth = true + break + } + } + if !hasServerAuth { + t.Error("expected certificate to have ExtKeyUsageServerAuth") + } +} + +func TestInternalReleaseRegistryLocalhostCertKeyGenerate(t *testing.T) { + ca := &InternalReleaseRegistrySignerCertKey{} + if err := ca.Generate(context.TODO(), asset.Parents{}); err != nil { + t.Fatalf("failed to generate CA: %v", err) + } + + parents := asset.Parents{} + parents.Add(ca) + + localhostCert := &InternalReleaseRegistryLocalhostCertKey{} + if err := localhostCert.Generate(context.TODO(), parents); err != nil { + t.Fatalf("failed to generate localhost certificate: %v", err) + } + + if len(localhostCert.Cert()) == 0 { + t.Error("expected certificate data") + } + if len(localhostCert.Key()) == 0 { + t.Error("expected key data") + } + + cert, err := PemToCertificate(localhostCert.Cert()) + if err != nil { + t.Fatalf("failed to parse certificate: %v", err) + } + + // Check SANs include only localhost + if len(cert.DNSNames) != 1 || cert.DNSNames[0] != "localhost" { + t.Errorf("expected DNSNames to be [localhost], got %v", cert.DNSNames) + } + + // Check IP addresses + expectedIPs := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")} + if len(cert.IPAddresses) != len(expectedIPs) { + t.Errorf("expected %d IP addresses, got %d", len(expectedIPs), len(cert.IPAddresses)) + } + + // Verify ExtKeyUsage includes ServerAuth + hasServerAuth := false + for _, usage := range cert.ExtKeyUsage { + if usage == x509.ExtKeyUsageServerAuth { + hasServerAuth = true + break + } + } + if !hasServerAuth { + t.Error("expected certificate to have ExtKeyUsageServerAuth") + } +} + +func TestInternalReleaseRegistrySignerCertKeyLoadFromDisk(t *testing.T) { + ca := &InternalReleaseRegistrySignerCertKey{} + + // Before loading, LoadedFromDisk should be false + if ca.LoadedFromDisk { + t.Error("expected LoadedFromDisk to be false before loading") + } + + // Generate a certificate + if err := ca.Generate(context.TODO(), asset.Parents{}); err != nil { + t.Fatalf("failed to generate CA: %v", err) + } + + // LoadedFromDisk should still be false after generation + if ca.LoadedFromDisk { + t.Error("expected LoadedFromDisk to be false after generation") + } + + // Note: Testing actual file loading would require a FileFetcher mock + // and is typically done in integration tests +}