From 3a6505e2a9d876dc4fc300066ffe606a995273e2 Mon Sep 17 00:00:00 2001 From: Manuel Dewald Date: Tue, 21 Jul 2026 16:34:43 +0200 Subject: [PATCH 1/2] feat(gcp): install k0s with k0sctl before Codesphere --- .../install_codesphere_dependencies.go | 58 ++++++---- cli/cmd/k0s/install_k0s.go | 4 +- docs/oms_install_k0s.md | 5 +- go.mod | 6 +- go.sum | 4 +- internal/bootstrap/gcp/gcp.go | 90 ++++++++++++---- internal/bootstrap/gcp/gcp_test.go | 62 +++++++++-- internal/bootstrap/gcp/install_config.go | 14 +-- internal/bootstrap/gcp/install_config_test.go | 3 +- .../installer/codesphere_prerequisites.go | 68 ++++++++++++ .../codesphere_prerequisites_test.go | 102 ++++++++++++++++++ internal/installer/download.go | 35 +++--- internal/installer/k0s.go | 29 ++++- internal/installer/k0s_test.go | 34 +++++- internal/installer/k0sctl.go | 47 +++++--- internal/installer/k0sctl_config.go | 32 +++--- internal/installer/k0sctl_config_test.go | 6 +- internal/installer/k0sctl_test.go | 89 +++++++++++++++ .../installer/vault/vault_encryption_test.go | 4 +- .../installer/vault/vault_secret_creator.go | 15 +++ .../vault/vault_secret_creator_test.go | 55 ++++++++++ internal/installer/vault/vault_suite_test.go | 16 +++ internal/util/slice.go | 25 +++++ internal/util/slice_test.go | 26 +++++ 24 files changed, 711 insertions(+), 118 deletions(-) create mode 100644 internal/installer/codesphere_prerequisites.go create mode 100644 internal/installer/codesphere_prerequisites_test.go create mode 100644 internal/installer/k0sctl_test.go create mode 100644 internal/installer/vault/vault_suite_test.go create mode 100644 internal/util/slice.go create mode 100644 internal/util/slice_test.go diff --git a/cli/cmd/codesphere/install_codesphere_dependencies.go b/cli/cmd/codesphere/install_codesphere_dependencies.go index 799b532d..62f47ac8 100644 --- a/cli/cmd/codesphere/install_codesphere_dependencies.go +++ b/cli/cmd/codesphere/install_codesphere_dependencies.go @@ -21,6 +21,7 @@ import ( "github.com/spf13/cobra" k8sruntime "k8s.io/apimachinery/pkg/runtime" k8sscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -59,13 +60,42 @@ func installCodesphereDepencies(opts *InstallCodesphereOpts, cfg files.RootConfi AutoApprove: opts.AutoApprove, } + installVault, restConfig, err := installer.VaultAndRESTConfig(opts.Vault, opts.PrivKey, cfg) + if err != nil { + return fmt.Errorf("failed to get vault and Kubernetes config: %w", err) + } + + scheme := k8sruntime.NewScheme() + if err := k8sscheme.AddToScheme(scheme); err != nil { + return fmt.Errorf("failed to add Kubernetes core scheme: %w", err) + } + + if err := argov1alpha1.AddToScheme(scheme); err != nil { + return fmt.Errorf("failed to add ArgoCD scheme: %w", err) + } + + kubeClient, err := ctrlclient.New(restConfig, ctrlclient.Options{Scheme: scheme}) + if err != nil { + return fmt.Errorf("failed to create Kubernetes client: %w", err) + } + + err = stlog.Step("Ensure Codesphere prerequisites", func() error { + return installer.EnsureCodespherePrerequisites(context.Background(), kubeClient) + }) + if err != nil { + return fmt.Errorf("failed to ensure Codesphere prerequisites: %w", err) + } + if !installer.IsStepSkipped(cfg, opts.SkipSteps, installer.ArgoCDStep) { - if err := ci.ExtractAndValidatePackage(pm); err != nil { + err = ci.ExtractAndValidatePackage(pm) + if err != nil { return fmt.Errorf("failed to extract and validate package: %w", err) } - if err := stlog.Step("Install ArgoCD pre-step", func() error { - return installArgoCDAndApps(opts, cfg, pm, stlog) - }); err != nil { + + err = stlog.Step("Install ArgoCD pre-step", func() error { + return installArgoCDAndApps(opts, cfg, pm, installVault, restConfig, kubeClient, stlog) + }) + if err != nil { return err } } @@ -78,13 +108,10 @@ func installCodesphereDepencies(opts *InstallCodesphereOpts, cfg files.RootConfi // installArgoCDAndApps runs ArgoCD install, vault secret sync, and pc-apps install // before the main dependency steps. -func installArgoCDAndApps(opts *InstallCodesphereOpts, cfg files.RootConfig, pm installer.PackageManager, stlog *bootstrap.StepLogger) error { +func installArgoCDAndApps(opts *InstallCodesphereOpts, cfg files.RootConfig, pm installer.PackageManager, installVault *files.InstallVault, restConfig *rest.Config, kubeClient ctrlclient.Client, stlog *bootstrap.StepLogger) error { var install *argocdinstaller.AppInstaller - if err := stlog.Substep("Load vault data", func() error { - installVault, restConfig, err := installer.VaultAndRESTConfig(opts.Vault, opts.PrivKey, cfg) - if err != nil { - return err - } + + if err := stlog.Substep("Initialize ArgoCD installer", func() error { registryPassword := "" if secret := installVault.GetSecret(files.SecretRegistryPassword); secret != nil && secret.Fields != nil { registryPassword = secret.Fields.Password @@ -92,17 +119,6 @@ func installArgoCDAndApps(opts *InstallCodesphereOpts, cfg files.RootConfig, pm if registryPassword == "" { return fmt.Errorf("registry password not found in vault (secret %q)", files.SecretRegistryPassword) } - scheme := k8sruntime.NewScheme() - if err := k8sscheme.AddToScheme(scheme); err != nil { - return fmt.Errorf("failed to add kubernetes core scheme: %w", err) - } - if err := argov1alpha1.AddToScheme(scheme); err != nil { - return fmt.Errorf("failed to add ArgoCD scheme: %w", err) - } - kubeClient, err := ctrlclient.New(restConfig, ctrlclient.Options{Scheme: scheme}) - if err != nil { - return fmt.Errorf("failed to create kubernetes client: %w", err) - } registryURL := opts.ArgoCDRegistryURL if registryURL == "" && cfg.Registry != nil { registryURL = cfg.Registry.Server + "/codesphere-cloud/charts" diff --git a/cli/cmd/k0s/install_k0s.go b/cli/cmd/k0s/install_k0s.go index 09b95f15..56a28fde 100644 --- a/cli/cmd/k0s/install_k0s.go +++ b/cli/cmd/k0s/install_k0s.go @@ -66,7 +66,7 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) { - Deploy k0s to all nodes defined in the install-config using k0sctl`), Example: util.FormatExamples("install k0s", []packageio.Example{ {Cmd: "--install-config ", Desc: "Path to Codesphere install-config file to generate k0s config from"}, - {Cmd: "--version ", Desc: "Version of k0s to install (e.g., v1.30.0+k0s.0)"}, + {Cmd: "--version ", Desc: "Version of k0s to install (e.g., v1.31.14+k0s.0)"}, {Cmd: "--k0sctl-version ", Desc: "Version of k0sctl to use (e.g., v0.17.4)"}, {Cmd: "--package ", Desc: "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from"}, {Cmd: "--ssh-key-path ", Desc: "SSH private key path for remote installation"}, @@ -78,7 +78,7 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) { Env: env.NewEnv(), FileWriter: intutil.NewFilesystemWriter(), } - k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", "", "Version of k0s to install") + k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", installer.DefaultK0sVersion, "Version of k0s to install") k0s.cmd.Flags().StringVar(&k0s.Opts.K0sctlVersion, "k0sctl-version", "", "Version of k0sctl to use") k0s.cmd.Flags().StringVarP(&k0s.Opts.Package, "package", "p", "", "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from") k0s.cmd.Flags().StringVar(&k0s.Opts.InstallConfig, "install-config", "", "Path to Codesphere install-config file (required)") diff --git a/docs/oms_install_k0s.md b/docs/oms_install_k0s.md index 89e22328..76705e15 100644 --- a/docs/oms_install_k0s.md +++ b/docs/oms_install_k0s.md @@ -22,7 +22,7 @@ oms install k0s [flags] # Path to Codesphere install-config file to generate k0s config from $ oms install k0s --install-config -# Version of k0s to install (e.g., v1.30.0+k0s.0) +# Version of k0s to install (e.g., v1.31.14+k0s.0) $ oms install k0s --version # Version of k0sctl to use (e.g., v0.17.4) @@ -54,10 +54,9 @@ $ oms install k0s --no-download --ssh-key-path string SSH private key path for remote installation --vault string Path to prod.vault.yaml to save the kubeconfig into (optional) --vault-priv-key string Path to the age private key to decrypt the vault (optional, for SOPS-encrypted vaults) - -v, --version string Version of k0s to install + -v, --version string Version of k0s to install (default "v1.31.14+k0s.0") ``` ### SEE ALSO * [oms install](oms_install.md) - Install Codesphere and other components - diff --git a/go.mod b/go.mod index 9d06d549..5c21d6d4 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.26.5 replace ( // GoReleaser pulls github.com/chrismellard/docker-credential-acr-env, // which imports github.com/Azure/azure-sdk-for-go/version. Azure SDK - // v68 removed that package, so keep the legacy monorepo on a verified - // version that still provides it. - github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + // v68 removed that package, so keep the legacy monorepo on the version + // requested by the credential helper, which still provides it. + github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v46.4.0+incompatible // argo-cd's go.mod resolves its nested gitops-engine module through a local // directory replace, which does not carry over to consumers, so pin it here. // The commit tagged for argo-cd v3.4.6 still imports the autoscaling v2beta* diff --git a/go.sum b/go.sum index 729219ca..8f9b3d1d 100644 --- a/go.sum +++ b/go.sum @@ -2711,8 +2711,8 @@ github.com/Antonboom/nilnil v1.1.2 h1:aNlFuJhaEseXe4fHO3xbjXlSeEiQVYa2lEkWD2s2hA github.com/Antonboom/nilnil v1.1.2/go.mod h1:0ynwvphOLmAuMwTNDyBnDZmSwZoDpcFXmUHmzoHH2WA= github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v46.4.0+incompatible h1:fCN6Pi+tEiEwFa8RSmtVlFHRXEZ+DJm9gfx/MKqYWw4= +github.com/Azure/azure-sdk-for-go v46.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 19eb5c30..540a8c32 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -351,6 +351,16 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" { + err = b.stlog.Step("Install k0s", b.InstallK0s) + if err != nil { + return fmt.Errorf("failed to install k0s: %w", err) + } + + err = b.stlog.Step("Wait for k0s nodes", b.WaitForK0sNodes) + if err != nil { + return fmt.Errorf("failed waiting for k0s nodes: %w", err) + } + err = b.stlog.Step("Install Codesphere", b.InstallCodesphere) if err != nil { return fmt.Errorf("failed to install Codesphere: %w", err) @@ -1058,52 +1068,89 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { } func (b *GCPBootstrapper) InstallCodesphere() error { - fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox() - if err != nil { + if err := b.ensureCodespherePackageOnJumpbox(); err != nil { return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err) } - err = b.runInstallCommand(fullPackageFilename) - if err != nil { + if err := b.runInstallCommand(b.codespherePackageFilename()); err != nil { return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) } return nil } -func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) { - packageFilename := "installer.tar.gz" +// InstallK0s deploys k0s with the native OMS installer and stores its +// kubeconfig in the encrypted install vault for the remaining installer steps. +func (b *GCPBootstrapper) InstallK0s() error { + // Reuse matching cached binaries and let k0sctl reconcile normally. Without + // --force, an unchanged cluster remains untouched on bootstrap retries. + installCmd := fmt.Sprintf("oms install k0s --version %s --install-config /etc/codesphere/config.yaml --vault %s --vault-priv-key %s/age_key.txt", + installer.DefaultK0sVersion, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), b.Env.SecretsDir) + if err := b.Env.Jumpbox.RunSSHCommand("root", installCmd); err != nil { + return fmt.Errorf("failed to install k0s from jumpbox: %w", err) + } + + return nil +} + +// WaitForK0sNodes restores the readiness barrier from the TypeScript +// Kubernetes setup. k0sctl apply completing is not sufficient for the +// Codesphere charts: all schedulable nodes must be Ready before gateway +// controllers and their admission webhooks are installed. +func (b *GCPBootstrapper) WaitForK0sNodes() error { + const command = "k0s kubectl wait --for=condition=Ready nodes --all --timeout=30m" + if err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", command); err != nil { + return fmt.Errorf("k0s nodes did not become ready: %w", err) + } + + return nil +} + +func (b *GCPBootstrapper) codespherePackageFilename() string { + packageFilename := b.codespherePackageArchiveName() + if b.Env.InstallLocal != "" { + return "local-" + packageFilename + } + + return portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFilename) +} + +func (b *GCPBootstrapper) codespherePackageArchiveName() string { if b.Env.RegistryType == RegistryTypeGitHub { - packageFilename = "installer-lite.tar.gz" + return "installer-lite.tar.gz" } + return "installer.tar.gz" +} + +func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() error { if b.Env.InstallLocal != "" { b.stlog.Logf("Copying local package %s to jumpbox...", b.Env.InstallLocal) - fullPackageFilename := fmt.Sprintf("local-%s", packageFilename) - err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallLocal, "/root/"+fullPackageFilename) + + err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallLocal, "/root/"+b.codespherePackageFilename()) if err != nil { - return "", fmt.Errorf("failed to copy local install package to jumpbox: %w", err) + return fmt.Errorf("failed to copy local install package to jumpbox: %w", err) } - return fullPackageFilename, nil + + return nil } if b.Env.InstallVersion == "" { - return "", errors.New("either install version or a local package must be specified to install Codesphere") + return errors.New("either install version or a local package must be specified to install Codesphere") } - fullPackageFilename := portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFilename) if b.Env.InstallHash == "" { - return "", fmt.Errorf("install hash must be set when install version is set") + return fmt.Errorf("install hash must be set when install version is set") } b.stlog.Logf("Downloading Codesphere package...") downloadCmd := fmt.Sprintf("oms download package -f %s -H %s %s", - packageFilename, b.Env.InstallHash, b.Env.InstallVersion) + b.codespherePackageArchiveName(), b.Env.InstallHash, b.Env.InstallVersion) err := b.Env.Jumpbox.RunSSHCommand("root", downloadCmd) if err != nil { - return "", fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err) + return fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err) } - return fullPackageFilename, nil + return nil } func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error { @@ -1114,9 +1161,13 @@ func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error { } func (b *GCPBootstrapper) generateSkipStepsArg() string { - skipSteps := b.Env.InstallSkipSteps + // k0s is installed by OMS before the TypeScript installer runs, so the + // TypeScript Kubernetes step must never run during GCP bootstrapping. + skipSteps := []string{"kubernetes"} + skipSteps = util.AppendUnique(skipSteps, b.Env.InstallSkipSteps...) + if b.Env.RegistryType == RegistryTypeGitHub { - skipSteps = append(skipSteps, "load-container-images") + skipSteps = util.AppendUnique(skipSteps, "load-container-images") } if len(skipSteps) == 0 { return "" @@ -1124,7 +1175,6 @@ func (b *GCPBootstrapper) generateSkipStepsArg() string { return " -s " + strings.Join(skipSteps, ",") } - func (b *GCPBootstrapper) GenerateK0sConfigScript() error { script := `#!/bin/bash diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index 49601814..55721ff5 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1395,7 +1395,7 @@ var _ = Describe("GCP Bootstrapper", func() { // Expect install codesphere nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", - "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer-lite.tar.gz -s load-container-images").Return(nil) + "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer-lite.tar.gz -s kubernetes,load-container-images").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1412,7 +1412,7 @@ var _ = Describe("GCP Bootstrapper", func() { nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H def9876543210 v1.2.3").Return(nil) // Expect install codesphere - nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-def9876543210-installer.tar.gz").Return(nil) + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-def9876543210-installer.tar.gz -s kubernetes").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1421,7 +1421,17 @@ var _ = Describe("GCP Bootstrapper", func() { It("downloads and installs codesphere with hash", func() { nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil) - nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz").Return(nil) + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s kubernetes").Return(nil) + + err := bs.InstallCodesphere() + Expect(err).NotTo(HaveOccurred()) + }) + + It("preserves requested skip steps without duplicating kubernetes", func() { + csEnv.InstallSkipSteps = []string{"postgres", "kubernetes"} + + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil) + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s kubernetes,postgres").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1440,7 +1450,7 @@ var _ = Describe("GCP Bootstrapper", func() { It("installs codesphere from local package", func() { nodeClient.EXPECT().CopyFile(mock.Anything, csEnv.InstallLocal, "/root/local-installer-lite.tar.gz").Return(nil) nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", - "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer-lite.tar.gz -s load-container-images").Return(nil) + "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer-lite.tar.gz -s kubernetes,load-container-images").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1454,7 +1464,7 @@ var _ = Describe("GCP Bootstrapper", func() { It("installs codesphere from local package", func() { nodeClient.EXPECT().CopyFile(mock.Anything, csEnv.InstallLocal, "/root/local-installer.tar.gz").Return(nil) nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", - "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer.tar.gz").Return(nil) + "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer.tar.gz -s kubernetes").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1498,7 +1508,7 @@ var _ = Describe("GCP Bootstrapper", func() { It("fails when install codesphere fails", func() { nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil).Once() - nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz").Return(fmt.Errorf("install error")).Once() + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s kubernetes").Return(fmt.Errorf("install error")).Once() err := bs.InstallCodesphere() Expect(err).To(HaveOccurred()) @@ -1507,6 +1517,46 @@ var _ = Describe("GCP Bootstrapper", func() { }) }) + Describe("InstallK0s", func() { + BeforeEach(func() { + csEnv.InstallVersion = "v1.2.3" + csEnv.InstallHash = "abc1234567890" + }) + + It("downloads k0s and lets k0sctl distribute it independently of the Codesphere package", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms install k0s --version v1.31.14+k0s.0 --install-config /etc/codesphere/config.yaml --vault /etc/codesphere/secrets/prod.vault.yaml --vault-priv-key /etc/codesphere/secrets/age_key.txt").Return(nil) + + err := bs.InstallK0s() + Expect(err).NotTo(HaveOccurred()) + }) + + It("reports a native k0s installation failure", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", mock.MatchedBy(func(command string) bool { + return strings.HasPrefix(command, "oms install k0s ") + })).Return(fmt.Errorf("k0s error")) + + err := bs.InstallK0s() + Expect(err).To(MatchError(ContainSubstring("failed to install k0s from jumpbox"))) + }) + }) + + Describe("WaitForK0sNodes", func() { + const command = "k0s kubectl wait --for=condition=Ready nodes --all --timeout=30m" + + It("waits for every node before installing cluster components", func() { + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", command).Return(nil) + + Expect(bs.WaitForK0sNodes()).To(Succeed()) + }) + + It("reports nodes that fail to become ready", func() { + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", command).Return(fmt.Errorf("timeout")) + + Expect(bs.WaitForK0sNodes()).To(MatchError(ContainSubstring("k0s nodes did not become ready"))) + }) + }) + Describe("GenerateK0sConfigScript", func() { Describe("Valid GenerateK0sConfigScript", func() { It("generates script", func() { diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 2775de59..774dbdc0 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -378,12 +378,14 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { b.applyExternalLokiConfig() b.applyPrometheusRemoteWriteConfig() - if !b.Env.ExistingConfigUsed { - err := b.icg.GenerateSecrets() - if err != nil { - return fmt.Errorf("failed to generate secrets: %w", err) - } - } else { + // Secret generation is idempotent and also backfills secrets introduced + // after an existing vault was created (for example the auth keys required by + // the ArgoCD pre-step). + if err := b.icg.GenerateSecrets(); err != nil { + return fmt.Errorf("failed to generate secrets: %w", err) + } + + if b.Env.ExistingConfigUsed { if err := b.regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname); err != nil { return err } diff --git a/internal/bootstrap/gcp/install_config_test.go b/internal/bootstrap/gcp/install_config_test.go index e5d0306d..38398963 100644 --- a/internal/bootstrap/gcp/install_config_test.go +++ b/internal/bootstrap/gcp/install_config_test.go @@ -1074,6 +1074,8 @@ var _ = Describe("Installconfig & Secrets", func() { Describe("ExistingConfigUsed", func() { BeforeEach(func() { csEnv.ExistingConfigUsed = true + + icg.EXPECT().GenerateSecrets().Return(nil) }) Context("with unchanged IP and existing key", func() { @@ -1103,7 +1105,6 @@ var _ = Describe("Installconfig & Secrets", func() { err := bs.UpdateInstallConfig() Expect(err).NotTo(HaveOccurred()) - icg.AssertNotCalled(GinkgoT(), "GenerateSecrets") Expect(vault.GetSecret(files.SecretPostgresPrimaryServerKeyPem).File.Content).To(Equal(origKey)) Expect(bs.Env.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem).To(Equal(origCert)) }) diff --git a/internal/installer/codesphere_prerequisites.go b/internal/installer/codesphere_prerequisites.go new file mode 100644 index 00000000..d66aed79 --- /dev/null +++ b/internal/installer/codesphere_prerequisites.go @@ -0,0 +1,68 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + codesphereNamespace = "codesphere" + dummyErrorPageServiceName = "error-page-server" +) + +// EnsureCodespherePrerequisites creates the resources required before the +// Codesphere dependency charts are installed. Existing resources are left +// untouched so Helm or a previous installation can continue to own them. +func EnsureCodespherePrerequisites(ctx context.Context, kubeClient ctrlclient.Client) error { + namespace := &corev1.Namespace{} + if err := kubeClient.Get(ctx, ctrlclient.ObjectKey{Name: codesphereNamespace}, namespace); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to check Codesphere namespace: %w", err) + } + + namespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: codesphereNamespace}, + } + if err := kubeClient.Create(ctx, namespace); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create Codesphere namespace: %w", err) + } + } + + service := &corev1.Service{} + + serviceKey := ctrlclient.ObjectKey{Name: dummyErrorPageServiceName, Namespace: codesphereNamespace} + if err := kubeClient.Get(ctx, serviceKey, service); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to check dummy error-page-server service: %w", err) + } + + service = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: dummyErrorPageServiceName, + Namespace: codesphereNamespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{{ + Port: 8080, + TargetPort: intstr.FromInt32(8080), + }}, + Selector: map[string]string{"app": dummyErrorPageServiceName}, + }, + } + if err := kubeClient.Create(ctx, service); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create dummy error-page-server service: %w", err) + } + } + + return nil +} diff --git a/internal/installer/codesphere_prerequisites_test.go b/internal/installer/codesphere_prerequisites_test.go new file mode 100644 index 00000000..9e175524 --- /dev/null +++ b/internal/installer/codesphere_prerequisites_test.go @@ -0,0 +1,102 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer_test + +import ( + "context" + "fmt" + + "github.com/codesphere-cloud/oms/internal/installer" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func newPrerequisitesClient(objects ...ctrlclient.Object) ctrlclient.Client { + return newPrerequisitesClientWithInterceptors(interceptor.Funcs{}, objects...) +} + +func newPrerequisitesClientWithInterceptors(interceptors interceptor.Funcs, objects ...ctrlclient.Object) ctrlclient.Client { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithInterceptorFuncs(interceptors). + Build() +} + +var _ = Describe("EnsureCodespherePrerequisites", func() { + It("creates the Codesphere namespace and dummy error-page service", func() { + kubeClient := newPrerequisitesClient() + + Expect(installer.EnsureCodespherePrerequisites(context.Background(), kubeClient)).To(Succeed()) + + namespace := &corev1.Namespace{} + Expect(kubeClient.Get(context.Background(), ctrlclient.ObjectKey{Name: "codesphere"}, namespace)).To(Succeed()) + + service := &corev1.Service{} + Expect(kubeClient.Get(context.Background(), ctrlclient.ObjectKey{Name: "error-page-server", Namespace: "codesphere"}, service)).To(Succeed()) + Expect(service.Spec.Ports).To(HaveLen(1)) + Expect(service.Spec.Ports[0].Port).To(Equal(int32(8080))) + }) + + It("leaves existing resources untouched", func() { + existingNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: "codesphere", + Labels: map[string]string{"owner": "existing-install"}, + }} + existingService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "error-page-server", Namespace: "codesphere"}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 9090}}}, + } + kubeClient := newPrerequisitesClientWithInterceptors(interceptor.Funcs{ + Create: func(context.Context, ctrlclient.WithWatch, ctrlclient.Object, ...ctrlclient.CreateOption) error { + return fmt.Errorf("existing resources must not be recreated") + }, + }, existingNamespace, existingService) + + Expect(installer.EnsureCodespherePrerequisites(context.Background(), kubeClient)).To(Succeed()) + + namespace := &corev1.Namespace{} + Expect(kubeClient.Get(context.Background(), ctrlclient.ObjectKey{Name: "codesphere"}, namespace)).To(Succeed()) + Expect(namespace.Labels).To(HaveKeyWithValue("owner", "existing-install")) + + service := &corev1.Service{} + Expect(kubeClient.Get(context.Background(), ctrlclient.ObjectKey{Name: "error-page-server", Namespace: "codesphere"}, service)).To(Succeed()) + Expect(service.Spec.Ports[0].Port).To(Equal(int32(9090))) + }) + + It("reports namespace creation failures", func() { + kubeClient := newPrerequisitesClientWithInterceptors(interceptor.Funcs{ + Create: func(ctx context.Context, client ctrlclient.WithWatch, obj ctrlclient.Object, opts ...ctrlclient.CreateOption) error { + if _, isNamespace := obj.(*corev1.Namespace); isNamespace { + return fmt.Errorf("kubectl error") + } + return client.Create(ctx, obj, opts...) + }, + }) + + Expect(installer.EnsureCodespherePrerequisites(context.Background(), kubeClient)).To(MatchError(ContainSubstring("failed to create Codesphere namespace"))) + }) + + It("reports service creation failures", func() { + kubeClient := newPrerequisitesClientWithInterceptors(interceptor.Funcs{ + Create: func(ctx context.Context, client ctrlclient.WithWatch, obj ctrlclient.Object, opts ...ctrlclient.CreateOption) error { + if _, isService := obj.(*corev1.Service); isService { + return fmt.Errorf("kubectl error") + } + return client.Create(ctx, obj, opts...) + }, + }) + + Expect(installer.EnsureCodespherePrerequisites(context.Background(), kubeClient)).To(MatchError(ContainSubstring("failed to create dummy error-page-server service"))) + }) +}) diff --git a/internal/installer/download.go b/internal/installer/download.go index 7a72373e..bb46b3d9 100644 --- a/internal/installer/download.go +++ b/internal/installer/download.go @@ -5,24 +5,13 @@ package installer import ( "fmt" - "path/filepath" + "strings" "github.com/codesphere-cloud/oms/internal/portal" "github.com/codesphere-cloud/oms/internal/util" ) -// downloadBinary downloads a binary from downloadURL into workdir/binaryName. -// It handles workdir creation, existing binary checks, file creation, download, and chmod. -func downloadBinary(fw util.FileIO, http portal.Http, workdir, binaryName, downloadURL string, force bool, quiet bool) (string, error) { - if err := fw.MkdirAll(workdir, 0755); err != nil { - return "", fmt.Errorf("failed to create workdir: %w", err) - } - - binaryPath := filepath.Join(workdir, binaryName) - if fw.Exists(binaryPath) && !force { - return "", fmt.Errorf("%s binary already exists at %s. Use --force to overwrite", binaryName, binaryPath) - } - +func downloadBinaryToPath(fw util.FileIO, http portal.Http, binaryPath, binaryName, downloadURL string, quiet bool) (string, error) { dstFile, err := fw.Create(binaryPath) if err != nil { return "", fmt.Errorf("failed to create %s binary file: %w", binaryName, err) @@ -39,3 +28,23 @@ func downloadBinary(fw util.FileIO, http portal.Http, workdir, binaryName, downl return binaryPath, nil } + +func localBinaryVersion(binaryPath string) (string, error) { + output, err := util.RunCommandWithOutput(binaryPath, []string{"version"}, "") + if err != nil { + return "", fmt.Errorf("failed to get version of application %s: %w", binaryPath, err) + } + + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if version, found := strings.CutPrefix(line, "version:"); found { + return strings.TrimSpace(version), nil + } + + if line != "" { + return line, nil + } + } + + return "", fmt.Errorf("version output is empty") +} diff --git a/internal/installer/k0s.go b/internal/installer/k0s.go index 6ba3bf97..ed052267 100644 --- a/internal/installer/k0s.go +++ b/internal/installer/k0s.go @@ -6,14 +6,20 @@ package installer import ( "fmt" "log" + "path/filepath" "runtime" "strings" + "github.com/codesphere-cloud/cs-go/pkg/io" "github.com/codesphere-cloud/oms/internal/env" "github.com/codesphere-cloud/oms/internal/portal" "github.com/codesphere-cloud/oms/internal/util" ) +// DefaultK0sVersion is the currently verified k0s version +// Use of newer versions should work in most cases but can't be guaranteed +const DefaultK0sVersion = "v1.31.14+k0s.0" + //mockery:generate: true type K0sManager interface { GetLatestVersion() (string, error) @@ -65,8 +71,29 @@ func (k *K0s) Download(version string, force bool, quiet bool) (string, error) { return "", fmt.Errorf("failed to determine cache directory: %w", err) } + if err := k.FileWriter.MkdirAll(cacheDir, 0755); err != nil { + return "", fmt.Errorf("failed to create workdir: %w", err) + } + + cachePath := filepath.Join(cacheDir, "k0s") + if k.FileWriter.Exists(cachePath) && !force { + cachedVersion, versionErr := localBinaryVersion(cachePath) + if versionErr == nil && cachedVersion == version { + io.Verbosef(!quiet, "Using cached k0s %s at %s", version, cachePath) + return cachePath, nil + } + + replaceReason := fmt.Sprintf("Cached k0s version %s does not match requested version %s; replacing it", cachedVersion, version) + if versionErr != nil { + replaceReason = "Cached k0s version could not be determined: " + versionErr.Error() + } + + io.Verbosef(!quiet, "Replacing existing k0s binary: %s", replaceReason) + } + downloadURL := fmt.Sprintf("https://github.com/k0sproject/k0s/releases/download/%s/k0s-%s-%s", version, version, k.Goarch) - path, err := downloadBinary(k.FileWriter, k.Http, cacheDir, "k0s", downloadURL, force, quiet) + + path, err := downloadBinaryToPath(k.FileWriter, k.Http, cachePath, "k0s", downloadURL, quiet) if err != nil { return "", err } diff --git a/internal/installer/k0s_test.go b/internal/installer/k0s_test.go index 48a5a0a1..f59be741 100644 --- a/internal/installer/k0s_test.go +++ b/internal/installer/k0s_test.go @@ -168,13 +168,37 @@ var _ = Describe("K0s", func() { mockFileWriter.EXPECT().MkdirAll(workDir, os.FileMode(0755)).Return(nil) }) - It("should fail when k0s binary exists and force is false", func() { + It("should reuse a cached k0s binary with the requested version", func() { + err := os.MkdirAll(workDir, 0755) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(k0sPath, []byte("#!/bin/sh\nprintf 'v1.29.1+k0s.0\\n'\n"), 0755) + Expect(err).ToNot(HaveOccurred()) mockFileWriter.EXPECT().Exists(k0sPath).Return(true) - _, err := k0s.Download("v1.29.1+k0s.0", false, false) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("k0s binary already exists")) - Expect(err.Error()).To(ContainSubstring("Use --force to overwrite")) + path, err := k0s.Download("v1.29.1+k0s.0", false, false) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(k0sPath)) + }) + + It("should replace a cached k0s binary with a different version", func() { + err := os.MkdirAll(workDir, 0755) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(k0sPath, []byte("#!/bin/sh\nprintf 'v1.28.0+k0s.0\\n'\n"), 0755) + Expect(err).ToNot(HaveOccurred()) + mockFileWriter.EXPECT().Exists(k0sPath).Return(true) + + realFile, err := os.Create(k0sPath) + Expect(err).ToNot(HaveOccurred()) + + defer util.CloseFileIgnoreError(realFile) + + mockFileWriter.EXPECT().Create(k0sPath).Return(realFile, nil) + mockHttp.EXPECT().Download("https://github.com/k0sproject/k0s/releases/download/v1.29.1+k0s.0/k0s-v1.29.1+k0s.0-amd64", realFile, false).Return(nil) + mockFileWriter.EXPECT().Chmod(k0sPath, os.FileMode(0755)).Return(nil) + + path, err := k0s.Download("v1.29.1+k0s.0", false, false) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(k0sPath)) }) It("should proceed when k0s binary exists and force is true", func() { diff --git a/internal/installer/k0sctl.go b/internal/installer/k0sctl.go index 93f17e6c..7ff1e677 100644 --- a/internal/installer/k0sctl.go +++ b/internal/installer/k0sctl.go @@ -7,9 +7,11 @@ import ( "encoding/json" "fmt" "log" + "path/filepath" "runtime" "strings" + "github.com/codesphere-cloud/cs-go/pkg/io" "github.com/codesphere-cloud/oms/internal/env" "github.com/codesphere-cloud/oms/internal/portal" "github.com/codesphere-cloud/oms/internal/util" @@ -66,42 +68,55 @@ func (k *K0sctl) GetLatestVersion() (string, error) { } func (k *K0sctl) Download(version string, force bool, quiet bool) (string, error) { + cacheDir, err := k.Env.GetOmsCacheDir() + if err != nil { + return "", fmt.Errorf("failed to determine cache directory: %w", err) + } + + if err := k.FileWriter.MkdirAll(cacheDir, 0755); err != nil { + return "", fmt.Errorf("failed to create workdir: %w", err) + } + if version == "" { var err error version, err = k.GetLatestVersion() if err != nil { return "", fmt.Errorf("failed to get latest version: %w", err) } - if !quiet { - log.Printf("Using latest k0sctl version: %s", version) - } + io.Verbosef(!quiet, "Using latest k0sctl version: %s", version) } - // Ensure version has v prefix for GitHub URL if !strings.HasPrefix(version, "v") { version = "v" + version } - binaryName := fmt.Sprintf("k0sctl-%s-%s", k.Goos, k.Goarch) - downloadURL := fmt.Sprintf("https://github.com/k0sproject/k0sctl/releases/download/%s/%s", version, binaryName) + cachePath := filepath.Join(cacheDir, "k0sctl") + if k.FileWriter.Exists(cachePath) && !force { + cachedVersion, versionErr := localBinaryVersion(cachePath) + if versionErr == nil && cachedVersion == version { + io.Verbosef(!quiet, "Using cached k0sctl %s at %s", version, cachePath) - if !quiet { - log.Printf("Downloading k0sctl %s from %s", version, downloadURL) - } + return cachePath, nil + } - cacheDir, err := k.Env.GetOmsCacheDir() - if err != nil { - return "", fmt.Errorf("failed to determine cache directory: %w", err) + if versionErr != nil { + io.Verbosef(!quiet, "Cached k0sctl version could not be determined; replacing it: %v", versionErr) + } else { + io.Verbosef(!quiet, "Cached k0sctl version %s does not match requested version %s; replacing it", cachedVersion, version) + } } - path, err := downloadBinary(k.FileWriter, k.Http, cacheDir, "k0sctl", downloadURL, force, quiet) + binaryName := fmt.Sprintf("k0sctl-%s-%s", k.Goos, k.Goarch) + downloadURL := fmt.Sprintf("https://github.com/k0sproject/k0sctl/releases/download/%s/%s", version, binaryName) + + io.Verbosef(!quiet, "Downloading k0sctl %s from %s", version, downloadURL) + + path, err := downloadBinaryToPath(k.FileWriter, k.Http, cachePath, "k0sctl", downloadURL, quiet) if err != nil { return "", err } - if !quiet { - log.Printf("k0sctl downloaded successfully to %s", path) - } + io.Verbosef(!quiet, "k0sctl downloaded successfully to %s", path) return path, nil } diff --git a/internal/installer/k0sctl_config.go b/internal/installer/k0sctl_config.go index b7a6dc88..30791b0d 100644 --- a/internal/installer/k0sctl_config.go +++ b/internal/installer/k0sctl_config.go @@ -5,6 +5,7 @@ package installer import ( "fmt" + "slices" "github.com/codesphere-cloud/oms/internal/installer/files" "gopkg.in/yaml.v3" @@ -68,7 +69,12 @@ type K0sctlApplyHooks struct { After []string `yaml:"after,omitempty"` } -func createK0sctlHost(node files.K8sNode, role string, installFlags []string, sshKeyPath string, k0sBinaryPath string) K0sctlHost { +func (k *K0sctlSpec) addUniqueK0sctlHost(node files.K8sNode, role string, installFlags []string, sshKeyPath string, k0sBinaryPath string) { + for _, host := range k.Hosts { + if host.PrivateAddress == node.IPAddress { + return + } + } host := K0sctlHost{ Role: role, SSH: K0sctlSSH{ @@ -89,7 +95,7 @@ func createK0sctlHost(node files.K8sNode, role string, installFlags []string, ss host.K0sBinaryPath = k0sBinaryPath } - return host + k.Hosts = append(k.Hosts, host) } // GenerateK0sctlConfig generates a k0sctl configuration from a Codesphere install-config @@ -123,24 +129,20 @@ func GenerateK0sctlConfig(installConfig *files.RootConfig, k0sVersion string, ss }, } - // Track added IPs to avoid duplicates - addedIPs := make(map[string]bool) - - // Add controller-only nodes from control planes + // Add control-plane nodes, enabling their worker role when configured. for _, cp := range installConfig.Kubernetes.ControlPlanes { - host := createK0sctlHost(cp, "controller", nil, sshKeyPath, k0sBinaryPath) - k0sctlConfig.Spec.Hosts = append(k0sctlConfig.Spec.Hosts, host) - addedIPs[cp.IPAddress] = true + var installFlags []string + // A node may intentionally be listed as both a control plane and a worker. + if slices.Contains(installConfig.Kubernetes.Workers, cp) { + installFlags = []string{"--enable-worker", "--no-taints=true"} + } + + k0sctlConfig.Spec.addUniqueK0sctlHost(cp, "controller", installFlags, sshKeyPath, k0sBinaryPath) } // Add dedicated worker nodes if present for _, worker := range installConfig.Kubernetes.Workers { - if addedIPs[worker.IPAddress] { - continue - } - host := createK0sctlHost(worker, "worker", nil, sshKeyPath, k0sBinaryPath) - k0sctlConfig.Spec.Hosts = append(k0sctlConfig.Spec.Hosts, host) - addedIPs[worker.IPAddress] = true + k0sctlConfig.Spec.addUniqueK0sctlHost(worker, "worker", nil, sshKeyPath, k0sBinaryPath) } return k0sctlConfig, nil diff --git a/internal/installer/k0sctl_config_test.go b/internal/installer/k0sctl_config_test.go index 72654d4f..55ca856a 100644 --- a/internal/installer/k0sctl_config_test.go +++ b/internal/installer/k0sctl_config_test.go @@ -69,7 +69,7 @@ var _ = Describe("K0sctlConfig", func() { Expect(k0sctlConfig.Spec.Hosts[2].Role).To(Equal("worker")) }) - It("should skip duplicate IPs between control planes and workers", func() { + It("should enable the worker role on control planes also listed as workers", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") installConfig.Kubernetes.Workers = []files.K8sNode{ {IPAddress: "10.0.1.10"}, // Duplicate @@ -79,10 +79,12 @@ var _ = Describe("K0sctlConfig", func() { k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") Expect(err).ToNot(HaveOccurred()) - // Should only have 2 hosts: 1 control plane + 1 unique worker + // The overlapping node is emitted once, but retains both roles in + // the same way as the TypeScript installer. Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(2)) Expect(k0sctlConfig.Spec.Hosts[0].SSH.Address).To(Equal("10.0.1.10")) Expect(k0sctlConfig.Spec.Hosts[0].Role).To(Equal("controller")) + Expect(k0sctlConfig.Spec.Hosts[0].InstallFlags).To(Equal([]string{"--enable-worker", "--no-taints=true"})) Expect(k0sctlConfig.Spec.Hosts[1].SSH.Address).To(Equal("10.0.2.10")) Expect(k0sctlConfig.Spec.Hosts[1].Role).To(Equal("worker")) }) diff --git a/internal/installer/k0sctl_test.go b/internal/installer/k0sctl_test.go new file mode 100644 index 00000000..8eb02d64 --- /dev/null +++ b/internal/installer/k0sctl_test.go @@ -0,0 +1,89 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer_test + +import ( + "os" + "path/filepath" + + "github.com/codesphere-cloud/oms/internal/env" + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/portal" + "github.com/codesphere-cloud/oms/internal/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("K0sctl", func() { + var ( + mockEnv *env.MockEnv + mockHTTP *portal.MockHttp + mockFileWriter *util.MockFileIO + cacheDir string + cachedPath string + k0sctl *installer.K0sctl + ) + + BeforeEach(func() { + mockEnv = env.NewMockEnv(GinkgoT()) + mockHTTP = portal.NewMockHttp(GinkgoT()) + mockFileWriter = util.NewMockFileIO(GinkgoT()) + cacheDir = GinkgoT().TempDir() + cachedPath = filepath.Join(cacheDir, "k0sctl") + + mockEnv.EXPECT().GetOmsCacheDir().Return(cacheDir, nil) + mockFileWriter.EXPECT().MkdirAll(cacheDir, os.FileMode(0755)).Return(nil) + + k0sctl = installer.NewK0sctl(mockHTTP, mockEnv, mockFileWriter) + k0sctl.Goos = "linux" + k0sctl.Goarch = "amd64" + }) + + writeCachedVersion := func(version string) { + script := "#!/bin/sh\nprintf 'version: " + version + "\\ncommit: test\\n'\n" + Expect(os.WriteFile(cachedPath, []byte(script), 0755)).To(Succeed()) + } + + expectDownload := func(version string) { + url := "https://github.com/k0sproject/k0sctl/releases/download/" + version + "/k0sctl-linux-amd64" + downloadFile, err := os.CreateTemp(cacheDir, "k0sctl-download") + Expect(err).NotTo(HaveOccurred()) + mockFileWriter.EXPECT().Create(cachedPath).Return(downloadFile, nil) + mockHTTP.EXPECT().Download(url, downloadFile, false).Return(nil) + mockFileWriter.EXPECT().Chmod(cachedPath, os.FileMode(0755)).Return(nil) + } + + It("reuses a cached binary with the requested version", func() { + writeCachedVersion("v0.32.1") + mockFileWriter.EXPECT().Exists(cachedPath).Return(true) + + path, err := k0sctl.Download("v0.32.1", false, false) + + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(cachedPath)) + }) + + It("replaces a cached binary with a different version without force", func() { + writeCachedVersion("v0.31.0") + mockFileWriter.EXPECT().Exists(cachedPath).Return(true) + expectDownload("v0.32.1") + + path, err := k0sctl.Download("v0.32.1", false, false) + + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(cachedPath)) + }) + + It("resolves an unpinned version and reuses the matching binary", func() { + writeCachedVersion("v0.32.1") + mockHTTP.EXPECT().Get("https://api.github.com/repos/k0sproject/k0sctl/releases/latest"). + Return([]byte(`{"tag_name":"v0.32.1"}`), nil) + mockFileWriter.EXPECT().Exists(cachedPath).Return(true) + + path, err := k0sctl.Download("", false, false) + + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(cachedPath)) + }) +}) diff --git a/internal/installer/vault/vault_encryption_test.go b/internal/installer/vault/vault_encryption_test.go index ba392b56..7045a68f 100644 --- a/internal/installer/vault/vault_encryption_test.go +++ b/internal/installer/vault/vault_encryption_test.go @@ -226,7 +226,7 @@ var _ = Describe("VaultEncryption", func() { plainYAML := "secrets:\n - name: test-secret\n fields:\n password: hunter2\n" Expect(os.WriteFile(vaultPath, []byte(plainYAML), 0644)).To(Succeed()) - vault, err := vault.LoadVaultData(vaultPath, "") + vault, err := vault.LoadUnencryptedVaultData(vaultPath) Expect(err).ToNot(HaveOccurred()) Expect(vault.Secrets).To(HaveLen(1)) Expect(vault.Secrets[0].Name).To(Equal("test-secret")) @@ -238,7 +238,7 @@ var _ = Describe("VaultEncryption", func() { wrappedYAML := "data: |\n secrets:\n - name: test-secret\n fields:\n password: hunter2\n" Expect(os.WriteFile(vaultPath, []byte(wrappedYAML), 0644)).To(Succeed()) - vault, err := vault.LoadVaultData(vaultPath, "") + vault, err := vault.LoadUnencryptedVaultData(vaultPath) Expect(err).ToNot(HaveOccurred()) Expect(vault.Secrets).To(HaveLen(1)) Expect(vault.Secrets[0].Name).To(Equal("test-secret")) diff --git a/internal/installer/vault/vault_secret_creator.go b/internal/installer/vault/vault_secret_creator.go index 67951a5a..5ac82548 100644 --- a/internal/installer/vault/vault_secret_creator.go +++ b/internal/installer/vault/vault_secret_creator.go @@ -65,6 +65,10 @@ func (v *VaultSecretCreator) CreateSecretFromVault(ctx context.Context, vault *f return err } + if err := v.ensureNamespace(ctx, namespace); err != nil { + return err + } + secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, @@ -85,6 +89,17 @@ func (v *VaultSecretCreator) CreateSecretFromVault(ctx context.Context, vault *f return nil } +func (v *VaultSecretCreator) ensureNamespace(ctx context.Context, namespace string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + } + if _, err := controllerutil.CreateOrUpdate(ctx, v.client, ns, func() error { return nil }); err != nil { + return fmt.Errorf("failed to ensure namespace %q: %w", namespace, err) + } + + return nil +} + // vaultToSecretData converts the entries of an InstallVault into Kubernetes secret data. // File entries produce a single key equal to the entry name containing the file content. // Field entries produce "entryName.password" and, when a username is present, "entryName.username". diff --git a/internal/installer/vault/vault_secret_creator_test.go b/internal/installer/vault/vault_secret_creator_test.go index a2dc9974..891ad1be 100644 --- a/internal/installer/vault/vault_secret_creator_test.go +++ b/internal/installer/vault/vault_secret_creator_test.go @@ -4,11 +4,66 @@ package vault import ( + "context" + "github.com/codesphere-cloud/oms/internal/installer/files" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +var _ = Describe("VaultSecretCreator", func() { + var ( + ctx context.Context + kubeClient ctrlclient.Client + creator *VaultSecretCreator + ) + + BeforeEach(func() { + ctx = context.Background() + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + kubeClient = fake.NewClientBuilder().WithScheme(scheme).Build() + creator = NewVaultSecretCreator(kubeClient) + }) + + It("creates the target namespace before syncing the vault secret", func() { + installVault := &files.InstallVault{Secrets: []files.SecretEntry{ + {Name: "registry", Fields: &files.SecretFields{Password: "first"}}, + }} + + Expect(creator.CreateSecretFromVault(ctx, installVault, VaultSecretNamespace, VaultSecretName)).To(Succeed()) + + namespace := &corev1.Namespace{} + Expect(kubeClient.Get(ctx, ctrlclient.ObjectKey{Name: VaultSecretNamespace}, namespace)).To(Succeed()) + + secret := &corev1.Secret{} + Expect(kubeClient.Get(ctx, ctrlclient.ObjectKey{Namespace: VaultSecretNamespace, Name: VaultSecretName}, secret)).To(Succeed()) + Expect(secret.Data).To(HaveKeyWithValue("registry.password", []byte("first"))) + }) + + It("keeps namespace creation idempotent when updating the vault secret", func() { + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: VaultSecretNamespace}} + Expect(kubeClient.Create(ctx, namespace)).To(Succeed()) + + installVault := &files.InstallVault{Secrets: []files.SecretEntry{ + {Name: "registry", Fields: &files.SecretFields{Password: "first"}}, + }} + Expect(creator.CreateSecretFromVault(ctx, installVault, VaultSecretNamespace, VaultSecretName)).To(Succeed()) + + installVault.SetSecret(files.SecretEntry{Name: "registry", Fields: &files.SecretFields{Password: "updated"}}) + Expect(creator.CreateSecretFromVault(ctx, installVault, VaultSecretNamespace, VaultSecretName)).To(Succeed()) + + secret := &corev1.Secret{} + Expect(kubeClient.Get(ctx, ctrlclient.ObjectKey{Namespace: VaultSecretNamespace, Name: VaultSecretName}, secret)).To(Succeed()) + Expect(secret.Data).To(HaveKeyWithValue("registry.password", []byte("updated"))) + }) +}) + var _ = Describe("vaultToSecretData", func() { It("stores file entry content under the entry name", func() { vault := &files.InstallVault{ diff --git a/internal/installer/vault/vault_suite_test.go b/internal/installer/vault/vault_suite_test.go new file mode 100644 index 00000000..7ad67417 --- /dev/null +++ b/internal/installer/vault/vault_suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package vault + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestVault(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Vault Suite") +} diff --git a/internal/util/slice.go b/internal/util/slice.go new file mode 100644 index 00000000..74d0bfc3 --- /dev/null +++ b/internal/util/slice.go @@ -0,0 +1,25 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package util + +// AppendUnique appends values that are not already present, preserving the +// order of the original slice and the additions. +func AppendUnique[T comparable](values []T, additions ...T) []T { + for _, addition := range additions { + found := false + + for _, value := range values { + if value == addition { + found = true + break + } + } + + if !found { + values = append(values, addition) + } + } + + return values +} diff --git a/internal/util/slice_test.go b/internal/util/slice_test.go new file mode 100644 index 00000000..19d3a1af --- /dev/null +++ b/internal/util/slice_test.go @@ -0,0 +1,26 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package util_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/util" +) + +var _ = Describe("AppendUnique", func() { + It("appends only values that are not already present", func() { + Expect(util.AppendUnique([]string{"one", "two"}, "two", "three", "three")). + To(Equal([]string{"one", "two", "three"})) + }) + + It("preserves a nil slice when there are no additions", func() { + Expect(util.AppendUnique[string](nil)).To(BeNil()) + }) + + It("supports comparable non-string values", func() { + Expect(util.AppendUnique([]int{1}, 2, 1)).To(Equal([]int{1, 2})) + }) +}) From 1b3366e17c0560ec5a8d9bb3d87fd8daa26598d8 Mon Sep 17 00:00:00 2001 From: NautiluX <2600004+NautiluX@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:39:06 +0000 Subject: [PATCH 2/2] chore(docs): Auto-update docs and licenses Signed-off-by: NautiluX <2600004+NautiluX@users.noreply.github.com> --- docs/oms_install_k0s.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/oms_install_k0s.md b/docs/oms_install_k0s.md index 76705e15..77777349 100644 --- a/docs/oms_install_k0s.md +++ b/docs/oms_install_k0s.md @@ -60,3 +60,4 @@ $ oms install k0s --no-download ### SEE ALSO * [oms install](oms_install.md) - Install Codesphere and other components +