feat(gcp): install k0s with k0sctl before Codesphere

cli/cmd/codesphere/install_codesphere_dependencies.go

@@ -21,6 +21,7 @@ import (
 	"github.com/spf13/cobra"
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
 	k8sscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -59,13 +60,42 @@ func installCodesphereDepencies(opts *InstallCodesphereOpts, cfg files.RootConfi
 		AutoApprove:      opts.AutoApprove,
 	}
 
+	installVault, restConfig, err := installer.VaultAndRESTConfig(opts.Vault, opts.PrivKey, cfg)
+	if err != nil {
+		return fmt.Errorf("failed to get vault and Kubernetes config: %w", err)
+	}
+
+	scheme := k8sruntime.NewScheme()
+	if err := k8sscheme.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("failed to add Kubernetes core scheme: %w", err)
+	}
+
+	if err := argov1alpha1.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("failed to add ArgoCD scheme: %w", err)
+	}
+
+	kubeClient, err := ctrlclient.New(restConfig, ctrlclient.Options{Scheme: scheme})
+	if err != nil {
+		return fmt.Errorf("failed to create Kubernetes client: %w", err)
+	}
+
+	err = stlog.Step("Ensure Codesphere prerequisites", func() error {
+		return installer.EnsureCodespherePrerequisites(context.Background(), kubeClient)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to ensure Codesphere prerequisites: %w", err)
+	}
+
 	if !installer.IsStepSkipped(cfg, opts.SkipSteps, installer.ArgoCDStep) {
-		if err := ci.ExtractAndValidatePackage(pm); err != nil {
+		err = ci.ExtractAndValidatePackage(pm)
+		if err != nil {
 			return fmt.Errorf("failed to extract and validate package: %w", err)
 		}
-		if err := stlog.Step("Install ArgoCD pre-step", func() error {
-			return installArgoCDAndApps(opts, cfg, pm, stlog)
-		}); err != nil {
+
+		err = stlog.Step("Install ArgoCD pre-step", func() error {
+			return installArgoCDAndApps(opts, cfg, pm, installVault, restConfig, kubeClient, stlog)
+		})
+		if err != nil {
 			return err
 		}
 	}
@@ -78,31 +108,17 @@ func installCodesphereDepencies(opts *InstallCodesphereOpts, cfg files.RootConfi
 
 // installArgoCDAndApps runs ArgoCD install, vault secret sync, and pc-apps install
 // before the main dependency steps.
-func installArgoCDAndApps(opts *InstallCodesphereOpts, cfg files.RootConfig, pm installer.PackageManager, stlog *bootstrap.StepLogger) error {
+func installArgoCDAndApps(opts *InstallCodesphereOpts, cfg files.RootConfig, pm installer.PackageManager, installVault *files.InstallVault, restConfig *rest.Config, kubeClient ctrlclient.Client, stlog *bootstrap.StepLogger) error {
 	var install *argocdinstaller.AppInstaller
-	if err := stlog.Substep("Load vault data", func() error {
-		installVault, restConfig, err := installer.VaultAndRESTConfig(opts.Vault, opts.PrivKey, cfg)
-		if err != nil {
-			return err
-		}
+
+	if err := stlog.Substep("Initialize ArgoCD installer", func() error {
 		registryPassword := ""
 		if secret := installVault.GetSecret(files.SecretRegistryPassword); secret != nil && secret.Fields != nil {
 			registryPassword = secret.Fields.Password
 		}
 		if registryPassword == "" {
 			return fmt.Errorf("registry password not found in vault (secret %q)", files.SecretRegistryPassword)
 		}
-		scheme := k8sruntime.NewScheme()
-		if err := k8sscheme.AddToScheme(scheme); err != nil {
-			return fmt.Errorf("failed to add kubernetes core scheme: %w", err)
-		}
-		if err := argov1alpha1.AddToScheme(scheme); err != nil {
-			return fmt.Errorf("failed to add ArgoCD scheme: %w", err)
-		}
-		kubeClient, err := ctrlclient.New(restConfig, ctrlclient.Options{Scheme: scheme})
-		if err != nil {
-			return fmt.Errorf("failed to create kubernetes client: %w", err)
-		}
 		registryURL := opts.ArgoCDRegistryURL
 		if registryURL == "" && cfg.Registry != nil {
 			registryURL = cfg.Registry.Server + "/codesphere-cloud/charts"

cli/cmd/k0s/install_k0s.go

@@ -66,7 +66,7 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) {
 			- Deploy k0s to all nodes defined in the install-config using k0sctl`),
 			Example: util.FormatExamples("install k0s", []packageio.Example{
 				{Cmd: "--install-config <path>", Desc: "Path to Codesphere install-config file to generate k0s config from"},
-				{Cmd: "--version <version>", Desc: "Version of k0s to install (e.g., v1.30.0+k0s.0)"},
+				{Cmd: "--version <version>", Desc: "Version of k0s to install (e.g., v1.31.14+k0s.0)"},
 				{Cmd: "--k0sctl-version <version>", Desc: "Version of k0sctl to use (e.g., v0.17.4)"},
 				{Cmd: "--package <file>", Desc: "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from"},
 				{Cmd: "--ssh-key-path <path>", Desc: "SSH private key path for remote installation"},
@@ -78,7 +78,7 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) {
 		Env:        env.NewEnv(),
 		FileWriter: intutil.NewFilesystemWriter(),
 	}
-	k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", "", "Version of k0s to install")
+	k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", installer.DefaultK0sVersion, "Version of k0s to install")
 	k0s.cmd.Flags().StringVar(&k0s.Opts.K0sctlVersion, "k0sctl-version", "", "Version of k0sctl to use")
 	k0s.cmd.Flags().StringVarP(&k0s.Opts.Package, "package", "p", "", "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from")
 	k0s.cmd.Flags().StringVar(&k0s.Opts.InstallConfig, "install-config", "", "Path to Codesphere install-config file (required)")

docs/oms_install_k0s.md

@@ -22,7 +22,7 @@ oms install k0s [flags]
 # Path to Codesphere install-config file to generate k0s config from
 $ oms install k0s --install-config <path>
 
-# Version of k0s to install (e.g., v1.30.0+k0s.0)
+# Version of k0s to install (e.g., v1.31.14+k0s.0)
 $ oms install k0s --version <version>
 
 # Version of k0sctl to use (e.g., v0.17.4)
@@ -54,7 +54,7 @@ $ oms install k0s --no-download
       --ssh-key-path string     SSH private key path for remote installation
       --vault string            Path to prod.vault.yaml to save the kubeconfig into (optional)
       --vault-priv-key string   Path to the age private key to decrypt the vault (optional, for SOPS-encrypted vaults)
-  -v, --version string          Version of k0s to install
+  -v, --version string          Version of k0s to install (default "v1.31.14+k0s.0")
 ```
 
 ### SEE ALSO

go.mod

@@ -5,9 +5,9 @@ go 1.26.5
 replace (
 	// GoReleaser pulls github.com/chrismellard/docker-credential-acr-env,
 	// which imports github.com/Azure/azure-sdk-for-go/version. Azure SDK
-	// v68 removed that package, so keep the legacy monorepo on a verified
-	// version that still provides it.
-	github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
+	// v68 removed that package, so keep the legacy monorepo on the version
+	// requested by the credential helper, which still provides it.
+	github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v46.4.0+incompatible
 	// argo-cd's go.mod resolves its nested gitops-engine module through a local
 	// directory replace, which does not carry over to consumers, so pin it here.
 	// The commit tagged for argo-cd v3.4.6 still imports the autoscaling v2beta*

go.sum

@@ -2711,8 +2711,8 @@ github.com/Antonboom/nilnil v1.1.2 h1:aNlFuJhaEseXe4fHO3xbjXlSeEiQVYa2lEkWD2s2hA
 github.com/Antonboom/nilnil v1.1.2/go.mod h1:0ynwvphOLmAuMwTNDyBnDZmSwZoDpcFXmUHmzoHH2WA=
 github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ=
 github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4=
-github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
-github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v46.4.0+incompatible h1:fCN6Pi+tEiEwFa8RSmtVlFHRXEZ+DJm9gfx/MKqYWw4=
+github.com/Azure/azure-sdk-for-go v46.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=

internal/bootstrap/gcp/gcp.go

@@ -351,6 +351,16 @@ func (b *GCPBootstrapper) Bootstrap() error {
 	}
 
 	if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" {
+		err = b.stlog.Step("Install k0s", b.InstallK0s)
+		if err != nil {
+			return fmt.Errorf("failed to install k0s: %w", err)
+		}
+
+		err = b.stlog.Step("Wait for k0s nodes", b.WaitForK0sNodes)
+		if err != nil {
+			return fmt.Errorf("failed waiting for k0s nodes: %w", err)
+		}
+
 		err = b.stlog.Step("Install Codesphere", b.InstallCodesphere)
 		if err != nil {
 			return fmt.Errorf("failed to install Codesphere: %w", err)
@@ -1058,52 +1068,89 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error {
 }
 
 func (b *GCPBootstrapper) InstallCodesphere() error {
-	fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox()
-	if err != nil {
+	if err := b.ensureCodespherePackageOnJumpbox(); err != nil {
 		return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err)
 	}
 
-	err = b.runInstallCommand(fullPackageFilename)
-	if err != nil {
+	if err := b.runInstallCommand(b.codespherePackageFilename()); err != nil {
 		return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err)
 	}
 
 	return nil
 }
 
-func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) {
-	packageFilename := "installer.tar.gz"
+// InstallK0s deploys k0s with the native OMS installer and stores its
+// kubeconfig in the encrypted install vault for the remaining installer steps.
+func (b *GCPBootstrapper) InstallK0s() error {
+	// Reuse matching cached binaries and let k0sctl reconcile normally. Without
+	// --force, an unchanged cluster remains untouched on bootstrap retries.
+	installCmd := fmt.Sprintf("oms install k0s --version %s --install-config /etc/codesphere/config.yaml --vault %s --vault-priv-key %s/age_key.txt",
+		installer.DefaultK0sVersion, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), b.Env.SecretsDir)
+	if err := b.Env.Jumpbox.RunSSHCommand("root", installCmd); err != nil {
+		return fmt.Errorf("failed to install k0s from jumpbox: %w", err)
+	}
+
+	return nil
+}
+
+// WaitForK0sNodes restores the readiness barrier from the TypeScript
+// Kubernetes setup. k0sctl apply completing is not sufficient for the
+// Codesphere charts: all schedulable nodes must be Ready before gateway
+// controllers and their admission webhooks are installed.
+func (b *GCPBootstrapper) WaitForK0sNodes() error {
+	const command = "k0s kubectl wait --for=condition=Ready nodes --all --timeout=30m"
+	if err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", command); err != nil {
+		return fmt.Errorf("k0s nodes did not become ready: %w", err)
+	}
+
+	return nil
+}
+
+func (b *GCPBootstrapper) codespherePackageFilename() string {
+	packageFilename := b.codespherePackageArchiveName()
+	if b.Env.InstallLocal != "" {
+		return "local-" + packageFilename
+	}
+
+	return portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFilename)
+}
+
+func (b *GCPBootstrapper) codespherePackageArchiveName() string {
 	if b.Env.RegistryType == RegistryTypeGitHub {
-		packageFilename = "installer-lite.tar.gz"
+		return "installer-lite.tar.gz"
 	}
 
+	return "installer.tar.gz"
+}
+
+func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() error {
 	if b.Env.InstallLocal != "" {
 		b.stlog.Logf("Copying local package %s to jumpbox...", b.Env.InstallLocal)
-		fullPackageFilename := fmt.Sprintf("local-%s", packageFilename)
-		err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallLocal, "/root/"+fullPackageFilename)
+
+		err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallLocal, "/root/"+b.codespherePackageFilename())
 		if err != nil {
-			return "", fmt.Errorf("failed to copy local install package to jumpbox: %w", err)
+			return fmt.Errorf("failed to copy local install package to jumpbox: %w", err)
 		}
-		return fullPackageFilename, nil
+
+		return nil
 	}
 
 	if b.Env.InstallVersion == "" {
-		return "", errors.New("either install version or a local package must be specified to install Codesphere")
+		return errors.New("either install version or a local package must be specified to install Codesphere")
 	}
 
-	fullPackageFilename := portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFilename)
 	if b.Env.InstallHash == "" {
-		return "", fmt.Errorf("install hash must be set when install version is set")
+		return fmt.Errorf("install hash must be set when install version is set")
 	}
 	b.stlog.Logf("Downloading Codesphere package...")
 	downloadCmd := fmt.Sprintf("oms download package -f %s -H %s %s",
-		packageFilename, b.Env.InstallHash, b.Env.InstallVersion)
+		b.codespherePackageArchiveName(), b.Env.InstallHash, b.Env.InstallVersion)
 	err := b.Env.Jumpbox.RunSSHCommand("root", downloadCmd)
 	if err != nil {
-		return "", fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err)
+		return fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err)
 	}
 
-	return fullPackageFilename, nil
+	return nil
 }
 
 func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error {
@@ -1114,17 +1161,20 @@ func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error {
 }
 
 func (b *GCPBootstrapper) generateSkipStepsArg() string {
-	skipSteps := b.Env.InstallSkipSteps
+	// k0s is installed by OMS before the TypeScript installer runs, so the
+	// TypeScript Kubernetes step must never run during GCP bootstrapping.
+	skipSteps := []string{"kubernetes"}
+	skipSteps = util.AppendUnique(skipSteps, b.Env.InstallSkipSteps...)
+
 	if b.Env.RegistryType == RegistryTypeGitHub {
-		skipSteps = append(skipSteps, "load-container-images")
+		skipSteps = util.AppendUnique(skipSteps, "load-container-images")
 	}
 	if len(skipSteps) == 0 {
 		return ""
 	}
 
 	return " -s " + strings.Join(skipSteps, ",")
 }
-
 func (b *GCPBootstrapper) GenerateK0sConfigScript() error {
 	script := `#!/bin/bash