From aacd865ca3f26262862ae3c774981bcaf0c7535b Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 14:59:46 +0200 Subject: [PATCH 01/11] refac(gcp): model the data centers of a bootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now a bootstrapped project was implicitly a single data center: its nodes, gateway IPs, config paths and domains all lived directly on CodesphereEnvironment. Multi-DC support needs more than one of each, so this introduces the DataCenter type that holds everything which must differ per data center, while project-level state (project, VPC, jumpbox, shared postgres node, registry) stays on the environment. BuildDataCenters derives the layout from the flags: one entry today, and with --multi-dc a second one that shares the first's PostgreSQL server. The primary data center keeps an empty resource-name suffix, so every name, path and domain a single-DC bootstrap produces is unchanged. Nothing consumes the layout yet — the callers are migrated in the following commits. Two mechanisms keep that migration safe: - ensureDataCenters derives the layout on first use and adopts state a caller passed through the legacy top-level environment fields, so every entry point works whether or not Bootstrap ran first, including infra files written before multi-DC support. - mirrorPrimaryDataCenter projects the primary data center back onto those fields before the infra file is written, so cleanup and restart-vms keep reading what they always have. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/datacenter.go | 268 ++++++++++++++++++++++ internal/bootstrap/gcp/datacenter_test.go | 105 +++++++++ internal/bootstrap/gcp/gcp.go | 68 ++++-- internal/bootstrap/gcp/infrafile.go | 6 + 4 files changed, 424 insertions(+), 23 deletions(-) create mode 100644 internal/bootstrap/gcp/datacenter.go create mode 100644 internal/bootstrap/gcp/datacenter_test.go diff --git a/internal/bootstrap/gcp/datacenter.go b/internal/bootstrap/gcp/datacenter.go new file mode 100644 index 00000000..7ef4fbcc --- /dev/null +++ b/internal/bootstrap/gcp/datacenter.go @@ -0,0 +1,268 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" +) + +// primaryDatacenterID is the ID of the first data center. It stays 1 in both modes so +// single-DC bootstraps keep their existing dataCenter.id. +const primaryDatacenterID = 1 + +// DataCenter holds the state of one Codesphere data center inside a bootstrapped GCP project. +// Project-level state (the GCP project, VPC, jumpbox, shared postgres node and container +// registry) lives on CodesphereEnvironment; everything that must differ between data centers +// lives here. +type DataCenter struct { + ID int `json:"id"` + Name string `json:"name"` + // Suffix is appended to data-center-scoped GCP resource names. It is empty for the primary + // data center, so single-DC bootstraps keep the resource names they have always used. + Suffix string `json:"suffix"` + + ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` + CephNodes []*node.Node `json:"ceph_nodes"` + + GatewayIP string `json:"gateway_ip"` + PublicGatewayIP string `json:"public_gateway_ip"` + SshProxyIP string `json:"ssh_proxy_ip"` + + // Local paths of the generated config and vault. + InstallConfigPath string `json:"-"` + SecretsFilePath string `json:"-"` + // Paths on the shared jumpbox. + RemoteConfigPath string `json:"remote_config_path"` + SecretsDir string `json:"secrets_dir"` + + WorkspaceHostingBaseDomain string `json:"workspace_hosting_base_domain"` + SshBaseDomain string `json:"ssh_base_domain"` + + // ExternalPostgres marks a data center that uses the primary data center's PostgreSQL + // server instead of installing its own. + ExternalPostgres bool `json:"external_postgres"` + + InstallConfig *files.RootConfig `json:"-"` + ExistingConfigUsed bool `json:"-"` + icg installer.InstallConfigManager `json:"-"` +} + +// IsPrimary reports whether this is the first data center of the installation. The primary data +// center owns the shared PostgreSQL server and the platform gateway that codesphere.domain +// resolves to. +func (dc *DataCenter) IsPrimary() bool { + return dc.Suffix == "" +} + +// ConfigManager returns the install config manager owning this data center's config and vault. +func (dc *DataCenter) ConfigManager() installer.InstallConfigManager { + return dc.icg +} + +// SetConfigManager assigns the install config manager for this data center. Exported for tests; +// Bootstrap assigns it via BuildDataCenters. +func (dc *DataCenter) SetConfigManager(icg installer.InstallConfigManager) { + dc.icg = icg +} + +// RemoteVaultPath returns the path of this data center's vault on the jumpbox. +func (dc *DataCenter) RemoteVaultPath() string { + return filepath.Join(dc.SecretsDir, "prod.vault.yaml") +} + +// RemoteAgeKeyPath returns the path of this data center's age identity on the jumpbox. +func (dc *DataCenter) RemoteAgeKeyPath() string { + return filepath.Join(dc.SecretsDir, "age_key.txt") +} + +// K0sConfigScriptPath returns the local filename of this data center's k0s configuration script. +func (dc *DataCenter) K0sConfigScriptPath() string { + return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix) +} + +// StepName qualifies a bootstrap step name with the data center it applies to. Single-DC +// bootstraps keep their unqualified step names. +func (dc *DataCenter) StepName(name string) string { + if dc.Suffix == "" { + return name + } + return fmt.Sprintf("%s (dc %d)", name, dc.ID) +} + +// BuildDataCenters derives the data center layout from the bootstrap environment: a single +// entry in single-DC mode, and two entries in multi-DC mode where the second one shares the +// first one's PostgreSQL server. +func BuildDataCenters(env *CodesphereEnvironment, newICG func() installer.InstallConfigManager) []*DataCenter { + if !env.MultiDC { + // A single data center keeps honouring --datacenter-id. In multi-DC mode the IDs are + // derived instead, because they drive the per-data-center domains; validateMultiDC + // rejects the combination. + id := env.DatacenterID + if id == 0 { + id = primaryDatacenterID + } + return []*DataCenter{newDataCenter(env, id, "", newICG)} + } + + return []*DataCenter{ + newDataCenter(env, primaryDatacenterID, "", newICG), + newDataCenter(env, primaryDatacenterID+1, "-dc2", newICG), + } +} + +// ensureDataCenters makes sure the environment has a usable data center layout. It derives the +// layout on first use and gives every data center an install config manager, so any entry point +// works whether or not Bootstrap ran first. +func (b *GCPBootstrapper) ensureDataCenters() { + if len(b.Env.DataCenters) > 0 { + b.ensureConfigManagers() + return + } + + b.Env.DataCenters = BuildDataCenters(b.Env, nil) + b.adoptLegacyEnvFields() + b.ensureConfigManagers() +} + +// ensureConfigManagers gives every data center an install config manager. The primary one reuses +// the bootstrapper's, so a single-DC bootstrap behaves exactly as it did before multi-DC support. +// Data centers restored from an infra file arrive without a manager, since it is not serialised. +func (b *GCPBootstrapper) ensureConfigManagers() { + newICG := b.NewConfigManager + if newICG == nil { + newICG = installer.NewInstallConfigManager + } + + for i, dc := range b.Env.DataCenters { + if dc.ConfigManager() != nil { + continue + } + if i == 0 && b.icg != nil { + dc.SetConfigManager(b.icg) + continue + } + dc.SetConfigManager(newICG()) + } +} + +// adoptLegacyEnvFields moves state that a caller supplied through the legacy top-level +// environment fields into the primary data center. Environments loaded from an infra file written +// before multi-DC support carry the primary data center's nodes and IPs there. +func (b *GCPBootstrapper) adoptLegacyEnvFields() { + primary := b.Env.DataCenters[0] + if len(primary.ControlPlaneNodes) == 0 { + primary.ControlPlaneNodes = b.Env.ControlPlaneNodes + } + if len(primary.CephNodes) == 0 { + primary.CephNodes = b.Env.CephNodes + } + if primary.GatewayIP == "" { + primary.GatewayIP = b.Env.GatewayIP + } + if primary.PublicGatewayIP == "" { + primary.PublicGatewayIP = b.Env.PublicGatewayIP + } + if primary.SshProxyIP == "" { + primary.SshProxyIP = b.Env.SshProxyIP + } + if primary.InstallConfig == nil { + primary.InstallConfig = b.Env.InstallConfig + } + // A caller that supplied a config through the environment also tells us whether it is an + // existing one, which decides between generating and regenerating secrets. + if b.Env.ExistingConfigUsed { + primary.ExistingConfigUsed = true + } +} + +// mirrorPrimaryDataCenter projects the primary data center's state onto the legacy top-level +// environment fields. Those are what the infra file exposes to `cleanup` and `restart-vms`, and +// what infra files written before multi-DC support contain. The projection is one-way and never +// read back into a DataCenter. +func (b *GCPBootstrapper) mirrorPrimaryDataCenter() { + if len(b.Env.DataCenters) == 0 { + return + } + + primary := b.primaryDC() + b.Env.ControlPlaneNodes = primary.ControlPlaneNodes + b.Env.CephNodes = primary.CephNodes + b.Env.GatewayIP = primary.GatewayIP + b.Env.PublicGatewayIP = primary.PublicGatewayIP + b.Env.SshProxyIP = primary.SshProxyIP + b.Env.InstallConfig = primary.InstallConfig + b.Env.ExistingConfigUsed = primary.ExistingConfigUsed +} + +// newDataCenter builds one data center, deriving its resource names, file paths and domains +// from the environment and the data-center suffix. +func newDataCenter(env *CodesphereEnvironment, id int, suffix string, newICG func() installer.InstallConfigManager) *DataCenter { + name := env.DatacenterName + if name == "" { + name = "dev" + } + if suffix != "" { + // The k0s cluster is named codesphere-, so the names must differ. + name += suffix + } + + dc := &DataCenter{ + ID: id, + Name: name, + Suffix: suffix, + InstallConfigPath: dcSuffixedPath(env.InstallConfigPath, suffix), + SecretsFilePath: dcSuffixedPath(env.SecretsFilePath, suffix), + RemoteConfigPath: dcSuffixedPath(remoteInstallConfigPath, suffix), + SecretsDir: env.SecretsDir + suffix, + WorkspaceHostingBaseDomain: workspaceHostingBaseDomain(env, id), + SshBaseDomain: sshBaseDomain(env, id), + ExternalPostgres: suffix != "", + } + if newICG != nil { + dc.icg = newICG() + } + + return dc +} + +// workspaceHostingBaseDomain returns the domain workspaces of the given data center are served +// from. Single-DC installations keep ws.; multi-DC installations prefix it with the +// data center ID so each data center's public gateway gets its own name. +func workspaceHostingBaseDomain(env *CodesphereEnvironment, id int) string { + if !env.MultiDC { + return "ws." + env.BaseDomain + } + return fmt.Sprintf("%d.ws.%s", id, env.BaseDomain) +} + +// sshBaseDomain returns the domain the workspace SSH proxy of the given data center is served +// from, following the same scheme as workspaceHostingBaseDomain. +func sshBaseDomain(env *CodesphereEnvironment, id int) string { + if !env.MultiDC { + return "ssh.cs." + env.BaseDomain + } + return fmt.Sprintf("%d.ssh.cs.%s", id, env.BaseDomain) +} + +// dcSuffixedPath inserts the data-center suffix before the file extension, turning +// config.yaml into config-dc2.yaml and prod.vault.yaml into prod-dc2.vault.yaml. +func dcSuffixedPath(path, suffix string) string { + if suffix == "" { + return path + } + + dir, file := filepath.Split(path) + base, ext := file, "" + if idx := strings.Index(file, "."); idx > 0 { + base, ext = file[:idx], file[idx:] + } + + return filepath.Join(dir, base+suffix+ext) +} diff --git a/internal/bootstrap/gcp/datacenter_test.go b/internal/bootstrap/gcp/datacenter_test.go new file mode 100644 index 00000000..76522bbf --- /dev/null +++ b/internal/bootstrap/gcp/datacenter_test.go @@ -0,0 +1,105 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" + "github.com/codesphere-cloud/oms/internal/installer" +) + +var _ = Describe("BuildDataCenters", func() { + newEnv := func(multiDC bool) *gcp.CodesphereEnvironment { + return &gcp.CodesphereEnvironment{ + MultiDC: multiDC, + BaseDomain: "example.com", + DatacenterName: "dev", + SecretsDir: "/etc/codesphere/secrets", + InstallConfigPath: "config.yaml", + SecretsFilePath: "prod.vault.yaml", + } + } + + Context("single data center", func() { + It("keeps the paths, secrets dir and domains a single-DC bootstrap has always used", func() { + dcs := gcp.BuildDataCenters(newEnv(false), installer.NewInstallConfigManager) + + Expect(dcs).To(HaveLen(1)) + dc := dcs[0] + Expect(dc.IsPrimary()).To(BeTrue()) + Expect(dc.ID).To(Equal(1)) + Expect(dc.Name).To(Equal("dev")) + Expect(dc.Suffix).To(BeEmpty()) + Expect(dc.InstallConfigPath).To(Equal("config.yaml")) + Expect(dc.SecretsFilePath).To(Equal("prod.vault.yaml")) + Expect(dc.RemoteConfigPath).To(Equal("/etc/codesphere/config.yaml")) + Expect(dc.SecretsDir).To(Equal("/etc/codesphere/secrets")) + Expect(dc.RemoteVaultPath()).To(Equal("/etc/codesphere/secrets/prod.vault.yaml")) + Expect(dc.RemoteAgeKeyPath()).To(Equal("/etc/codesphere/secrets/age_key.txt")) + Expect(dc.K0sConfigScriptPath()).To(Equal("configure-k0s.sh")) + Expect(dc.WorkspaceHostingBaseDomain).To(Equal("ws.example.com")) + Expect(dc.SshBaseDomain).To(Equal("ssh.cs.example.com")) + Expect(dc.ExternalPostgres).To(BeFalse()) + Expect(dc.StepName("Encrypt vault")).To(Equal("Encrypt vault")) + }) + }) + + Context("multi data center", func() { + var dcs []*gcp.DataCenter + + BeforeEach(func() { + dcs = gcp.BuildDataCenters(newEnv(true), installer.NewInstallConfigManager) + }) + + It("builds two data centers with the second sharing the first's postgres", func() { + Expect(dcs).To(HaveLen(2)) + Expect(dcs[0].ExternalPostgres).To(BeFalse()) + Expect(dcs[1].ExternalPostgres).To(BeTrue()) + }) + + It("leaves the primary data center's resource names unsuffixed", func() { + Expect(dcs[0].Suffix).To(BeEmpty()) + Expect(dcs[0].InstallConfigPath).To(Equal("config.yaml")) + Expect(dcs[0].SecretsDir).To(Equal("/etc/codesphere/secrets")) + }) + + It("gives the secondary data center its own name, paths and secrets dir", func() { + dc := dcs[1] + Expect(dc.IsPrimary()).To(BeFalse()) + Expect(dc.ID).To(Equal(2)) + // The k0s cluster is named codesphere-, so the names must differ. + Expect(dc.Name).To(Equal("dev-dc2")) + Expect(dc.InstallConfigPath).To(Equal("config-dc2.yaml")) + Expect(dc.SecretsFilePath).To(Equal("prod-dc2.vault.yaml")) + Expect(dc.RemoteConfigPath).To(Equal("/etc/codesphere/config-dc2.yaml")) + // A separate secrets dir, so the installer cannot overwrite the primary's kubeconfig + // and ceph credentials through config.secrets.baseDir. + Expect(dc.SecretsDir).To(Equal("/etc/codesphere/secrets-dc2")) + Expect(dc.RemoteVaultPath()).To(Equal("/etc/codesphere/secrets-dc2/prod.vault.yaml")) + Expect(dc.RemoteAgeKeyPath()).To(Equal("/etc/codesphere/secrets-dc2/age_key.txt")) + Expect(dc.K0sConfigScriptPath()).To(Equal("configure-k0s-dc2.sh")) + Expect(dc.StepName("Encrypt vault")).To(Equal("Encrypt vault (dc 2)")) + }) + + It("scopes the workspace and ssh domains per data center", func() { + Expect(dcs[0].WorkspaceHostingBaseDomain).To(Equal("1.ws.example.com")) + Expect(dcs[0].SshBaseDomain).To(Equal("1.ssh.cs.example.com")) + Expect(dcs[1].WorkspaceHostingBaseDomain).To(Equal("2.ws.example.com")) + Expect(dcs[1].SshBaseDomain).To(Equal("2.ssh.cs.example.com")) + }) + + It("gives each data center its own config manager", func() { + Expect(dcs[0].ConfigManager()).NotTo(BeIdenticalTo(dcs[1].ConfigManager())) + }) + }) + + It("falls back to the dev datacenter name", func() { + env := newEnv(false) + env.DatacenterName = "" + + Expect(gcp.BuildDataCenters(env, installer.NewInstallConfigManager)[0].Name).To(Equal("dev")) + }) +}) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 19eb5c30..e8ef40ce 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -96,18 +96,36 @@ type GCPBootstrapper struct { NodeClient node.NodeClient PortalClient portal.Portal GitHubClient github.GitHubClient + // NewConfigManager creates the install config manager of a data center. Each data center + // owns its own config and vault, so multi-DC bootstraps need more than one. + NewConfigManager func() installer.InstallConfigManager +} + +// primaryDC returns the first data center, which owns the shared PostgreSQL server and the +// platform gateway that codesphere.domain resolves to. +func (b *GCPBootstrapper) primaryDC() *DataCenter { + return b.Env.DataCenters[0] } type CodesphereEnvironment struct { - ProjectID string `json:"project_id"` - ProjectTTL string `json:"project_ttl"` - ProjectName string `json:"project_name"` - DNSProjectID string `json:"dns_project_id"` - Jumpbox *node.Node `json:"jumpbox"` - PostgreSQLNode *node.Node `json:"postgres_node"` - ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` - CephNodes []*node.Node `json:"ceph_nodes"` - ContainerRegistryURL string `json:"-"` + ProjectID string `json:"project_id"` + ProjectTTL string `json:"project_ttl"` + ProjectName string `json:"project_name"` + DNSProjectID string `json:"dns_project_id"` + Jumpbox *node.Node `json:"jumpbox"` + PostgreSQLNode *node.Node `json:"postgres_node"` + // MultiDC bootstraps two data centers that share the PostgreSQL server but run separate + // Kubernetes and Ceph clusters. + MultiDC bool `json:"multi_dc"` + // DataCenters holds the per-data-center state. It always has at least one entry. + DataCenters []*DataCenter `json:"datacenters"` + // Mirrors of DataCenters[0], written for infra files consumed by cleanup and restart-vms. + ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` + CephNodes []*node.Node `json:"ceph_nodes"` + // ContainerRegistryURL is the resolved registry server all data centers pull images from. + ContainerRegistryURL string `json:"container_registry_url,omitempty"` + RegistryUsername string `json:"-"` + RegistryPassword string `json:"-"` ExistingConfigUsed bool `json:"-"` InstallVersion string `json:"install_version"` InstallLocal string `json:"install_local"` @@ -181,10 +199,13 @@ type CodesphereEnvironment struct { SSHPrivateKeyPath string `json:"-"` DatacenterID int `json:"-"` DatacenterName string `json:"-"` - CustomPgIP string `json:"custom_pg_ip"` - Region string `json:"region"` - Zone string `json:"zone"` - DNSZoneName string `json:"dns_zone_name"` + // DatacenterIDExplicit records whether --datacenter-id was set on the command line. The + // value alone cannot distinguish the default 1 from an explicit 1. + DatacenterIDExplicit bool `json:"-"` + CustomPgIP string `json:"custom_pg_ip"` + Region string `json:"region"` + Zone string `json:"zone"` + DNSZoneName string `json:"dns_zone_name"` // Test user creation CreateTestUser bool `json:"-"` @@ -208,16 +229,17 @@ func NewGCPBootstrapper( gitHubClient github.GitHubClient, ) (*GCPBootstrapper, error) { return &GCPBootstrapper{ - ctx: ctx, - stlog: stlog, - fw: fw, - icg: icg, - GCPClient: gcpClient, - Env: CodesphereEnv, - NodeClient: sshRunner, - PortalClient: portalClient, - Time: time, - GitHubClient: gitHubClient, + ctx: ctx, + stlog: stlog, + fw: fw, + icg: icg, + GCPClient: gcpClient, + Env: CodesphereEnv, + NodeClient: sshRunner, + PortalClient: portalClient, + Time: time, + GitHubClient: gitHubClient, + NewConfigManager: installer.NewInstallConfigManager, }, nil } diff --git a/internal/bootstrap/gcp/infrafile.go b/internal/bootstrap/gcp/infrafile.go index 72378f9f..f7f6d855 100644 --- a/internal/bootstrap/gcp/infrafile.go +++ b/internal/bootstrap/gcp/infrafile.go @@ -40,6 +40,12 @@ func LoadInfraFile(fw util.FileIO, infraFilePath string) (CodesphereEnvironment, // WriteInfraFile writes details about the bootstrapped codesphere environment into a file. func (b *GCPBootstrapper) WriteInfraFile() error { + b.ensureDataCenters() + + // The legacy top-level node and IP fields are what cleanup and restart-vms read, so keep + // them in sync with the primary data center before serialising. + b.mirrorPrimaryDataCenter() + envBytes, err := json.MarshalIndent(b.Env, "", " ") if err != nil { return fmt.Errorf("failed to marshal codesphere env: %w", err) From a7638463c8d7a6095a0eb23ac4bc1ad54e1c1bbc Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:00:29 +0200 Subject: [PATCH 02/11] feat(gcp): derive the VM definitions per data center MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The VM list was a package-level global describing one data center. It becomes a function of the environment: the project-shared jumpbox and postgres VMs plus each data center's three Ceph and three k0s nodes, whose names carry that data center's suffix. A single-DC bootstrap therefore still gets exactly ceph-1..3 and k0s-1..3. EnsureComputeInstances routes each instance into its own data center by the definition's DataCenterID instead of inferring placement from the VM's tag, and sorts each data center's nodes independently — the install config assigns roles by index. restart-vms resolves the valid VM names from the infra file's data center layout, so a node of a second data center can be restarted by name. It now also reads the infra file when --project-id and --zone are given, best-effort, since only the file knows the layout. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- cli/cmd/bootstrap_gcp_restart_vms.go | 53 ++++--- docs/oms_beta_bootstrap-gcp_restart-vms.md | 3 + internal/bootstrap/gcp/gce.go | 153 +++++++++++++++----- internal/bootstrap/gcp/gce_test.go | 55 +++++++ internal/bootstrap/gcp/test_helpers_test.go | 9 ++ 5 files changed, 212 insertions(+), 61 deletions(-) diff --git a/cli/cmd/bootstrap_gcp_restart_vms.go b/cli/cmd/bootstrap_gcp_restart_vms.go index 900b9353..a35ffebb 100644 --- a/cli/cmd/bootstrap_gcp_restart_vms.go +++ b/cli/cmd/bootstrap_gcp_restart_vms.go @@ -28,32 +28,46 @@ type BootstrapGcpRestartVMsOpts struct { Name string } -// resolveProjectAndZone returns the project ID and zone from flags or the infra file. -// If both flags are set they are used directly; if neither is set, the infra file is read. -// Providing only one of --project-id / --zone is an error. -func (c *BootstrapGcpRestartVMsCmd) resolveProjectAndZone(fw intutil.FileIO) (string, string, error) { +// resolveEnvironment returns the environment to restart VMs in. Project ID and zone come from +// the flags or, when neither is set, from the infra file. Providing only one of +// --project-id / --zone is an error. +// +// The data center layout always comes from the infra file, since it determines the VM names. It +// is read best-effort when the flags supply project and zone, in which case a missing file just +// means single-data-center names. +func (c *BootstrapGcpRestartVMsCmd) resolveEnvironment(fw intutil.FileIO) (*gcp.CodesphereEnvironment, error) { projectID := c.Opts.ProjectID zone := c.Opts.Zone if (projectID == "") != (zone == "") { - return "", "", fmt.Errorf("--project-id and --zone must be provided together") - } - if projectID != "" { - return projectID, zone, nil + return nil, fmt.Errorf("--project-id and --zone must be provided together") } infraFilePath := gcp.GetInfraFilePath() infraEnv, exists, err := gcp.LoadInfraFile(fw, infraFilePath) if err != nil { - return "", "", fmt.Errorf("failed to load infra file: %w", err) - } - if !exists { - return "", "", fmt.Errorf("infra file not found at %s; use --project-id and --zone flags", infraFilePath) + if projectID == "" { + return nil, fmt.Errorf("failed to load infra file: %w", err) + } + log.Printf("Warning: %v", err) } - if infraEnv.ProjectID == "" || infraEnv.Zone == "" { - return "", "", fmt.Errorf("infra file is missing project ID or zone; use --project-id and --zone flags") + + if projectID == "" { + if !exists { + return nil, fmt.Errorf("infra file not found at %s; use --project-id and --zone flags", infraFilePath) + } + if infraEnv.ProjectID == "" || infraEnv.Zone == "" { + return nil, fmt.Errorf("infra file is missing project ID or zone; use --project-id and --zone flags") + } + projectID, zone = infraEnv.ProjectID, infraEnv.Zone } - return infraEnv.ProjectID, infraEnv.Zone, nil + + return &gcp.CodesphereEnvironment{ + ProjectID: projectID, + Zone: zone, + MultiDC: infraEnv.MultiDC, + DataCenters: infraEnv.DataCenters, + }, nil } func (c *BootstrapGcpRestartVMsCmd) RunE(_ *cobra.Command, _ []string) error { @@ -61,18 +75,14 @@ func (c *BootstrapGcpRestartVMsCmd) RunE(_ *cobra.Command, _ []string) error { stlog := bootstrap.NewStepLogger(false) fw := intutil.NewFilesystemWriter() - projectID, zone, err := c.resolveProjectAndZone(fw) + csEnv, err := c.resolveEnvironment(fw) if err != nil { return err } + projectID, zone := csEnv.ProjectID, csEnv.Zone gcpClient := gcp.NewGCPClient(ctx, stlog, os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")) - csEnv := &gcp.CodesphereEnvironment{ - ProjectID: projectID, - Zone: zone, - } - bs, err := gcp.NewGCPBootstrapper( ctx, nil, stlog, csEnv, nil, gcpClient, fw, nil, nil, intutil.NewTime(), nil, @@ -112,6 +122,7 @@ func AddBootstrapGcpRestartVMsCmd(bootstrapGcp *cobra.Command, opts *util.Global {Desc: "Restart all VMs using project info from the local infra file"}, {Cmd: "--name jumpbox", Desc: "Restart only the jumpbox VM"}, {Cmd: "--name k0s-1", Desc: "Restart a specific k0s node"}, + {Cmd: "--name k0s-1-dc2", Desc: "Restart a node of the second data center of a --multi-dc bootstrap"}, {Cmd: "--project-id my-project --zone us-central1-a", Desc: "Restart all VMs with explicit project and zone"}, {Cmd: "--project-id my-project --zone us-central1-a --name ceph-1", Desc: "Restart a specific VM with explicit project and zone"}, }), diff --git a/docs/oms_beta_bootstrap-gcp_restart-vms.md b/docs/oms_beta_bootstrap-gcp_restart-vms.md index 9eeb152e..0f723624 100644 --- a/docs/oms_beta_bootstrap-gcp_restart-vms.md +++ b/docs/oms_beta_bootstrap-gcp_restart-vms.md @@ -26,6 +26,9 @@ $ oms beta bootstrap-gcp restart-vms --name jumpbox # Restart a specific k0s node $ oms beta bootstrap-gcp restart-vms --name k0s-1 +# Restart a node of the second data center of a --multi-dc bootstrap +$ oms beta bootstrap-gcp restart-vms --name k0s-1-dc2 + # Restart all VMs with explicit project and zone $ oms beta bootstrap-gcp restart-vms --project-id my-project --zone us-central1-a diff --git a/internal/bootstrap/gcp/gce.go b/internal/bootstrap/gcp/gce.go index a7b7fc66..7314749d 100644 --- a/internal/bootstrap/gcp/gce.go +++ b/internal/bootstrap/gcp/gce.go @@ -24,18 +24,69 @@ type VMDef struct { Tags []string AdditionalDisks []int64 ExternalIP bool + // DataCenterID is the data center the VM belongs to, or 0 for the project-shared VMs + // (jumpbox and postgres) that every data center uses. + DataCenterID int } -// Example VM definitions (expand as needed) -var vmDefs = []VMDef{ - {"jumpbox", "e2-medium", []string{"jumpbox", "ssh"}, []int64{}, true}, - {"postgres", "e2-standard-2", []string{"postgres"}, []int64{}, true}, - {"ceph-1", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"ceph-2", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"ceph-3", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"k0s-1", "e2-standard-8", []string{"k0s"}, []int64{}, false}, - {"k0s-2", "e2-standard-8", []string{"k0s"}, []int64{}, false}, - {"k0s-3", "e2-standard-8", []string{"k0s"}, []int64{}, false}, +// cephNodesPerDataCenter and k0sNodesPerDataCenter are the per-data-center node counts. Three +// Ceph nodes are the minimum for replication; three k0s nodes give one control plane and three +// workers, as written into the install config. +const ( + cephNodesPerDataCenter = 3 + k0sNodesPerDataCenter = 3 +) + +// sharedVMDefs returns the VMs that exist once per project, regardless of how many data centers +// are bootstrapped. The postgres node hosts the database both data centers share. +func sharedVMDefs() []VMDef { + return []VMDef{ + {Name: "jumpbox", MachineType: "e2-medium", Tags: []string{"jumpbox", "ssh"}, AdditionalDisks: []int64{}, ExternalIP: true}, + {Name: "postgres", MachineType: "e2-standard-2", Tags: []string{"postgres"}, AdditionalDisks: []int64{}, ExternalIP: true}, + } +} + +// dataCenterVMDefs returns the Ceph and k0s VMs of one data center. The suffix is empty for the +// primary data center, so single-DC bootstraps keep the names ceph-1..3 and k0s-1..3. +func dataCenterVMDefs(dcID int, suffix string) []VMDef { + defs := make([]VMDef, 0, cephNodesPerDataCenter+k0sNodesPerDataCenter) + for i := 1; i <= cephNodesPerDataCenter; i++ { + defs = append(defs, VMDef{ + Name: fmt.Sprintf("ceph-%d%s", i, suffix), + MachineType: "e2-standard-8", + Tags: []string{"ceph"}, + AdditionalDisks: []int64{10, 100}, + DataCenterID: dcID, + }) + } + for i := 1; i <= k0sNodesPerDataCenter; i++ { + defs = append(defs, VMDef{ + Name: fmt.Sprintf("k0s-%d%s", i, suffix), + MachineType: "e2-standard-8", + Tags: []string{"k0s"}, + AdditionalDisks: []int64{}, + DataCenterID: dcID, + }) + } + + return defs +} + +// VMDefsForEnv returns every VM definition of the environment: the project-shared VMs plus the +// Ceph and k0s VMs of each data center. When the environment carries no data centers — as with +// an infra file written before multi-DC support — it falls back to a single unsuffixed one. +func VMDefsForEnv(env *CodesphereEnvironment) []VMDef { + defs := sharedVMDefs() + + dcs := env.DataCenters + if len(dcs) == 0 { + dcs = []*DataCenter{{ID: primaryDatacenterID}} + } + for _, dc := range dcs { + defs = append(defs, dataCenterVMDefs(dc.ID, dc.Suffix)...) + } + + return defs } // validateVMProvisioningOptions checks that spot and preemptible options are not both set @@ -51,16 +102,20 @@ type vmResult struct { name string externalIP string internalIP string + dcID int } // EnsureComputeInstances ensures that all required compute instances are present and running. func (b *GCPBootstrapper) EnsureComputeInstances() error { + b.ensureDataCenters() + + vms := VMDefsForEnv(b.Env) wg := sync.WaitGroup{} - errCh := make(chan error, len(vmDefs)) - resultCh := make(chan vmResult, len(vmDefs)) - logCh := make(chan string, len(vmDefs)) + errCh := make(chan error, len(vms)) + resultCh := make(chan vmResult, len(vms)) + logCh := make(chan string, len(vms)) - for _, vm := range vmDefs { + for _, vm := range vms { wg.Add(1) go func(vm VMDef) { defer wg.Done() @@ -95,6 +150,12 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error { NodeClient: b.NodeClient, FileIO: b.fw, } + dcByID := map[int]*DataCenter{} + for _, dc := range b.Env.DataCenters { + dc.CephNodes = nil + dc.ControlPlaneNodes = nil + dcByID[dc.ID] = dc + } for result := range resultCh { switch result.vmType { case "jumpbox": @@ -102,22 +163,31 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error { case "postgres": b.Env.PostgreSQLNode = b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) case "ceph": - node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) - b.Env.CephNodes = append(b.Env.CephNodes, node) + dc, ok := dcByID[result.dcID] + if !ok { + return fmt.Errorf("instance %s belongs to unknown data center %d", result.name, result.dcID) + } + dc.CephNodes = append(dc.CephNodes, b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)) case "k0s": - node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) - b.Env.ControlPlaneNodes = append(b.Env.ControlPlaneNodes, node) + dc, ok := dcByID[result.dcID] + if !ok { + return fmt.Errorf("instance %s belongs to unknown data center %d", result.name, result.dcID) + } + dc.ControlPlaneNodes = append(dc.ControlPlaneNodes, b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)) } } - //sort ceph nodes by name to ensure consistent ordering - sort.Slice(b.Env.CephNodes, func(i, j int) bool { - return b.Env.CephNodes[i].GetName() < b.Env.CephNodes[j].GetName() - }) - //sort control plane nodes by name to ensure consistent ordering - sort.Slice(b.Env.ControlPlaneNodes, func(i, j int) bool { - return b.Env.ControlPlaneNodes[i].GetName() < b.Env.ControlPlaneNodes[j].GetName() - }) + // Sort each data center's nodes by name to ensure consistent ordering, since the install + // config assigns roles by index. + for _, dc := range b.Env.DataCenters { + sort.Slice(dc.CephNodes, func(i, j int) bool { + return dc.CephNodes[i].GetName() < dc.CephNodes[j].GetName() + }) + sort.Slice(dc.ControlPlaneNodes, func(i, j int) bool { + return dc.ControlPlaneNodes[i].GetName() < dc.ControlPlaneNodes[j].GetName() + }) + } + b.mirrorPrimaryDataCenter() return nil } @@ -163,6 +233,7 @@ func (b *GCPBootstrapper) ensureVM(vm VMDef, rootDiskSize int64, logCh chan<- st name: vm.Name, externalIP: externalIP, internalIP: internalIP, + dcID: vm.DataCenterID, }, nil } @@ -362,30 +433,32 @@ func (b *GCPBootstrapper) waitForInstanceRunning(projectID, zone, name string, n name, pollInterval*time.Duration(maxAttempts)) } -// findVMDef looks up a VM definition by name. Returns nil if not found. -func findVMDef(name string) *VMDef { - for _, vm := range vmDefs { - if vm.Name == name { - return &vm +// findVMDef looks up a VM definition by name among the given definitions. Returns nil if not +// found. +func findVMDef(defs []VMDef, name string) *VMDef { + for i := range defs { + if defs[i].Name == name { + return &defs[i] } } return nil } -// validVMNames returns the list of known VM names from vmDefs. -func validVMNames() []string { - names := make([]string, len(vmDefs)) - for i, vm := range vmDefs { +// validVMNames returns the names of the given VM definitions. +func validVMNames(defs []VMDef) []string { + names := make([]string, len(defs)) + for i, vm := range defs { names[i] = vm.Name } return names } -// RestartVM restarts a single stopped or terminated VM by a name that is defined in vmDefs. +// RestartVM restarts a single stopped or terminated VM by a name defined for this environment. func (b *GCPBootstrapper) RestartVM(name string) error { - vm := findVMDef(name) + defs := VMDefsForEnv(b.Env) + vm := findVMDef(defs, name) if vm == nil { - return fmt.Errorf("unknown VM name %q; valid names are: %s", name, strings.Join(validVMNames(), ", ")) + return fmt.Errorf("unknown VM name %q; valid names are: %s", name, strings.Join(validVMNames(defs), ", ")) } projectID := b.Env.ProjectID @@ -424,10 +497,10 @@ func (b *GCPBootstrapper) RestartVM(name string) error { return nil } -// RestartVMs restarts all stopped or terminated VMs defined in vmDefs. +// RestartVMs restarts all stopped or terminated VMs of the environment, across every data center. func (b *GCPBootstrapper) RestartVMs() error { var errs []error - for _, vm := range vmDefs { + for _, vm := range VMDefsForEnv(b.Env) { if err := b.RestartVM(vm.Name); err != nil { errs = append(errs, err) } diff --git a/internal/bootstrap/gcp/gce_test.go b/internal/bootstrap/gcp/gce_test.go index 4d6a1f5f..73dee10c 100644 --- a/internal/bootstrap/gcp/gce_test.go +++ b/internal/bootstrap/gcp/gce_test.go @@ -23,6 +23,61 @@ import ( var _ = Describe("GCE", func() { + Describe("VMDefsForEnv", func() { + It("keeps the names a single-data-center bootstrap has always used", func() { + env := &gcp.CodesphereEnvironment{} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + defs := gcp.VMDefsForEnv(env) + + Expect(vmNames(defs)).To(Equal([]string{ + "jumpbox", "postgres", + "ceph-1", "ceph-2", "ceph-3", + "k0s-1", "k0s-2", "k0s-3", + })) + }) + + It("adds suffixed ceph and k0s nodes per additional data center", func() { + env := &gcp.CodesphereEnvironment{MultiDC: true} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + defs := gcp.VMDefsForEnv(env) + + Expect(vmNames(defs)).To(Equal([]string{ + "jumpbox", "postgres", + "ceph-1", "ceph-2", "ceph-3", + "k0s-1", "k0s-2", "k0s-3", + "ceph-1-dc2", "ceph-2-dc2", "ceph-3-dc2", + "k0s-1-dc2", "k0s-2-dc2", "k0s-3-dc2", + })) + }) + + It("assigns the shared VMs to no data center and the rest to theirs", func() { + env := &gcp.CodesphereEnvironment{MultiDC: true} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + byName := map[string]int{} + for _, def := range gcp.VMDefsForEnv(env) { + byName[def.Name] = def.DataCenterID + } + + Expect(byName["jumpbox"]).To(BeZero()) + Expect(byName["postgres"]).To(BeZero()) + Expect(byName["ceph-1"]).To(Equal(1)) + Expect(byName["k0s-3"]).To(Equal(1)) + Expect(byName["ceph-1-dc2"]).To(Equal(2)) + Expect(byName["k0s-3-dc2"]).To(Equal(2)) + }) + + // Infra files written before multi-DC support carry no data center list. + It("falls back to a single unsuffixed data center when the environment has none", func() { + defs := gcp.VMDefsForEnv(&gcp.CodesphereEnvironment{}) + + Expect(vmNames(defs)).To(ContainElement("k0s-1")) + Expect(vmNames(defs)).To(HaveLen(8)) + }) + }) + Describe("IsNotFoundError", func() { Context("when error is nil", func() { It("should return false", func() { diff --git a/internal/bootstrap/gcp/test_helpers_test.go b/internal/bootstrap/gcp/test_helpers_test.go index e7131a4b..2d9629c6 100644 --- a/internal/bootstrap/gcp/test_helpers_test.go +++ b/internal/bootstrap/gcp/test_helpers_test.go @@ -24,6 +24,15 @@ import ( func protoString(s string) *string { return &s } +// vmNames returns the names of the given VM definitions, in order. +func vmNames(defs []gcp.VMDef) []string { + names := make([]string, len(defs)) + for i, def := range defs { + names[i] = def.Name + } + return names +} + // makeInstance creates a computepb.Instance with the given status and IPs. func makeInstance(status, internalIP, externalIP string) *computepb.Instance { inst := &computepb.Instance{ From 58f659e78da94ac50aca45d5a360bccb72648120 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:03:33 +0200 Subject: [PATCH 03/11] feat(gcp): configure hosts, gateways and installs per data center MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrates the infrastructure steps from the single implicit data center to the layout: root login and host configuration run over every data center's nodes, each data center reserves its own gateway, public gateway and SSH proxy IP under a suffixed name, and each gets its own k0s configuration script patching its own gateway services. InstallCodesphere and RunK0sConfigScript now loop over the data centers in ascending order and log a step per data center. The order matters once there is more than one: the primary data center's install creates the database, roles and schema the others reuse, and a k0s script can only patch gateway services an install has already created. The install command moves into the exported InstallCommand, since the CLI prints it for the operator when the bootstrap does not install itself, and it now names the data center's own config, vault and age key. EnsureHostsConfigured also creates /etc/codesphere/secrets up front on every node: the installer uploads a data center's age key to that fixed path but only creates its own configured secrets.baseDir, so for a data center whose baseDir differs the upload target would not exist. Behaviour for a single data center is unchanged — same VM names, same IP names, same script, same install command. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/gcp.go | 211 +++++++++++++++++++++++------ internal/bootstrap/gcp/gcp_test.go | 65 +++++++++ 2 files changed, 233 insertions(+), 43 deletions(-) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index e8ef40ce..72aaa895 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "log" - "path/filepath" "slices" "strings" "time" @@ -36,6 +35,24 @@ const ( RegistryTypeGitHub RegistryType = "github" ) +// remoteK0sConfigScriptPath is where each data center's k0s configuration script is placed on +// that data center's first control plane node. Data centers have separate nodes, so the path can +// be the same for all of them. +const remoteK0sConfigScriptPath = "/root/configure-k0s.sh" + +// installerNodeSecretsDir is where the Codesphere installer uploads a data center's age key on +// every one of that data center's nodes. The path is fixed even though the installer reads the key +// from the data center's own secrets.baseDir on the jumpbox, and the installer only creates +// baseDir on the node — so for a data center whose baseDir differs, the upload target would not +// exist. Creating it up front is harmless: data centers have separate nodes, so a node only ever +// holds its own data center's key. +const installerNodeSecretsDir = "/etc/codesphere/secrets" + +// vpcSubnetCIDR is the range of the project's single subnet, shared by all data centers. It is +// also each data center's ceph.nodesSubnet; their Ceph clusters stay separate because each has +// its own hosts, monitors and FSID. +const vpcSubnetCIDR = "10.10.0.0/20" + // CheckOMSManagedLabel checks if the given labels map indicates an OMS-managed project. // A project is considered OMS-managed if it has the 'oms-managed' label set to "true". func CheckOMSManagedLabel(labels map[string]string) bool { @@ -107,6 +124,30 @@ func (b *GCPBootstrapper) primaryDC() *DataCenter { return b.Env.DataCenters[0] } +// allNodes returns every node of the project: the jumpbox, the shared postgres node and all +// data centers' Ceph and k0s nodes. +func (b *GCPBootstrapper) allNodes() []*node.Node { + nodes := []*node.Node{b.Env.Jumpbox, b.Env.PostgreSQLNode} + for _, dc := range b.Env.DataCenters { + nodes = append(nodes, dc.ControlPlaneNodes...) + nodes = append(nodes, dc.CephNodes...) + } + + return nodes +} + +// clusterNodes returns every Ceph and k0s node of all data centers, i.e. all nodes except the +// jumpbox and the shared postgres node. +func (b *GCPBootstrapper) clusterNodes() []*node.Node { + nodes := []*node.Node{} + for _, dc := range b.Env.DataCenters { + nodes = append(nodes, dc.ControlPlaneNodes...) + nodes = append(nodes, dc.CephNodes...) + } + + return nodes +} + type CodesphereEnvironment struct { ProjectID string `json:"project_id"` ProjectTTL string `json:"project_ttl"` @@ -373,12 +414,14 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" { - err = b.stlog.Step("Install Codesphere", b.InstallCodesphere) + err = b.InstallCodesphere() if err != nil { return fmt.Errorf("failed to install Codesphere: %w", err) } - err = b.stlog.Step("Run k0s config script", b.RunK0sConfigScript) + // Every data center is installed before any k0s script runs, so a script never patches + // gateway services an install has yet to create. + err = b.RunK0sConfigScript() if err != nil { return fmt.Errorf("failed to run k0s config script: %w", err) } @@ -393,8 +436,11 @@ func (b *GCPBootstrapper) Bootstrap() error { return nil } -// createTestUser creates a test user in the PostgreSQL instance using the testuser package and logs the credentials. +// createTestUser creates a test user in the shared PostgreSQL instance using the testuser package +// and logs the credentials. The user's team is homed in the primary data center. func (b *GCPBootstrapper) createTestUser() error { + b.ensureDataCenters() + if b.Env.PostgreSQLNode == nil { return fmt.Errorf("postgres node not found in bootstrap environment") } @@ -404,10 +450,11 @@ func (b *GCPBootstrapper) createTestUser() error { return fmt.Errorf("postgres node has no external IP") } - if b.Env.InstallConfig == nil { + primary := b.primaryDC() + if primary.InstallConfig == nil { return fmt.Errorf("install config not found in bootstrap environment") } - pgPasswordSecret := b.icg.GetVault().GetSecret(files.SecretPostgresPassword) + pgPasswordSecret := primary.ConfigManager().GetVault().GetSecret(files.SecretPostgresPassword) if pgPasswordSecret == nil || pgPasswordSecret.Fields == nil { return fmt.Errorf("postgres admin password not found in vault") } @@ -420,7 +467,7 @@ func (b *GCPBootstrapper) createTestUser() error { Password: pgPassword, DBName: testuser.DefaultDBName, SSLMode: "require", - DatacenterID: b.Env.DatacenterID, + DatacenterID: primary.ID, }) if err != nil { return err @@ -707,7 +754,7 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error { Allowed: []*computepb.Allowed{ {IPProtocol: protoString("all")}, }, - SourceRanges: []string{"10.10.0.0/20"}, + SourceRanges: []string{vpcSubnetCIDR}, Description: protoString("Allow all internal traffic"), } err = b.GCPClient.CreateFirewallRule(b.Env.ProjectID, internalRule) @@ -770,19 +817,34 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error { return nil } -// EnsureGatewayIPAddresses reserves the static external IP addresses for the ingress -// controllers of the cluster (gateway and public gateway) and the SSH workspace proxy. +// EnsureGatewayIPAddresses reserves the static external IP addresses of every data center: the +// ingress controllers of its cluster (gateway and public gateway) and its SSH workspace proxy. func (b *GCPBootstrapper) EnsureGatewayIPAddresses() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + if err := b.ensureGatewayIPAddresses(dc); err != nil { + return err + } + } + b.mirrorPrimaryDataCenter() + + return nil +} + +// ensureGatewayIPAddresses reserves one data center's static external IP addresses. Their names +// carry the data-center suffix, so the primary data center keeps the unsuffixed names. +func (b *GCPBootstrapper) ensureGatewayIPAddresses(dc *DataCenter) error { var err error - b.Env.GatewayIP, err = b.EnsureExternalIP("gateway") + dc.GatewayIP, err = b.EnsureExternalIP("gateway" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure gateway IP: %w", err) } - b.Env.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway") + dc.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure public gateway IP: %w", err) } - b.Env.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy") + dc.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure ssh proxy IP: %w", err) } @@ -823,14 +885,9 @@ func (b *GCPBootstrapper) EnsureExternalIP(name string) (string, error) { } func (b *GCPBootstrapper) EnsureRootLoginEnabled() error { - allNodes := []*node.Node{ - b.Env.Jumpbox, - } - allNodes = append(allNodes, b.Env.ControlPlaneNodes...) - allNodes = append(allNodes, b.Env.PostgreSQLNode) - allNodes = append(allNodes, b.Env.CephNodes...) + b.ensureDataCenters() - for _, node := range allNodes { + for _, node := range b.allNodes() { err := b.stlog.Substep(fmt.Sprintf("Ensuring root login enabled on %s", node.GetName()), func() error { return b.ensureRootLoginEnabledInNode(node) }) @@ -916,8 +973,9 @@ func (b *GCPBootstrapper) EnsureOmsInstalled() (err error) { } func (b *GCPBootstrapper) EnsureHostsConfigured() error { - allNodes := append(b.Env.ControlPlaneNodes, b.Env.PostgreSQLNode) - allNodes = append(allNodes, b.Env.CephNodes...) + b.ensureDataCenters() + + allNodes := append([]*node.Node{b.Env.PostgreSQLNode}, b.clusterNodes()...) for _, node := range allNodes { if !node.HasInotifyWatchesConfigured() { @@ -932,6 +990,25 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error { return fmt.Errorf("failed to configure memory map on %s: %w", node.GetName(), err) } } + err := node.RunSSHCommand("root", "mkdir -p "+installerNodeSecretsDir) + if err != nil { + return fmt.Errorf("failed to create secrets directory on %s: %w", node.GetName(), err) + } + } + + // A secondary data center's secrets directory differs from the fixed path above, so create that + // one too on its own nodes. Nodes belong to exactly one data center, so no node gets a foreign + // data center's directory. + for _, dc := range b.Env.DataCenters { + if dc.SecretsDir == installerNodeSecretsDir { + continue + } + for _, n := range append(append([]*node.Node{}, dc.ControlPlaneNodes...), dc.CephNodes...) { + err := n.RunSSHCommand("root", "mkdir -p "+dc.SecretsDir) + if err != nil { + return fmt.Errorf("failed to create secrets directory on %s: %w", n.GetName(), err) + } + } } return nil @@ -1079,15 +1156,42 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return nil } +// InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in +// ascending data center order. The order matters: the primary data center's install creates the +// database, roles and schema that the secondary ones reuse. func (b *GCPBootstrapper) InstallCodesphere() error { + b.ensureDataCenters() + fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox() if err != nil { return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err) } - err = b.runInstallCommand(fullPackageFilename) - if err != nil { - return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) + for _, dc := range b.Env.DataCenters { + err = b.stlog.Step(dc.StepName("Install Codesphere"), func() error { + return b.runInstallCommand(dc, fullPackageFilename) + }) + if err != nil { + return fmt.Errorf("failed to install Codesphere from jumpbox (data center %d): %w", dc.ID, err) + } + } + + return nil +} + +// RunK0sConfigScript runs every data center's k0s configuration script on its first control +// plane node. It requires that data center's Codesphere install to have completed, since the +// script patches the gateway services the install creates. +func (b *GCPBootstrapper) RunK0sConfigScript() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + err := b.stlog.Step(dc.StepName("Run k0s config script"), func() error { + return b.runK0sConfigScript(dc) + }) + if err != nil { + return err + } } return nil @@ -1128,11 +1232,16 @@ func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) { return fullPackageFilename, nil } -func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error { - b.stlog.Logf("Installing Codesphere...") - installCmd := fmt.Sprintf("oms install codesphere -c /etc/codesphere/config.yaml -k %s/age_key.txt --vault %s -p %s%s", - b.Env.SecretsDir, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), packageFilename, b.generateSkipStepsArg()) - return b.Env.Jumpbox.RunSSHCommand("root", installCmd) +func (b *GCPBootstrapper) runInstallCommand(dc *DataCenter, packageFilename string) error { + b.stlog.Logf("Installing Codesphere in data center %d...", dc.ID) + return b.Env.Jumpbox.RunSSHCommand("root", b.InstallCommand(dc, packageFilename)) +} + +// InstallCommand returns the command that installs Codesphere into the given data center from +// the jumpbox. It is also printed for the operator when the bootstrap does not install itself. +func (b *GCPBootstrapper) InstallCommand(dc *DataCenter, packageFilename string) string { + return fmt.Sprintf("oms install codesphere -c %s -k %s --vault %s -p %s%s", + dc.RemoteConfigPath, dc.RemoteAgeKeyPath(), dc.RemoteVaultPath(), packageFilename, b.generateSkipStepsArg()) } func (b *GCPBootstrapper) generateSkipStepsArg() string { @@ -1147,7 +1256,21 @@ func (b *GCPBootstrapper) generateSkipStepsArg() string { return " -s " + strings.Join(skipSteps, ",") } +// GenerateK0sConfigScript writes and uploads the k0s cloud-provider configuration script of +// every data center to that data center's first control plane node. func (b *GCPBootstrapper) GenerateK0sConfigScript() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + if err := b.generateK0sConfigScript(dc); err != nil { + return err + } + } + + return nil +} + +func (b *GCPBootstrapper) generateK0sConfigScript(dc *DataCenter) error { script := `#!/bin/bash cat < cloud.conf @@ -1215,14 +1338,14 @@ $KUBECTL apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-gc $KUBECTL apply -f cc-deployment.yaml # set loadBalancerIP for public-gateway-controller and gateway-controller -$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.PublicGatewayIP + `'"}}' -$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.GatewayIP + `'"}}' +$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.PublicGatewayIP + `'"}}' +$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.GatewayIP + `'"}}' sed -i 's/k0scontroller/k0scontroller --enable-cloud-provider/g' /etc/systemd/system/k0scontroller.service -ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" +ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" -ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" +ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" systemctl daemon-reload systemctl restart k0scontroller @@ -1231,25 +1354,27 @@ systemctl restart k0scontroller // --enable-cloud-provider on worker nodes systemd file /etc/systemd/system/k0sworker.service // in addition on the first node: /etc/systemd/system/k0scontroller.service the flag --enable-cloud-provider - err := b.fw.WriteFile("configure-k0s.sh", []byte(script), 0755) + localScript := dc.K0sConfigScriptPath() + err := b.fw.WriteFile(localScript, []byte(script), 0755) if err != nil { - return fmt.Errorf("failed to write configure-k0s.sh: %w", err) + return fmt.Errorf("failed to write %s: %w", localScript, err) } - err = b.Env.ControlPlaneNodes[0].NodeClient.CopyFile(b.Env.ControlPlaneNodes[0], "configure-k0s.sh", "/root/configure-k0s.sh") + controller := dc.ControlPlaneNodes[0] + err = controller.NodeClient.CopyFile(controller, localScript, remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to copy configure-k0s.sh to control plane node: %w", err) + return fmt.Errorf("failed to copy %s to control plane node: %w", localScript, err) } - err = b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "chmod +x /root/configure-k0s.sh") + err = controller.RunSSHCommand("root", "chmod +x "+remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to make configure-k0s.sh executable on control plane node: %w", err) + return fmt.Errorf("failed to make %s executable: %w", localScript, err) } return nil } -func (b *GCPBootstrapper) RunK0sConfigScript() error { - err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "/root/configure-k0s.sh") +func (b *GCPBootstrapper) runK0sConfigScript(dc *DataCenter) error { + err := dc.ControlPlaneNodes[0].RunSSHCommand("root", remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) + return fmt.Errorf("failed to configure k0s in data center %d: %w", dc.ID, err) } return nil diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index d0d6ff15..073ef283 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1235,9 +1235,74 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureHostsConfigured() Expect(err).NotTo(HaveOccurred()) }) + + It("creates the directory the installer uploads the age key to on every node", func() { + mkdirs := map[string]int{} + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(n *node.Node, _ string, command string) error { + if command == "mkdir -p /etc/codesphere/secrets" { + mkdirs[n.GetName()]++ + } + return nil + }) + + Expect(bs.EnsureHostsConfigured()).To(Succeed()) + // The postgres node plus every cluster node of every data center. + Expect(mkdirs).To(Equal(map[string]int{ + "postgres": 1, + "k0s-1": 1, "k0s-2": 1, "k0s-3": 1, + "ceph-1": 1, "ceph-2": 1, "ceph-3": 1, + })) + }) + + It("creates a secondary data center's own secrets directory on its nodes only", func() { + secondary := &gcp.DataCenter{ID: 2, Suffix: "-dc2", SecretsDir: "/etc/codesphere/secrets-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, + SecretsDir: "/etc/codesphere/secrets", + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + + mkdirs := map[string][]string{} + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(n *node.Node, _ string, command string) error { + if dir, found := strings.CutPrefix(command, "mkdir -p "); found { + mkdirs[n.GetName()] = append(mkdirs[n.GetName()], dir) + } + return nil + }) + + Expect(bs.EnsureHostsConfigured()).To(Succeed()) + // Both paths on the secondary's nodes, because the installer's fixed path is + // created everywhere; the secondary's path on nobody else's. + Expect(mkdirs["k0s-1-dc2"]).To(ConsistOf("/etc/codesphere/secrets", "/etc/codesphere/secrets-dc2")) + Expect(mkdirs["ceph-1-dc2"]).To(ConsistOf("/etc/codesphere/secrets", "/etc/codesphere/secrets-dc2")) + Expect(mkdirs["k0s-1"]).To(ConsistOf("/etc/codesphere/secrets")) + Expect(mkdirs["postgres"]).To(ConsistOf("/etc/codesphere/secrets")) + }) }) Describe("Invalid cases", func() { + It("fails when the secrets directory cannot be created", func() { + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(_ *node.Node, _ string, command string) error { + if command == "mkdir -p /etc/codesphere/secrets" { + return fmt.Errorf("ouch") + } + return nil + }) + + err := bs.EnsureHostsConfigured() + Expect(err).To(MatchError(ContainSubstring("failed to create secrets directory on postgres"))) + }) + It("fails when ConfigureInotifyWatches fails", func() { nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything).Return(fmt.Errorf("ouch")) From d1de6c993ef0afe6701a216da47b66010c9e1177 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:06:24 +0200 Subject: [PATCH 04/11] fix(gcp): give every data center its own DNS records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workspaces resolve per data center, and so does the platform: the frontend asks . for the configuration of the data center a workspace lives in. OMS only created cs. and its wildcard, both pointing at the first data center's gateway, so with more than one data center the second one's endpoint resolved to the first's gateway, which has no route for that host — every browser request for it was reset, and since the frontend fetches that config before rendering, the whole UI failed. EnsureDNSRecords now creates, per data center, its workspace hosting names and SSH proxy name pointing at its own public gateway and SSH proxy, plus .cs. and its wildcard pointing at its own platform gateway. The per-data-center platform names are only created when there is more than one data center: a single one is the primary, which cs. already resolves to. The records that were created are recorded in the infra file, so cleanup deletes exactly those. DeleteDNSRecordSets therefore takes the record list instead of a base domain, and cleanup falls back to deriving the names for infra files written before this and for a cleanup driven only by --project-id. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- cli/cmd/bootstrap_gcp_cleanup_test.go | 33 +++++- internal/bootstrap/gcp/cleanup.go | 15 ++- internal/bootstrap/gcp/datacenter.go | 10 ++ internal/bootstrap/gcp/datacenter_test.go | 27 +++++ internal/bootstrap/gcp/gcp.go | 116 ++++++++++++++-------- internal/bootstrap/gcp/gcp_client.go | 8 +- internal/bootstrap/gcp/gcp_test.go | 35 +++++++ internal/bootstrap/gcp/mocks.go | 22 ++-- 8 files changed, 210 insertions(+), 56 deletions(-) diff --git a/cli/cmd/bootstrap_gcp_cleanup_test.go b/cli/cmd/bootstrap_gcp_cleanup_test.go index 3337fd4c..75c4da29 100644 --- a/cli/cmd/bootstrap_gcp_cleanup_test.go +++ b/cli/cmd/bootstrap_gcp_cleanup_test.go @@ -322,7 +322,38 @@ var _ = Describe("BootstrapGcpCleanupCmd", func() { mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true) mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil) - mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", "example.com").Return(nil) + // An infra file without a recorded record list predates multi-DC support, so + // cleanup falls back to the single-data-center record names. + mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", gcp.GetDNSRecordNames("example.com")).Return(nil) + mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil) + mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil) + + err := cleanupCmd.ExecuteCleanup(deps) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when the infra file recorded the DNS records it created", func() { + It("should delete exactly those records", func() { + cleanupCmd.Opts.ProjectID = "test-project" + cleanupCmd.Opts.Force = true + + recorded := []gcp.DNSRecordName{ + {Name: "cs.example.com.", Rtype: "A"}, + {Name: "2.ws.example.com.", Rtype: "A"}, + } + validEnv := gcp.CodesphereEnvironment{ + ProjectID: "test-project", + BaseDomain: "example.com", + DNSZoneName: "test-zone", + MultiDC: true, + DNSRecords: recorded, + } + envData, _ := json.Marshal(validEnv) + + mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true) + mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil) + mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", recorded).Return(nil) mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil) mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil) diff --git a/internal/bootstrap/gcp/cleanup.go b/internal/bootstrap/gcp/cleanup.go index f30e59a7..f16f3885 100644 --- a/internal/bootstrap/gcp/cleanup.go +++ b/internal/bootstrap/gcp/cleanup.go @@ -180,7 +180,20 @@ func (e *CleanupExecutor) CleanupDNSRecords() error { log.Printf("Skipping DNS cleanup: missing base domain or DNS zone name (provide --base-domain/--dns-zone-name or use --skip-dns-cleanup)") return nil } - return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.BaseDomain) + return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.dnsRecords()) +} + +// dnsRecords returns the DNS records to delete. The bootstrap records what it created in the +// infra file, which is authoritative. Older infra files predate that, and a cleanup driven only +// by --project-id has no infra file at all, so both fall back to deriving the names. +func (e *CleanupExecutor) dnsRecords() []DNSRecordName { + if len(e.InfraEnv.DNSRecords) > 0 { + return e.InfraEnv.DNSRecords + } + if len(e.InfraEnv.DataCenters) > 0 { + return DataCenterDNSRecordNames(e.BaseDomain, e.InfraEnv.DataCenters) + } + return GetDNSRecordNames(e.BaseDomain) } // RemoveDNSIAMBinding removes the cloud-controller service account's IAM binding diff --git a/internal/bootstrap/gcp/datacenter.go b/internal/bootstrap/gcp/datacenter.go index 7ef4fbcc..d263137f 100644 --- a/internal/bootstrap/gcp/datacenter.go +++ b/internal/bootstrap/gcp/datacenter.go @@ -82,6 +82,16 @@ func (dc *DataCenter) RemoteAgeKeyPath() string { return filepath.Join(dc.SecretsDir, "age_key.txt") } +// PlatformDomain returns the host this data center serves its own platform services on. The +// platform derives it from the data center ID and codesphere.domain, which OMS sets to +// cs., and the frontend calls it directly — the browser asks +// .cs. for the config of the data center a workspace lives in. So it has to +// resolve to this data center's platform gateway, not to the primary's, which is what +// cs. and its wildcard point at. +func (dc *DataCenter) PlatformDomain(baseDomain string) string { + return fmt.Sprintf("%d.cs.%s", dc.ID, baseDomain) +} + // K0sConfigScriptPath returns the local filename of this data center's k0s configuration script. func (dc *DataCenter) K0sConfigScriptPath() string { return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix) diff --git a/internal/bootstrap/gcp/datacenter_test.go b/internal/bootstrap/gcp/datacenter_test.go index 76522bbf..f822dbb4 100644 --- a/internal/bootstrap/gcp/datacenter_test.go +++ b/internal/bootstrap/gcp/datacenter_test.go @@ -103,3 +103,30 @@ var _ = Describe("BuildDataCenters", func() { Expect(gcp.BuildDataCenters(env, installer.NewInstallConfigManager)[0].Name).To(Equal("dev")) }) }) + +var _ = Describe("DataCenterDNSRecordNames", func() { + It("returns the single-DC records for one data center", func() { + dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{BaseDomain: "example.com"}, nil) + + Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(ConsistOf(gcp.GetDNSRecordNames("example.com"))) + }) + + It("shares the platform names and scopes the workspace names per data center", func() { + dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{MultiDC: true, BaseDomain: "example.com"}, nil) + + Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(Equal([]gcp.DNSRecordName{ + {Name: "cs.example.com.", Rtype: "A"}, + {Name: "*.cs.example.com.", Rtype: "A"}, + {Name: "1.ws.example.com.", Rtype: "A"}, + {Name: "*.1.ws.example.com.", Rtype: "A"}, + {Name: "*.1.ssh.cs.example.com.", Rtype: "A"}, + {Name: "1.cs.example.com.", Rtype: "A"}, + {Name: "*.1.cs.example.com.", Rtype: "A"}, + {Name: "2.ws.example.com.", Rtype: "A"}, + {Name: "*.2.ws.example.com.", Rtype: "A"}, + {Name: "*.2.ssh.cs.example.com.", Rtype: "A"}, + {Name: "2.cs.example.com.", Rtype: "A"}, + {Name: "*.2.cs.example.com.", Rtype: "A"}, + })) + }) +}) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 72aaa895..d6a2d19f 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -63,15 +63,17 @@ func CheckOMSManagedLabel(labels map[string]string) bool { return exists && value == "true" } -// GetDNSRecordNames returns the DNS record names that OMS creates for a given base domain. -func GetDNSRecordNames(baseDomain string) []struct { - Name string - Rtype string -} { - return []struct { - Name string - Rtype string - }{ +// DNSRecordName identifies a DNS record set that OMS manages. +type DNSRecordName struct { + Name string `json:"name"` + Rtype string `json:"rtype"` +} + +// GetDNSRecordNames returns the DNS record names a single-data-center bootstrap creates for a +// given base domain. It is the fallback for infra files written before multi-DC support, which +// do not record the created records. +func GetDNSRecordNames(baseDomain string) []DNSRecordName { + return []DNSRecordName{ {fmt.Sprintf("cs.%s.", baseDomain), "A"}, {fmt.Sprintf("*.cs.%s.", baseDomain), "A"}, {fmt.Sprintf("ws.%s.", baseDomain), "A"}, @@ -80,6 +82,30 @@ func GetDNSRecordNames(baseDomain string) []struct { } } +// DataCenterDNSRecordNames returns every DNS record OMS creates for the given data center +// layout: the shared platform gateway names plus each data center's workspace and SSH names. +func DataCenterDNSRecordNames(baseDomain string, dcs []*DataCenter) []DNSRecordName { + records := []DNSRecordName{ + {fmt.Sprintf("cs.%s.", baseDomain), "A"}, + {fmt.Sprintf("*.cs.%s.", baseDomain), "A"}, + } + for _, dc := range dcs { + records = append(records, + DNSRecordName{fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.SshBaseDomain), "A"}, + ) + if len(dcs) > 1 { + records = append(records, + DNSRecordName{fmt.Sprintf("%s.", dc.PlatformDomain(baseDomain)), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.PlatformDomain(baseDomain)), "A"}, + ) + } + } + + return records +} + // This should ALWAYS be empty. Internal flags are for internal feature // development and not intended for customer use. // Atm. it's not empty as the internal flags below are likely preview or @@ -160,6 +186,9 @@ type CodesphereEnvironment struct { MultiDC bool `json:"multi_dc"` // DataCenters holds the per-data-center state. It always has at least one entry. DataCenters []*DataCenter `json:"datacenters"` + // DNSRecords records the DNS records the bootstrap created, so cleanup deletes exactly + // those instead of recomputing the list. + DNSRecords []DNSRecordName `json:"dns_records,omitempty"` // Mirrors of DataCenters[0], written for infra files consumed by cleanup and restart-vms. ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` CephNodes []*node.Node `json:"ceph_nodes"` @@ -1104,6 +1133,8 @@ func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error { } func (b *GCPBootstrapper) EnsureDNSRecords() error { + b.ensureDataCenters() + gcpProject := b.Env.DNSProjectID if b.Env.DNSProjectID == "" { gcpProject = b.Env.ProjectID @@ -1115,37 +1146,30 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return fmt.Errorf("failed to ensure DNS managed zone: %w", err) } + // The platform is served from one domain shared by all data centers, pointing at the + // primary data center's gateway. records := []*dns.ResourceRecordSet{ - { - Name: fmt.Sprintf("cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.GatewayIP}, - }, - { - Name: fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.GatewayIP}, - }, - { - Name: fmt.Sprintf("*.ws.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.PublicGatewayIP}, - }, - { - Name: fmt.Sprintf("ws.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.PublicGatewayIP}, - }, - { - Name: fmt.Sprintf("*.ssh.cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.SshProxyIP}, - }, + dnsARecord(fmt.Sprintf("cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP), + dnsARecord(fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP), + } + // Workspaces and their SSH endpoints resolve per data center, so each one gets its own + // names pointing at its own public gateway and SSH proxy. + for _, dc := range b.Env.DataCenters { + records = append(records, + dnsARecord(fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", dc.SshBaseDomain), dc.SshProxyIP), + ) + // The platform calls each data center's own services at .cs., which + // the wildcard above would send to the primary data center's gateway. A single data + // center is that primary, so it needs no record of its own. + if len(b.Env.DataCenters) > 1 { + platformDomain := dc.PlatformDomain(b.Env.BaseDomain) + records = append(records, + dnsARecord(fmt.Sprintf("%s.", platformDomain), dc.GatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", platformDomain), dc.GatewayIP), + ) + } } err = b.GCPClient.EnsureDNSRecordSets(gcpProject, zoneName, records) @@ -1153,9 +1177,23 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return fmt.Errorf("failed to ensure DNS record sets: %w", err) } + // Record what was created so cleanup deletes exactly these records instead of recomputing + // the list from the base domain. + b.Env.DNSRecords = DataCenterDNSRecordNames(b.Env.BaseDomain, b.Env.DataCenters) + return nil } +// dnsARecord builds a short-TTL A record set, as used during initial setup. +func dnsARecord(name, ip string) *dns.ResourceRecordSet { + return &dns.ResourceRecordSet{ + Name: name, + Type: "A", + Ttl: 300, + Rrdatas: []string{ip}, + } +} + // InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in // ascending data center order. The order matters: the primary data center's install creates the // database, roles and schema that the secondary ones reuse. diff --git a/internal/bootstrap/gcp/gcp_client.go b/internal/bootstrap/gcp/gcp_client.go index 7b764de3..cb7a9c94 100644 --- a/internal/bootstrap/gcp/gcp_client.go +++ b/internal/bootstrap/gcp/gcp_client.go @@ -61,7 +61,7 @@ type GCPClientManager interface { GetAddress(projectID, region, addressName string) (*computepb.Address, error) EnsureDNSManagedZone(projectID, zoneName, dnsName, description string) error EnsureDNSRecordSets(projectID, zoneName string, records []*dns.ResourceRecordSet) error - DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error + DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error CreatePublicCAExternalAccountKey(projectID string) (keyID, b64MacKey string, err error) } @@ -828,15 +828,15 @@ func (c *GCPClient) EnsureDNSRecordSets(projectID, zoneName string, records []*d return nil } -// DeleteDNSRecordSets deletes DNS record sets created by OMS for the given base domain. -func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error { +// DeleteDNSRecordSets deletes the given DNS record sets, ignoring those that no longer exist. +func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error { service, err := dns.NewService(c.ctx) if err != nil { return fmt.Errorf("failed to create DNS service: %w", err) } var deletions []*dns.ResourceRecordSet - for _, record := range GetDNSRecordNames(baseDomain) { + for _, record := range records { existing, err := service.ResourceRecordSets.Get(projectID, zoneName, record.Name, record.Rtype).Context(c.ctx).Do() if IsNotFoundError(err) { continue diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index 073ef283..a47e8754 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1419,6 +1419,41 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureDNSRecords() Expect(err).NotTo(HaveOccurred()) }) + + It("points each data center's platform host at its own gateway", func() { + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, GatewayIP: "1.1.1.1", PublicGatewayIP: "1.1.1.2", SshProxyIP: "1.1.1.3", + WorkspaceHostingBaseDomain: "1.ws.example.com", SshBaseDomain: "1.ssh.cs.example.com", + }, + { + ID: 2, Suffix: "-dc2", GatewayIP: "2.2.2.1", PublicGatewayIP: "2.2.2.2", SshProxyIP: "2.2.2.3", + WorkspaceHostingBaseDomain: "2.ws.example.com", SshBaseDomain: "2.ssh.cs.example.com", + }, + } + + gc.EXPECT().EnsureDNSManagedZone(csEnv.DNSProjectID, csEnv.DNSZoneName, csEnv.BaseDomain+".", mock.Anything).Return(nil) + targets := map[string]string{} + gc.EXPECT().EnsureDNSRecordSets(csEnv.DNSProjectID, csEnv.DNSZoneName, mock.Anything). + RunAndReturn(func(_ string, _ string, records []*dns.ResourceRecordSet) error { + for _, r := range records { + targets[r.Name] = r.Rrdatas[0] + } + return nil + }) + + Expect(bs.EnsureDNSRecords()).To(Succeed()) + // The shared platform name stays on the primary data center's gateway, but the + // per-data-center platform host the frontend calls resolves to its own gateway. + Expect(targets["cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["*.cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["1.cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["2.cs.example.com."]).To(Equal("2.2.2.1")) + Expect(targets["*.2.cs.example.com."]).To(Equal("2.2.2.1")) + // Workspaces and SSH keep pointing at the public gateway and SSH proxy. + Expect(targets["2.ws.example.com."]).To(Equal("2.2.2.2")) + Expect(targets["*.2.ssh.cs.example.com."]).To(Equal("2.2.2.3")) + }) }) Describe("Invalid cases", func() { diff --git a/internal/bootstrap/gcp/mocks.go b/internal/bootstrap/gcp/mocks.go index 5b371325..74b64038 100644 --- a/internal/bootstrap/gcp/mocks.go +++ b/internal/bootstrap/gcp/mocks.go @@ -796,16 +796,16 @@ func (_c *MockGCPClientManager_CreateVPC_Call) RunAndReturn(run func(projectID s } // DeleteDNSRecordSets provides a mock function for the type MockGCPClientManager -func (_mock *MockGCPClientManager) DeleteDNSRecordSets(projectID string, zoneName string, baseDomain string) error { - ret := _mock.Called(projectID, zoneName, baseDomain) +func (_mock *MockGCPClientManager) DeleteDNSRecordSets(projectID string, zoneName string, records []DNSRecordName) error { + ret := _mock.Called(projectID, zoneName, records) if len(ret) == 0 { panic("no return value specified for DeleteDNSRecordSets") } var r0 error - if returnFunc, ok := ret.Get(0).(func(string, string, string) error); ok { - r0 = returnFunc(projectID, zoneName, baseDomain) + if returnFunc, ok := ret.Get(0).(func(string, string, []DNSRecordName) error); ok { + r0 = returnFunc(projectID, zoneName, records) } else { r0 = ret.Error(0) } @@ -820,12 +820,12 @@ type MockGCPClientManager_DeleteDNSRecordSets_Call struct { // DeleteDNSRecordSets is a helper method to define mock.On call // - projectID string // - zoneName string -// - baseDomain string -func (_e *MockGCPClientManager_Expecter) DeleteDNSRecordSets(projectID any, zoneName any, baseDomain any) *MockGCPClientManager_DeleteDNSRecordSets_Call { - return &MockGCPClientManager_DeleteDNSRecordSets_Call{Call: _e.mock.On("DeleteDNSRecordSets", projectID, zoneName, baseDomain)} +// - records []DNSRecordName +func (_e *MockGCPClientManager_Expecter) DeleteDNSRecordSets(projectID any, zoneName any, records any) *MockGCPClientManager_DeleteDNSRecordSets_Call { + return &MockGCPClientManager_DeleteDNSRecordSets_Call{Call: _e.mock.On("DeleteDNSRecordSets", projectID, zoneName, records)} } -func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID string, zoneName string, baseDomain string)) *MockGCPClientManager_DeleteDNSRecordSets_Call { +func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID string, zoneName string, records []DNSRecordName)) *MockGCPClientManager_DeleteDNSRecordSets_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { @@ -835,9 +835,9 @@ func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID if args[1] != nil { arg1 = args[1].(string) } - var arg2 string + var arg2 []DNSRecordName if args[2] != nil { - arg2 = args[2].(string) + arg2 = args[2].([]DNSRecordName) } run( arg0, @@ -853,7 +853,7 @@ func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Return(err error) *Mock return _c } -func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) RunAndReturn(run func(projectID string, zoneName string, baseDomain string) error) *MockGCPClientManager_DeleteDNSRecordSets_Call { +func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) RunAndReturn(run func(projectID string, zoneName string, records []DNSRecordName) error) *MockGCPClientManager_DeleteDNSRecordSets_Call { _c.Call.Return(run) return _c } From 9c65a79eb92efceb25c7c3400ffc5bcc05252c36 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:08:56 +0200 Subject: [PATCH 05/11] refac(gcp): thread the data center through install config generation Config and vault generation worked on b.Env.InstallConfig and the single b.icg. Both become per data center: every step takes the DataCenter it generates for and reads its nodes, IPs, paths and config manager from there. The exported Ensure*/Update* entry points keep their signatures and operate on the primary data center, so step names and existing callers are unchanged. Two details worth pointing out: - secrets.baseDir is now the data center's own directory. The installer resolves the vault it reads and writes back from that path, so sharing it between data centers would let one data center's kubernetes and ceph steps overwrite the other's kubeconfig and Ceph credentials. - the local config, vault and remote paths come from the data center, so a second data center writes config-dc2.yaml rather than overwriting config.yaml. Pure refactoring: a single-data-center bootstrap produces the same config and vault as before. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/install_config.go | 359 +++++++++++++---------- 1 file changed, 203 insertions(+), 156 deletions(-) diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 83ebdf0a..4a78fb32 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -16,47 +16,55 @@ const ( remoteInstallConfigPath string = "/etc/codesphere/config.yaml" ) -// EnsureInstallConfig uses the local config or recovers it from an existing jumpbox if desired. -// Else it applies the minimal profile to a new config. +// EnsureInstallConfig prepares the primary data center's install config. func (b *GCPBootstrapper) EnsureInstallConfig() error { + b.ensureDataCenters() + + return b.ensureInstallConfig(b.primaryDC()) +} + +// ensureInstallConfig uses the data center's local config or recovers it from an existing +// jumpbox if desired. Else it applies the minimal profile to a new config. +func (b *GCPBootstrapper) ensureInstallConfig(dc *DataCenter) error { // recovery will overwrite local config or create a new file if b.Env.RecoverConfig { - err := b.recoverConfig() + err := b.recoverConfig(dc) if err != nil { return fmt.Errorf("failed to recover config: %w", err) } } - if b.fw.Exists(b.Env.InstallConfigPath) { - if err := b.loadVaultForConfigTemplating(); err != nil { + if b.fw.Exists(dc.InstallConfigPath) { + if err := b.loadVaultForConfigTemplating(dc); err != nil { return fmt.Errorf("failed to load vault templating: %w", err) } - err := b.icg.LoadInstallConfigFromFile(b.Env.InstallConfigPath) + err := dc.icg.LoadInstallConfigFromFile(dc.InstallConfigPath) if err != nil { return fmt.Errorf("failed to load config file: %w", err) } - b.Env.ExistingConfigUsed = true + dc.ExistingConfigUsed = true } else { - err := b.icg.ApplyProfile("minimal") + err := dc.icg.ApplyProfile("minimal") if err != nil { return fmt.Errorf("failed to apply profile: %w", err) } } - b.Env.InstallConfig = b.icg.GetInstallConfig() + dc.InstallConfig = dc.icg.GetInstallConfig() + b.mirrorPrimaryDataCenter() return nil } -func (b *GCPBootstrapper) loadVaultForConfigTemplating() error { - if !b.fw.Exists(b.Env.SecretsFilePath) { +func (b *GCPBootstrapper) loadVaultForConfigTemplating(dc *DataCenter) error { + if !b.fw.Exists(dc.SecretsFilePath) { return nil } // during bootstrapping, the vault is not yet encrpyted - if err := b.icg.LoadVaultFromUnecryptedFile(b.Env.SecretsFilePath); err != nil { + if err := dc.icg.LoadVaultFromUnecryptedFile(dc.SecretsFilePath); err != nil { return fmt.Errorf("failed to load vault from file: %w", err) } @@ -66,7 +74,7 @@ func (b *GCPBootstrapper) loadVaultForConfigTemplating() error { // recoverConfig downloads the config and secrets from the jumpbox if it exists. // Since recovery is done when the project or VMs are not ensured, we need to search for the jumpbox IP first. // Returns an error if project or jumpbox does not exist or downloading fails. -func (b *GCPBootstrapper) recoverConfig() error { +func (b *GCPBootstrapper) recoverConfig(dc *DataCenter) error { existingProject, err := b.GCPClient.GetProjectByName(b.Env.FolderID, b.Env.ProjectName) if err != nil { return fmt.Errorf("failed to find gcp project for config recovery: %w", err) @@ -79,12 +87,12 @@ func (b *GCPBootstrapper) recoverConfig() error { } b.Env.Jumpbox = jumpbox - err = b.Env.Jumpbox.NodeClient.DownloadFile(jumpbox, remoteInstallConfigPath, b.Env.InstallConfigPath) + err = b.Env.Jumpbox.NodeClient.DownloadFile(jumpbox, dc.RemoteConfigPath, dc.InstallConfigPath) if err != nil { return fmt.Errorf("failed to download install config from jumpbox: %w", err) } - err = b.recoverVault() + err = b.recoverVault(dc) if err != nil { return fmt.Errorf("failed to recover vault: %w", err) } @@ -93,8 +101,8 @@ func (b *GCPBootstrapper) recoverConfig() error { } // recoverVault unencrypts the secrets file on the jumpbox and download the file to the local destination -func (b *GCPBootstrapper) recoverVault() error { - const vaultCopyPath string = "/tmp/prod.vault.yaml" +func (b *GCPBootstrapper) recoverVault(dc *DataCenter) error { + vaultCopyPath := fmt.Sprintf("/tmp/prod%s.vault.yaml", dc.Suffix) defer func() { err := b.Env.Jumpbox.RunSSHCommand("root", "rm -f "+vaultCopyPath) if err != nil { @@ -102,12 +110,12 @@ func (b *GCPBootstrapper) recoverVault() error { } }() - err := b.decryptVault(vaultCopyPath) + err := b.decryptVault(dc, vaultCopyPath) if err != nil { return fmt.Errorf("failed to create decrypted vault for recovery: %w", err) } - err = b.Env.Jumpbox.NodeClient.DownloadFile(b.Env.Jumpbox, vaultCopyPath, b.Env.SecretsFilePath) + err = b.Env.Jumpbox.NodeClient.DownloadFile(b.Env.Jumpbox, vaultCopyPath, dc.SecretsFilePath) if err != nil { return fmt.Errorf("failed to download secrets file from jumpbox: %w", err) } @@ -115,50 +123,60 @@ func (b *GCPBootstrapper) recoverVault() error { return nil } +// UpdateInstallConfig writes the bootstrapped infrastructure into the primary data center's +// install config. func (b *GCPBootstrapper) UpdateInstallConfig() error { + b.ensureDataCenters() + + return b.updateInstallConfig(b.primaryDC()) +} + +func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { // Update install config with necessary values - b.Env.InstallConfig.Datacenter.ID = b.Env.DatacenterID - if b.Env.DatacenterName == "" { - b.Env.DatacenterName = "dev" - } - b.Env.InstallConfig.Datacenter.Name = b.Env.DatacenterName - b.Env.InstallConfig.Datacenter.City = "Karlsruhe" - b.Env.InstallConfig.Datacenter.CountryCode = "DE" - b.Env.InstallConfig.Secrets.BaseDir = b.Env.SecretsDir + dc.InstallConfig.Datacenter.ID = dc.ID + dc.InstallConfig.Datacenter.Name = dc.Name + dc.InstallConfig.Datacenter.City = "Karlsruhe" + dc.InstallConfig.Datacenter.CountryCode = "DE" + // Each data center reads and writes its own vault. The installer resolves the vault from + // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes + // steps overwrite another's credentials. + dc.InstallConfig.Secrets.BaseDir = dc.SecretsDir if b.Env.RegistryType != RegistryTypeGitHub { - b.Env.InstallConfig.Registry.ReplaceImagesInBom = true - b.Env.InstallConfig.Registry.LoadContainerImages = true + dc.InstallConfig.Registry.ReplaceImagesInBom = true + dc.InstallConfig.Registry.LoadContainerImages = true } - if b.Env.InstallConfig.Postgres.Primary == nil { - b.Env.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ + if dc.InstallConfig.Postgres.Primary == nil { + dc.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ Hostname: b.Env.PostgreSQLNode.GetName(), } } - previousPrimaryIP := b.Env.InstallConfig.Postgres.Primary.IP - previousPrimaryHostname := b.Env.InstallConfig.Postgres.Primary.Hostname - b.Env.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() - b.Env.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() + previousPrimaryIP := dc.InstallConfig.Postgres.Primary.IP + previousPrimaryHostname := dc.InstallConfig.Postgres.Primary.Hostname + dc.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() + dc.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() - b.Env.InstallConfig.Ceph.CsiKubeletDir = "/var/lib/k0s/kubelet" - b.Env.InstallConfig.Ceph.NodesSubnet = "10.10.0.0/20" - b.Env.InstallConfig.Ceph.Hosts = []files.CephHost{ + dc.InstallConfig.Ceph.CsiKubeletDir = "/var/lib/k0s/kubelet" + // All data centers share the project's subnet; their Ceph clusters stay separate because + // each has its own hosts, monitors and FSID. + dc.InstallConfig.Ceph.NodesSubnet = vpcSubnetCIDR + dc.InstallConfig.Ceph.Hosts = []files.CephHost{ { - Hostname: b.Env.CephNodes[0].GetName(), + Hostname: dc.CephNodes[0].GetName(), IsMaster: true, - IPAddress: b.Env.CephNodes[0].GetInternalIP(), + IPAddress: dc.CephNodes[0].GetInternalIP(), }, { - Hostname: b.Env.CephNodes[1].GetName(), - IPAddress: b.Env.CephNodes[1].GetInternalIP(), + Hostname: dc.CephNodes[1].GetName(), + IPAddress: dc.CephNodes[1].GetInternalIP(), }, { - Hostname: b.Env.CephNodes[2].GetName(), - IPAddress: b.Env.CephNodes[2].GetInternalIP(), + Hostname: dc.CephNodes[2].GetName(), + IPAddress: dc.CephNodes[2].GetInternalIP(), }, } - b.Env.InstallConfig.Ceph.OSDs = []files.CephOSD{ + dc.InstallConfig.Ceph.OSDs = []files.CephOSD{ { SpecID: "default", Placement: files.CephPlacement{ @@ -175,48 +193,48 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { }, } - b.Env.InstallConfig.Kubernetes = files.KubernetesConfig{ + dc.InstallConfig.Kubernetes = files.KubernetesConfig{ ManagedByCodesphere: true, - APIServerHost: b.Env.ControlPlaneNodes[0].GetInternalIP(), + APIServerHost: dc.ControlPlaneNodes[0].GetInternalIP(), ControlPlanes: []files.K8sNode{ { - IPAddress: b.Env.ControlPlaneNodes[0].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[0].GetInternalIP(), }, }, Workers: []files.K8sNode{ { - IPAddress: b.Env.ControlPlaneNodes[0].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[0].GetInternalIP(), }, { - IPAddress: b.Env.ControlPlaneNodes[1].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[1].GetInternalIP(), }, { - IPAddress: b.Env.ControlPlaneNodes[2].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[2].GetInternalIP(), }, }, } - b.Env.InstallConfig.Cluster.Kyverno = &files.KyvernoConfig{ + dc.InstallConfig.Cluster.Kyverno = &files.KyvernoConfig{ Enabled: false, } - b.Env.InstallConfig.Cluster.Gateway.ServiceType = "LoadBalancer" - b.Env.InstallConfig.Cluster.Gateway.Annotations = map[string]string{ - "cloud.google.com/load-balancer-ipv4": b.Env.GatewayIP, + dc.InstallConfig.Cluster.Gateway.ServiceType = "LoadBalancer" + dc.InstallConfig.Cluster.Gateway.Annotations = map[string]string{ + "cloud.google.com/load-balancer-ipv4": dc.GatewayIP, } - b.Env.InstallConfig.Cluster.PublicGateway.ServiceType = "LoadBalancer" - b.Env.InstallConfig.Cluster.PublicGateway.Annotations = map[string]string{ - "cloud.google.com/load-balancer-ipv4": b.Env.PublicGatewayIP, + dc.InstallConfig.Cluster.PublicGateway.ServiceType = "LoadBalancer" + dc.InstallConfig.Cluster.PublicGateway.Annotations = map[string]string{ + "cloud.google.com/load-balancer-ipv4": dc.PublicGatewayIP, } - b.applySshProxyConfig() + b.applySshProxyConfig(dc) dnsProject := b.Env.DNSProjectID if b.Env.DNSProjectID == "" { dnsProject = b.Env.ProjectID } - b.Env.InstallConfig.Cluster.Certificates.Override = map[string]interface{}{ + dc.InstallConfig.Cluster.Certificates.Override = map[string]interface{}{ "issuers": map[string]interface{}{ "letsEncryptHttp": map[string]interface{}{ "enabled": !b.Env.GoogleACMEIssuer, @@ -248,26 +266,28 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { } acmeConfig.Server = "https://dv.acme-v02.api.pki.goog/directory" acmeConfig.EABKeyID = keyID - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAcmeEabMacKey, Fields: &files.SecretFields{Password: b64MacKey}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAcmeEabMacKey, Fields: &files.SecretFields{Password: b64MacKey}}) } - b.Env.InstallConfig.Codesphere.CertIssuer = files.CertIssuerConfig{ + dc.InstallConfig.Codesphere.CertIssuer = files.CertIssuerConfig{ Type: "acme", Acme: acmeConfig, } - b.Env.InstallConfig.Codesphere.Domain = "cs." + b.Env.BaseDomain - b.Env.InstallConfig.Codesphere.WorkspaceHostingBaseDomain = "ws." + b.Env.BaseDomain - b.Env.InstallConfig.Codesphere.PublicIP = b.Env.ControlPlaneNodes[1].GetExternalIP() - b.Env.InstallConfig.Codesphere.CustomDomains = files.CustomDomainsConfig{ - CNameBaseDomain: "ws." + b.Env.BaseDomain, + // The platform is served from one domain shared by all data centers, while workspaces and + // custom domains resolve to the data center hosting them. + dc.InstallConfig.Codesphere.Domain = "cs." + b.Env.BaseDomain + dc.InstallConfig.Codesphere.WorkspaceHostingBaseDomain = dc.WorkspaceHostingBaseDomain + dc.InstallConfig.Codesphere.CustomDomains = files.CustomDomainsConfig{ + CNameBaseDomain: dc.WorkspaceHostingBaseDomain, } - b.Env.InstallConfig.Codesphere.DNSServers = []string{"8.8.8.8"} - b.Env.InstallConfig.Codesphere.DeployConfig = bootstrap.DefaultCodesphereDeployConfig() - b.Env.InstallConfig.Codesphere.Plans = bootstrap.DefaultCodespherePlans() + dc.InstallConfig.Codesphere.PublicIP = dc.ControlPlaneNodes[1].GetExternalIP() + dc.InstallConfig.Codesphere.DNSServers = []string{"8.8.8.8"} + dc.InstallConfig.Codesphere.DeployConfig = bootstrap.DefaultCodesphereDeployConfig() + dc.InstallConfig.Codesphere.Plans = bootstrap.DefaultCodespherePlans() - b.Env.InstallConfig.Codesphere.GitProviders = &files.GitProvidersConfig{} + dc.InstallConfig.Codesphere.GitProviders = &files.GitProvidersConfig{} if b.Env.GitHubAppName != "" && b.Env.GitHubAppClientID != "" && b.Env.GitHubAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.GitHub = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.GitHub = &files.GitProviderConfig{ Enabled: true, URL: "https://github.com", API: files.APIConfig{ @@ -282,11 +302,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { InstallationURI: "https://github.com/apps/" + b.Env.GitHubAppName + "/installations/new", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientId, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientId, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientSecret}}) } if b.Env.GitLabAppClientID != "" && b.Env.GitLabAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.GitLab = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.GitLab = &files.GitProviderConfig{ Enabled: true, URL: "https://gitlab.com", API: files.APIConfig{ @@ -300,11 +320,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { RedirectURI: "https://cs." + b.Env.BaseDomain + "/ide/auth/gitlab/callback", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientId, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientSecret, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientId, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientSecret, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientSecret}}) } if b.Env.BitbucketAppClientID != "" && b.Env.BitbucketAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.Bitbucket = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.Bitbucket = &files.GitProviderConfig{ Enabled: true, URL: "https://bitbucket.org", API: files.APIConfig{ @@ -318,11 +338,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { RedirectURI: "https://cs." + b.Env.BaseDomain + "/ide/auth/bitbucket/callback", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientId, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientId, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientSecret}}) } if b.Env.AzureDevOpsAppClientID != "" && b.Env.AzureDevOpsAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.AzureDevOps = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.AzureDevOps = &files.GitProviderConfig{ Enabled: true, URL: "https://dev.azure.com", API: files.APIConfig{ @@ -337,15 +357,15 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { Scope: "openid offline_access https://app.vssps.visualstudio.com/vso.code_full", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientId, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientSecret, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientId, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientSecret, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientSecret}}) } if b.Env.OidcIssuerURL != "" && b.Env.OidcClientID != "" && b.Env.OidcClientSecret != "" { name := b.Env.OidcProviderName if name == "" { name = "OIDC" } - b.Env.InstallConfig.Codesphere.OAuth = &files.OAuthProvidersConfig{ + dc.InstallConfig.Codesphere.OAuth = &files.OAuthProvidersConfig{ Oidc: &files.OidcOAuthProvider{ Type: "oidc", Enabled: true, @@ -354,12 +374,12 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { Scopes: []string{"openid", "profile", "email"}, }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientId, Fields: &files.SecretFields{Password: b.Env.OidcClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientSecret, Fields: &files.SecretFields{Password: b.Env.OidcClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientId, Fields: &files.SecretFields{Password: b.Env.OidcClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientSecret, Fields: &files.SecretFields{Password: b.Env.OidcClientSecret}}) } if b.Env.CentralOtelPassword != "" || b.Env.LocalTraceEndpoint != "" { - b.Env.InstallConfig.Codesphere.TelemetryExport = &files.TelemetryExport{ + dc.InstallConfig.Codesphere.TelemetryExport = &files.TelemetryExport{ RemoteEndpoint: b.Env.CentralOtelEndpoint, RemoteExport: b.Env.CentralOtelPassword != "", Traces: b.Env.LocalTraceEndpoint != "", @@ -368,71 +388,75 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { } } - b.Env.InstallConfig.Codesphere.Internal = b.Env.InternalFlags - b.Env.InstallConfig.Codesphere.Preview = util.StringSliceToBoolMap(b.Env.PreviewFlags) - b.Env.InstallConfig.Codesphere.Features = util.StringSliceToBoolMap(b.Env.FeatureFlags) + dc.InstallConfig.Codesphere.Internal = b.Env.InternalFlags + dc.InstallConfig.Codesphere.Preview = util.StringSliceToBoolMap(b.Env.PreviewFlags) + dc.InstallConfig.Codesphere.Features = util.StringSliceToBoolMap(b.Env.FeatureFlags) // Only set when the flag is provided so a recovered config keeps its value on re-runs. if b.Env.ClusterAdminEmail != "" { - b.Env.InstallConfig.Codesphere.ClusterAdminEmail = b.Env.ClusterAdminEmail + dc.InstallConfig.Codesphere.ClusterAdminEmail = b.Env.ClusterAdminEmail } - b.applyExternalLokiConfig() - b.applyPrometheusRemoteWriteConfig() + b.applyExternalLokiConfig(dc) + b.applyPrometheusRemoteWriteConfig(dc) - if !b.Env.ExistingConfigUsed { - err := b.icg.GenerateSecrets() + if !dc.ExistingConfigUsed { + err := dc.icg.GenerateSecrets() if err != nil { return fmt.Errorf("failed to generate secrets: %w", err) } } else { - if err := b.regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname); err != nil { + if err := b.regeneratePostgresCerts(dc, previousPrimaryIP, previousPrimaryHostname); err != nil { return err } } if b.Env.CentralOtelUsername != "" && b.Env.CentralOtelPassword != "" { - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - b.Env.InstallConfig.Cluster.Monitoring.CentralOtelExport = &files.CentralOtelConfig{ + dc.InstallConfig.Cluster.Monitoring.CentralOtelExport = &files.CentralOtelConfig{ Enabled: true, Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretCentralOtelCreds, Fields: &files.SecretFields{Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretCentralOtelCreds, Fields: &files.SecretFields{Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword}}) } if b.Env.OpenBaoURI != "" { - b.Env.InstallConfig.Codesphere.OpenBao = &files.OpenBaoConfig{ + dc.InstallConfig.Codesphere.OpenBao = &files.OpenBaoConfig{ Engine: b.Env.OpenBaoEngine, URI: b.Env.OpenBaoURI, User: b.Env.OpenBaoUser, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOpenBaoPassword, Fields: &files.SecretFields{Password: b.Env.OpenBaoPassword}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOpenBaoPassword, Fields: &files.SecretFields{Password: b.Env.OpenBaoPassword}}) } - if err := b.icg.WriteInstallConfig(b.Env.InstallConfigPath, true); err != nil { + if err := dc.icg.WriteInstallConfig(dc.InstallConfigPath, true); err != nil { return fmt.Errorf("failed to write config file: %w", err) } - if err := b.icg.WriteVault(b.Env.SecretsFilePath, true); err != nil { + if err := dc.icg.WriteVault(dc.SecretsFilePath, true); err != nil { return fmt.Errorf("failed to write vault file: %w", err) } - err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallConfigPath, remoteInstallConfigPath) + // CopyFile creates the destination directory, so a secondary data center's secrets + // directory does not need to exist yet. + err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, dc.InstallConfigPath, dc.RemoteConfigPath) if err != nil { return fmt.Errorf("failed to copy install config to jumpbox: %w", err) } - err = b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.SecretsFilePath, b.Env.SecretsDir+"/prod.vault.yaml") + err = b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, dc.SecretsFilePath, dc.RemoteVaultPath()) if err != nil { return fmt.Errorf("failed to copy secrets file to jumpbox: %w", err) } + b.mirrorPrimaryDataCenter() + return nil } -func (b *GCPBootstrapper) applySshProxyConfig() { - b.Env.InstallConfig.PcApps = util.DeepMergeMaps(b.Env.InstallConfig.PcApps, files.ChartValues{ +func (b *GCPBootstrapper) applySshProxyConfig(dc *DataCenter) { + dc.InstallConfig.PcApps = util.DeepMergeMaps(dc.InstallConfig.PcApps, files.ChartValues{ "applications": map[string]any{ "ssh-workspace-proxy": map[string]any{ "enabled": true, @@ -440,9 +464,9 @@ func (b *GCPBootstrapper) applySshProxyConfig() { "service": map[string]any{ "enabled": true, "type": "LoadBalancer", - "loadBalancerIP": b.Env.SshProxyIP, + "loadBalancerIP": dc.SshProxyIP, "annotations": map[string]any{ - "cloud.google.com/load-balancer-ipv4": b.Env.SshProxyIP, + "cloud.google.com/load-balancer-ipv4": dc.SshProxyIP, }, }, }, @@ -451,16 +475,16 @@ func (b *GCPBootstrapper) applySshProxyConfig() { }) } -func (b *GCPBootstrapper) applyExternalLokiConfig() { +func (b *GCPBootstrapper) applyExternalLokiConfig(dc *DataCenter) { if b.Env.ExternalLokiEndpoint == "" { return } - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy == nil { - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy = &files.GrafanaAlloyConfig{} + if dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy == nil { + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy = &files.GrafanaAlloyConfig{} } loki := &files.LokiConnectionConfig{ @@ -469,42 +493,42 @@ func (b *GCPBootstrapper) applyExternalLokiConfig() { Password: b.Env.ExternalLokiSecret, } - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Enabled = true - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Loki = loki - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretLokiGatewayBasicAuthPassword, Fields: &files.SecretFields{Password: b.Env.ExternalLokiSecret}}) + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Enabled = true + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Loki = loki + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretLokiGatewayBasicAuthPassword, Fields: &files.SecretFields{Password: b.Env.ExternalLokiSecret}}) } -func (b *GCPBootstrapper) applyPrometheusRemoteWriteConfig() { +func (b *GCPBootstrapper) applyPrometheusRemoteWriteConfig(dc *DataCenter) { if b.Env.PrometheusRemoteWriteURL == "" { return } - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.Prometheus == nil { - b.Env.InstallConfig.Cluster.Monitoring.Prometheus = &files.PrometheusConfig{} + if dc.InstallConfig.Cluster.Monitoring.Prometheus == nil { + dc.InstallConfig.Cluster.Monitoring.Prometheus = &files.PrometheusConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite == nil { - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite = &files.RemoteWriteConfig{} + if dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite == nil { + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite = &files.RemoteWriteConfig{} } - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Enabled = true - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Url = b.Env.PrometheusRemoteWriteURL - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.ClusterName = b.Env.DatacenterName - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Username = b.Env.PrometheusRemoteWriteUser - b.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWritePassword", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWritePassword}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWriteUser", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWriteUser}}) + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Enabled = true + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Url = b.Env.PrometheusRemoteWriteURL + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.ClusterName = dc.Name + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Username = b.Env.PrometheusRemoteWriteUser + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWritePassword", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWritePassword}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWriteUser", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWriteUser}}) } // regeneratePostgresCerts regenerates PostgreSQL TLS certificates when the IP/hostname // changed or no private key was loaded from the vault. -func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname string) error { - vault := b.icg.GetVault() +func (b *GCPBootstrapper) regeneratePostgresCerts(dc *DataCenter, previousPrimaryIP, previousPrimaryHostname string) error { + vault := dc.icg.GetVault() primaryKeySecret := vault.GetSecret(files.SecretPostgresPrimaryServerKeyPem) primaryNeedsRegen := primaryKeySecret == nil || primaryKeySecret.File == nil || - previousPrimaryIP != b.Env.InstallConfig.Postgres.Primary.IP || - previousPrimaryHostname != b.Env.InstallConfig.Postgres.Primary.Hostname + previousPrimaryIP != dc.InstallConfig.Postgres.Primary.IP || + previousPrimaryHostname != dc.InstallConfig.Postgres.Primary.Hostname if primaryNeedsRegen { caSecret := vault.GetSecret(files.SecretPostgresCaKeyPem) @@ -513,9 +537,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri } primaryKeyPEM, primaryCertPEM, err := secrets.GenerateServerCertificate( caSecret.File.Content, - b.Env.InstallConfig.Postgres.CACertPem, - b.Env.InstallConfig.Postgres.Primary.Hostname, - []string{b.Env.InstallConfig.Postgres.Primary.IP}) + dc.InstallConfig.Postgres.CACertPem, + dc.InstallConfig.Postgres.Primary.Hostname, + []string{dc.InstallConfig.Postgres.Primary.IP}) if err != nil { return fmt.Errorf("failed to generate primary server certificate: %w", err) } @@ -523,9 +547,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri return fmt.Errorf("primary PostgreSQL cert/key validation failed: %w", err) } vault.SetSecret(files.SecretEntry{Name: files.SecretPostgresPrimaryServerKeyPem, File: &files.SecretFile{Name: "primary.key", Content: primaryKeyPEM}}) - b.Env.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem = primaryCertPEM + dc.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem = primaryCertPEM } - if b.Env.InstallConfig.Postgres.Replica != nil { + if dc.InstallConfig.Postgres.Replica != nil { replicaKeySecret := vault.GetSecret(files.SecretPostgresReplicaServerKeyPem) if replicaKeySecret == nil || replicaKeySecret.File == nil { caSecret := vault.GetSecret(files.SecretPostgresCaKeyPem) @@ -534,9 +558,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri } replicaKeyPEM, replicaCertPEM, err := secrets.GenerateServerCertificate( caSecret.File.Content, - b.Env.InstallConfig.Postgres.CACertPem, - b.Env.InstallConfig.Postgres.Replica.Name, - []string{b.Env.InstallConfig.Postgres.Replica.IP}) + dc.InstallConfig.Postgres.CACertPem, + dc.InstallConfig.Postgres.Replica.Name, + []string{dc.InstallConfig.Postgres.Replica.IP}) if err != nil { return fmt.Errorf("failed to generate replica server certificate: %w", err) } @@ -544,19 +568,25 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri return fmt.Errorf("replica PostgreSQL cert/key validation failed: %w", err) } vault.SetSecret(files.SecretEntry{Name: files.SecretPostgresReplicaServerKeyPem, File: &files.SecretFile{Name: "replica.key", Content: replicaKeyPEM}}) - b.Env.InstallConfig.Postgres.Replica.SSLConfig.ServerCertPem = replicaCertPEM + dc.InstallConfig.Postgres.Replica.SSLConfig.ServerCertPem = replicaCertPEM } } return nil } +// EnsureAgeKey generates the primary data center's age identity on the jumpbox. func (b *GCPBootstrapper) EnsureAgeKey() error { - hasKey := b.Env.Jumpbox.NodeClient.HasFile(b.Env.Jumpbox, b.Env.SecretsDir+"/age_key.txt") - if hasKey { + b.ensureDataCenters() + + return b.ensureAgeKey(b.primaryDC()) +} + +func (b *GCPBootstrapper) ensureAgeKey(dc *DataCenter) error { + if b.Env.Jumpbox.NodeClient.HasFile(b.Env.Jumpbox, dc.RemoteAgeKeyPath()) { return nil } - err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("mkdir -p %s; age-keygen -o %s/age_key.txt", b.Env.SecretsDir, b.Env.SecretsDir)) + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("mkdir -p %s; age-keygen -o %s", dc.SecretsDir, dc.RemoteAgeKeyPath())) if err != nil { return fmt.Errorf("failed to generate age key on jumpbox: %w", err) } @@ -564,24 +594,41 @@ func (b *GCPBootstrapper) EnsureAgeKey() error { return nil } +// EnsureSecrets loads the primary data center's vault if it already exists locally. func (b *GCPBootstrapper) EnsureSecrets() error { - if b.fw.Exists(b.Env.SecretsFilePath) { - err := b.icg.LoadVaultFromUnecryptedFile(b.Env.SecretsFilePath) + b.ensureDataCenters() + + return b.ensureSecrets(b.primaryDC()) +} + +func (b *GCPBootstrapper) ensureSecrets(dc *DataCenter) error { + if b.fw.Exists(dc.SecretsFilePath) { + err := dc.icg.LoadVaultFromUnecryptedFile(dc.SecretsFilePath) if err != nil { return fmt.Errorf("failed to load vault file: %w", err) } } - b.Env.Secrets = b.icg.GetVault() + if dc.IsPrimary() { + b.Env.Secrets = dc.icg.GetVault() + } + b.mirrorPrimaryDataCenter() return nil } +// EncryptVault encrypts the primary data center's vault on the jumpbox. func (b *GCPBootstrapper) EncryptVault() error { - err := b.Env.Jumpbox.RunSSHCommand("root", "cp "+b.Env.SecretsDir+"/prod.vault.yaml{,.bak}") + b.ensureDataCenters() + + return b.encryptVault(b.primaryDC()) +} + +func (b *GCPBootstrapper) encryptVault(dc *DataCenter) error { + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("cp %s{,.bak}", dc.RemoteVaultPath())) if err != nil { return fmt.Errorf("failed backup vault on jumpbox: %w", err) } - err = b.Env.Jumpbox.RunSSHCommand("root", "sops --encrypt --in-place --age $(age-keygen -y "+b.Env.SecretsDir+"/age_key.txt) "+b.Env.SecretsDir+"/prod.vault.yaml") + err = b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("sops --encrypt --in-place --age $(age-keygen -y %s) %s", dc.RemoteAgeKeyPath(), dc.RemoteVaultPath())) if err != nil { return fmt.Errorf("failed to encrypt vault on jumpbox: %w", err) } @@ -589,10 +636,10 @@ func (b *GCPBootstrapper) EncryptVault() error { return nil } -// decryptVault creates an unencrypted copy of the vault in dst on the jumpbox +// decryptVault creates an unencrypted copy of the data center's vault in dst on the jumpbox. // Make sure to delete the unencrypted file when not needed anymore. -func (b *GCPBootstrapper) decryptVault(dst string) error { - err := b.Env.Jumpbox.RunSSHCommand("root", "cp "+b.Env.SecretsDir+"/prod.vault.yaml "+dst) +func (b *GCPBootstrapper) decryptVault(dc *DataCenter, dst string) error { + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("cp %s %s", dc.RemoteVaultPath(), dst)) if err != nil { return fmt.Errorf("failed to create tmp vault on jumpbox: %w", err) } @@ -602,7 +649,7 @@ func (b *GCPBootstrapper) decryptVault(dst string) error { return fmt.Errorf("failed to make vault file readable only for root on jumpbox: %w", err) } - err = b.Env.Jumpbox.RunSSHCommand("root", "SOPS_AGE_KEY_FILE="+b.Env.SecretsDir+"/age_key.txt sops --decrypt --in-place "+dst) + err = b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("SOPS_AGE_KEY_FILE=%s sops --decrypt --in-place %s", dc.RemoteAgeKeyPath(), dst)) if err != nil { return fmt.Errorf("failed to decrypt vault on jumpbox: %w", err) } From c5f430fb8f83b148bd328928c6e2726f77702a2c Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:14:11 +0200 Subject: [PATCH 06/11] refac(gcp): resolve the container registry once for all data centers The registry was written straight into the single install config, so which registry the nodes pull from was decided per config rather than per project. It now resolves onto the environment (ContainerRegistryURL plus credentials) and updateInstallConfig applies it to every data center's config and vault. All three registry types go through the same field, which also fixes the artifact registry never recording its URI on the create path. EnsureLocalContainerRegistry is split in two, because it early-returned when the registry was already running and thereby skipped distributing the registry certificate. A re-run that adds a data center hits exactly that path, and its nodes would then fail every pull with "certificate signed by unknown authority". Starting the registry stays conditional; distributing the certificate now always runs, over every data center's cluster nodes, which is safe because it is idempotent. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/gcp.go | 64 +++++++++++++++------- internal/bootstrap/gcp/gcp_test.go | 69 +++++++++++++++++++----- internal/bootstrap/gcp/install_config.go | 12 ++++- 3 files changed, 112 insertions(+), 33 deletions(-) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index d6a2d19f..632c28f7 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -711,7 +711,7 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { repo, err := b.GCPClient.GetArtifactRegistry(b.Env.ProjectID, b.Env.Region, repoName) if err == nil && repo != nil { - b.Env.InstallConfig.Registry.Server = repo.GetRegistryUri() + b.Env.ContainerRegistryURL = repo.GetRegistryUri() return nil } @@ -719,6 +719,7 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { if err != nil || repo == nil { return fmt.Errorf("failed to create artifact registry: %w, repo: %v", err, repo) } + b.Env.ContainerRegistryURL = repo.GetRegistryUri() return nil } @@ -1043,33 +1044,52 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error { return nil } -// EnsureLocalContainerRegistry installs a docker registry on the postgres node to speed up image loading time +// EnsureLocalContainerRegistry installs a container registry on the postgres node to speed up +// image loading time, and makes every cluster node of every data center trust its certificate. func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { + b.ensureDataCenters() + + registryServer, err := b.ensureRegistryRunning() + if err != nil { + return err + } + b.Env.ContainerRegistryURL = registryServer + + // The certificate must be distributed on every run, not only when the registry was just + // created: a re-run that adds a data center finds the registry already up, and that data + // center's nodes would otherwise not trust it. + return b.distributeRegistryCert(b.clusterNodes()) +} + +// ensureRegistryRunning starts the container registry on the postgres node and generates its +// credentials when it is not already serving. Returns the registry server address. +func (b *GCPBootstrapper) ensureRegistryRunning() (string, error) { localRegistryServer := b.Env.PostgreSQLNode.GetInternalIP() + ":5000" // Figure out if registry is already running b.stlog.Logf("Checking if local container registry is already running on postgres node") checkCommand := `test "$(podman ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` err := b.Env.PostgreSQLNode.RunSSHCommand("root", checkCommand) + vault := b.primaryDC().ConfigManager().GetVault() registryUsername := "" registryPassword := "" - if s := b.icg.GetVault().GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { registryUsername = s.Fields.Password } - if s := b.icg.GetVault().GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { registryPassword = s.Fields.Password } - if err == nil && b.Env.InstallConfig.Registry != nil && b.Env.InstallConfig.Registry.Server == localRegistryServer && - registryUsername != "" && registryPassword != "" { + if err == nil && registryUsername != "" && registryPassword != "" { b.stlog.Logf("Local container registry already running on postgres node") - return nil + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword + return localRegistryServer, nil } - b.Env.InstallConfig.Registry.Server = localRegistryServer registryUsername = "custom-registry" registryPassword = shortuuid.New() - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: registryUsername}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: registryPassword}}) + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword commands := []string{ "apt-get update", @@ -1089,19 +1109,24 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { -v /root/registry.crt:/certs/registry.crt \ -v /root/registry.key:/certs/registry.key \ registry:2`, - `mkdir -p /etc/docker/certs.d/` + b.Env.InstallConfig.Registry.Server, - `cp /root/registry.crt /etc/docker/certs.d/` + b.Env.InstallConfig.Registry.Server + `/ca.crt`, + `mkdir -p /etc/docker/certs.d/` + localRegistryServer, + `cp /root/registry.crt /etc/docker/certs.d/` + localRegistryServer + `/ca.crt`, } for _, cmd := range commands { b.stlog.Logf("Running command on postgres node: %s", util.Truncate(cmd, 12)) err := b.Env.PostgreSQLNode.RunSSHCommand("root", cmd) if err != nil { - return fmt.Errorf("failed to run command on postgres node: %w", err) + return "", fmt.Errorf("failed to run command on postgres node: %w", err) } } - allNodes := append(b.Env.ControlPlaneNodes, b.Env.CephNodes...) - for _, node := range allNodes { + return localRegistryServer, nil +} + +// distributeRegistryCert installs the local registry's self-signed certificate on the given +// nodes. It is idempotent, so it is safe — and required — to re-run for an additional data center. +func (b *GCPBootstrapper) distributeRegistryCert(nodes []*node.Node) error { + for _, node := range nodes { b.stlog.Logf("Configuring node '%s' to trust local registry certificate", node.GetName()) err := b.Env.PostgreSQLNode.RunSSHCommand("root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.GetInternalIP()+":/usr/local/share/ca-certificates/registry.crt") if err != nil { @@ -1120,15 +1145,14 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { return nil } +// EnsureGitHubAccessConfigured resolves ghcr.io as the registry all data centers pull from. func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error { if b.Env.GitHubPAT == "" { return fmt.Errorf("GitHub PAT is not set") } - b.Env.InstallConfig.Registry.Server = "ghcr.io" - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUser}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.GitHubPAT}}) - b.Env.InstallConfig.Registry.ReplaceImagesInBom = false - b.Env.InstallConfig.Registry.LoadContainerImages = false + b.Env.ContainerRegistryURL = "ghcr.io" + b.Env.RegistryUsername = b.Env.RegistryUser + b.Env.RegistryPassword = b.Env.GitHubPAT return nil } diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index a47e8754..0ba80054 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -851,8 +851,7 @@ var _ = Describe("GCP Bootstrapper", func() { Describe("EnsureLocalContainerRegistry", func() { Describe("Valid EnsureLocalContainerRegistry", func() { It("installs local registry", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) + icg.EXPECT().GetVault().Return(&files.InstallVault{}) // Setup mocked node // Check if running - return error to simulate not running @@ -868,7 +867,56 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureLocalContainerRegistry() Expect(err).NotTo(HaveOccurred()) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal("custom-registry")) + Expect(bs.Env.RegistryUsername).To(Equal("custom-registry")) + Expect(bs.Env.RegistryPassword).NotTo(BeEmpty()) + Expect(bs.Env.ContainerRegistryURL).To(Equal(bs.Env.PostgreSQLNode.GetInternalIP() + ":5000")) + }) + + // A re-run that adds a data center finds the registry already up. Its nodes still + // need the registry's self-signed certificate, or every image pull fails. + It("distributes the registry certificate even when the registry is already running", func() { + vault := &files.InstallVault{} + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: "custom-registry"}}) + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: "existing-password"}}) + icg.EXPECT().GetVault().Return(vault) + + bs.Env.MultiDC = true + bs.Env.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1", nodeClient)} + bs.Env.CephNodes = []*node.Node{fakeNode("ceph-1", nodeClient)} + secondary := &gcp.DataCenter{ID: 2, Suffix: "-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + // Registry is already running with credentials in the vault. + nodeClient.EXPECT().RunCommand(bs.Env.PostgreSQLNode, "root", mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "podman ps") + })).Return(nil) + + scpTargets := []string{} + nodeClient.EXPECT().RunCommand(bs.Env.PostgreSQLNode, "root", mock.MatchedBy(func(cmd string) bool { + return strings.HasPrefix(cmd, "scp ") + })).RunAndReturn(func(_ *node.Node, _ string, cmd string) error { + scpTargets = append(scpTargets, cmd) + return nil + }).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "update-ca-certificates").Return(nil).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "systemctl restart docker.service || true").Return(nil).Times(4) + + // Register the second data center only after ensureDataCenters would have run, + // mirroring what EnsureComputeInstances produces for a --multi-dc bootstrap. + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + bs.Env.DataCenters[0].SetConfigManager(icg) + + Expect(bs.EnsureLocalContainerRegistry()).To(Succeed()) + Expect(scpTargets).To(HaveLen(4)) + Expect(bs.Env.RegistryPassword).To(Equal("existing-password")) }) }) @@ -988,17 +1036,14 @@ var _ = Describe("GCP Bootstrapper", func() { csEnv.GitHubPAT = "fake-pat" csEnv.RegistryUser = "custom-registry" }) - It("sets configuration options in installconfig", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) - + // The resolved registry and its credentials live on the environment; every data center's + // config picks them up in updateInstallConfig. + It("resolves ghcr.io as the registry for all data centers", func() { err := bs.EnsureGitHubAccessConfigured() Expect(err).NotTo(HaveOccurred()) - Expect(bs.Env.InstallConfig.Registry.Server).To(Equal("ghcr.io")) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal(csEnv.RegistryUser)) - Expect(vault.GetSecret(files.SecretRegistryPassword).Fields.Password).To(Equal(csEnv.GitHubPAT)) - Expect(bs.Env.InstallConfig.Registry.LoadContainerImages).To(BeFalse()) - Expect(bs.Env.InstallConfig.Registry.ReplaceImagesInBom).To(BeFalse()) + Expect(bs.Env.ContainerRegistryURL).To(Equal("ghcr.io")) + Expect(bs.Env.RegistryUsername).To(Equal(csEnv.RegistryUser)) + Expect(bs.Env.RegistryPassword).To(Equal(csEnv.GitHubPAT)) }) Context("When GitHub PAT is missing", func() { diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 4a78fb32..4c80f872 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -141,7 +141,17 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes // steps overwrite another's credentials. dc.InstallConfig.Secrets.BaseDir = dc.SecretsDir - if b.Env.RegistryType != RegistryTypeGitHub { + if b.Env.ContainerRegistryURL != "" { + dc.InstallConfig.Registry.Server = b.Env.ContainerRegistryURL + } + if b.Env.RegistryUsername != "" || b.Env.RegistryPassword != "" { + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUsername}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.RegistryPassword}}) + } + if b.Env.RegistryType == RegistryTypeGitHub { + dc.InstallConfig.Registry.ReplaceImagesInBom = false + dc.InstallConfig.Registry.LoadContainerImages = false + } else { dc.InstallConfig.Registry.ReplaceImagesInBom = true dc.InstallConfig.Registry.LoadContainerImages = true } From 86fc8cb05bd6f0d8321b08d109997e8d38aee7c2 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:19:00 +0200 Subject: [PATCH 07/11] feat(gcp): bootstrap two data centers with --multi-dc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires up multi-data-center support: --multi-dc bootstraps a second data center in the same project, sharing the VPC, jumpbox and PostgreSQL server but running its own Kubernetes and Ceph cluster. The secondary data center's config and vault are derived from the primary's after its secrets exist, which is why Bootstrap handles it in a loop after the primary's config is written: - the config is cloned and its data-center-scoped fields cleared, so the installer's dataCenters topology, the domains and the shared registry are inherited while the ingress CA and cephadm key are regenerated; - the vault is derived through DeriveDataCenterVault, keeping the postgres roles and token keys and dropping the per-cluster secrets; - postgres becomes mode: external pointing at the shared server's internal IP, which is what its certificate's only SAN carries, and "postgres" is appended to operations.skip in the config so manual re-runs on the jumpbox skip it too. A post-generation check fails the bootstrap if secrets that must match across data centers diverged, or if a per-cluster secret was inherited instead of regenerated — both would otherwise surface long after the fact, against a live shared database. Installs run strictly sequentially in ascending data center order, since the primary's install creates the database, roles and schema the others reuse. validateMultiDC rejects the combinations that cannot work (--write-config=false, an explicit --datacenter-id, an empty datacenter name), and the CLI prints one install command per data center in the required order. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- .gitignore | 4 + cli/cmd/bootstrap_gcp.go | 18 +- docs/oms_beta_bootstrap-gcp.md | 1 + internal/bootstrap/gcp/gcp.go | 80 +++++- internal/bootstrap/gcp/install_config.go | 268 ++++++++++++++++-- internal/bootstrap/gcp/install_config_test.go | 4 + 6 files changed, 344 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index eb2c4d4b..011d5c24 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,10 @@ internal/util/testdata/ config.yaml prod.vault.yaml configure-k0s.sh +# ... and their per-data-center variants written by a --multi-dc bootstrap +config-dc*.yaml +prod-dc*.vault.yaml +configure-k0s-dc*.sh # Debugger files __debug* diff --git a/cli/cmd/bootstrap_gcp.go b/cli/cmd/bootstrap_gcp.go index 8f87eae5..b9f557eb 100644 --- a/cli/cmd/bootstrap_gcp.go +++ b/cli/cmd/bootstrap_gcp.go @@ -89,6 +89,7 @@ func AddBootstrapGcpCmd(parent *cobra.Command, opts *util.GlobalOptions) { flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.SpotVMs, "spot-vms", false, "Use Spot VMs for Codesphere infrastructure. Falls back to standard VMs if spot capacity unavailable. Mutually exclusive with --preemptible (default: false)") flags.IntVar(&bootstrapGcpCmd.CodesphereEnv.DatacenterID, "datacenter-id", 1, "Datacenter ID (default: 1)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DatacenterName, "datacenter-name", "dev", "Datacenter name (default: dev)") + flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.MultiDC, "multi-dc", false, "Bootstrap two data centers that share one PostgreSQL server but run separate Kubernetes and Ceph clusters. Doubles the Ceph and k0s nodes to 14 VMs (~100 vCPUs) and reserves 6 static IPs, so the region's CPU quota may need raising. Cannot be combined with --datacenter-id. (default: false)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.CustomPgIP, "custom-pg-ip", "", "Custom PostgreSQL IP (optional)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.Region, "region", "europe-west4", "GCP Region (default: europe-west4)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.Zone, "zone", "europe-west4-a", "GCP Zone (default: europe-west4-a)") @@ -174,6 +175,8 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error { c.CodesphereEnv.RegistryType = gcp.RegistryType(c.InputRegistryType) c.CodesphereEnv.OmsWorkdir = c.Env.GetOmsWorkdir() + // The value alone cannot distinguish the default 1 from an explicit --datacenter-id=1. + c.CodesphereEnv.DatacenterIDExplicit = c.cmd.Flags().Changed("datacenter-id") if c.CodesphereEnv.GitHubPAT != "" { c.CodesphereEnv.RegistryType = gcp.RegistryTypeGitHub if c.CodesphereEnv.RegistryUser == "" { @@ -208,18 +211,27 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error { if bs.Env.InstallVersion != "" { log.Printf("Access Codesphere in your web browser at https://cs.%s", bs.Env.BaseDomain) + for _, dc := range bs.Env.DataCenters { + log.Printf("Data center %d hosts workspaces under %s", dc.ID, dc.WorkspaceHostingBaseDomain) + } return nil } packageName := "-installer" - installCmd := "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml" if gcp.RegistryType(bs.Env.RegistryType) == gcp.RegistryTypeGitHub { log.Printf("You set a GitHub PAT for direct image access. Make sure to use a lite package, as VM root disk sizes are reduced.") - installCmd += " -s load-container-images" packageName += "-lite" } - log.Printf("example install command (run from jumpbox):\n%s -p %s.tar.gz", installCmd, packageName) + packageFile := packageName + ".tar.gz" + if len(bs.Env.DataCenters) > 1 { + log.Printf("example install commands (run from jumpbox). Run the data center 1 command to completion first — the other data centers share its database:") + } else { + log.Printf("example install command (run from jumpbox):") + } + for _, dc := range bs.Env.DataCenters { + log.Printf("# data center %d\n%s", dc.ID, bs.InstallCommand(dc, packageFile)) + } return nil } diff --git a/docs/oms_beta_bootstrap-gcp.md b/docs/oms_beta_bootstrap-gcp.md index b19a359f..eb3facf0 100644 --- a/docs/oms_beta_bootstrap-gcp.md +++ b/docs/oms_beta_bootstrap-gcp.md @@ -57,6 +57,7 @@ oms beta bootstrap-gcp [flags] --install-version string Codesphere version to install (default: none) --internal-flags stringArray Internal flags to enable in Codesphere installation (optional) (default [headless-services,vcluster,custom-service-image,ms-in-ls]) --local-trace-endpoint string Endpoint for exporting traces to an in-cluster storage (optional) + --multi-dc Bootstrap two data centers that share one PostgreSQL server but run separate Kubernetes and Ceph clusters. Doubles the Ceph and k0s nodes to 14 VMs (~100 vCPUs) and reserves 6 static IPs, so the region's CPU quota may need raising. Cannot be combined with --datacenter-id. (default: false) --oidc-client-id string OIDC OAuth provider Client ID (optional) --oidc-client-secret string OIDC OAuth provider Client Secret (optional) --oidc-issuer-url string OIDC OAuth provider issuer URL (optional) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 632c28f7..bcb40227 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -150,6 +150,11 @@ func (b *GCPBootstrapper) primaryDC() *DataCenter { return b.Env.DataCenters[0] } +// secondaryDCs returns every data center apart from the primary one. +func (b *GCPBootstrapper) secondaryDCs() []*DataCenter { + return b.Env.DataCenters[1:] +} + // allNodes returns every node of the project: the jumpbox, the shared postgres node and all // data centers' Ceph and k0s nodes. func (b *GCPBootstrapper) allNodes() []*node.Node { @@ -416,19 +421,41 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.WriteConfig { - err = b.stlog.Step("Update install config", b.UpdateInstallConfig) + err = b.writeDataCenterConfig(b.primaryDC()) if err != nil { - return fmt.Errorf("failed to update install config: %w", err) + return err + } + } + + // Secondary data centers fall back to deriving their config and vault from the primary one, + // so this has to run after the primary's secrets were generated above. + for _, dc := range b.secondaryDCs() { + err = b.stlog.Step(dc.StepName("Ensure install config"), func() error { + return b.ensureInstallConfig(dc) + }) + if err != nil { + return fmt.Errorf("failed to ensure install config of data center %d: %w", dc.ID, err) } - err = b.stlog.Step("Ensure age key", b.EnsureAgeKey) + err = b.stlog.Step(dc.StepName("Ensure secrets"), func() error { + return b.ensureSecrets(dc) + }) if err != nil { - return fmt.Errorf("failed to ensure age key: %w", err) + return fmt.Errorf("failed to ensure secrets of data center %d: %w", dc.ID, err) } - err = b.stlog.Step("Encrypt vault", b.EncryptVault) + err = b.stlog.Step(dc.StepName("Derive config and vault"), func() error { + return b.seedSecondaryDataCenter(b.primaryDC(), dc) + }) if err != nil { - return fmt.Errorf("failed to encrypt vault: %w", err) + return fmt.Errorf("failed to derive data center %d: %w", dc.ID, err) + } + + if b.Env.WriteConfig { + err = b.writeDataCenterConfig(dc) + if err != nil { + return err + } } } @@ -554,9 +581,50 @@ func (b *GCPBootstrapper) ValidateInput() error { return err } + err = b.validateMultiDC() + if err != nil { + return err + } + return b.validateTelemetryExportParams() } +// validateMultiDC rejects flag combinations a multi-data-center bootstrap cannot satisfy. +func (b *GCPBootstrapper) validateMultiDC() error { + if !b.Env.MultiDC { + return nil + } + + // A secondary data center's config and vault are derived from the primary's, which only + // happens when configs are written. + if !b.Env.WriteConfig { + return fmt.Errorf("multi-dc requires write-config to be enabled") + } + + // The data center IDs are derived (1 and 2) and drive the workspace hosting domains. + if b.Env.DatacenterIDExplicit { + return fmt.Errorf("datacenter-id cannot be combined with multi-dc, the IDs are derived") + } + + // The k0s cluster is named codesphere-, so both names must be set and + // distinct. BuildDataCenters derives the second one by suffixing the first. + if b.Env.DatacenterName == "" { + return fmt.Errorf("datacenter-name is required with multi-dc") + } + + // Every data center gets its own local config and vault, derived by suffixing these paths. + for name, path := range map[string]string{ + "install-config": b.Env.InstallConfigPath, + "secrets-file": b.Env.SecretsFilePath, + } { + if path == "" { + return fmt.Errorf("cannot derive a per-data-center path: %s is empty", name) + } + } + + return nil +} + func (b *GCPBootstrapper) validateClusterAdminEmail() error { if b.Env.ClusterAdminEmail == "" { return nil diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 4c80f872..b2438aa4 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -4,9 +4,13 @@ package gcp import ( + "errors" "fmt" + "reflect" + "slices" "github.com/codesphere-cloud/oms/internal/bootstrap" + "github.com/codesphere-cloud/oms/internal/codesphere" "github.com/codesphere-cloud/oms/internal/installer/files" "github.com/codesphere-cloud/oms/internal/installer/secrets" "github.com/codesphere-cloud/oms/internal/util" @@ -14,9 +18,17 @@ import ( const ( remoteInstallConfigPath string = "/etc/codesphere/config.yaml" + // sharedPostgresPort is the port the shared PostgreSQL server listens on. Secondary data + // centers need it spelled out because they connect to it as an external server. + sharedPostgresPort int = 5432 ) -// EnsureInstallConfig prepares the primary data center's install config. +// errRemoteConfigMissing reports that the jumpbox holds no config for a data center. Recovering +// a secondary data center tolerates this, since --multi-dc can add one to an existing project. +var errRemoteConfigMissing = errors.New("no install config found on the jumpbox") + +// EnsureInstallConfig prepares the primary data center's install config. Secondary data centers +// are handled separately in Bootstrap, after the primary's secrets exist. func (b *GCPBootstrapper) EnsureInstallConfig() error { b.ensureDataCenters() @@ -29,7 +41,12 @@ func (b *GCPBootstrapper) ensureInstallConfig(dc *DataCenter) error { // recovery will overwrite local config or create a new file if b.Env.RecoverConfig { err := b.recoverConfig(dc) - if err != nil { + if errors.Is(err, errRemoteConfigMissing) && !dc.IsPrimary() { + // A secondary data center may not exist on the jumpbox yet, which is the case when + // --multi-dc is used to add one to an existing single-DC project. Its config is + // derived from the primary's instead. + b.stlog.Logf("No config found on the jumpbox for data center %d, generating a new one", dc.ID) + } else if err != nil { return fmt.Errorf("failed to recover config: %w", err) } } @@ -45,14 +62,17 @@ func (b *GCPBootstrapper) ensureInstallConfig(dc *DataCenter) error { } dc.ExistingConfigUsed = true - } else { + dc.InstallConfig = dc.icg.GetInstallConfig() + } else if dc.IsPrimary() { err := dc.icg.ApplyProfile("minimal") if err != nil { return fmt.Errorf("failed to apply profile: %w", err) } + dc.InstallConfig = dc.icg.GetInstallConfig() } + // A secondary data center without a config of its own is left unset here, so + // seedSecondaryDataCenter can derive it from the primary data center instead of the profile. - dc.InstallConfig = dc.icg.GetInstallConfig() b.mirrorPrimaryDataCenter() return nil @@ -87,6 +107,12 @@ func (b *GCPBootstrapper) recoverConfig(dc *DataCenter) error { } b.Env.Jumpbox = jumpbox + // Only a secondary data center may legitimately be absent from the jumpbox, so only there is + // it worth probing first; the primary's missing config stays a download failure. + if !dc.IsPrimary() && !b.Env.Jumpbox.NodeClient.HasFile(jumpbox, dc.RemoteConfigPath) { + return fmt.Errorf("%w at %s", errRemoteConfigMissing, dc.RemoteConfigPath) + } + err = b.Env.Jumpbox.NodeClient.DownloadFile(jumpbox, dc.RemoteConfigPath, dc.InstallConfigPath) if err != nil { return fmt.Errorf("failed to download install config from jumpbox: %w", err) @@ -124,7 +150,8 @@ func (b *GCPBootstrapper) recoverVault(dc *DataCenter) error { } // UpdateInstallConfig writes the bootstrapped infrastructure into the primary data center's -// install config. +// install config. Secondary data centers go through updateInstallConfig directly, after their +// config and vault have been derived from the primary's. func (b *GCPBootstrapper) UpdateInstallConfig() error { b.ensureDataCenters() @@ -133,10 +160,7 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { // Update install config with necessary values - dc.InstallConfig.Datacenter.ID = dc.ID - dc.InstallConfig.Datacenter.Name = dc.Name - dc.InstallConfig.Datacenter.City = "Karlsruhe" - dc.InstallConfig.Datacenter.CountryCode = "DE" + dc.InstallConfig.Datacenter = datacenterConfig(dc) // Each data center reads and writes its own vault. The installer resolves the vault from // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes // steps overwrite another's credentials. @@ -156,16 +180,8 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { dc.InstallConfig.Registry.LoadContainerImages = true } - if dc.InstallConfig.Postgres.Primary == nil { - dc.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ - Hostname: b.Env.PostgreSQLNode.GetName(), - } - } - - previousPrimaryIP := dc.InstallConfig.Postgres.Primary.IP - previousPrimaryHostname := dc.InstallConfig.Postgres.Primary.Hostname - dc.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() - dc.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() + previousPrimaryIP, previousPrimaryHostname := b.applyPostgresConfig(dc) + b.applyDataCenterTopology(dc) dc.InstallConfig.Ceph.CsiKubeletDir = "/var/lib/k0s/kubelet" // All data centers share the project's subnet; their Ceph clusters stay separate because @@ -290,7 +306,11 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { dc.InstallConfig.Codesphere.CustomDomains = files.CustomDomainsConfig{ CNameBaseDomain: dc.WorkspaceHostingBaseDomain, } - dc.InstallConfig.Codesphere.PublicIP = dc.ControlPlaneNodes[1].GetExternalIP() + if b.Env.MultiDC { + dc.InstallConfig.Codesphere.PublicIP = dc.PublicGatewayIP + } else { + dc.InstallConfig.Codesphere.PublicIP = dc.ControlPlaneNodes[1].GetExternalIP() + } dc.InstallConfig.Codesphere.DNSServers = []string{"8.8.8.8"} dc.InstallConfig.Codesphere.DeployConfig = bootstrap.DefaultCodesphereDeployConfig() dc.InstallConfig.Codesphere.Plans = bootstrap.DefaultCodespherePlans() @@ -408,7 +428,9 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { b.applyExternalLokiConfig(dc) b.applyPrometheusRemoteWriteConfig(dc) - if !dc.ExistingConfigUsed { + // A secondary data center always generates: its vault was seeded from the primary's, so the + // sentinels stop everything except its own ingress CA and cephadm key from being regenerated. + if !dc.ExistingConfigUsed || !dc.IsPrimary() { err := dc.icg.GenerateSecrets() if err != nil { return fmt.Errorf("failed to generate secrets: %w", err) @@ -419,6 +441,12 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { } } + if !dc.IsPrimary() { + if err := b.verifySecondaryDataCenterSecrets(b.primaryDC(), dc); err != nil { + return err + } + } + if b.Env.CentralOtelUsername != "" && b.Env.CentralOtelPassword != "" { if dc.InstallConfig.Cluster.Monitoring == nil { dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} @@ -465,6 +493,108 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { return nil } +// datacenterConfig describes a data center the way the install config does. All data centers of a +// bootstrapped instance live in the same GCP region, so city and country code are the same for all +// of them. +func datacenterConfig(dc *DataCenter) files.DatacenterConfig { + return files.DatacenterConfig{ + ID: dc.ID, + Name: dc.Name, + City: "Karlsruhe", + CountryCode: "DE", + } +} + +// applyDataCenterTopology tells the platform about every data center of the installation. Without +// it, the installer defaults the list to the local data center, so each data center of a multi-DC +// instance would render as a single-data-center one. The list is identical in every data center's +// config; dataCenter stays the local one, so the platform still knows which data center it runs in. +// +// A single data center keeps the list unset and relies on the installer's default, so its config is +// unchanged from what OMS has always written. +func (b *GCPBootstrapper) applyDataCenterTopology(dc *DataCenter) { + if len(b.Env.DataCenters) < 2 { + return + } + + all := make([]files.DatacenterConfig, 0, len(b.Env.DataCenters)) + for _, other := range b.Env.DataCenters { + all = append(all, datacenterConfig(other)) + } + dc.InstallConfig.DataCenters = all + // Clients land in the primary data center, which is also the one cs. resolves to. + dc.InstallConfig.DefaultDataCenterID = b.primaryDC().ID + + b.dropAvailableDataCentersOverride(dc) +} + +// dropAvailableDataCentersOverride removes the chart override OMS used to write before the +// installer supported dataCenters in the config. A recovered config may still carry it, where it +// would shadow the list above with a bare ID list. Any other override content is left alone. +func (b *GCPBootstrapper) dropAvailableDataCentersOverride(dc *DataCenter) { + global, ok := dc.InstallConfig.Codesphere.Override["global"].(map[string]interface{}) + if !ok { + return + } + + delete(global, "availableDataCenters") + if len(global) == 0 { + delete(dc.InstallConfig.Codesphere.Override, "global") + } + if len(dc.InstallConfig.Codesphere.Override) == 0 { + dc.InstallConfig.Codesphere.Override = nil + } +} + +// applyPostgresConfig points the data center at its PostgreSQL server. The primary data center +// installs the server on its own node; every other data center connects to that same server as +// an external one and skips the postgres install step. +// +// Returns the primary IP and hostname the config held before, so regeneratePostgresCerts can +// tell whether the server's identity changed. +func (b *GCPBootstrapper) applyPostgresConfig(dc *DataCenter) (previousIP, previousHostname string) { + if dc.ExternalPostgres { + dc.InstallConfig.Postgres = files.PostgresConfig{ + Mode: "external", + // The server certificate carries only an IP SAN, so the address must be the IP. + ServerAddress: b.Env.PostgreSQLNode.GetInternalIP(), + Port: sharedPostgresPort, + // The CA certificate lives in the config, not the vault, and cannot be re-derived + // from the CA key — so it has to be copied from the primary data center. + CACertPem: b.primaryDC().InstallConfig.Postgres.CACertPem, + Primary: nil, + Replica: nil, + } + b.skipInstallerStep(dc, "postgres") + + return "", "" + } + + if dc.InstallConfig.Postgres.Primary == nil { + dc.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ + Hostname: b.Env.PostgreSQLNode.GetName(), + } + } + + previousIP = dc.InstallConfig.Postgres.Primary.IP + previousHostname = dc.InstallConfig.Postgres.Primary.Hostname + dc.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() + dc.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() + + return previousIP, previousHostname +} + +// skipInstallerStep persists a skipped installer step in the data center's config, so manual +// `oms install codesphere` re-runs on the jumpbox skip it too. +func (b *GCPBootstrapper) skipInstallerStep(dc *DataCenter, step string) { + if dc.InstallConfig.Operations == nil { + dc.InstallConfig.Operations = &files.OperationsConfig{} + } + if !slices.Contains(dc.InstallConfig.Operations.Skip, step) { + dc.InstallConfig.Operations.Skip = append(dc.InstallConfig.Operations.Skip, step) + } +} + func (b *GCPBootstrapper) applySshProxyConfig(dc *DataCenter) { dc.InstallConfig.PcApps = util.DeepMergeMaps(dc.InstallConfig.PcApps, files.ChartValues{ "applications": map[string]any{ @@ -532,8 +662,13 @@ func (b *GCPBootstrapper) applyPrometheusRemoteWriteConfig(dc *DataCenter) { } // regeneratePostgresCerts regenerates PostgreSQL TLS certificates when the IP/hostname -// changed or no private key was loaded from the vault. +// changed or no private key was loaded from the vault. It is a no-op for a data center that +// connects to an external server, since that server owns its own certificates. func (b *GCPBootstrapper) regeneratePostgresCerts(dc *DataCenter, previousPrimaryIP, previousPrimaryHostname string) error { + if dc.InstallConfig.Postgres.Primary == nil { + return nil + } + vault := dc.icg.GetVault() primaryKeySecret := vault.GetSecret(files.SecretPostgresPrimaryServerKeyPem) primaryNeedsRegen := primaryKeySecret == nil || primaryKeySecret.File == nil || @@ -666,3 +801,92 @@ func (b *GCPBootstrapper) decryptVault(dc *DataCenter, dst string) error { return nil } + +// writeDataCenterConfig writes the data center's install config and vault, then places the +// encrypted vault and its age identity on the jumpbox. +func (b *GCPBootstrapper) writeDataCenterConfig(dc *DataCenter) error { + err := b.stlog.Step(dc.StepName("Update install config"), func() error { + return b.updateInstallConfig(dc) + }) + if err != nil { + return fmt.Errorf("failed to update install config of data center %d: %w", dc.ID, err) + } + + err = b.stlog.Step(dc.StepName("Ensure age key"), func() error { + return b.ensureAgeKey(dc) + }) + if err != nil { + return fmt.Errorf("failed to ensure age key of data center %d: %w", dc.ID, err) + } + + err = b.stlog.Step(dc.StepName("Encrypt vault"), func() error { + return b.encryptVault(dc) + }) + if err != nil { + return fmt.Errorf("failed to encrypt vault of data center %d: %w", dc.ID, err) + } + + return nil +} + +// seedSecondaryDataCenter derives a secondary data center's config and vault from the primary +// one's, for the parts it does not already have. Secrets tied to the shared database and to +// cross-data-center authentication are copied verbatim; the per-cluster ones are dropped so +// GenerateSecrets regenerates them for this data center. +// +// Anything the data center already loaded from its own files is kept, so re-runs do not rotate a +// live installation's secrets. +func (b *GCPBootstrapper) seedSecondaryDataCenter(primary, dc *DataCenter) error { + if dc.InstallConfig == nil { + config, err := secrets.DeriveDataCenterConfig(primary.InstallConfig) + if err != nil { + return fmt.Errorf("failed to derive config from data center %d: %w", primary.ID, err) + } + dc.icg.SetInstallConfig(config) + dc.InstallConfig = dc.icg.GetInstallConfig() + } + + if len(dc.icg.GetVault().Secrets) == 0 { + dc.icg.SetVault(secrets.DeriveDataCenterVault(primary.icg.GetVault())) + } + + return nil +} + +// verifySecondaryDataCenterSecrets fails the bootstrap if a secondary data center's secrets +// diverged from the primary's where they must match. Divergent PostgreSQL roles would let one +// data center's install rotate credentials the other one is using, and a divergent token key +// would break cross-data-center authentication — both only visible long after the fact. +func (b *GCPBootstrapper) verifySecondaryDataCenterSecrets(primary, dc *DataCenter) error { + primaryVault := primary.icg.GetVault() + vault := dc.icg.GetVault() + + shared := []string{files.SecretTokenPrivateKey, files.SecretPostgresPassword} + for _, svc := range codesphere.PostgresServices { + shared = append(shared, files.PostgresUserSecretName(svc.Name), files.PostgresPasswordSecretName(svc.Name)) + } + for _, name := range shared { + expected := primaryVault.GetSecret(name) + if expected == nil { + continue + } + if !reflect.DeepEqual(vault.GetSecret(name), expected) { + return fmt.Errorf("secret %q of data center %d differs from the primary data center, but both use the same database", name, dc.ID) + } + } + + if dc.InstallConfig.Postgres.CACertPem == "" { + return fmt.Errorf("data center %d has no postgres CA certificate and could not verify the shared server", dc.ID) + } + + for _, name := range []string{files.SecretKubeConfig, files.SecretCephSshPrivateKey} { + if vault.GetSecret(name) == nil { + continue + } + if reflect.DeepEqual(vault.GetSecret(name), primaryVault.GetSecret(name)) { + return fmt.Errorf("secret %q of data center %d is the primary data center's, but they run separate clusters", name, dc.ID) + } + } + + return nil +} diff --git a/internal/bootstrap/gcp/install_config_test.go b/internal/bootstrap/gcp/install_config_test.go index 6ccd50d0..3e4b0c1d 100644 --- a/internal/bootstrap/gcp/install_config_test.go +++ b/internal/bootstrap/gcp/install_config_test.go @@ -344,6 +344,10 @@ var _ = Describe("Installconfig & Secrets", func() { Expect(bs.Env.InstallConfig.Datacenter.ID).To(Equal(1)) Expect(bs.Env.InstallConfig.Datacenter.Name).To(Equal("dev")) + // A single data center relies on the installer defaulting the topology to the + // local data center, so the config stays as it was before multi-DC support. + Expect(bs.Env.InstallConfig.DataCenters).To(BeEmpty()) + Expect(bs.Env.InstallConfig.DefaultDataCenterID).To(BeZero()) Expect(bs.Env.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) Expect(bs.Env.InstallConfig.Codesphere.Features).To(Equal(map[string]bool{})) Expect(bs.Env.InstallConfig.Codesphere.Internal).To(Equal(gcp.DefaultInternalFlags)) From 5a9f70e3fc0ac43c002b821843f54e8e5c7c7a71 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:20:07 +0200 Subject: [PATCH 08/11] test(gcp): end-to-end multi-data-center bootstrap coverage and docs Covers a full --multi-dc bootstrap in one spec: two data centers with their own nodes, IPs, configs and vaults, the second one pointed at the first's PostgreSQL server, both told about the topology, and the secrets partitioned the way a shared database and separate clusters require. This is the test that pins the invariants the individual commits only enforce locally. Also documents multi-DC on GCP in the installation guide: the quota footprint, the install order, the two config/vault/secrets-dir paths, the topology keys and the DNS scheme, including the per-data-center platform records the frontend depends on. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- ...nstall-codesphere-on-any-infrastructure.md | 109 ++++++ internal/bootstrap/gcp/multi_dc_test.go | 327 ++++++++++++++++++ 2 files changed, 436 insertions(+) create mode 100644 internal/bootstrap/gcp/multi_dc_test.go diff --git a/docs/install-codesphere-on-any-infrastructure.md b/docs/install-codesphere-on-any-infrastructure.md index 6ff6b46c..597776b8 100644 --- a/docs/install-codesphere-on-any-infrastructure.md +++ b/docs/install-codesphere-on-any-infrastructure.md @@ -463,3 +463,112 @@ Run the Codesphere smoke test when an API key for the environment is available: ```bash oms smoketest codesphere --help ``` + +## Appendix: multiple data centers + +A Codesphere instance can span several data centers. They share one PostgreSQL server and one +platform domain, but each runs its own Kubernetes and Ceph cluster and hosts its own workspaces. +The steps above describe one data center; repeat them per data center with these differences. + +**Configuration.** All data centers keep the same `codesphere.domain`, `codesphere.plans`, +`codesphere.deployConfig`, `codesphere.managedServices`, feature flags and Git provider settings — +that data is stored in the shared database, so it must not diverge. Per data center: + +| Setting | Per data center | +| --- | --- | +| `dataCenter.id` / `dataCenter.name` | Distinct. The k0s cluster is named `codesphere-`, so the names must differ. | +| `codesphere.workspaceHostingBaseDomain` | `.`, resolving to that data center's workspace gateway. | +| `codesphere.publicIp` | That data center's public address. | +| `secrets.baseDir` | A directory of its own. The installer resolves the vault from `secrets.baseDir` and writes `kubeConfig` and the Ceph credentials back into it, so sharing a directory lets one data center overwrite another's. | +| `ceph.*`, `kubernetes.*`, gateway annotations | That data center's own hosts and addresses. | +| `cluster.monitoring.prometheus.remoteWrite.clusterName` | That data center's name. | + +**Topology.** Every data center's config lists all of them, so the platform knows which data centers +a user can pick from: + +```yaml +dataCenters: + - id: 1 + name: multidc + city: Karlsruhe + countryCode: DE + - id: 2 + name: multidc-dc2 + city: Karlsruhe + countryCode: DE +defaultDataCenterId: 1 +``` + +The list is identical in every data center's config, while `dataCenter` stays the local one. +Leaving `dataCenters` out defaults it to the local data center alone — correct for a single data +center, but it makes a multi-data-center instance render as single. + +**Shared PostgreSQL.** The first data center installs the server. Every other one sets +`postgres.mode: external` with `postgres.serverAddress` pointing at it, omits `postgres.primary` +and `postgres.replica`, copies `postgres.caCertPem` from the first data center's config, and adds +`postgres` to `operations.skip`. Use the server's IP address, not its hostname: the generated +server certificate carries an IP SAN only. + +**Secrets.** Derive each additional data center's vault from the first one's, keeping everything +that both must agree on and regenerating only what belongs to a single cluster: + +| Keep identical | Why | +| --- | --- | +| `postgresPassword`, `postgresReplicaPassword`, `postgresCaKeyPem`, and every `postgresUser*` / `postgresPassword*` pair | The roles live on the shared server. Divergent values make one data center's pods fail authentication, or let its install rotate credentials the other is using. | +| `tokenPrivateKey`, `tokenPublicKey` | A session token minted in one data center is presented to services in the other. | +| `domainAuthPrivateKey` / `PublicKey`, `mounterHmacSecret`, `mongoDbPasswordEncryptionKey` | They sign or encrypt rows in the shared database. | +| Registry, OAuth, OIDC and OpenBao credentials | The same external services. | + +| Regenerate per data center | Paired config field | +| --- | --- | +| `selfSignedCaKeyPem` | `cluster.certificates.ca.certPem` | +| `cephSshPrivateKey` | `ceph.cephAdmSshKey.publicKey` | +| `acmeEabMacKey` | `codesphere.certIssuer.acme.eabKeyId` | +| `kubeConfig`, and everything prefixed `ceph`, `csi` or `rgw` | written by the installer's `ceph` and `kubernetes` steps | + +Always clear the paired config field together with the vault secret. The generators are gated on +the vault entry, so a config field left in place keeps a stale value next to a fresh key. + +**DNS.** The platform name stays shared; the per-data-center platform, workspace and SSH names are +not: + +| Record | Target | +| --- | --- | +| ``, `*.` | The first data center's platform gateway | +| `.`, `*..` | **That** data center's platform gateway | +| `.`, `*..` | That data center's workspace gateway | +| `*..ssh.` | That data center's workspace SSH proxy | + +The second row is easy to miss. The platform builds each data center's service endpoint as +`.` and the browser calls it directly — before rendering any UI, it asks +the endpoint of the data center a workspace lives in for its configuration. Left to the +`*.` wildcard, that name resolves to the first data center's gateway, which has +no route for it, and the whole UI fails to load with a connection error. + +**Install order.** Install the first data center to completion before starting the next. Its +install creates the database, roles and schema that the others reuse. Every data center runs the +`ceph`, `kubernetes`, `set-up-cluster`, `codesphere` and `ms-backends` steps — the `codesphere` +step is what registers the data center in the configmap. + +### On GCP, for testing + +`oms beta bootstrap-gcp --multi-dc=true` builds a two-data-center instance in one GCP project, +applying everything above automatically. It shares the project's VPC, jumpbox and PostgreSQL VM, +and gives each data center three Ceph nodes, three k0s nodes and three static IPs of its own: + +```bash +oms beta bootstrap-gcp \ + --project-name multidc-test --billing-account "$BILLING" \ + --base-domain oms-testing.example.com \ + --multi-dc=true --datacenter-name multidc \ + --install-version +``` + +The second data center's resources are suffixed `-dc2`: VMs `ceph-1-dc2` … `k0s-3-dc2`, static IPs +`gateway-dc2`, `public-gateway-dc2`, `ssh-proxy-dc2`, local files `config-dc2.yaml` and +`prod-dc2.vault.yaml`, and `/etc/codesphere/config-dc2.yaml` plus +`/etc/codesphere/secrets-dc2/` on the shared jumpbox. Workspaces resolve under +`1.ws.` and `2.ws.`. + +Two data centers mean 14 VMs — roughly 100 vCPUs — and 6 regional static addresses. `--multi-dc` cannot be combined with +`--datacenter-id`, since the IDs are derived, and requires `--write-config`. diff --git a/internal/bootstrap/gcp/multi_dc_test.go b/internal/bootstrap/gcp/multi_dc_test.go new file mode 100644 index 00000000..3f62f3fc --- /dev/null +++ b/internal/bootstrap/gcp/multi_dc_test.go @@ -0,0 +1,327 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp_test + +import ( + "context" + "fmt" + "os" + + "cloud.google.com/go/compute/apiv1/computepb" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + "google.golang.org/api/cloudbilling/v1" + + "github.com/codesphere-cloud/oms/internal/bootstrap" + "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" + "github.com/codesphere-cloud/oms/internal/codesphere" + "github.com/codesphere-cloud/oms/internal/env" + "github.com/codesphere-cloud/oms/internal/github" + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" + "github.com/codesphere-cloud/oms/internal/portal" + "github.com/codesphere-cloud/oms/internal/util" +) + +// realConfigManager returns an install config manager that keeps its files in memory, so tests +// can inspect the config and vault a data center would actually be installed with. +func realConfigManager(writes map[string][]byte) installer.InstallConfigManager { + icm := installer.NewInstallConfigManager() + icm.(*installer.InstallConfig).SetFileIO(&recordingFileIO{writes: writes}) + return icm +} + +// recordingFileIO records written files and reports every other path as missing, so a bootstrap +// run behaves as if nothing existed locally beforehand. +type recordingFileIO struct { + util.FilesystemWriter + writes map[string][]byte +} + +func (f *recordingFileIO) CreateAndWrite(path string, content []byte, _ string) error { + f.writes[path] = content + return nil +} + +var _ = Describe("Multi-DC bootstrap", func() { + var ( + nodeClient *node.MockNodeClient + csEnv *gcp.CodesphereEnvironment + gc *gcp.MockGCPClientManager + fw *util.MockFileIO + writes map[string][]byte + primaryICG installer.InstallConfigManager + + bs *gcp.GCPBootstrapper + ) + + BeforeEach(func() { + nodeClient = node.NewMockNodeClient(GinkgoT()) + gc = gcp.NewMockGCPClientManager(GinkgoT()) + fw = util.NewMockFileIO(GinkgoT()) + writes = map[string][]byte{} + primaryICG = realConfigManager(writes) + + csEnv = &gcp.CodesphereEnvironment{ + MultiDC: true, + ProjectName: "test-project", + BillingAccount: "test-billing-account", + BaseDomain: "example.com", + Region: "us-central1", + Zone: "us-central1-a", + DNSProjectID: "dns-project", + DNSZoneName: "test-zone", + ProjectTTL: "1h", + SecretsDir: "/etc/codesphere/secrets", + DatacenterName: "multidc", + InstallConfigPath: "config.yaml", + SecretsFilePath: "prod.vault.yaml", + WriteConfig: true, + RegistryType: gcp.RegistryTypeGitHub, + RegistryUser: "registry-user", + GitHubPAT: "fake-pat", + SSHPublicKeyPath: "key.pub", + RootDiskSize: 50, + InternalFlags: gcp.DefaultInternalFlags, + PreviewFlags: gcp.DefaultPreviewFlags, + FeatureFlags: gcp.DefaultFeatureFlags, + } + }) + + JustBeforeEach(func() { + var err error + bs, err = gcp.NewGCPBootstrapper( + context.Background(), + env.NewEnv(), + bootstrap.NewStepLogger(false), + csEnv, + primaryICG, + gc, + fw, + nodeClient, + portal.NewMockPortal(GinkgoT()), + util.NewFakeTime(), + github.NewMockGitHubClient(GinkgoT()), + ) + Expect(err).NotTo(HaveOccurred()) + bs.NewConfigManager = func() installer.InstallConfigManager { return realConfigManager(writes) } + }) + + // expectBootstrapMocks sets up the GCP, SSH and file mocks for a full two-data-center run. + expectBootstrapMocks := func(projectID string) { + const vmCount = 14 + + fw.EXPECT().Exists(mock.Anything).Return(false) + fw.EXPECT().MkdirAll(mock.Anything, os.FileMode(0755)).Return(nil) + fw.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + fw.EXPECT().ReadFile(mock.Anything).Return([]byte("ssh-rsa AAA..."), nil).Times(vmCount) + + // EnsureProject only creates a project when the lookup fails with this exact message. + gc.EXPECT().GetProjectByName(mock.Anything, "test-project").Return(nil, fmt.Errorf("project not found: test-project")) + gc.EXPECT().CreateProjectID("test-project").Return(projectID) + gc.EXPECT().CreateProject(mock.Anything, mock.Anything, "test-project", mock.Anything).Return(mock.Anything, nil) + gc.EXPECT().GetBillingInfo(projectID).Return(&cloudbilling.ProjectBillingInfo{BillingEnabled: false}, nil) + gc.EXPECT().EnableBilling(projectID, "test-billing-account").Return(nil) + gc.EXPECT().EnableAPIs(projectID, mock.Anything).Return(nil) + gc.EXPECT().CreateServiceAccount(projectID, "cloud-controller", "cloud-controller").Return("cc@p.iam.gserviceaccount.com", false, nil) + gc.EXPECT().AssignIAMRole(projectID, "cloud-controller", projectID, []string{"roles/compute.admin"}).Return(nil) + gc.EXPECT().AssignIAMRole("dns-project", "cloud-controller", projectID, []string{"roles/dns.admin"}).Return(nil) + gc.EXPECT().CreateVPC(projectID, "us-central1", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + gc.EXPECT().CreateFirewallRule(projectID, mock.Anything).Return(nil).Times(5) + + mockGetInstanceNotFoundThenRunning(gc, projectID, "us-central1-a", makeRunningInstance("10.10.0.2", "1.2.3.4"), vmCount) + gc.EXPECT().CreateInstance(projectID, "us-central1-a", mock.Anything).Return(nil).Times(vmCount) + + // Two data centers reserve three addresses each, suffixed for the second one. + for _, name := range []string{ + "gateway", "public-gateway", "ssh-proxy", + "gateway-dc2", "public-gateway-dc2", "ssh-proxy-dc2", + } { + ip := fmt.Sprintf("203.0.113.%d", len(name)) + gc.EXPECT().GetAddress(projectID, "us-central1", name).Return(nil, fmt.Errorf("not found")).Once() + gc.EXPECT().CreateAddress(projectID, "us-central1", mock.MatchedBy(func(addr *computepb.Address) bool { + return addr.GetName() == name + })).Return(ip, nil).Once() + } + + gc.EXPECT().EnsureDNSManagedZone("dns-project", "test-zone", "example.com.", mock.Anything).Return(nil) + gc.EXPECT().EnsureDNSRecordSets("dns-project", "test-zone", mock.Anything).Return(nil) + + nodeClient.EXPECT().WaitReady(mock.Anything, mock.Anything).Return(nil) + nodeClient.EXPECT().HasFile(mock.Anything, mock.Anything).Return(false) + nodeClient.EXPECT().RunCommand(mock.Anything, mock.Anything, mock.Anything).Return(nil) + nodeClient.EXPECT().CopyFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + } + + It("bootstraps two data centers sharing one database", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + Expect(bs.Env.DataCenters).To(HaveLen(2)) + primary, secondary := bs.Env.DataCenters[0], bs.Env.DataCenters[1] + + By("giving each data center its own ceph and k0s nodes") + Expect(vmNamesOfNodes(primary.CephNodes)).To(Equal([]string{"ceph-1", "ceph-2", "ceph-3"})) + Expect(vmNamesOfNodes(primary.ControlPlaneNodes)).To(Equal([]string{"k0s-1", "k0s-2", "k0s-3"})) + Expect(vmNamesOfNodes(secondary.CephNodes)).To(Equal([]string{"ceph-1-dc2", "ceph-2-dc2", "ceph-3-dc2"})) + Expect(vmNamesOfNodes(secondary.ControlPlaneNodes)).To(Equal([]string{"k0s-1-dc2", "k0s-2-dc2", "k0s-3-dc2"})) + + By("sharing the platform domain and scoping the workspace domain") + Expect(primary.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) + Expect(secondary.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) + Expect(primary.InstallConfig.Codesphere.WorkspaceHostingBaseDomain).To(Equal("1.ws.example.com")) + Expect(secondary.InstallConfig.Codesphere.WorkspaceHostingBaseDomain).To(Equal("2.ws.example.com")) + Expect(secondary.InstallConfig.Codesphere.CustomDomains.CNameBaseDomain).To(Equal("2.ws.example.com")) + + By("giving each data center a distinct ID and name") + Expect(primary.InstallConfig.Datacenter.ID).To(Equal(1)) + Expect(secondary.InstallConfig.Datacenter.ID).To(Equal(2)) + // The k0s cluster is named codesphere-, so the names must differ. + Expect(primary.InstallConfig.Datacenter.Name).To(Equal("multidc")) + Expect(secondary.InstallConfig.Datacenter.Name).To(Equal("multidc-dc2")) + + By("installing postgres in the primary data center and reusing it in the secondary") + Expect(primary.InstallConfig.Postgres.Mode).To(Equal("install")) + Expect(primary.InstallConfig.Postgres.Primary.IP).To(Equal("10.10.0.2")) + Expect(secondary.InstallConfig.Postgres.Mode).To(Equal("external")) + // The server certificate carries only an IP SAN, so the address must be the IP. + Expect(secondary.InstallConfig.Postgres.ServerAddress).To(Equal("10.10.0.2")) + Expect(secondary.InstallConfig.Postgres.Primary).To(BeNil()) + Expect(secondary.InstallConfig.Postgres.Replica).To(BeNil()) + Expect(secondary.InstallConfig.Postgres.CACertPem).To(Equal(primary.InstallConfig.Postgres.CACertPem)) + Expect(secondary.InstallConfig.Postgres.CACertPem).NotTo(BeEmpty()) + + By("skipping the postgres install step in the secondary data center only") + Expect(secondary.InstallConfig.Operations.Skip).To(ContainElement("postgres")) + // Both data centers install their own ceph and kubernetes. + Expect(secondary.InstallConfig.Operations.Skip).NotTo(ContainElement("ceph")) + Expect(secondary.InstallConfig.Operations.Skip).NotTo(ContainElement("kubernetes")) + if primary.InstallConfig.Operations != nil { + Expect(primary.InstallConfig.Operations.Skip).NotTo(ContainElement("postgres")) + } + + By("telling both data centers about every data center of the installation") + // The installer defaults dataCenters to the local one, so without the list each data + // center renders as a single-data-center instance. + topology := []files.DatacenterConfig{ + {ID: 1, Name: "multidc", City: "Karlsruhe", CountryCode: "DE"}, + {ID: 2, Name: "multidc-dc2", City: "Karlsruhe", CountryCode: "DE"}, + } + for _, dc := range bs.Env.DataCenters { + Expect(dc.InstallConfig.DataCenters).To(Equal(topology), "data center %d", dc.ID) + Expect(dc.InstallConfig.DefaultDataCenterID).To(Equal(1), "data center %d", dc.ID) + // The local data center is still the one this config installs. + Expect(dc.InstallConfig.Datacenter.ID).To(Equal(dc.ID)) + } + + By("giving each data center its own vault directory on the shared jumpbox") + Expect(primary.InstallConfig.Secrets.BaseDir).To(Equal("/etc/codesphere/secrets")) + Expect(secondary.InstallConfig.Secrets.BaseDir).To(Equal("/etc/codesphere/secrets-dc2")) + Expect(writes).To(HaveKey("config.yaml")) + Expect(writes).To(HaveKey("config-dc2.yaml")) + Expect(writes).To(HaveKey("prod.vault.yaml")) + Expect(writes).To(HaveKey("prod-dc2.vault.yaml")) + + primaryVault := primary.ConfigManager().GetVault() + secondaryVault := secondary.ConfigManager().GetVault() + + By("sharing every secret both data centers need to agree on") + shared := []string{ + files.SecretPostgresPassword, + files.SecretPostgresReplicaPassword, + files.SecretPostgresCaKeyPem, + files.SecretTokenPrivateKey, + files.SecretTokenPublicKey, + files.SecretDomainAuthPrivateKey, + files.SecretMounterHmacSecret, + files.SecretMongoDbPasswordEncryptionKey, + } + for _, svc := range codesphere.PostgresServices { + shared = append(shared, files.PostgresUserSecretName(svc.Name), files.PostgresPasswordSecretName(svc.Name)) + } + for _, name := range shared { + Expect(primaryVault.GetSecret(name)).NotTo(BeNil(), "primary should have %s", name) + Expect(secondaryVault.GetSecret(name)).To(Equal(primaryVault.GetSecret(name)), "%s must be shared", name) + } + + By("regenerating the per-cluster secrets for the secondary data center") + for _, name := range []string{files.SecretSelfSignedCaKeyPem, files.SecretCephSshPrivateKey} { + Expect(secondaryVault.GetSecret(name)).NotTo(BeNil(), "secondary should have %s", name) + Expect(secondaryVault.GetSecret(name).File.Content). + NotTo(Equal(primaryVault.GetSecret(name).File.Content), "%s must be per data center", name) + } + Expect(secondary.InstallConfig.Cluster.Certificates.CA.CertPem). + NotTo(Equal(primary.InstallConfig.Cluster.Certificates.CA.CertPem)) + Expect(secondary.InstallConfig.Ceph.CephAdmSSHKey.PublicKey). + NotTo(Equal(primary.InstallConfig.Ceph.CephAdmSSHKey.PublicKey)) + + By("recording the DNS records it created so cleanup can delete them") + Expect(bs.Env.DNSRecords).To(Equal(gcp.DataCenterDNSRecordNames("example.com", bs.Env.DataCenters))) + + By("pointing each data center's install command at its own config and vault") + Expect(bs.InstallCommand(primary, "pkg.tar.gz")).To(ContainSubstring("-c /etc/codesphere/config.yaml")) + Expect(bs.InstallCommand(primary, "pkg.tar.gz")).To(ContainSubstring("--vault /etc/codesphere/secrets/prod.vault.yaml")) + Expect(bs.InstallCommand(secondary, "pkg.tar.gz")).To(ContainSubstring("-c /etc/codesphere/config-dc2.yaml")) + Expect(bs.InstallCommand(secondary, "pkg.tar.gz")).To(ContainSubstring("--vault /etc/codesphere/secrets-dc2/prod.vault.yaml")) + + By("producing postgres blocks the installer's validation accepts") + for _, dc := range bs.Env.DataCenters { + for _, problem := range dc.ConfigManager().ValidateInstallConfig() { + Expect(problem).NotTo(ContainSubstring("postgres"), "data center %d", dc.ID) + } + } + }) + + It("mirrors the primary data center onto the legacy infra file fields", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + primary := bs.Env.DataCenters[0] + Expect(bs.Env.ControlPlaneNodes).To(Equal(primary.ControlPlaneNodes)) + Expect(bs.Env.CephNodes).To(Equal(primary.CephNodes)) + Expect(bs.Env.GatewayIP).To(Equal(primary.GatewayIP)) + Expect(bs.Env.PublicGatewayIP).To(Equal(primary.PublicGatewayIP)) + Expect(bs.Env.SshProxyIP).To(Equal(primary.SshProxyIP)) + }) + + Describe("validateMultiDC", func() { + DescribeTable("rejects flag combinations it cannot satisfy", + func(mutate func(), wantErr string) { + mutate() + Expect(bs.ValidateInput()).To(MatchError(ContainSubstring(wantErr))) + }, + Entry("without write-config", func() { csEnv.WriteConfig = false }, "multi-dc requires write-config"), + Entry("with an explicit datacenter ID", func() { csEnv.DatacenterIDExplicit = true }, "datacenter-id cannot be combined with multi-dc"), + Entry("without a datacenter name", func() { csEnv.DatacenterName = "" }, "datacenter-name is required with multi-dc"), + Entry("without a config path", func() { csEnv.InstallConfigPath = "" }, "cannot derive a per-data-center path"), + Entry("without a secrets path", func() { csEnv.SecretsFilePath = "" }, "cannot derive a per-data-center path"), + ) + + It("accepts the default multi-dc flags", func() { + Expect(bs.ValidateInput()).To(Succeed()) + }) + + It("does not constrain a single-data-center bootstrap", func() { + csEnv.MultiDC = false + csEnv.WriteConfig = false + csEnv.DatacenterName = "" + csEnv.DatacenterIDExplicit = true + + Expect(bs.ValidateInput()).To(Succeed()) + }) + }) +}) + +// vmNamesOfNodes returns the names of the given nodes, in order. +func vmNamesOfNodes(nodes []*node.Node) []string { + names := make([]string, len(nodes)) + for i, n := range nodes { + names[i] = n.GetName() + } + return names +} From 355417a64038f5cdb5e162ab0bdc040d244c9db6 Mon Sep 17 00:00:00 2001 From: NJona <25478046+NJona@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:51:25 +0000 Subject: [PATCH 09/11] chore(docs): Auto-update docs and licenses Signed-off-by: NJona <25478046+NJona@users.noreply.github.com> --- ...nstall-codesphere-on-any-infrastructure.md | 109 ------------------ 1 file changed, 109 deletions(-) diff --git a/docs/install-codesphere-on-any-infrastructure.md b/docs/install-codesphere-on-any-infrastructure.md index 597776b8..6ff6b46c 100644 --- a/docs/install-codesphere-on-any-infrastructure.md +++ b/docs/install-codesphere-on-any-infrastructure.md @@ -463,112 +463,3 @@ Run the Codesphere smoke test when an API key for the environment is available: ```bash oms smoketest codesphere --help ``` - -## Appendix: multiple data centers - -A Codesphere instance can span several data centers. They share one PostgreSQL server and one -platform domain, but each runs its own Kubernetes and Ceph cluster and hosts its own workspaces. -The steps above describe one data center; repeat them per data center with these differences. - -**Configuration.** All data centers keep the same `codesphere.domain`, `codesphere.plans`, -`codesphere.deployConfig`, `codesphere.managedServices`, feature flags and Git provider settings — -that data is stored in the shared database, so it must not diverge. Per data center: - -| Setting | Per data center | -| --- | --- | -| `dataCenter.id` / `dataCenter.name` | Distinct. The k0s cluster is named `codesphere-`, so the names must differ. | -| `codesphere.workspaceHostingBaseDomain` | `.`, resolving to that data center's workspace gateway. | -| `codesphere.publicIp` | That data center's public address. | -| `secrets.baseDir` | A directory of its own. The installer resolves the vault from `secrets.baseDir` and writes `kubeConfig` and the Ceph credentials back into it, so sharing a directory lets one data center overwrite another's. | -| `ceph.*`, `kubernetes.*`, gateway annotations | That data center's own hosts and addresses. | -| `cluster.monitoring.prometheus.remoteWrite.clusterName` | That data center's name. | - -**Topology.** Every data center's config lists all of them, so the platform knows which data centers -a user can pick from: - -```yaml -dataCenters: - - id: 1 - name: multidc - city: Karlsruhe - countryCode: DE - - id: 2 - name: multidc-dc2 - city: Karlsruhe - countryCode: DE -defaultDataCenterId: 1 -``` - -The list is identical in every data center's config, while `dataCenter` stays the local one. -Leaving `dataCenters` out defaults it to the local data center alone — correct for a single data -center, but it makes a multi-data-center instance render as single. - -**Shared PostgreSQL.** The first data center installs the server. Every other one sets -`postgres.mode: external` with `postgres.serverAddress` pointing at it, omits `postgres.primary` -and `postgres.replica`, copies `postgres.caCertPem` from the first data center's config, and adds -`postgres` to `operations.skip`. Use the server's IP address, not its hostname: the generated -server certificate carries an IP SAN only. - -**Secrets.** Derive each additional data center's vault from the first one's, keeping everything -that both must agree on and regenerating only what belongs to a single cluster: - -| Keep identical | Why | -| --- | --- | -| `postgresPassword`, `postgresReplicaPassword`, `postgresCaKeyPem`, and every `postgresUser*` / `postgresPassword*` pair | The roles live on the shared server. Divergent values make one data center's pods fail authentication, or let its install rotate credentials the other is using. | -| `tokenPrivateKey`, `tokenPublicKey` | A session token minted in one data center is presented to services in the other. | -| `domainAuthPrivateKey` / `PublicKey`, `mounterHmacSecret`, `mongoDbPasswordEncryptionKey` | They sign or encrypt rows in the shared database. | -| Registry, OAuth, OIDC and OpenBao credentials | The same external services. | - -| Regenerate per data center | Paired config field | -| --- | --- | -| `selfSignedCaKeyPem` | `cluster.certificates.ca.certPem` | -| `cephSshPrivateKey` | `ceph.cephAdmSshKey.publicKey` | -| `acmeEabMacKey` | `codesphere.certIssuer.acme.eabKeyId` | -| `kubeConfig`, and everything prefixed `ceph`, `csi` or `rgw` | written by the installer's `ceph` and `kubernetes` steps | - -Always clear the paired config field together with the vault secret. The generators are gated on -the vault entry, so a config field left in place keeps a stale value next to a fresh key. - -**DNS.** The platform name stays shared; the per-data-center platform, workspace and SSH names are -not: - -| Record | Target | -| --- | --- | -| ``, `*.` | The first data center's platform gateway | -| `.`, `*..` | **That** data center's platform gateway | -| `.`, `*..` | That data center's workspace gateway | -| `*..ssh.` | That data center's workspace SSH proxy | - -The second row is easy to miss. The platform builds each data center's service endpoint as -`.` and the browser calls it directly — before rendering any UI, it asks -the endpoint of the data center a workspace lives in for its configuration. Left to the -`*.` wildcard, that name resolves to the first data center's gateway, which has -no route for it, and the whole UI fails to load with a connection error. - -**Install order.** Install the first data center to completion before starting the next. Its -install creates the database, roles and schema that the others reuse. Every data center runs the -`ceph`, `kubernetes`, `set-up-cluster`, `codesphere` and `ms-backends` steps — the `codesphere` -step is what registers the data center in the configmap. - -### On GCP, for testing - -`oms beta bootstrap-gcp --multi-dc=true` builds a two-data-center instance in one GCP project, -applying everything above automatically. It shares the project's VPC, jumpbox and PostgreSQL VM, -and gives each data center three Ceph nodes, three k0s nodes and three static IPs of its own: - -```bash -oms beta bootstrap-gcp \ - --project-name multidc-test --billing-account "$BILLING" \ - --base-domain oms-testing.example.com \ - --multi-dc=true --datacenter-name multidc \ - --install-version -``` - -The second data center's resources are suffixed `-dc2`: VMs `ceph-1-dc2` … `k0s-3-dc2`, static IPs -`gateway-dc2`, `public-gateway-dc2`, `ssh-proxy-dc2`, local files `config-dc2.yaml` and -`prod-dc2.vault.yaml`, and `/etc/codesphere/config-dc2.yaml` plus -`/etc/codesphere/secrets-dc2/` on the shared jumpbox. Workspaces resolve under -`1.ws.` and `2.ws.`. - -Two data centers mean 14 VMs — roughly 100 vCPUs — and 6 regional static addresses. `--multi-dc` cannot be combined with -`--datacenter-id`, since the IDs are derived, and requires `--write-config`. From becb7ca113e452fd6beef5df1dde9e439db1dd82 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 7 Aug 2026 14:20:14 +0200 Subject: [PATCH 10/11] feat(openfga): add psk auth to config Signed-off-by: Jona Neef --- internal/installer/argocd/install_and_apps.go | 7 +- internal/installer/config_manager.go | 40 ++++++++++ internal/installer/config_manager_test.go | 53 ++++++++++++- internal/installer/files/config_yaml.go | 33 ++++++++ internal/installer/files/secret_names.go | 1 + internal/installer/openfga_pc_apps.go | 59 ++++++++++++++ internal/installer/openfga_pc_apps_test.go | 78 +++++++++++++++++++ internal/installer/secrets/datacenter.go | 21 +++++ internal/installer/secrets/datacenter_test.go | 38 +++++++++ internal/installer/secrets/secrets.go | 25 ++++++ internal/installer/secrets/secrets_test.go | 33 ++++++++ 11 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 internal/installer/openfga_pc_apps.go create mode 100644 internal/installer/openfga_pc_apps_test.go diff --git a/internal/installer/argocd/install_and_apps.go b/internal/installer/argocd/install_and_apps.go index 625b2a9b..1b9f8236 100644 --- a/internal/installer/argocd/install_and_apps.go +++ b/internal/installer/argocd/install_and_apps.go @@ -11,6 +11,7 @@ import ( "github.com/codesphere-cloud/oms/internal/installer/files" "github.com/codesphere-cloud/oms/internal/installer/secrets" "github.com/codesphere-cloud/oms/internal/installer/vault" + "github.com/codesphere-cloud/oms/internal/util" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -69,13 +70,17 @@ func (i *AppInstaller) SyncVaultSecret(ctx context.Context) error { // InstallPCApps installs or upgrades pc-applications using the version from // the supplied installer BOM. func (i *AppInstaller) InstallPCApps(ctx context.Context, bomPath string) error { + // Values derived from the install config form the base; an explicit pcApps block in + // config.yaml wins over them, and the --pc-apps-values files win over both. + values := util.DeepMergeMaps(installer.OpenFgaPcAppsValues(&i.cfg.Config), i.cfg.Config.PcApps) + pcApps, err := installer.NewPcAppsFromBom( i.cfg.KubeClient, i.cfg.RESTConfig, bomPath, DefaultNamespace, i.cfg.PCAppsValues, - i.cfg.Config.PcApps, + values, ) if err != nil { return fmt.Errorf("failed to initialize pc-apps installer: %w", err) diff --git a/internal/installer/config_manager.go b/internal/installer/config_manager.go index d55f65fc..cf0e41d6 100644 --- a/internal/installer/config_manager.go +++ b/internal/installer/config_manager.go @@ -192,6 +192,36 @@ func (g *InstallConfig) ValidateInstallConfig() []string { } } + errors = append(errors, validateOpenFga(g.Config.Codesphere.OpenFga)...) + + return errors +} + +// validateOpenFga checks the codesphere.openFga block. A data center that does not deploy +// OpenFGA has nowhere to fall back to, so it must name the instance it uses; a data center that +// exposes one must say under which host. +func validateOpenFga(config *files.OpenFgaConfig) []string { + if config == nil { + return nil + } + + errors := []string{} + if !config.DeploysOpenFga() && config.APIURL == "" { + errors = append(errors, "OpenFGA apiUrl is required when codesphere.openFga.deploy is false") + } + if config.APIURL != "" { + if _, err := url.ParseRequestURI(config.APIURL); err != nil { + errors = append(errors, "OpenFGA apiUrl must be a valid URL") + } + } + if config.ExposesOpenFga() { + if config.Expose.Host == "" { + errors = append(errors, "OpenFGA expose host is required when codesphere.openFga.expose.enabled is true") + } + if !config.DeploysOpenFga() { + errors = append(errors, "OpenFGA cannot be exposed by a data center that does not deploy it") + } + } return errors } @@ -232,6 +262,16 @@ func (g *InstallConfig) ValidateVault() []string { } } + // Generated by EnsureSecrets for a fresh installation, but a data center joining an + // existing installation must carry over the primary data center's key — the services + // and the OpenFGA instance they share authenticate with the same one. + if !foundSecrets[files.SecretOpenFgaPresharedKey] { + errors = append(errors, fmt.Sprintf( + "required secret missing: %s (in a multi-data-center installation, copy it from the data center that deploys OpenFGA)", + files.SecretOpenFgaPresharedKey, + )) + } + return errors } diff --git a/internal/installer/config_manager_test.go b/internal/installer/config_manager_test.go index 1fd403d4..e2a00f2a 100644 --- a/internal/installer/config_manager_test.go +++ b/internal/installer/config_manager_test.go @@ -298,6 +298,55 @@ var _ = Describe("ConfigManager", func() { }) }) + Context("openFga validation", func() { + It("should accept an absent openFga block", func() { + configManager.Config.Codesphere.OpenFga = nil + errors := configManager.ValidateInstallConfig() + Expect(errors).NotTo(ContainElement(ContainSubstring("OpenFGA"))) + }) + + It("should require an apiUrl when the data center does not deploy OpenFGA", func() { + deploy := false + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{Deploy: &deploy} + errors := configManager.ValidateInstallConfig() + Expect(errors).To(ContainElement(ContainSubstring("OpenFGA apiUrl is required"))) + }) + + It("should validate the apiUrl format", func() { + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{APIURL: "not-a-valid-url"} + errors := configManager.ValidateInstallConfig() + Expect(errors).To(ContainElement(ContainSubstring("OpenFGA apiUrl must be a valid URL"))) + }) + + It("should require a host when exposing OpenFGA", func() { + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true}, + } + errors := configManager.ValidateInstallConfig() + Expect(errors).To(ContainElement(ContainSubstring("OpenFGA expose host is required"))) + }) + + It("should reject exposing an OpenFGA the data center does not deploy", func() { + deploy := false + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{ + Deploy: &deploy, + APIURL: "https://openfga.1.cs.example.com", + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.2.cs.example.com"}, + } + errors := configManager.ValidateInstallConfig() + Expect(errors).To(ContainElement(ContainSubstring("cannot be exposed by a data center that does not deploy it"))) + }) + + It("should accept a data center that deploys and exposes OpenFGA", func() { + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{ + APIURL: "https://openfga.1.cs.example.com", + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + } + errors := configManager.ValidateInstallConfig() + Expect(errors).NotTo(ContainElement(ContainSubstring("OpenFGA"))) + }) + }) + Context("ceph validation", func() { It("should require at least one Ceph host", func() { configManager.Config.Ceph.Hosts = []files.CephHost{} @@ -424,6 +473,7 @@ var _ = Describe("ConfigManager", func() { {Name: "selfSignedCaKeyPem"}, {Name: "domainAuthPrivateKey"}, {Name: "domainAuthPublicKey"}, + {Name: "openFgaPresharedKey"}, }, } }) @@ -443,9 +493,10 @@ var _ = Describe("ConfigManager", func() { }, } errors := configManager.ValidateVault() - Expect(errors).To(HaveLen(4)) + Expect(errors).To(HaveLen(5)) Expect(errors).To(ContainElement(ContainSubstring("domainAuthPrivateKey"))) Expect(errors).To(ContainElement(ContainSubstring("domainAuthPublicKey"))) + Expect(errors).To(ContainElement(ContainSubstring("openFgaPresharedKey"))) }) }) }) diff --git a/internal/installer/files/config_yaml.go b/internal/installer/files/config_yaml.go index b459920f..01b1db8e 100644 --- a/internal/installer/files/config_yaml.go +++ b/internal/installer/files/config_yaml.go @@ -360,6 +360,7 @@ type CodesphereConfig struct { OAuth *OAuthProvidersConfig `yaml:"oauth,omitempty"` ManagedServices []ManagedServiceConfig `yaml:"managedServices,omitempty"` OpenBao *OpenBaoConfig `yaml:"openBao,omitempty"` + OpenFga *OpenFgaConfig `yaml:"openFga,omitempty"` Migration *MigrationConfig `yaml:"migration,omitempty"` TelemetryExport *TelemetryExport `yaml:"telemetryExport,omitempty"` Override ChartOverride `yaml:"override,omitempty"` @@ -382,6 +383,38 @@ type OpenBaoConfig struct { User string `yaml:"user,omitempty"` } +// OpenFgaConfig configures the authorization store. One OpenFGA instance serves a whole +// installation, so in a multi-data-center setup exactly one data center deploys and exposes +// it (Deploy + Expose) and every other one only points at it (APIURL). +type OpenFgaConfig struct { + // Deploy controls whether pc-applications deploys OpenFGA in this data center. + // Defaults to true when unset, matching the pc-applications chart. + Deploy *bool `yaml:"deploy,omitempty"` + // APIURL is the URL the Codesphere services reach OpenFGA at. Defaults to the + // in-cluster service of a locally deployed OpenFGA; required when Deploy is false. + APIURL string `yaml:"apiUrl,omitempty"` + // Expose publishes the deployed OpenFGA through the Codesphere gateway so the other + // data centers can reach it. + Expose *OpenFgaExposeConfig `yaml:"expose,omitempty"` +} + +type OpenFgaExposeConfig struct { + Enabled bool `yaml:"enabled"` + // Host OpenFGA is served under. Must resolve to this data center's public IP and + // is what the other data centers put in their APIURL. + Host string `yaml:"host,omitempty"` +} + +// DeploysOpenFga reports whether pc-applications should deploy OpenFGA in this data center. +func (c *OpenFgaConfig) DeploysOpenFga() bool { + return c == nil || c.Deploy == nil || *c.Deploy +} + +// ExposesOpenFga reports whether the deployed OpenFGA is published through the gateway. +func (c *OpenFgaConfig) ExposesOpenFga() bool { + return c != nil && c.Expose != nil && c.Expose.Enabled +} + type OAuthProvidersConfig struct { Oidc *OidcOAuthProvider `yaml:"oidc,omitempty"` } diff --git a/internal/installer/files/secret_names.go b/internal/installer/files/secret_names.go index 41dfd305..c4791f43 100644 --- a/internal/installer/files/secret_names.go +++ b/internal/installer/files/secret_names.go @@ -44,6 +44,7 @@ const ( SecretOpenBaoPassword = "openBaoPassword" // OpenFGA + SecretOpenFgaPresharedKey = "openFgaPresharedKey" SecretOpenfgaDbBackupAccessKeyId = "openfgaDbBackupAccessKeyId" SecretOpenfgaDbBackupSecretAccessKey = "openfgaDbBackupSecretAccessKey" diff --git a/internal/installer/openfga_pc_apps.go b/internal/installer/openfga_pc_apps.go new file mode 100644 index 00000000..2841efa2 --- /dev/null +++ b/internal/installer/openfga_pc_apps.go @@ -0,0 +1,59 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +// OpenFgaPcAppsValues translates the customer-facing codesphere.openFga block of the install +// config into pc-applications values. +// +// OpenFGA is deployed by pc-applications, but whether a data center runs its own instance and +// whether that instance is published is an installation-level decision, not a chart detail — so +// operators configure it in config.yaml and this derives the chart values from it. The result is +// the *base* of the pc-apps values: an explicit `pcApps` block in config.yaml and any +// --pc-apps-values file still override it. +// +// Returns nil when the config says nothing about OpenFGA, leaving the pc-applications chart +// defaults untouched. +func OpenFgaPcAppsValues(config *files.RootConfig) files.ChartValues { + fga := config.Codesphere.OpenFga + if fga == nil { + return nil + } + + openfga := files.ChartValues{"enabled": fga.DeploysOpenFga()} + + if fga.Expose != nil { + gateway := files.ChartValues{"enabled": fga.Expose.Enabled} + if fga.Expose.Host != "" { + gateway["host"] = fga.Expose.Host + } + // The cert-manager ClusterIssuer the cluster step creates is named after the + // configured issuer type, so the gateway certificate follows the same issuer as + // the Codesphere frontend gateway. + gateway["tls"] = files.ChartValues{ + "certificate": files.ChartValues{ + "issuerRef": files.ChartValues{"name": certIssuerName(config)}, + }, + } + openfga["valuesObject"] = files.ChartValues{"gateway": gateway} + } + + return files.ChartValues{ + "applications": files.ChartValues{ + "openfga": openfga, + }, + } +} + +// certIssuerName returns the name of the ClusterIssuer for this installation, matching the +// naming the cluster step uses (the issuer type is the issuer name). +func certIssuerName(config *files.RootConfig) string { + if config.Codesphere.CertIssuer.Type != "" { + return string(config.Codesphere.CertIssuer.Type) + } + return string(files.CertIssuerTypeSelfSigned) +} diff --git a/internal/installer/openfga_pc_apps_test.go b/internal/installer/openfga_pc_apps_test.go new file mode 100644 index 00000000..9b704d59 --- /dev/null +++ b/internal/installer/openfga_pc_apps_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +var _ = Describe("OpenFgaPcAppsValues", func() { + configWith := func(fga *files.OpenFgaConfig, issuer files.CertIssuerType) *files.RootConfig { + config := &files.RootConfig{} + config.Codesphere.OpenFga = fga + config.Codesphere.CertIssuer.Type = issuer + return config + } + + // The application entry of the rendered values, or nil if there is none. + openfgaValues := func(values files.ChartValues) files.ChartValues { + apps, ok := values["applications"].(files.ChartValues) + Expect(ok).To(BeTrue(), "expected an applications map") + return apps["openfga"].(files.ChartValues) + } + + It("leaves the chart defaults alone when the config says nothing", func() { + Expect(installer.OpenFgaPcAppsValues(configWith(nil, ""))).To(BeNil()) + }) + + It("disables the application for a data center that uses a remote OpenFGA", func() { + deploy := false + values := installer.OpenFgaPcAppsValues(configWith(&files.OpenFgaConfig{ + Deploy: &deploy, + APIURL: "https://openfga.1.cs.example.com", + }, files.CertIssuerTypeACME)) + + fga := openfgaValues(values) + Expect(fga["enabled"]).To(BeFalse()) + // Nothing to expose, so the gateway is not configured at all. + Expect(fga).NotTo(HaveKey("valuesObject")) + }) + + It("defaults to deploying when only the exposure is configured", func() { + values := installer.OpenFgaPcAppsValues(configWith(&files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + }, files.CertIssuerTypeACME)) + + fga := openfgaValues(values) + Expect(fga["enabled"]).To(BeTrue()) + + gateway := fga["valuesObject"].(files.ChartValues)["gateway"].(files.ChartValues) + Expect(gateway["enabled"]).To(BeTrue()) + Expect(gateway["host"]).To(Equal("openfga.1.cs.example.com")) + }) + + It("issues the gateway certificate with the installation's cert issuer", func() { + values := installer.OpenFgaPcAppsValues(configWith(&files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + }, files.CertIssuerTypeACME)) + + gateway := openfgaValues(values)["valuesObject"].(files.ChartValues)["gateway"].(files.ChartValues) + issuerRef := gateway["tls"].(files.ChartValues)["certificate"].(files.ChartValues)["issuerRef"].(files.ChartValues) + Expect(issuerRef["name"]).To(Equal("acme")) + }) + + It("falls back to the self-signed issuer when no cert issuer is configured", func() { + values := installer.OpenFgaPcAppsValues(configWith(&files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + }, "")) + + gateway := openfgaValues(values)["valuesObject"].(files.ChartValues)["gateway"].(files.ChartValues) + issuerRef := gateway["tls"].(files.ChartValues)["certificate"].(files.ChartValues)["issuerRef"].(files.ChartValues) + Expect(issuerRef["name"]).To(Equal("self-signed")) + }) +}) diff --git a/internal/installer/secrets/datacenter.go b/internal/installer/secrets/datacenter.go index b60c48d1..194b6b3b 100644 --- a/internal/installer/secrets/datacenter.go +++ b/internal/installer/secrets/datacenter.go @@ -87,10 +87,31 @@ func DeriveDataCenterConfig(primary *files.RootConfig) (*files.RootConfig, error return nil, fmt.Errorf("clone primary data center config: %w", err) } clearDataCenterScopedConfig(derived) + deriveOpenFgaConfig(primary, derived) return derived, nil } +// deriveOpenFgaConfig points the derived data center at the primary's OpenFGA instead of letting +// it deploy one of its own. An installation has a single authorization store, so a second +// deployment would mean a second, empty set of permissions. +// +// Only possible when the primary exposes OpenFGA — that exposed host is the derived data center's +// only route to it. When it does not, the block is left as the primary wrote it, and completing it +// (exposing the primary, or pointing this data center at some other reachable instance) is up to +// whoever set up the config. +func deriveOpenFgaConfig(primary, derived *files.RootConfig) { + if !primary.Codesphere.OpenFga.ExposesOpenFga() { + return + } + + deploy := false + derived.Codesphere.OpenFga = &files.OpenFgaConfig{ + Deploy: &deploy, + APIURL: "https://" + primary.Codesphere.OpenFga.Expose.Host, + } +} + // clearDataCenterScopedConfig resets the config fields that are written by the Ensure* functions // alongside a data-center-scoped vault secret. Those pairs must always be mutated together: the // Ensure* functions are gated on the vault entry, so a config field left in place would keep a diff --git a/internal/installer/secrets/datacenter_test.go b/internal/installer/secrets/datacenter_test.go index 6c49b61f..281d2451 100644 --- a/internal/installer/secrets/datacenter_test.go +++ b/internal/installer/secrets/datacenter_test.go @@ -190,6 +190,44 @@ var _ = Describe("DeriveDataCenterConfig", func() { Expect(derived.Codesphere.CertIssuer.Acme).To(BeNil()) Expect(derived.Cluster.Certificates.CA.CertPem).To(BeEmpty()) }) + + It("points the derived data center at the primary's exposed OpenFGA", func() { + primary.Codesphere.OpenFga = &files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + } + + derived, err := secrets.DeriveDataCenterConfig(primary) + Expect(err).NotTo(HaveOccurred()) + + Expect(derived.Codesphere.OpenFga.DeploysOpenFga()).To(BeFalse()) + Expect(derived.Codesphere.OpenFga.APIURL).To(Equal("https://openfga.1.cs.example.com")) + // The derived data center has nothing of its own to expose. + Expect(derived.Codesphere.OpenFga.ExposesOpenFga()).To(BeFalse()) + // The primary keeps deploying and exposing it. + Expect(primary.Codesphere.OpenFga.DeploysOpenFga()).To(BeTrue()) + Expect(primary.Codesphere.OpenFga.ExposesOpenFga()).To(BeTrue()) + }) + + It("leaves the OpenFGA block alone when the primary does not expose it", func() { + primary.Codesphere.OpenFga = &files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: false}, + } + + derived, err := secrets.DeriveDataCenterConfig(primary) + Expect(err).NotTo(HaveOccurred()) + + Expect(derived.Codesphere.OpenFga.DeploysOpenFga()).To(BeTrue()) + Expect(derived.Codesphere.OpenFga.APIURL).To(BeEmpty()) + }) + + It("handles a config without an OpenFGA block", func() { + primary.Codesphere.OpenFga = nil + + derived, err := secrets.DeriveDataCenterConfig(primary) + Expect(err).NotTo(HaveOccurred()) + + Expect(derived.Codesphere.OpenFga).To(BeNil()) + }) }) var _ = Describe("secondary data center secret generation", func() { diff --git a/internal/installer/secrets/secrets.go b/internal/installer/secrets/secrets.go index d75d7e30..2224563b 100644 --- a/internal/installer/secrets/secrets.go +++ b/internal/installer/secrets/secrets.go @@ -49,6 +49,9 @@ func EnsureSecrets(vault *files.InstallVault, config *files.RootConfig) error { if err := EnsureMounterHmacSecret(vault); err != nil { return fmt.Errorf("ensure hmac secret: %w", err) } + if err := EnsureOpenFgaPresharedKey(vault); err != nil { + return fmt.Errorf("ensure openfga preshared key: %w", err) + } if err := EnsureDefaultSecrets(vault); err != nil { return fmt.Errorf("ensure default secrets: %w", err) } @@ -172,6 +175,28 @@ func EnsureMounterHmacSecret(vault *files.InstallVault) error { return nil } +// EnsureOpenFgaPresharedKey generates the preshared key that OpenFGA and the Codesphere +// services authenticate with, as 64 hex characters. Idempotent. +// +// Deliberately *not* a data-center-scoped secret (see datacenter.go): every data center's +// services talk to the same OpenFGA instance, so DeriveDataCenterVault must carry this key +// over to derived vaults unchanged. A data center whose vault is not derived from the +// primary one needs the key copied in by hand. +func EnsureOpenFgaPresharedKey(vault *files.InstallVault) error { + if vault.GetSecret(files.SecretOpenFgaPresharedKey) != nil { + return nil + } + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return fmt.Errorf("read random bytes: %w", err) + } + vault.SetSecret(files.SecretEntry{ + Name: files.SecretOpenFgaPresharedKey, + Fields: &files.SecretFields{Password: hex.EncodeToString(b)}, + }) + return nil +} + // EnsureNixSigningKeys generates an Ed25519 signing key pair for nix-cache in the // format "host:hexKey" if not already present. Idempotent. func EnsureNixSigningKeys(vault *files.InstallVault, host string) error { diff --git a/internal/installer/secrets/secrets_test.go b/internal/installer/secrets/secrets_test.go index 3d46c264..b11d8e7c 100644 --- a/internal/installer/secrets/secrets_test.go +++ b/internal/installer/secrets/secrets_test.go @@ -125,6 +125,39 @@ var _ = Describe("EnsureMounterHmacSecret", func() { }) }) +var _ = Describe("EnsureOpenFgaPresharedKey", func() { + It("creates a 64-character hex secret", func() { + vault := newVault() + Expect(secrets.EnsureOpenFgaPresharedKey(vault)).To(Succeed()) + + secret := vault.GetSecret("openFgaPresharedKey") + Expect(secret).NotTo(BeNil()) + Expect(secret.Fields).NotTo(BeNil()) + Expect(secret.Fields.Password).To(HaveLen(64)) + Expect(secret.Fields.Password).To(MatchRegexp("^[0-9a-f]+$")) + }) + + It("is idempotent", func() { + vault := newVault() + Expect(secrets.EnsureOpenFgaPresharedKey(vault)).To(Succeed()) + original := vault.GetSecret("openFgaPresharedKey").Fields.Password + + Expect(secrets.EnsureOpenFgaPresharedKey(vault)).To(Succeed()) + Expect(vault.GetSecret("openFgaPresharedKey").Fields.Password).To(Equal(original)) + }) + + It("is shared across data centers", func() { + vault := newVault() + Expect(secrets.EnsureOpenFgaPresharedKey(vault)).To(Succeed()) + primary := vault.GetSecret("openFgaPresharedKey").Fields.Password + + derived := secrets.DeriveDataCenterVault(vault) + Expect(secrets.EnsureOpenFgaPresharedKey(derived)).To(Succeed()) + + Expect(derived.GetSecret("openFgaPresharedKey").Fields.Password).To(Equal(primary)) + }) +}) + var _ = Describe("EnsureNixSigningKeys", func() { It("creates priv/pub keys in host:hexKey format", func() { vault := newVault() From a7f42af31e1725536f7d1fb46bf0570ead8e16ac Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 7 Aug 2026 17:16:40 +0200 Subject: [PATCH 11/11] fix error message in install config Signed-off-by: Jona Neef --- internal/bootstrap/gcp/install_config.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index b2438aa4..14e4d273 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -861,7 +861,10 @@ func (b *GCPBootstrapper) verifySecondaryDataCenterSecrets(primary, dc *DataCent primaryVault := primary.icg.GetVault() vault := dc.icg.GetVault() - shared := []string{files.SecretTokenPrivateKey, files.SecretPostgresPassword} + // openFgaPresharedKey is in here because a secondary data center that already has a vault is + // not re-derived, but GenerateSecrets still runs for it — so a vault predating the key gets a + // freshly generated one that the shared OpenFGA instance rejects. + shared := []string{files.SecretTokenPrivateKey, files.SecretPostgresPassword, files.SecretOpenFgaPresharedKey} for _, svc := range codesphere.PostgresServices { shared = append(shared, files.PostgresUserSecretName(svc.Name), files.PostgresPasswordSecretName(svc.Name)) } @@ -871,7 +874,7 @@ func (b *GCPBootstrapper) verifySecondaryDataCenterSecrets(primary, dc *DataCent continue } if !reflect.DeepEqual(vault.GetSecret(name), expected) { - return fmt.Errorf("secret %q of data center %d differs from the primary data center, but both use the same database", name, dc.ID) + return fmt.Errorf("secret %q of data center %d differs from the primary data center, but both data centers share it", name, dc.ID) } }