diff --git a/.gitignore b/.gitignore index eb2c4d4b..011d5c24 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,10 @@ internal/util/testdata/ config.yaml prod.vault.yaml configure-k0s.sh +# ... and their per-data-center variants written by a --multi-dc bootstrap +config-dc*.yaml +prod-dc*.vault.yaml +configure-k0s-dc*.sh # Debugger files __debug* diff --git a/cli/cmd/bootstrap_gcp.go b/cli/cmd/bootstrap_gcp.go index 8f87eae5..b9f557eb 100644 --- a/cli/cmd/bootstrap_gcp.go +++ b/cli/cmd/bootstrap_gcp.go @@ -89,6 +89,7 @@ func AddBootstrapGcpCmd(parent *cobra.Command, opts *util.GlobalOptions) { flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.SpotVMs, "spot-vms", false, "Use Spot VMs for Codesphere infrastructure. Falls back to standard VMs if spot capacity unavailable. Mutually exclusive with --preemptible (default: false)") flags.IntVar(&bootstrapGcpCmd.CodesphereEnv.DatacenterID, "datacenter-id", 1, "Datacenter ID (default: 1)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DatacenterName, "datacenter-name", "dev", "Datacenter name (default: dev)") + flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.MultiDC, "multi-dc", false, "Bootstrap two data centers that share one PostgreSQL server but run separate Kubernetes and Ceph clusters. Doubles the Ceph and k0s nodes to 14 VMs (~100 vCPUs) and reserves 6 static IPs, so the region's CPU quota may need raising. Cannot be combined with --datacenter-id. (default: false)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.CustomPgIP, "custom-pg-ip", "", "Custom PostgreSQL IP (optional)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.Region, "region", "europe-west4", "GCP Region (default: europe-west4)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.Zone, "zone", "europe-west4-a", "GCP Zone (default: europe-west4-a)") @@ -174,6 +175,8 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error { c.CodesphereEnv.RegistryType = gcp.RegistryType(c.InputRegistryType) c.CodesphereEnv.OmsWorkdir = c.Env.GetOmsWorkdir() + // The value alone cannot distinguish the default 1 from an explicit --datacenter-id=1. + c.CodesphereEnv.DatacenterIDExplicit = c.cmd.Flags().Changed("datacenter-id") if c.CodesphereEnv.GitHubPAT != "" { c.CodesphereEnv.RegistryType = gcp.RegistryTypeGitHub if c.CodesphereEnv.RegistryUser == "" { @@ -208,18 +211,27 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error { if bs.Env.InstallVersion != "" { log.Printf("Access Codesphere in your web browser at https://cs.%s", bs.Env.BaseDomain) + for _, dc := range bs.Env.DataCenters { + log.Printf("Data center %d hosts workspaces under %s", dc.ID, dc.WorkspaceHostingBaseDomain) + } return nil } packageName := "-installer" - installCmd := "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml" if gcp.RegistryType(bs.Env.RegistryType) == gcp.RegistryTypeGitHub { log.Printf("You set a GitHub PAT for direct image access. Make sure to use a lite package, as VM root disk sizes are reduced.") - installCmd += " -s load-container-images" packageName += "-lite" } - log.Printf("example install command (run from jumpbox):\n%s -p %s.tar.gz", installCmd, packageName) + packageFile := packageName + ".tar.gz" + if len(bs.Env.DataCenters) > 1 { + log.Printf("example install commands (run from jumpbox). Run the data center 1 command to completion first — the other data centers share its database:") + } else { + log.Printf("example install command (run from jumpbox):") + } + for _, dc := range bs.Env.DataCenters { + log.Printf("# data center %d\n%s", dc.ID, bs.InstallCommand(dc, packageFile)) + } return nil } diff --git a/cli/cmd/bootstrap_gcp_cleanup_test.go b/cli/cmd/bootstrap_gcp_cleanup_test.go index 3337fd4c..75c4da29 100644 --- a/cli/cmd/bootstrap_gcp_cleanup_test.go +++ b/cli/cmd/bootstrap_gcp_cleanup_test.go @@ -322,7 +322,38 @@ var _ = Describe("BootstrapGcpCleanupCmd", func() { mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true) mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil) - mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", "example.com").Return(nil) + // An infra file without a recorded record list predates multi-DC support, so + // cleanup falls back to the single-data-center record names. + mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", gcp.GetDNSRecordNames("example.com")).Return(nil) + mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil) + mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil) + + err := cleanupCmd.ExecuteCleanup(deps) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when the infra file recorded the DNS records it created", func() { + It("should delete exactly those records", func() { + cleanupCmd.Opts.ProjectID = "test-project" + cleanupCmd.Opts.Force = true + + recorded := []gcp.DNSRecordName{ + {Name: "cs.example.com.", Rtype: "A"}, + {Name: "2.ws.example.com.", Rtype: "A"}, + } + validEnv := gcp.CodesphereEnvironment{ + ProjectID: "test-project", + BaseDomain: "example.com", + DNSZoneName: "test-zone", + MultiDC: true, + DNSRecords: recorded, + } + envData, _ := json.Marshal(validEnv) + + mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true) + mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil) + mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", recorded).Return(nil) mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil) mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil) diff --git a/cli/cmd/bootstrap_gcp_restart_vms.go b/cli/cmd/bootstrap_gcp_restart_vms.go index 900b9353..a35ffebb 100644 --- a/cli/cmd/bootstrap_gcp_restart_vms.go +++ b/cli/cmd/bootstrap_gcp_restart_vms.go @@ -28,32 +28,46 @@ type BootstrapGcpRestartVMsOpts struct { Name string } -// resolveProjectAndZone returns the project ID and zone from flags or the infra file. -// If both flags are set they are used directly; if neither is set, the infra file is read. -// Providing only one of --project-id / --zone is an error. -func (c *BootstrapGcpRestartVMsCmd) resolveProjectAndZone(fw intutil.FileIO) (string, string, error) { +// resolveEnvironment returns the environment to restart VMs in. Project ID and zone come from +// the flags or, when neither is set, from the infra file. Providing only one of +// --project-id / --zone is an error. +// +// The data center layout always comes from the infra file, since it determines the VM names. It +// is read best-effort when the flags supply project and zone, in which case a missing file just +// means single-data-center names. +func (c *BootstrapGcpRestartVMsCmd) resolveEnvironment(fw intutil.FileIO) (*gcp.CodesphereEnvironment, error) { projectID := c.Opts.ProjectID zone := c.Opts.Zone if (projectID == "") != (zone == "") { - return "", "", fmt.Errorf("--project-id and --zone must be provided together") - } - if projectID != "" { - return projectID, zone, nil + return nil, fmt.Errorf("--project-id and --zone must be provided together") } infraFilePath := gcp.GetInfraFilePath() infraEnv, exists, err := gcp.LoadInfraFile(fw, infraFilePath) if err != nil { - return "", "", fmt.Errorf("failed to load infra file: %w", err) - } - if !exists { - return "", "", fmt.Errorf("infra file not found at %s; use --project-id and --zone flags", infraFilePath) + if projectID == "" { + return nil, fmt.Errorf("failed to load infra file: %w", err) + } + log.Printf("Warning: %v", err) } - if infraEnv.ProjectID == "" || infraEnv.Zone == "" { - return "", "", fmt.Errorf("infra file is missing project ID or zone; use --project-id and --zone flags") + + if projectID == "" { + if !exists { + return nil, fmt.Errorf("infra file not found at %s; use --project-id and --zone flags", infraFilePath) + } + if infraEnv.ProjectID == "" || infraEnv.Zone == "" { + return nil, fmt.Errorf("infra file is missing project ID or zone; use --project-id and --zone flags") + } + projectID, zone = infraEnv.ProjectID, infraEnv.Zone } - return infraEnv.ProjectID, infraEnv.Zone, nil + + return &gcp.CodesphereEnvironment{ + ProjectID: projectID, + Zone: zone, + MultiDC: infraEnv.MultiDC, + DataCenters: infraEnv.DataCenters, + }, nil } func (c *BootstrapGcpRestartVMsCmd) RunE(_ *cobra.Command, _ []string) error { @@ -61,18 +75,14 @@ func (c *BootstrapGcpRestartVMsCmd) RunE(_ *cobra.Command, _ []string) error { stlog := bootstrap.NewStepLogger(false) fw := intutil.NewFilesystemWriter() - projectID, zone, err := c.resolveProjectAndZone(fw) + csEnv, err := c.resolveEnvironment(fw) if err != nil { return err } + projectID, zone := csEnv.ProjectID, csEnv.Zone gcpClient := gcp.NewGCPClient(ctx, stlog, os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")) - csEnv := &gcp.CodesphereEnvironment{ - ProjectID: projectID, - Zone: zone, - } - bs, err := gcp.NewGCPBootstrapper( ctx, nil, stlog, csEnv, nil, gcpClient, fw, nil, nil, intutil.NewTime(), nil, @@ -112,6 +122,7 @@ func AddBootstrapGcpRestartVMsCmd(bootstrapGcp *cobra.Command, opts *util.Global {Desc: "Restart all VMs using project info from the local infra file"}, {Cmd: "--name jumpbox", Desc: "Restart only the jumpbox VM"}, {Cmd: "--name k0s-1", Desc: "Restart a specific k0s node"}, + {Cmd: "--name k0s-1-dc2", Desc: "Restart a node of the second data center of a --multi-dc bootstrap"}, {Cmd: "--project-id my-project --zone us-central1-a", Desc: "Restart all VMs with explicit project and zone"}, {Cmd: "--project-id my-project --zone us-central1-a --name ceph-1", Desc: "Restart a specific VM with explicit project and zone"}, }), diff --git a/docs/oms_beta_bootstrap-gcp.md b/docs/oms_beta_bootstrap-gcp.md index b19a359f..eb3facf0 100644 --- a/docs/oms_beta_bootstrap-gcp.md +++ b/docs/oms_beta_bootstrap-gcp.md @@ -57,6 +57,7 @@ oms beta bootstrap-gcp [flags] --install-version string Codesphere version to install (default: none) --internal-flags stringArray Internal flags to enable in Codesphere installation (optional) (default [headless-services,vcluster,custom-service-image,ms-in-ls]) --local-trace-endpoint string Endpoint for exporting traces to an in-cluster storage (optional) + --multi-dc Bootstrap two data centers that share one PostgreSQL server but run separate Kubernetes and Ceph clusters. Doubles the Ceph and k0s nodes to 14 VMs (~100 vCPUs) and reserves 6 static IPs, so the region's CPU quota may need raising. Cannot be combined with --datacenter-id. (default: false) --oidc-client-id string OIDC OAuth provider Client ID (optional) --oidc-client-secret string OIDC OAuth provider Client Secret (optional) --oidc-issuer-url string OIDC OAuth provider issuer URL (optional) diff --git a/docs/oms_beta_bootstrap-gcp_restart-vms.md b/docs/oms_beta_bootstrap-gcp_restart-vms.md index 9eeb152e..0f723624 100644 --- a/docs/oms_beta_bootstrap-gcp_restart-vms.md +++ b/docs/oms_beta_bootstrap-gcp_restart-vms.md @@ -26,6 +26,9 @@ $ oms beta bootstrap-gcp restart-vms --name jumpbox # Restart a specific k0s node $ oms beta bootstrap-gcp restart-vms --name k0s-1 +# Restart a node of the second data center of a --multi-dc bootstrap +$ oms beta bootstrap-gcp restart-vms --name k0s-1-dc2 + # Restart all VMs with explicit project and zone $ oms beta bootstrap-gcp restart-vms --project-id my-project --zone us-central1-a diff --git a/internal/bootstrap/gcp/cleanup.go b/internal/bootstrap/gcp/cleanup.go index f30e59a7..f16f3885 100644 --- a/internal/bootstrap/gcp/cleanup.go +++ b/internal/bootstrap/gcp/cleanup.go @@ -180,7 +180,20 @@ func (e *CleanupExecutor) CleanupDNSRecords() error { log.Printf("Skipping DNS cleanup: missing base domain or DNS zone name (provide --base-domain/--dns-zone-name or use --skip-dns-cleanup)") return nil } - return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.BaseDomain) + return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.dnsRecords()) +} + +// dnsRecords returns the DNS records to delete. The bootstrap records what it created in the +// infra file, which is authoritative. Older infra files predate that, and a cleanup driven only +// by --project-id has no infra file at all, so both fall back to deriving the names. +func (e *CleanupExecutor) dnsRecords() []DNSRecordName { + if len(e.InfraEnv.DNSRecords) > 0 { + return e.InfraEnv.DNSRecords + } + if len(e.InfraEnv.DataCenters) > 0 { + return DataCenterDNSRecordNames(e.BaseDomain, e.InfraEnv.DataCenters) + } + return GetDNSRecordNames(e.BaseDomain) } // RemoveDNSIAMBinding removes the cloud-controller service account's IAM binding diff --git a/internal/bootstrap/gcp/datacenter.go b/internal/bootstrap/gcp/datacenter.go new file mode 100644 index 00000000..d263137f --- /dev/null +++ b/internal/bootstrap/gcp/datacenter.go @@ -0,0 +1,278 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" +) + +// primaryDatacenterID is the ID of the first data center. It stays 1 in both modes so +// single-DC bootstraps keep their existing dataCenter.id. +const primaryDatacenterID = 1 + +// DataCenter holds the state of one Codesphere data center inside a bootstrapped GCP project. +// Project-level state (the GCP project, VPC, jumpbox, shared postgres node and container +// registry) lives on CodesphereEnvironment; everything that must differ between data centers +// lives here. +type DataCenter struct { + ID int `json:"id"` + Name string `json:"name"` + // Suffix is appended to data-center-scoped GCP resource names. It is empty for the primary + // data center, so single-DC bootstraps keep the resource names they have always used. + Suffix string `json:"suffix"` + + ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` + CephNodes []*node.Node `json:"ceph_nodes"` + + GatewayIP string `json:"gateway_ip"` + PublicGatewayIP string `json:"public_gateway_ip"` + SshProxyIP string `json:"ssh_proxy_ip"` + + // Local paths of the generated config and vault. + InstallConfigPath string `json:"-"` + SecretsFilePath string `json:"-"` + // Paths on the shared jumpbox. + RemoteConfigPath string `json:"remote_config_path"` + SecretsDir string `json:"secrets_dir"` + + WorkspaceHostingBaseDomain string `json:"workspace_hosting_base_domain"` + SshBaseDomain string `json:"ssh_base_domain"` + + // ExternalPostgres marks a data center that uses the primary data center's PostgreSQL + // server instead of installing its own. + ExternalPostgres bool `json:"external_postgres"` + + InstallConfig *files.RootConfig `json:"-"` + ExistingConfigUsed bool `json:"-"` + icg installer.InstallConfigManager `json:"-"` +} + +// IsPrimary reports whether this is the first data center of the installation. The primary data +// center owns the shared PostgreSQL server and the platform gateway that codesphere.domain +// resolves to. +func (dc *DataCenter) IsPrimary() bool { + return dc.Suffix == "" +} + +// ConfigManager returns the install config manager owning this data center's config and vault. +func (dc *DataCenter) ConfigManager() installer.InstallConfigManager { + return dc.icg +} + +// SetConfigManager assigns the install config manager for this data center. Exported for tests; +// Bootstrap assigns it via BuildDataCenters. +func (dc *DataCenter) SetConfigManager(icg installer.InstallConfigManager) { + dc.icg = icg +} + +// RemoteVaultPath returns the path of this data center's vault on the jumpbox. +func (dc *DataCenter) RemoteVaultPath() string { + return filepath.Join(dc.SecretsDir, "prod.vault.yaml") +} + +// RemoteAgeKeyPath returns the path of this data center's age identity on the jumpbox. +func (dc *DataCenter) RemoteAgeKeyPath() string { + return filepath.Join(dc.SecretsDir, "age_key.txt") +} + +// PlatformDomain returns the host this data center serves its own platform services on. The +// platform derives it from the data center ID and codesphere.domain, which OMS sets to +// cs., and the frontend calls it directly — the browser asks +// .cs. for the config of the data center a workspace lives in. So it has to +// resolve to this data center's platform gateway, not to the primary's, which is what +// cs. and its wildcard point at. +func (dc *DataCenter) PlatformDomain(baseDomain string) string { + return fmt.Sprintf("%d.cs.%s", dc.ID, baseDomain) +} + +// K0sConfigScriptPath returns the local filename of this data center's k0s configuration script. +func (dc *DataCenter) K0sConfigScriptPath() string { + return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix) +} + +// StepName qualifies a bootstrap step name with the data center it applies to. Single-DC +// bootstraps keep their unqualified step names. +func (dc *DataCenter) StepName(name string) string { + if dc.Suffix == "" { + return name + } + return fmt.Sprintf("%s (dc %d)", name, dc.ID) +} + +// BuildDataCenters derives the data center layout from the bootstrap environment: a single +// entry in single-DC mode, and two entries in multi-DC mode where the second one shares the +// first one's PostgreSQL server. +func BuildDataCenters(env *CodesphereEnvironment, newICG func() installer.InstallConfigManager) []*DataCenter { + if !env.MultiDC { + // A single data center keeps honouring --datacenter-id. In multi-DC mode the IDs are + // derived instead, because they drive the per-data-center domains; validateMultiDC + // rejects the combination. + id := env.DatacenterID + if id == 0 { + id = primaryDatacenterID + } + return []*DataCenter{newDataCenter(env, id, "", newICG)} + } + + return []*DataCenter{ + newDataCenter(env, primaryDatacenterID, "", newICG), + newDataCenter(env, primaryDatacenterID+1, "-dc2", newICG), + } +} + +// ensureDataCenters makes sure the environment has a usable data center layout. It derives the +// layout on first use and gives every data center an install config manager, so any entry point +// works whether or not Bootstrap ran first. +func (b *GCPBootstrapper) ensureDataCenters() { + if len(b.Env.DataCenters) > 0 { + b.ensureConfigManagers() + return + } + + b.Env.DataCenters = BuildDataCenters(b.Env, nil) + b.adoptLegacyEnvFields() + b.ensureConfigManagers() +} + +// ensureConfigManagers gives every data center an install config manager. The primary one reuses +// the bootstrapper's, so a single-DC bootstrap behaves exactly as it did before multi-DC support. +// Data centers restored from an infra file arrive without a manager, since it is not serialised. +func (b *GCPBootstrapper) ensureConfigManagers() { + newICG := b.NewConfigManager + if newICG == nil { + newICG = installer.NewInstallConfigManager + } + + for i, dc := range b.Env.DataCenters { + if dc.ConfigManager() != nil { + continue + } + if i == 0 && b.icg != nil { + dc.SetConfigManager(b.icg) + continue + } + dc.SetConfigManager(newICG()) + } +} + +// adoptLegacyEnvFields moves state that a caller supplied through the legacy top-level +// environment fields into the primary data center. Environments loaded from an infra file written +// before multi-DC support carry the primary data center's nodes and IPs there. +func (b *GCPBootstrapper) adoptLegacyEnvFields() { + primary := b.Env.DataCenters[0] + if len(primary.ControlPlaneNodes) == 0 { + primary.ControlPlaneNodes = b.Env.ControlPlaneNodes + } + if len(primary.CephNodes) == 0 { + primary.CephNodes = b.Env.CephNodes + } + if primary.GatewayIP == "" { + primary.GatewayIP = b.Env.GatewayIP + } + if primary.PublicGatewayIP == "" { + primary.PublicGatewayIP = b.Env.PublicGatewayIP + } + if primary.SshProxyIP == "" { + primary.SshProxyIP = b.Env.SshProxyIP + } + if primary.InstallConfig == nil { + primary.InstallConfig = b.Env.InstallConfig + } + // A caller that supplied a config through the environment also tells us whether it is an + // existing one, which decides between generating and regenerating secrets. + if b.Env.ExistingConfigUsed { + primary.ExistingConfigUsed = true + } +} + +// mirrorPrimaryDataCenter projects the primary data center's state onto the legacy top-level +// environment fields. Those are what the infra file exposes to `cleanup` and `restart-vms`, and +// what infra files written before multi-DC support contain. The projection is one-way and never +// read back into a DataCenter. +func (b *GCPBootstrapper) mirrorPrimaryDataCenter() { + if len(b.Env.DataCenters) == 0 { + return + } + + primary := b.primaryDC() + b.Env.ControlPlaneNodes = primary.ControlPlaneNodes + b.Env.CephNodes = primary.CephNodes + b.Env.GatewayIP = primary.GatewayIP + b.Env.PublicGatewayIP = primary.PublicGatewayIP + b.Env.SshProxyIP = primary.SshProxyIP + b.Env.InstallConfig = primary.InstallConfig + b.Env.ExistingConfigUsed = primary.ExistingConfigUsed +} + +// newDataCenter builds one data center, deriving its resource names, file paths and domains +// from the environment and the data-center suffix. +func newDataCenter(env *CodesphereEnvironment, id int, suffix string, newICG func() installer.InstallConfigManager) *DataCenter { + name := env.DatacenterName + if name == "" { + name = "dev" + } + if suffix != "" { + // The k0s cluster is named codesphere-, so the names must differ. + name += suffix + } + + dc := &DataCenter{ + ID: id, + Name: name, + Suffix: suffix, + InstallConfigPath: dcSuffixedPath(env.InstallConfigPath, suffix), + SecretsFilePath: dcSuffixedPath(env.SecretsFilePath, suffix), + RemoteConfigPath: dcSuffixedPath(remoteInstallConfigPath, suffix), + SecretsDir: env.SecretsDir + suffix, + WorkspaceHostingBaseDomain: workspaceHostingBaseDomain(env, id), + SshBaseDomain: sshBaseDomain(env, id), + ExternalPostgres: suffix != "", + } + if newICG != nil { + dc.icg = newICG() + } + + return dc +} + +// workspaceHostingBaseDomain returns the domain workspaces of the given data center are served +// from. Single-DC installations keep ws.; multi-DC installations prefix it with the +// data center ID so each data center's public gateway gets its own name. +func workspaceHostingBaseDomain(env *CodesphereEnvironment, id int) string { + if !env.MultiDC { + return "ws." + env.BaseDomain + } + return fmt.Sprintf("%d.ws.%s", id, env.BaseDomain) +} + +// sshBaseDomain returns the domain the workspace SSH proxy of the given data center is served +// from, following the same scheme as workspaceHostingBaseDomain. +func sshBaseDomain(env *CodesphereEnvironment, id int) string { + if !env.MultiDC { + return "ssh.cs." + env.BaseDomain + } + return fmt.Sprintf("%d.ssh.cs.%s", id, env.BaseDomain) +} + +// dcSuffixedPath inserts the data-center suffix before the file extension, turning +// config.yaml into config-dc2.yaml and prod.vault.yaml into prod-dc2.vault.yaml. +func dcSuffixedPath(path, suffix string) string { + if suffix == "" { + return path + } + + dir, file := filepath.Split(path) + base, ext := file, "" + if idx := strings.Index(file, "."); idx > 0 { + base, ext = file[:idx], file[idx:] + } + + return filepath.Join(dir, base+suffix+ext) +} diff --git a/internal/bootstrap/gcp/datacenter_test.go b/internal/bootstrap/gcp/datacenter_test.go new file mode 100644 index 00000000..f822dbb4 --- /dev/null +++ b/internal/bootstrap/gcp/datacenter_test.go @@ -0,0 +1,132 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" + "github.com/codesphere-cloud/oms/internal/installer" +) + +var _ = Describe("BuildDataCenters", func() { + newEnv := func(multiDC bool) *gcp.CodesphereEnvironment { + return &gcp.CodesphereEnvironment{ + MultiDC: multiDC, + BaseDomain: "example.com", + DatacenterName: "dev", + SecretsDir: "/etc/codesphere/secrets", + InstallConfigPath: "config.yaml", + SecretsFilePath: "prod.vault.yaml", + } + } + + Context("single data center", func() { + It("keeps the paths, secrets dir and domains a single-DC bootstrap has always used", func() { + dcs := gcp.BuildDataCenters(newEnv(false), installer.NewInstallConfigManager) + + Expect(dcs).To(HaveLen(1)) + dc := dcs[0] + Expect(dc.IsPrimary()).To(BeTrue()) + Expect(dc.ID).To(Equal(1)) + Expect(dc.Name).To(Equal("dev")) + Expect(dc.Suffix).To(BeEmpty()) + Expect(dc.InstallConfigPath).To(Equal("config.yaml")) + Expect(dc.SecretsFilePath).To(Equal("prod.vault.yaml")) + Expect(dc.RemoteConfigPath).To(Equal("/etc/codesphere/config.yaml")) + Expect(dc.SecretsDir).To(Equal("/etc/codesphere/secrets")) + Expect(dc.RemoteVaultPath()).To(Equal("/etc/codesphere/secrets/prod.vault.yaml")) + Expect(dc.RemoteAgeKeyPath()).To(Equal("/etc/codesphere/secrets/age_key.txt")) + Expect(dc.K0sConfigScriptPath()).To(Equal("configure-k0s.sh")) + Expect(dc.WorkspaceHostingBaseDomain).To(Equal("ws.example.com")) + Expect(dc.SshBaseDomain).To(Equal("ssh.cs.example.com")) + Expect(dc.ExternalPostgres).To(BeFalse()) + Expect(dc.StepName("Encrypt vault")).To(Equal("Encrypt vault")) + }) + }) + + Context("multi data center", func() { + var dcs []*gcp.DataCenter + + BeforeEach(func() { + dcs = gcp.BuildDataCenters(newEnv(true), installer.NewInstallConfigManager) + }) + + It("builds two data centers with the second sharing the first's postgres", func() { + Expect(dcs).To(HaveLen(2)) + Expect(dcs[0].ExternalPostgres).To(BeFalse()) + Expect(dcs[1].ExternalPostgres).To(BeTrue()) + }) + + It("leaves the primary data center's resource names unsuffixed", func() { + Expect(dcs[0].Suffix).To(BeEmpty()) + Expect(dcs[0].InstallConfigPath).To(Equal("config.yaml")) + Expect(dcs[0].SecretsDir).To(Equal("/etc/codesphere/secrets")) + }) + + It("gives the secondary data center its own name, paths and secrets dir", func() { + dc := dcs[1] + Expect(dc.IsPrimary()).To(BeFalse()) + Expect(dc.ID).To(Equal(2)) + // The k0s cluster is named codesphere-, so the names must differ. + Expect(dc.Name).To(Equal("dev-dc2")) + Expect(dc.InstallConfigPath).To(Equal("config-dc2.yaml")) + Expect(dc.SecretsFilePath).To(Equal("prod-dc2.vault.yaml")) + Expect(dc.RemoteConfigPath).To(Equal("/etc/codesphere/config-dc2.yaml")) + // A separate secrets dir, so the installer cannot overwrite the primary's kubeconfig + // and ceph credentials through config.secrets.baseDir. + Expect(dc.SecretsDir).To(Equal("/etc/codesphere/secrets-dc2")) + Expect(dc.RemoteVaultPath()).To(Equal("/etc/codesphere/secrets-dc2/prod.vault.yaml")) + Expect(dc.RemoteAgeKeyPath()).To(Equal("/etc/codesphere/secrets-dc2/age_key.txt")) + Expect(dc.K0sConfigScriptPath()).To(Equal("configure-k0s-dc2.sh")) + Expect(dc.StepName("Encrypt vault")).To(Equal("Encrypt vault (dc 2)")) + }) + + It("scopes the workspace and ssh domains per data center", func() { + Expect(dcs[0].WorkspaceHostingBaseDomain).To(Equal("1.ws.example.com")) + Expect(dcs[0].SshBaseDomain).To(Equal("1.ssh.cs.example.com")) + Expect(dcs[1].WorkspaceHostingBaseDomain).To(Equal("2.ws.example.com")) + Expect(dcs[1].SshBaseDomain).To(Equal("2.ssh.cs.example.com")) + }) + + It("gives each data center its own config manager", func() { + Expect(dcs[0].ConfigManager()).NotTo(BeIdenticalTo(dcs[1].ConfigManager())) + }) + }) + + It("falls back to the dev datacenter name", func() { + env := newEnv(false) + env.DatacenterName = "" + + Expect(gcp.BuildDataCenters(env, installer.NewInstallConfigManager)[0].Name).To(Equal("dev")) + }) +}) + +var _ = Describe("DataCenterDNSRecordNames", func() { + It("returns the single-DC records for one data center", func() { + dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{BaseDomain: "example.com"}, nil) + + Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(ConsistOf(gcp.GetDNSRecordNames("example.com"))) + }) + + It("shares the platform names and scopes the workspace names per data center", func() { + dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{MultiDC: true, BaseDomain: "example.com"}, nil) + + Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(Equal([]gcp.DNSRecordName{ + {Name: "cs.example.com.", Rtype: "A"}, + {Name: "*.cs.example.com.", Rtype: "A"}, + {Name: "1.ws.example.com.", Rtype: "A"}, + {Name: "*.1.ws.example.com.", Rtype: "A"}, + {Name: "*.1.ssh.cs.example.com.", Rtype: "A"}, + {Name: "1.cs.example.com.", Rtype: "A"}, + {Name: "*.1.cs.example.com.", Rtype: "A"}, + {Name: "2.ws.example.com.", Rtype: "A"}, + {Name: "*.2.ws.example.com.", Rtype: "A"}, + {Name: "*.2.ssh.cs.example.com.", Rtype: "A"}, + {Name: "2.cs.example.com.", Rtype: "A"}, + {Name: "*.2.cs.example.com.", Rtype: "A"}, + })) + }) +}) diff --git a/internal/bootstrap/gcp/gce.go b/internal/bootstrap/gcp/gce.go index a7b7fc66..7314749d 100644 --- a/internal/bootstrap/gcp/gce.go +++ b/internal/bootstrap/gcp/gce.go @@ -24,18 +24,69 @@ type VMDef struct { Tags []string AdditionalDisks []int64 ExternalIP bool + // DataCenterID is the data center the VM belongs to, or 0 for the project-shared VMs + // (jumpbox and postgres) that every data center uses. + DataCenterID int } -// Example VM definitions (expand as needed) -var vmDefs = []VMDef{ - {"jumpbox", "e2-medium", []string{"jumpbox", "ssh"}, []int64{}, true}, - {"postgres", "e2-standard-2", []string{"postgres"}, []int64{}, true}, - {"ceph-1", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"ceph-2", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"ceph-3", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"k0s-1", "e2-standard-8", []string{"k0s"}, []int64{}, false}, - {"k0s-2", "e2-standard-8", []string{"k0s"}, []int64{}, false}, - {"k0s-3", "e2-standard-8", []string{"k0s"}, []int64{}, false}, +// cephNodesPerDataCenter and k0sNodesPerDataCenter are the per-data-center node counts. Three +// Ceph nodes are the minimum for replication; three k0s nodes give one control plane and three +// workers, as written into the install config. +const ( + cephNodesPerDataCenter = 3 + k0sNodesPerDataCenter = 3 +) + +// sharedVMDefs returns the VMs that exist once per project, regardless of how many data centers +// are bootstrapped. The postgres node hosts the database both data centers share. +func sharedVMDefs() []VMDef { + return []VMDef{ + {Name: "jumpbox", MachineType: "e2-medium", Tags: []string{"jumpbox", "ssh"}, AdditionalDisks: []int64{}, ExternalIP: true}, + {Name: "postgres", MachineType: "e2-standard-2", Tags: []string{"postgres"}, AdditionalDisks: []int64{}, ExternalIP: true}, + } +} + +// dataCenterVMDefs returns the Ceph and k0s VMs of one data center. The suffix is empty for the +// primary data center, so single-DC bootstraps keep the names ceph-1..3 and k0s-1..3. +func dataCenterVMDefs(dcID int, suffix string) []VMDef { + defs := make([]VMDef, 0, cephNodesPerDataCenter+k0sNodesPerDataCenter) + for i := 1; i <= cephNodesPerDataCenter; i++ { + defs = append(defs, VMDef{ + Name: fmt.Sprintf("ceph-%d%s", i, suffix), + MachineType: "e2-standard-8", + Tags: []string{"ceph"}, + AdditionalDisks: []int64{10, 100}, + DataCenterID: dcID, + }) + } + for i := 1; i <= k0sNodesPerDataCenter; i++ { + defs = append(defs, VMDef{ + Name: fmt.Sprintf("k0s-%d%s", i, suffix), + MachineType: "e2-standard-8", + Tags: []string{"k0s"}, + AdditionalDisks: []int64{}, + DataCenterID: dcID, + }) + } + + return defs +} + +// VMDefsForEnv returns every VM definition of the environment: the project-shared VMs plus the +// Ceph and k0s VMs of each data center. When the environment carries no data centers — as with +// an infra file written before multi-DC support — it falls back to a single unsuffixed one. +func VMDefsForEnv(env *CodesphereEnvironment) []VMDef { + defs := sharedVMDefs() + + dcs := env.DataCenters + if len(dcs) == 0 { + dcs = []*DataCenter{{ID: primaryDatacenterID}} + } + for _, dc := range dcs { + defs = append(defs, dataCenterVMDefs(dc.ID, dc.Suffix)...) + } + + return defs } // validateVMProvisioningOptions checks that spot and preemptible options are not both set @@ -51,16 +102,20 @@ type vmResult struct { name string externalIP string internalIP string + dcID int } // EnsureComputeInstances ensures that all required compute instances are present and running. func (b *GCPBootstrapper) EnsureComputeInstances() error { + b.ensureDataCenters() + + vms := VMDefsForEnv(b.Env) wg := sync.WaitGroup{} - errCh := make(chan error, len(vmDefs)) - resultCh := make(chan vmResult, len(vmDefs)) - logCh := make(chan string, len(vmDefs)) + errCh := make(chan error, len(vms)) + resultCh := make(chan vmResult, len(vms)) + logCh := make(chan string, len(vms)) - for _, vm := range vmDefs { + for _, vm := range vms { wg.Add(1) go func(vm VMDef) { defer wg.Done() @@ -95,6 +150,12 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error { NodeClient: b.NodeClient, FileIO: b.fw, } + dcByID := map[int]*DataCenter{} + for _, dc := range b.Env.DataCenters { + dc.CephNodes = nil + dc.ControlPlaneNodes = nil + dcByID[dc.ID] = dc + } for result := range resultCh { switch result.vmType { case "jumpbox": @@ -102,22 +163,31 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error { case "postgres": b.Env.PostgreSQLNode = b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) case "ceph": - node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) - b.Env.CephNodes = append(b.Env.CephNodes, node) + dc, ok := dcByID[result.dcID] + if !ok { + return fmt.Errorf("instance %s belongs to unknown data center %d", result.name, result.dcID) + } + dc.CephNodes = append(dc.CephNodes, b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)) case "k0s": - node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) - b.Env.ControlPlaneNodes = append(b.Env.ControlPlaneNodes, node) + dc, ok := dcByID[result.dcID] + if !ok { + return fmt.Errorf("instance %s belongs to unknown data center %d", result.name, result.dcID) + } + dc.ControlPlaneNodes = append(dc.ControlPlaneNodes, b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)) } } - //sort ceph nodes by name to ensure consistent ordering - sort.Slice(b.Env.CephNodes, func(i, j int) bool { - return b.Env.CephNodes[i].GetName() < b.Env.CephNodes[j].GetName() - }) - //sort control plane nodes by name to ensure consistent ordering - sort.Slice(b.Env.ControlPlaneNodes, func(i, j int) bool { - return b.Env.ControlPlaneNodes[i].GetName() < b.Env.ControlPlaneNodes[j].GetName() - }) + // Sort each data center's nodes by name to ensure consistent ordering, since the install + // config assigns roles by index. + for _, dc := range b.Env.DataCenters { + sort.Slice(dc.CephNodes, func(i, j int) bool { + return dc.CephNodes[i].GetName() < dc.CephNodes[j].GetName() + }) + sort.Slice(dc.ControlPlaneNodes, func(i, j int) bool { + return dc.ControlPlaneNodes[i].GetName() < dc.ControlPlaneNodes[j].GetName() + }) + } + b.mirrorPrimaryDataCenter() return nil } @@ -163,6 +233,7 @@ func (b *GCPBootstrapper) ensureVM(vm VMDef, rootDiskSize int64, logCh chan<- st name: vm.Name, externalIP: externalIP, internalIP: internalIP, + dcID: vm.DataCenterID, }, nil } @@ -362,30 +433,32 @@ func (b *GCPBootstrapper) waitForInstanceRunning(projectID, zone, name string, n name, pollInterval*time.Duration(maxAttempts)) } -// findVMDef looks up a VM definition by name. Returns nil if not found. -func findVMDef(name string) *VMDef { - for _, vm := range vmDefs { - if vm.Name == name { - return &vm +// findVMDef looks up a VM definition by name among the given definitions. Returns nil if not +// found. +func findVMDef(defs []VMDef, name string) *VMDef { + for i := range defs { + if defs[i].Name == name { + return &defs[i] } } return nil } -// validVMNames returns the list of known VM names from vmDefs. -func validVMNames() []string { - names := make([]string, len(vmDefs)) - for i, vm := range vmDefs { +// validVMNames returns the names of the given VM definitions. +func validVMNames(defs []VMDef) []string { + names := make([]string, len(defs)) + for i, vm := range defs { names[i] = vm.Name } return names } -// RestartVM restarts a single stopped or terminated VM by a name that is defined in vmDefs. +// RestartVM restarts a single stopped or terminated VM by a name defined for this environment. func (b *GCPBootstrapper) RestartVM(name string) error { - vm := findVMDef(name) + defs := VMDefsForEnv(b.Env) + vm := findVMDef(defs, name) if vm == nil { - return fmt.Errorf("unknown VM name %q; valid names are: %s", name, strings.Join(validVMNames(), ", ")) + return fmt.Errorf("unknown VM name %q; valid names are: %s", name, strings.Join(validVMNames(defs), ", ")) } projectID := b.Env.ProjectID @@ -424,10 +497,10 @@ func (b *GCPBootstrapper) RestartVM(name string) error { return nil } -// RestartVMs restarts all stopped or terminated VMs defined in vmDefs. +// RestartVMs restarts all stopped or terminated VMs of the environment, across every data center. func (b *GCPBootstrapper) RestartVMs() error { var errs []error - for _, vm := range vmDefs { + for _, vm := range VMDefsForEnv(b.Env) { if err := b.RestartVM(vm.Name); err != nil { errs = append(errs, err) } diff --git a/internal/bootstrap/gcp/gce_test.go b/internal/bootstrap/gcp/gce_test.go index 4d6a1f5f..73dee10c 100644 --- a/internal/bootstrap/gcp/gce_test.go +++ b/internal/bootstrap/gcp/gce_test.go @@ -23,6 +23,61 @@ import ( var _ = Describe("GCE", func() { + Describe("VMDefsForEnv", func() { + It("keeps the names a single-data-center bootstrap has always used", func() { + env := &gcp.CodesphereEnvironment{} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + defs := gcp.VMDefsForEnv(env) + + Expect(vmNames(defs)).To(Equal([]string{ + "jumpbox", "postgres", + "ceph-1", "ceph-2", "ceph-3", + "k0s-1", "k0s-2", "k0s-3", + })) + }) + + It("adds suffixed ceph and k0s nodes per additional data center", func() { + env := &gcp.CodesphereEnvironment{MultiDC: true} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + defs := gcp.VMDefsForEnv(env) + + Expect(vmNames(defs)).To(Equal([]string{ + "jumpbox", "postgres", + "ceph-1", "ceph-2", "ceph-3", + "k0s-1", "k0s-2", "k0s-3", + "ceph-1-dc2", "ceph-2-dc2", "ceph-3-dc2", + "k0s-1-dc2", "k0s-2-dc2", "k0s-3-dc2", + })) + }) + + It("assigns the shared VMs to no data center and the rest to theirs", func() { + env := &gcp.CodesphereEnvironment{MultiDC: true} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + byName := map[string]int{} + for _, def := range gcp.VMDefsForEnv(env) { + byName[def.Name] = def.DataCenterID + } + + Expect(byName["jumpbox"]).To(BeZero()) + Expect(byName["postgres"]).To(BeZero()) + Expect(byName["ceph-1"]).To(Equal(1)) + Expect(byName["k0s-3"]).To(Equal(1)) + Expect(byName["ceph-1-dc2"]).To(Equal(2)) + Expect(byName["k0s-3-dc2"]).To(Equal(2)) + }) + + // Infra files written before multi-DC support carry no data center list. + It("falls back to a single unsuffixed data center when the environment has none", func() { + defs := gcp.VMDefsForEnv(&gcp.CodesphereEnvironment{}) + + Expect(vmNames(defs)).To(ContainElement("k0s-1")) + Expect(vmNames(defs)).To(HaveLen(8)) + }) + }) + Describe("IsNotFoundError", func() { Context("when error is nil", func() { It("should return false", func() { diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 19eb5c30..bcb40227 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "log" - "path/filepath" "slices" "strings" "time" @@ -36,6 +35,24 @@ const ( RegistryTypeGitHub RegistryType = "github" ) +// remoteK0sConfigScriptPath is where each data center's k0s configuration script is placed on +// that data center's first control plane node. Data centers have separate nodes, so the path can +// be the same for all of them. +const remoteK0sConfigScriptPath = "/root/configure-k0s.sh" + +// installerNodeSecretsDir is where the Codesphere installer uploads a data center's age key on +// every one of that data center's nodes. The path is fixed even though the installer reads the key +// from the data center's own secrets.baseDir on the jumpbox, and the installer only creates +// baseDir on the node — so for a data center whose baseDir differs, the upload target would not +// exist. Creating it up front is harmless: data centers have separate nodes, so a node only ever +// holds its own data center's key. +const installerNodeSecretsDir = "/etc/codesphere/secrets" + +// vpcSubnetCIDR is the range of the project's single subnet, shared by all data centers. It is +// also each data center's ceph.nodesSubnet; their Ceph clusters stay separate because each has +// its own hosts, monitors and FSID. +const vpcSubnetCIDR = "10.10.0.0/20" + // CheckOMSManagedLabel checks if the given labels map indicates an OMS-managed project. // A project is considered OMS-managed if it has the 'oms-managed' label set to "true". func CheckOMSManagedLabel(labels map[string]string) bool { @@ -46,15 +63,17 @@ func CheckOMSManagedLabel(labels map[string]string) bool { return exists && value == "true" } -// GetDNSRecordNames returns the DNS record names that OMS creates for a given base domain. -func GetDNSRecordNames(baseDomain string) []struct { - Name string - Rtype string -} { - return []struct { - Name string - Rtype string - }{ +// DNSRecordName identifies a DNS record set that OMS manages. +type DNSRecordName struct { + Name string `json:"name"` + Rtype string `json:"rtype"` +} + +// GetDNSRecordNames returns the DNS record names a single-data-center bootstrap creates for a +// given base domain. It is the fallback for infra files written before multi-DC support, which +// do not record the created records. +func GetDNSRecordNames(baseDomain string) []DNSRecordName { + return []DNSRecordName{ {fmt.Sprintf("cs.%s.", baseDomain), "A"}, {fmt.Sprintf("*.cs.%s.", baseDomain), "A"}, {fmt.Sprintf("ws.%s.", baseDomain), "A"}, @@ -63,6 +82,30 @@ func GetDNSRecordNames(baseDomain string) []struct { } } +// DataCenterDNSRecordNames returns every DNS record OMS creates for the given data center +// layout: the shared platform gateway names plus each data center's workspace and SSH names. +func DataCenterDNSRecordNames(baseDomain string, dcs []*DataCenter) []DNSRecordName { + records := []DNSRecordName{ + {fmt.Sprintf("cs.%s.", baseDomain), "A"}, + {fmt.Sprintf("*.cs.%s.", baseDomain), "A"}, + } + for _, dc := range dcs { + records = append(records, + DNSRecordName{fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.SshBaseDomain), "A"}, + ) + if len(dcs) > 1 { + records = append(records, + DNSRecordName{fmt.Sprintf("%s.", dc.PlatformDomain(baseDomain)), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.PlatformDomain(baseDomain)), "A"}, + ) + } + } + + return records +} + // This should ALWAYS be empty. Internal flags are for internal feature // development and not intended for customer use. // Atm. it's not empty as the internal flags below are likely preview or @@ -96,18 +139,68 @@ type GCPBootstrapper struct { NodeClient node.NodeClient PortalClient portal.Portal GitHubClient github.GitHubClient + // NewConfigManager creates the install config manager of a data center. Each data center + // owns its own config and vault, so multi-DC bootstraps need more than one. + NewConfigManager func() installer.InstallConfigManager +} + +// primaryDC returns the first data center, which owns the shared PostgreSQL server and the +// platform gateway that codesphere.domain resolves to. +func (b *GCPBootstrapper) primaryDC() *DataCenter { + return b.Env.DataCenters[0] +} + +// secondaryDCs returns every data center apart from the primary one. +func (b *GCPBootstrapper) secondaryDCs() []*DataCenter { + return b.Env.DataCenters[1:] +} + +// allNodes returns every node of the project: the jumpbox, the shared postgres node and all +// data centers' Ceph and k0s nodes. +func (b *GCPBootstrapper) allNodes() []*node.Node { + nodes := []*node.Node{b.Env.Jumpbox, b.Env.PostgreSQLNode} + for _, dc := range b.Env.DataCenters { + nodes = append(nodes, dc.ControlPlaneNodes...) + nodes = append(nodes, dc.CephNodes...) + } + + return nodes +} + +// clusterNodes returns every Ceph and k0s node of all data centers, i.e. all nodes except the +// jumpbox and the shared postgres node. +func (b *GCPBootstrapper) clusterNodes() []*node.Node { + nodes := []*node.Node{} + for _, dc := range b.Env.DataCenters { + nodes = append(nodes, dc.ControlPlaneNodes...) + nodes = append(nodes, dc.CephNodes...) + } + + return nodes } type CodesphereEnvironment struct { - ProjectID string `json:"project_id"` - ProjectTTL string `json:"project_ttl"` - ProjectName string `json:"project_name"` - DNSProjectID string `json:"dns_project_id"` - Jumpbox *node.Node `json:"jumpbox"` - PostgreSQLNode *node.Node `json:"postgres_node"` - ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` - CephNodes []*node.Node `json:"ceph_nodes"` - ContainerRegistryURL string `json:"-"` + ProjectID string `json:"project_id"` + ProjectTTL string `json:"project_ttl"` + ProjectName string `json:"project_name"` + DNSProjectID string `json:"dns_project_id"` + Jumpbox *node.Node `json:"jumpbox"` + PostgreSQLNode *node.Node `json:"postgres_node"` + // MultiDC bootstraps two data centers that share the PostgreSQL server but run separate + // Kubernetes and Ceph clusters. + MultiDC bool `json:"multi_dc"` + // DataCenters holds the per-data-center state. It always has at least one entry. + DataCenters []*DataCenter `json:"datacenters"` + // DNSRecords records the DNS records the bootstrap created, so cleanup deletes exactly + // those instead of recomputing the list. + DNSRecords []DNSRecordName `json:"dns_records,omitempty"` + // Mirrors of DataCenters[0], written for infra files consumed by cleanup and restart-vms. + ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` + CephNodes []*node.Node `json:"ceph_nodes"` + // ContainerRegistryURL is the resolved registry server all data centers pull images from. + ContainerRegistryURL string `json:"container_registry_url,omitempty"` + RegistryUsername string `json:"-"` + RegistryPassword string `json:"-"` ExistingConfigUsed bool `json:"-"` InstallVersion string `json:"install_version"` InstallLocal string `json:"install_local"` @@ -181,10 +274,13 @@ type CodesphereEnvironment struct { SSHPrivateKeyPath string `json:"-"` DatacenterID int `json:"-"` DatacenterName string `json:"-"` - CustomPgIP string `json:"custom_pg_ip"` - Region string `json:"region"` - Zone string `json:"zone"` - DNSZoneName string `json:"dns_zone_name"` + // DatacenterIDExplicit records whether --datacenter-id was set on the command line. The + // value alone cannot distinguish the default 1 from an explicit 1. + DatacenterIDExplicit bool `json:"-"` + CustomPgIP string `json:"custom_pg_ip"` + Region string `json:"region"` + Zone string `json:"zone"` + DNSZoneName string `json:"dns_zone_name"` // Test user creation CreateTestUser bool `json:"-"` @@ -208,16 +304,17 @@ func NewGCPBootstrapper( gitHubClient github.GitHubClient, ) (*GCPBootstrapper, error) { return &GCPBootstrapper{ - ctx: ctx, - stlog: stlog, - fw: fw, - icg: icg, - GCPClient: gcpClient, - Env: CodesphereEnv, - NodeClient: sshRunner, - PortalClient: portalClient, - Time: time, - GitHubClient: gitHubClient, + ctx: ctx, + stlog: stlog, + fw: fw, + icg: icg, + GCPClient: gcpClient, + Env: CodesphereEnv, + NodeClient: sshRunner, + PortalClient: portalClient, + Time: time, + GitHubClient: gitHubClient, + NewConfigManager: installer.NewInstallConfigManager, }, nil } @@ -324,19 +421,41 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.WriteConfig { - err = b.stlog.Step("Update install config", b.UpdateInstallConfig) + err = b.writeDataCenterConfig(b.primaryDC()) if err != nil { - return fmt.Errorf("failed to update install config: %w", err) + return err + } + } + + // Secondary data centers fall back to deriving their config and vault from the primary one, + // so this has to run after the primary's secrets were generated above. + for _, dc := range b.secondaryDCs() { + err = b.stlog.Step(dc.StepName("Ensure install config"), func() error { + return b.ensureInstallConfig(dc) + }) + if err != nil { + return fmt.Errorf("failed to ensure install config of data center %d: %w", dc.ID, err) } - err = b.stlog.Step("Ensure age key", b.EnsureAgeKey) + err = b.stlog.Step(dc.StepName("Ensure secrets"), func() error { + return b.ensureSecrets(dc) + }) if err != nil { - return fmt.Errorf("failed to ensure age key: %w", err) + return fmt.Errorf("failed to ensure secrets of data center %d: %w", dc.ID, err) } - err = b.stlog.Step("Encrypt vault", b.EncryptVault) + err = b.stlog.Step(dc.StepName("Derive config and vault"), func() error { + return b.seedSecondaryDataCenter(b.primaryDC(), dc) + }) if err != nil { - return fmt.Errorf("failed to encrypt vault: %w", err) + return fmt.Errorf("failed to derive data center %d: %w", dc.ID, err) + } + + if b.Env.WriteConfig { + err = b.writeDataCenterConfig(dc) + if err != nil { + return err + } } } @@ -351,12 +470,14 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" { - err = b.stlog.Step("Install Codesphere", b.InstallCodesphere) + err = b.InstallCodesphere() if err != nil { return fmt.Errorf("failed to install Codesphere: %w", err) } - err = b.stlog.Step("Run k0s config script", b.RunK0sConfigScript) + // Every data center is installed before any k0s script runs, so a script never patches + // gateway services an install has yet to create. + err = b.RunK0sConfigScript() if err != nil { return fmt.Errorf("failed to run k0s config script: %w", err) } @@ -371,8 +492,11 @@ func (b *GCPBootstrapper) Bootstrap() error { return nil } -// createTestUser creates a test user in the PostgreSQL instance using the testuser package and logs the credentials. +// createTestUser creates a test user in the shared PostgreSQL instance using the testuser package +// and logs the credentials. The user's team is homed in the primary data center. func (b *GCPBootstrapper) createTestUser() error { + b.ensureDataCenters() + if b.Env.PostgreSQLNode == nil { return fmt.Errorf("postgres node not found in bootstrap environment") } @@ -382,10 +506,11 @@ func (b *GCPBootstrapper) createTestUser() error { return fmt.Errorf("postgres node has no external IP") } - if b.Env.InstallConfig == nil { + primary := b.primaryDC() + if primary.InstallConfig == nil { return fmt.Errorf("install config not found in bootstrap environment") } - pgPasswordSecret := b.icg.GetVault().GetSecret(files.SecretPostgresPassword) + pgPasswordSecret := primary.ConfigManager().GetVault().GetSecret(files.SecretPostgresPassword) if pgPasswordSecret == nil || pgPasswordSecret.Fields == nil { return fmt.Errorf("postgres admin password not found in vault") } @@ -398,7 +523,7 @@ func (b *GCPBootstrapper) createTestUser() error { Password: pgPassword, DBName: testuser.DefaultDBName, SSLMode: "require", - DatacenterID: b.Env.DatacenterID, + DatacenterID: primary.ID, }) if err != nil { return err @@ -456,9 +581,50 @@ func (b *GCPBootstrapper) ValidateInput() error { return err } + err = b.validateMultiDC() + if err != nil { + return err + } + return b.validateTelemetryExportParams() } +// validateMultiDC rejects flag combinations a multi-data-center bootstrap cannot satisfy. +func (b *GCPBootstrapper) validateMultiDC() error { + if !b.Env.MultiDC { + return nil + } + + // A secondary data center's config and vault are derived from the primary's, which only + // happens when configs are written. + if !b.Env.WriteConfig { + return fmt.Errorf("multi-dc requires write-config to be enabled") + } + + // The data center IDs are derived (1 and 2) and drive the workspace hosting domains. + if b.Env.DatacenterIDExplicit { + return fmt.Errorf("datacenter-id cannot be combined with multi-dc, the IDs are derived") + } + + // The k0s cluster is named codesphere-, so both names must be set and + // distinct. BuildDataCenters derives the second one by suffixing the first. + if b.Env.DatacenterName == "" { + return fmt.Errorf("datacenter-name is required with multi-dc") + } + + // Every data center gets its own local config and vault, derived by suffixing these paths. + for name, path := range map[string]string{ + "install-config": b.Env.InstallConfigPath, + "secrets-file": b.Env.SecretsFilePath, + } { + if path == "" { + return fmt.Errorf("cannot derive a per-data-center path: %s is empty", name) + } + } + + return nil +} + func (b *GCPBootstrapper) validateClusterAdminEmail() error { if b.Env.ClusterAdminEmail == "" { return nil @@ -613,7 +779,7 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { repo, err := b.GCPClient.GetArtifactRegistry(b.Env.ProjectID, b.Env.Region, repoName) if err == nil && repo != nil { - b.Env.InstallConfig.Registry.Server = repo.GetRegistryUri() + b.Env.ContainerRegistryURL = repo.GetRegistryUri() return nil } @@ -621,6 +787,7 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { if err != nil || repo == nil { return fmt.Errorf("failed to create artifact registry: %w, repo: %v", err, repo) } + b.Env.ContainerRegistryURL = repo.GetRegistryUri() return nil } @@ -685,7 +852,7 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error { Allowed: []*computepb.Allowed{ {IPProtocol: protoString("all")}, }, - SourceRanges: []string{"10.10.0.0/20"}, + SourceRanges: []string{vpcSubnetCIDR}, Description: protoString("Allow all internal traffic"), } err = b.GCPClient.CreateFirewallRule(b.Env.ProjectID, internalRule) @@ -748,19 +915,34 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error { return nil } -// EnsureGatewayIPAddresses reserves the static external IP addresses for the ingress -// controllers of the cluster (gateway and public gateway) and the SSH workspace proxy. +// EnsureGatewayIPAddresses reserves the static external IP addresses of every data center: the +// ingress controllers of its cluster (gateway and public gateway) and its SSH workspace proxy. func (b *GCPBootstrapper) EnsureGatewayIPAddresses() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + if err := b.ensureGatewayIPAddresses(dc); err != nil { + return err + } + } + b.mirrorPrimaryDataCenter() + + return nil +} + +// ensureGatewayIPAddresses reserves one data center's static external IP addresses. Their names +// carry the data-center suffix, so the primary data center keeps the unsuffixed names. +func (b *GCPBootstrapper) ensureGatewayIPAddresses(dc *DataCenter) error { var err error - b.Env.GatewayIP, err = b.EnsureExternalIP("gateway") + dc.GatewayIP, err = b.EnsureExternalIP("gateway" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure gateway IP: %w", err) } - b.Env.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway") + dc.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure public gateway IP: %w", err) } - b.Env.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy") + dc.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure ssh proxy IP: %w", err) } @@ -801,14 +983,9 @@ func (b *GCPBootstrapper) EnsureExternalIP(name string) (string, error) { } func (b *GCPBootstrapper) EnsureRootLoginEnabled() error { - allNodes := []*node.Node{ - b.Env.Jumpbox, - } - allNodes = append(allNodes, b.Env.ControlPlaneNodes...) - allNodes = append(allNodes, b.Env.PostgreSQLNode) - allNodes = append(allNodes, b.Env.CephNodes...) + b.ensureDataCenters() - for _, node := range allNodes { + for _, node := range b.allNodes() { err := b.stlog.Substep(fmt.Sprintf("Ensuring root login enabled on %s", node.GetName()), func() error { return b.ensureRootLoginEnabledInNode(node) }) @@ -894,8 +1071,9 @@ func (b *GCPBootstrapper) EnsureOmsInstalled() (err error) { } func (b *GCPBootstrapper) EnsureHostsConfigured() error { - allNodes := append(b.Env.ControlPlaneNodes, b.Env.PostgreSQLNode) - allNodes = append(allNodes, b.Env.CephNodes...) + b.ensureDataCenters() + + allNodes := append([]*node.Node{b.Env.PostgreSQLNode}, b.clusterNodes()...) for _, node := range allNodes { if !node.HasInotifyWatchesConfigured() { @@ -910,38 +1088,76 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error { return fmt.Errorf("failed to configure memory map on %s: %w", node.GetName(), err) } } + err := node.RunSSHCommand("root", "mkdir -p "+installerNodeSecretsDir) + if err != nil { + return fmt.Errorf("failed to create secrets directory on %s: %w", node.GetName(), err) + } + } + + // A secondary data center's secrets directory differs from the fixed path above, so create that + // one too on its own nodes. Nodes belong to exactly one data center, so no node gets a foreign + // data center's directory. + for _, dc := range b.Env.DataCenters { + if dc.SecretsDir == installerNodeSecretsDir { + continue + } + for _, n := range append(append([]*node.Node{}, dc.ControlPlaneNodes...), dc.CephNodes...) { + err := n.RunSSHCommand("root", "mkdir -p "+dc.SecretsDir) + if err != nil { + return fmt.Errorf("failed to create secrets directory on %s: %w", n.GetName(), err) + } + } } return nil } -// EnsureLocalContainerRegistry installs a docker registry on the postgres node to speed up image loading time +// EnsureLocalContainerRegistry installs a container registry on the postgres node to speed up +// image loading time, and makes every cluster node of every data center trust its certificate. func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { + b.ensureDataCenters() + + registryServer, err := b.ensureRegistryRunning() + if err != nil { + return err + } + b.Env.ContainerRegistryURL = registryServer + + // The certificate must be distributed on every run, not only when the registry was just + // created: a re-run that adds a data center finds the registry already up, and that data + // center's nodes would otherwise not trust it. + return b.distributeRegistryCert(b.clusterNodes()) +} + +// ensureRegistryRunning starts the container registry on the postgres node and generates its +// credentials when it is not already serving. Returns the registry server address. +func (b *GCPBootstrapper) ensureRegistryRunning() (string, error) { localRegistryServer := b.Env.PostgreSQLNode.GetInternalIP() + ":5000" // Figure out if registry is already running b.stlog.Logf("Checking if local container registry is already running on postgres node") checkCommand := `test "$(podman ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` err := b.Env.PostgreSQLNode.RunSSHCommand("root", checkCommand) + vault := b.primaryDC().ConfigManager().GetVault() registryUsername := "" registryPassword := "" - if s := b.icg.GetVault().GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { registryUsername = s.Fields.Password } - if s := b.icg.GetVault().GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { registryPassword = s.Fields.Password } - if err == nil && b.Env.InstallConfig.Registry != nil && b.Env.InstallConfig.Registry.Server == localRegistryServer && - registryUsername != "" && registryPassword != "" { + if err == nil && registryUsername != "" && registryPassword != "" { b.stlog.Logf("Local container registry already running on postgres node") - return nil + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword + return localRegistryServer, nil } - b.Env.InstallConfig.Registry.Server = localRegistryServer registryUsername = "custom-registry" registryPassword = shortuuid.New() - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: registryUsername}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: registryPassword}}) + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword commands := []string{ "apt-get update", @@ -961,19 +1177,24 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { -v /root/registry.crt:/certs/registry.crt \ -v /root/registry.key:/certs/registry.key \ registry:2`, - `mkdir -p /etc/docker/certs.d/` + b.Env.InstallConfig.Registry.Server, - `cp /root/registry.crt /etc/docker/certs.d/` + b.Env.InstallConfig.Registry.Server + `/ca.crt`, + `mkdir -p /etc/docker/certs.d/` + localRegistryServer, + `cp /root/registry.crt /etc/docker/certs.d/` + localRegistryServer + `/ca.crt`, } for _, cmd := range commands { b.stlog.Logf("Running command on postgres node: %s", util.Truncate(cmd, 12)) err := b.Env.PostgreSQLNode.RunSSHCommand("root", cmd) if err != nil { - return fmt.Errorf("failed to run command on postgres node: %w", err) + return "", fmt.Errorf("failed to run command on postgres node: %w", err) } } - allNodes := append(b.Env.ControlPlaneNodes, b.Env.CephNodes...) - for _, node := range allNodes { + return localRegistryServer, nil +} + +// distributeRegistryCert installs the local registry's self-signed certificate on the given +// nodes. It is idempotent, so it is safe — and required — to re-run for an additional data center. +func (b *GCPBootstrapper) distributeRegistryCert(nodes []*node.Node) error { + for _, node := range nodes { b.stlog.Logf("Configuring node '%s' to trust local registry certificate", node.GetName()) err := b.Env.PostgreSQLNode.RunSSHCommand("root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.GetInternalIP()+":/usr/local/share/ca-certificates/registry.crt") if err != nil { @@ -992,19 +1213,20 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { return nil } +// EnsureGitHubAccessConfigured resolves ghcr.io as the registry all data centers pull from. func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error { if b.Env.GitHubPAT == "" { return fmt.Errorf("GitHub PAT is not set") } - b.Env.InstallConfig.Registry.Server = "ghcr.io" - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUser}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.GitHubPAT}}) - b.Env.InstallConfig.Registry.ReplaceImagesInBom = false - b.Env.InstallConfig.Registry.LoadContainerImages = false + b.Env.ContainerRegistryURL = "ghcr.io" + b.Env.RegistryUsername = b.Env.RegistryUser + b.Env.RegistryPassword = b.Env.GitHubPAT return nil } func (b *GCPBootstrapper) EnsureDNSRecords() error { + b.ensureDataCenters() + gcpProject := b.Env.DNSProjectID if b.Env.DNSProjectID == "" { gcpProject = b.Env.ProjectID @@ -1016,37 +1238,30 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return fmt.Errorf("failed to ensure DNS managed zone: %w", err) } + // The platform is served from one domain shared by all data centers, pointing at the + // primary data center's gateway. records := []*dns.ResourceRecordSet{ - { - Name: fmt.Sprintf("cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.GatewayIP}, - }, - { - Name: fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.GatewayIP}, - }, - { - Name: fmt.Sprintf("*.ws.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.PublicGatewayIP}, - }, - { - Name: fmt.Sprintf("ws.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.PublicGatewayIP}, - }, - { - Name: fmt.Sprintf("*.ssh.cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.SshProxyIP}, - }, + dnsARecord(fmt.Sprintf("cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP), + dnsARecord(fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP), + } + // Workspaces and their SSH endpoints resolve per data center, so each one gets its own + // names pointing at its own public gateway and SSH proxy. + for _, dc := range b.Env.DataCenters { + records = append(records, + dnsARecord(fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", dc.SshBaseDomain), dc.SshProxyIP), + ) + // The platform calls each data center's own services at .cs., which + // the wildcard above would send to the primary data center's gateway. A single data + // center is that primary, so it needs no record of its own. + if len(b.Env.DataCenters) > 1 { + platformDomain := dc.PlatformDomain(b.Env.BaseDomain) + records = append(records, + dnsARecord(fmt.Sprintf("%s.", platformDomain), dc.GatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", platformDomain), dc.GatewayIP), + ) + } } err = b.GCPClient.EnsureDNSRecordSets(gcpProject, zoneName, records) @@ -1054,18 +1269,59 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return fmt.Errorf("failed to ensure DNS record sets: %w", err) } + // Record what was created so cleanup deletes exactly these records instead of recomputing + // the list from the base domain. + b.Env.DNSRecords = DataCenterDNSRecordNames(b.Env.BaseDomain, b.Env.DataCenters) + return nil } +// dnsARecord builds a short-TTL A record set, as used during initial setup. +func dnsARecord(name, ip string) *dns.ResourceRecordSet { + return &dns.ResourceRecordSet{ + Name: name, + Type: "A", + Ttl: 300, + Rrdatas: []string{ip}, + } +} + +// InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in +// ascending data center order. The order matters: the primary data center's install creates the +// database, roles and schema that the secondary ones reuse. func (b *GCPBootstrapper) InstallCodesphere() error { + b.ensureDataCenters() + fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox() if err != nil { return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err) } - err = b.runInstallCommand(fullPackageFilename) - if err != nil { - return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) + for _, dc := range b.Env.DataCenters { + err = b.stlog.Step(dc.StepName("Install Codesphere"), func() error { + return b.runInstallCommand(dc, fullPackageFilename) + }) + if err != nil { + return fmt.Errorf("failed to install Codesphere from jumpbox (data center %d): %w", dc.ID, err) + } + } + + return nil +} + +// RunK0sConfigScript runs every data center's k0s configuration script on its first control +// plane node. It requires that data center's Codesphere install to have completed, since the +// script patches the gateway services the install creates. +func (b *GCPBootstrapper) RunK0sConfigScript() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + err := b.stlog.Step(dc.StepName("Run k0s config script"), func() error { + return b.runK0sConfigScript(dc) + }) + if err != nil { + return err + } } return nil @@ -1106,11 +1362,16 @@ func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) { return fullPackageFilename, nil } -func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error { - b.stlog.Logf("Installing Codesphere...") - installCmd := fmt.Sprintf("oms install codesphere -c /etc/codesphere/config.yaml -k %s/age_key.txt --vault %s -p %s%s", - b.Env.SecretsDir, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), packageFilename, b.generateSkipStepsArg()) - return b.Env.Jumpbox.RunSSHCommand("root", installCmd) +func (b *GCPBootstrapper) runInstallCommand(dc *DataCenter, packageFilename string) error { + b.stlog.Logf("Installing Codesphere in data center %d...", dc.ID) + return b.Env.Jumpbox.RunSSHCommand("root", b.InstallCommand(dc, packageFilename)) +} + +// InstallCommand returns the command that installs Codesphere into the given data center from +// the jumpbox. It is also printed for the operator when the bootstrap does not install itself. +func (b *GCPBootstrapper) InstallCommand(dc *DataCenter, packageFilename string) string { + return fmt.Sprintf("oms install codesphere -c %s -k %s --vault %s -p %s%s", + dc.RemoteConfigPath, dc.RemoteAgeKeyPath(), dc.RemoteVaultPath(), packageFilename, b.generateSkipStepsArg()) } func (b *GCPBootstrapper) generateSkipStepsArg() string { @@ -1125,7 +1386,21 @@ func (b *GCPBootstrapper) generateSkipStepsArg() string { return " -s " + strings.Join(skipSteps, ",") } +// GenerateK0sConfigScript writes and uploads the k0s cloud-provider configuration script of +// every data center to that data center's first control plane node. func (b *GCPBootstrapper) GenerateK0sConfigScript() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + if err := b.generateK0sConfigScript(dc); err != nil { + return err + } + } + + return nil +} + +func (b *GCPBootstrapper) generateK0sConfigScript(dc *DataCenter) error { script := `#!/bin/bash cat < cloud.conf @@ -1193,14 +1468,14 @@ $KUBECTL apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-gc $KUBECTL apply -f cc-deployment.yaml # set loadBalancerIP for public-gateway-controller and gateway-controller -$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.PublicGatewayIP + `'"}}' -$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.GatewayIP + `'"}}' +$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.PublicGatewayIP + `'"}}' +$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.GatewayIP + `'"}}' sed -i 's/k0scontroller/k0scontroller --enable-cloud-provider/g' /etc/systemd/system/k0scontroller.service -ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" +ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" -ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" +ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" systemctl daemon-reload systemctl restart k0scontroller @@ -1209,25 +1484,27 @@ systemctl restart k0scontroller // --enable-cloud-provider on worker nodes systemd file /etc/systemd/system/k0sworker.service // in addition on the first node: /etc/systemd/system/k0scontroller.service the flag --enable-cloud-provider - err := b.fw.WriteFile("configure-k0s.sh", []byte(script), 0755) + localScript := dc.K0sConfigScriptPath() + err := b.fw.WriteFile(localScript, []byte(script), 0755) if err != nil { - return fmt.Errorf("failed to write configure-k0s.sh: %w", err) + return fmt.Errorf("failed to write %s: %w", localScript, err) } - err = b.Env.ControlPlaneNodes[0].NodeClient.CopyFile(b.Env.ControlPlaneNodes[0], "configure-k0s.sh", "/root/configure-k0s.sh") + controller := dc.ControlPlaneNodes[0] + err = controller.NodeClient.CopyFile(controller, localScript, remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to copy configure-k0s.sh to control plane node: %w", err) + return fmt.Errorf("failed to copy %s to control plane node: %w", localScript, err) } - err = b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "chmod +x /root/configure-k0s.sh") + err = controller.RunSSHCommand("root", "chmod +x "+remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to make configure-k0s.sh executable on control plane node: %w", err) + return fmt.Errorf("failed to make %s executable: %w", localScript, err) } return nil } -func (b *GCPBootstrapper) RunK0sConfigScript() error { - err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "/root/configure-k0s.sh") +func (b *GCPBootstrapper) runK0sConfigScript(dc *DataCenter) error { + err := dc.ControlPlaneNodes[0].RunSSHCommand("root", remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) + return fmt.Errorf("failed to configure k0s in data center %d: %w", dc.ID, err) } return nil diff --git a/internal/bootstrap/gcp/gcp_client.go b/internal/bootstrap/gcp/gcp_client.go index 7b764de3..cb7a9c94 100644 --- a/internal/bootstrap/gcp/gcp_client.go +++ b/internal/bootstrap/gcp/gcp_client.go @@ -61,7 +61,7 @@ type GCPClientManager interface { GetAddress(projectID, region, addressName string) (*computepb.Address, error) EnsureDNSManagedZone(projectID, zoneName, dnsName, description string) error EnsureDNSRecordSets(projectID, zoneName string, records []*dns.ResourceRecordSet) error - DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error + DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error CreatePublicCAExternalAccountKey(projectID string) (keyID, b64MacKey string, err error) } @@ -828,15 +828,15 @@ func (c *GCPClient) EnsureDNSRecordSets(projectID, zoneName string, records []*d return nil } -// DeleteDNSRecordSets deletes DNS record sets created by OMS for the given base domain. -func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error { +// DeleteDNSRecordSets deletes the given DNS record sets, ignoring those that no longer exist. +func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error { service, err := dns.NewService(c.ctx) if err != nil { return fmt.Errorf("failed to create DNS service: %w", err) } var deletions []*dns.ResourceRecordSet - for _, record := range GetDNSRecordNames(baseDomain) { + for _, record := range records { existing, err := service.ResourceRecordSets.Get(projectID, zoneName, record.Name, record.Rtype).Context(c.ctx).Do() if IsNotFoundError(err) { continue diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index d0d6ff15..0ba80054 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -851,8 +851,7 @@ var _ = Describe("GCP Bootstrapper", func() { Describe("EnsureLocalContainerRegistry", func() { Describe("Valid EnsureLocalContainerRegistry", func() { It("installs local registry", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) + icg.EXPECT().GetVault().Return(&files.InstallVault{}) // Setup mocked node // Check if running - return error to simulate not running @@ -868,7 +867,56 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureLocalContainerRegistry() Expect(err).NotTo(HaveOccurred()) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal("custom-registry")) + Expect(bs.Env.RegistryUsername).To(Equal("custom-registry")) + Expect(bs.Env.RegistryPassword).NotTo(BeEmpty()) + Expect(bs.Env.ContainerRegistryURL).To(Equal(bs.Env.PostgreSQLNode.GetInternalIP() + ":5000")) + }) + + // A re-run that adds a data center finds the registry already up. Its nodes still + // need the registry's self-signed certificate, or every image pull fails. + It("distributes the registry certificate even when the registry is already running", func() { + vault := &files.InstallVault{} + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: "custom-registry"}}) + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: "existing-password"}}) + icg.EXPECT().GetVault().Return(vault) + + bs.Env.MultiDC = true + bs.Env.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1", nodeClient)} + bs.Env.CephNodes = []*node.Node{fakeNode("ceph-1", nodeClient)} + secondary := &gcp.DataCenter{ID: 2, Suffix: "-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + // Registry is already running with credentials in the vault. + nodeClient.EXPECT().RunCommand(bs.Env.PostgreSQLNode, "root", mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "podman ps") + })).Return(nil) + + scpTargets := []string{} + nodeClient.EXPECT().RunCommand(bs.Env.PostgreSQLNode, "root", mock.MatchedBy(func(cmd string) bool { + return strings.HasPrefix(cmd, "scp ") + })).RunAndReturn(func(_ *node.Node, _ string, cmd string) error { + scpTargets = append(scpTargets, cmd) + return nil + }).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "update-ca-certificates").Return(nil).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "systemctl restart docker.service || true").Return(nil).Times(4) + + // Register the second data center only after ensureDataCenters would have run, + // mirroring what EnsureComputeInstances produces for a --multi-dc bootstrap. + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + bs.Env.DataCenters[0].SetConfigManager(icg) + + Expect(bs.EnsureLocalContainerRegistry()).To(Succeed()) + Expect(scpTargets).To(HaveLen(4)) + Expect(bs.Env.RegistryPassword).To(Equal("existing-password")) }) }) @@ -988,17 +1036,14 @@ var _ = Describe("GCP Bootstrapper", func() { csEnv.GitHubPAT = "fake-pat" csEnv.RegistryUser = "custom-registry" }) - It("sets configuration options in installconfig", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) - + // The resolved registry and its credentials live on the environment; every data center's + // config picks them up in updateInstallConfig. + It("resolves ghcr.io as the registry for all data centers", func() { err := bs.EnsureGitHubAccessConfigured() Expect(err).NotTo(HaveOccurred()) - Expect(bs.Env.InstallConfig.Registry.Server).To(Equal("ghcr.io")) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal(csEnv.RegistryUser)) - Expect(vault.GetSecret(files.SecretRegistryPassword).Fields.Password).To(Equal(csEnv.GitHubPAT)) - Expect(bs.Env.InstallConfig.Registry.LoadContainerImages).To(BeFalse()) - Expect(bs.Env.InstallConfig.Registry.ReplaceImagesInBom).To(BeFalse()) + Expect(bs.Env.ContainerRegistryURL).To(Equal("ghcr.io")) + Expect(bs.Env.RegistryUsername).To(Equal(csEnv.RegistryUser)) + Expect(bs.Env.RegistryPassword).To(Equal(csEnv.GitHubPAT)) }) Context("When GitHub PAT is missing", func() { @@ -1235,9 +1280,74 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureHostsConfigured() Expect(err).NotTo(HaveOccurred()) }) + + It("creates the directory the installer uploads the age key to on every node", func() { + mkdirs := map[string]int{} + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(n *node.Node, _ string, command string) error { + if command == "mkdir -p /etc/codesphere/secrets" { + mkdirs[n.GetName()]++ + } + return nil + }) + + Expect(bs.EnsureHostsConfigured()).To(Succeed()) + // The postgres node plus every cluster node of every data center. + Expect(mkdirs).To(Equal(map[string]int{ + "postgres": 1, + "k0s-1": 1, "k0s-2": 1, "k0s-3": 1, + "ceph-1": 1, "ceph-2": 1, "ceph-3": 1, + })) + }) + + It("creates a secondary data center's own secrets directory on its nodes only", func() { + secondary := &gcp.DataCenter{ID: 2, Suffix: "-dc2", SecretsDir: "/etc/codesphere/secrets-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, + SecretsDir: "/etc/codesphere/secrets", + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + + mkdirs := map[string][]string{} + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(n *node.Node, _ string, command string) error { + if dir, found := strings.CutPrefix(command, "mkdir -p "); found { + mkdirs[n.GetName()] = append(mkdirs[n.GetName()], dir) + } + return nil + }) + + Expect(bs.EnsureHostsConfigured()).To(Succeed()) + // Both paths on the secondary's nodes, because the installer's fixed path is + // created everywhere; the secondary's path on nobody else's. + Expect(mkdirs["k0s-1-dc2"]).To(ConsistOf("/etc/codesphere/secrets", "/etc/codesphere/secrets-dc2")) + Expect(mkdirs["ceph-1-dc2"]).To(ConsistOf("/etc/codesphere/secrets", "/etc/codesphere/secrets-dc2")) + Expect(mkdirs["k0s-1"]).To(ConsistOf("/etc/codesphere/secrets")) + Expect(mkdirs["postgres"]).To(ConsistOf("/etc/codesphere/secrets")) + }) }) Describe("Invalid cases", func() { + It("fails when the secrets directory cannot be created", func() { + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(_ *node.Node, _ string, command string) error { + if command == "mkdir -p /etc/codesphere/secrets" { + return fmt.Errorf("ouch") + } + return nil + }) + + err := bs.EnsureHostsConfigured() + Expect(err).To(MatchError(ContainSubstring("failed to create secrets directory on postgres"))) + }) + It("fails when ConfigureInotifyWatches fails", func() { nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything).Return(fmt.Errorf("ouch")) @@ -1354,6 +1464,41 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureDNSRecords() Expect(err).NotTo(HaveOccurred()) }) + + It("points each data center's platform host at its own gateway", func() { + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, GatewayIP: "1.1.1.1", PublicGatewayIP: "1.1.1.2", SshProxyIP: "1.1.1.3", + WorkspaceHostingBaseDomain: "1.ws.example.com", SshBaseDomain: "1.ssh.cs.example.com", + }, + { + ID: 2, Suffix: "-dc2", GatewayIP: "2.2.2.1", PublicGatewayIP: "2.2.2.2", SshProxyIP: "2.2.2.3", + WorkspaceHostingBaseDomain: "2.ws.example.com", SshBaseDomain: "2.ssh.cs.example.com", + }, + } + + gc.EXPECT().EnsureDNSManagedZone(csEnv.DNSProjectID, csEnv.DNSZoneName, csEnv.BaseDomain+".", mock.Anything).Return(nil) + targets := map[string]string{} + gc.EXPECT().EnsureDNSRecordSets(csEnv.DNSProjectID, csEnv.DNSZoneName, mock.Anything). + RunAndReturn(func(_ string, _ string, records []*dns.ResourceRecordSet) error { + for _, r := range records { + targets[r.Name] = r.Rrdatas[0] + } + return nil + }) + + Expect(bs.EnsureDNSRecords()).To(Succeed()) + // The shared platform name stays on the primary data center's gateway, but the + // per-data-center platform host the frontend calls resolves to its own gateway. + Expect(targets["cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["*.cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["1.cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["2.cs.example.com."]).To(Equal("2.2.2.1")) + Expect(targets["*.2.cs.example.com."]).To(Equal("2.2.2.1")) + // Workspaces and SSH keep pointing at the public gateway and SSH proxy. + Expect(targets["2.ws.example.com."]).To(Equal("2.2.2.2")) + Expect(targets["*.2.ssh.cs.example.com."]).To(Equal("2.2.2.3")) + }) }) Describe("Invalid cases", func() { diff --git a/internal/bootstrap/gcp/infrafile.go b/internal/bootstrap/gcp/infrafile.go index 72378f9f..f7f6d855 100644 --- a/internal/bootstrap/gcp/infrafile.go +++ b/internal/bootstrap/gcp/infrafile.go @@ -40,6 +40,12 @@ func LoadInfraFile(fw util.FileIO, infraFilePath string) (CodesphereEnvironment, // WriteInfraFile writes details about the bootstrapped codesphere environment into a file. func (b *GCPBootstrapper) WriteInfraFile() error { + b.ensureDataCenters() + + // The legacy top-level node and IP fields are what cleanup and restart-vms read, so keep + // them in sync with the primary data center before serialising. + b.mirrorPrimaryDataCenter() + envBytes, err := json.MarshalIndent(b.Env, "", " ") if err != nil { return fmt.Errorf("failed to marshal codesphere env: %w", err) diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 83ebdf0a..14e4d273 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -4,9 +4,13 @@ package gcp import ( + "errors" "fmt" + "reflect" + "slices" "github.com/codesphere-cloud/oms/internal/bootstrap" + "github.com/codesphere-cloud/oms/internal/codesphere" "github.com/codesphere-cloud/oms/internal/installer/files" "github.com/codesphere-cloud/oms/internal/installer/secrets" "github.com/codesphere-cloud/oms/internal/util" @@ -14,49 +18,73 @@ import ( const ( remoteInstallConfigPath string = "/etc/codesphere/config.yaml" + // sharedPostgresPort is the port the shared PostgreSQL server listens on. Secondary data + // centers need it spelled out because they connect to it as an external server. + sharedPostgresPort int = 5432 ) -// EnsureInstallConfig uses the local config or recovers it from an existing jumpbox if desired. -// Else it applies the minimal profile to a new config. +// errRemoteConfigMissing reports that the jumpbox holds no config for a data center. Recovering +// a secondary data center tolerates this, since --multi-dc can add one to an existing project. +var errRemoteConfigMissing = errors.New("no install config found on the jumpbox") + +// EnsureInstallConfig prepares the primary data center's install config. Secondary data centers +// are handled separately in Bootstrap, after the primary's secrets exist. func (b *GCPBootstrapper) EnsureInstallConfig() error { + b.ensureDataCenters() + + return b.ensureInstallConfig(b.primaryDC()) +} + +// ensureInstallConfig uses the data center's local config or recovers it from an existing +// jumpbox if desired. Else it applies the minimal profile to a new config. +func (b *GCPBootstrapper) ensureInstallConfig(dc *DataCenter) error { // recovery will overwrite local config or create a new file if b.Env.RecoverConfig { - err := b.recoverConfig() - if err != nil { + err := b.recoverConfig(dc) + if errors.Is(err, errRemoteConfigMissing) && !dc.IsPrimary() { + // A secondary data center may not exist on the jumpbox yet, which is the case when + // --multi-dc is used to add one to an existing single-DC project. Its config is + // derived from the primary's instead. + b.stlog.Logf("No config found on the jumpbox for data center %d, generating a new one", dc.ID) + } else if err != nil { return fmt.Errorf("failed to recover config: %w", err) } } - if b.fw.Exists(b.Env.InstallConfigPath) { - if err := b.loadVaultForConfigTemplating(); err != nil { + if b.fw.Exists(dc.InstallConfigPath) { + if err := b.loadVaultForConfigTemplating(dc); err != nil { return fmt.Errorf("failed to load vault templating: %w", err) } - err := b.icg.LoadInstallConfigFromFile(b.Env.InstallConfigPath) + err := dc.icg.LoadInstallConfigFromFile(dc.InstallConfigPath) if err != nil { return fmt.Errorf("failed to load config file: %w", err) } - b.Env.ExistingConfigUsed = true - } else { - err := b.icg.ApplyProfile("minimal") + dc.ExistingConfigUsed = true + dc.InstallConfig = dc.icg.GetInstallConfig() + } else if dc.IsPrimary() { + err := dc.icg.ApplyProfile("minimal") if err != nil { return fmt.Errorf("failed to apply profile: %w", err) } + dc.InstallConfig = dc.icg.GetInstallConfig() } + // A secondary data center without a config of its own is left unset here, so + // seedSecondaryDataCenter can derive it from the primary data center instead of the profile. - b.Env.InstallConfig = b.icg.GetInstallConfig() + b.mirrorPrimaryDataCenter() return nil } -func (b *GCPBootstrapper) loadVaultForConfigTemplating() error { - if !b.fw.Exists(b.Env.SecretsFilePath) { +func (b *GCPBootstrapper) loadVaultForConfigTemplating(dc *DataCenter) error { + if !b.fw.Exists(dc.SecretsFilePath) { return nil } // during bootstrapping, the vault is not yet encrpyted - if err := b.icg.LoadVaultFromUnecryptedFile(b.Env.SecretsFilePath); err != nil { + if err := dc.icg.LoadVaultFromUnecryptedFile(dc.SecretsFilePath); err != nil { return fmt.Errorf("failed to load vault from file: %w", err) } @@ -66,7 +94,7 @@ func (b *GCPBootstrapper) loadVaultForConfigTemplating() error { // recoverConfig downloads the config and secrets from the jumpbox if it exists. // Since recovery is done when the project or VMs are not ensured, we need to search for the jumpbox IP first. // Returns an error if project or jumpbox does not exist or downloading fails. -func (b *GCPBootstrapper) recoverConfig() error { +func (b *GCPBootstrapper) recoverConfig(dc *DataCenter) error { existingProject, err := b.GCPClient.GetProjectByName(b.Env.FolderID, b.Env.ProjectName) if err != nil { return fmt.Errorf("failed to find gcp project for config recovery: %w", err) @@ -79,12 +107,18 @@ func (b *GCPBootstrapper) recoverConfig() error { } b.Env.Jumpbox = jumpbox - err = b.Env.Jumpbox.NodeClient.DownloadFile(jumpbox, remoteInstallConfigPath, b.Env.InstallConfigPath) + // Only a secondary data center may legitimately be absent from the jumpbox, so only there is + // it worth probing first; the primary's missing config stays a download failure. + if !dc.IsPrimary() && !b.Env.Jumpbox.NodeClient.HasFile(jumpbox, dc.RemoteConfigPath) { + return fmt.Errorf("%w at %s", errRemoteConfigMissing, dc.RemoteConfigPath) + } + + err = b.Env.Jumpbox.NodeClient.DownloadFile(jumpbox, dc.RemoteConfigPath, dc.InstallConfigPath) if err != nil { return fmt.Errorf("failed to download install config from jumpbox: %w", err) } - err = b.recoverVault() + err = b.recoverVault(dc) if err != nil { return fmt.Errorf("failed to recover vault: %w", err) } @@ -93,8 +127,8 @@ func (b *GCPBootstrapper) recoverConfig() error { } // recoverVault unencrypts the secrets file on the jumpbox and download the file to the local destination -func (b *GCPBootstrapper) recoverVault() error { - const vaultCopyPath string = "/tmp/prod.vault.yaml" +func (b *GCPBootstrapper) recoverVault(dc *DataCenter) error { + vaultCopyPath := fmt.Sprintf("/tmp/prod%s.vault.yaml", dc.Suffix) defer func() { err := b.Env.Jumpbox.RunSSHCommand("root", "rm -f "+vaultCopyPath) if err != nil { @@ -102,12 +136,12 @@ func (b *GCPBootstrapper) recoverVault() error { } }() - err := b.decryptVault(vaultCopyPath) + err := b.decryptVault(dc, vaultCopyPath) if err != nil { return fmt.Errorf("failed to create decrypted vault for recovery: %w", err) } - err = b.Env.Jumpbox.NodeClient.DownloadFile(b.Env.Jumpbox, vaultCopyPath, b.Env.SecretsFilePath) + err = b.Env.Jumpbox.NodeClient.DownloadFile(b.Env.Jumpbox, vaultCopyPath, dc.SecretsFilePath) if err != nil { return fmt.Errorf("failed to download secrets file from jumpbox: %w", err) } @@ -115,50 +149,60 @@ func (b *GCPBootstrapper) recoverVault() error { return nil } +// UpdateInstallConfig writes the bootstrapped infrastructure into the primary data center's +// install config. Secondary data centers go through updateInstallConfig directly, after their +// config and vault have been derived from the primary's. func (b *GCPBootstrapper) UpdateInstallConfig() error { - // Update install config with necessary values - b.Env.InstallConfig.Datacenter.ID = b.Env.DatacenterID - if b.Env.DatacenterName == "" { - b.Env.DatacenterName = "dev" - } - b.Env.InstallConfig.Datacenter.Name = b.Env.DatacenterName - b.Env.InstallConfig.Datacenter.City = "Karlsruhe" - b.Env.InstallConfig.Datacenter.CountryCode = "DE" - b.Env.InstallConfig.Secrets.BaseDir = b.Env.SecretsDir - if b.Env.RegistryType != RegistryTypeGitHub { - b.Env.InstallConfig.Registry.ReplaceImagesInBom = true - b.Env.InstallConfig.Registry.LoadContainerImages = true - } + b.ensureDataCenters() - if b.Env.InstallConfig.Postgres.Primary == nil { - b.Env.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ - Hostname: b.Env.PostgreSQLNode.GetName(), - } + return b.updateInstallConfig(b.primaryDC()) +} + +func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { + // Update install config with necessary values + dc.InstallConfig.Datacenter = datacenterConfig(dc) + // Each data center reads and writes its own vault. The installer resolves the vault from + // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes + // steps overwrite another's credentials. + dc.InstallConfig.Secrets.BaseDir = dc.SecretsDir + if b.Env.ContainerRegistryURL != "" { + dc.InstallConfig.Registry.Server = b.Env.ContainerRegistryURL + } + if b.Env.RegistryUsername != "" || b.Env.RegistryPassword != "" { + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUsername}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.RegistryPassword}}) + } + if b.Env.RegistryType == RegistryTypeGitHub { + dc.InstallConfig.Registry.ReplaceImagesInBom = false + dc.InstallConfig.Registry.LoadContainerImages = false + } else { + dc.InstallConfig.Registry.ReplaceImagesInBom = true + dc.InstallConfig.Registry.LoadContainerImages = true } - previousPrimaryIP := b.Env.InstallConfig.Postgres.Primary.IP - previousPrimaryHostname := b.Env.InstallConfig.Postgres.Primary.Hostname - b.Env.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() - b.Env.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() + previousPrimaryIP, previousPrimaryHostname := b.applyPostgresConfig(dc) + b.applyDataCenterTopology(dc) - b.Env.InstallConfig.Ceph.CsiKubeletDir = "/var/lib/k0s/kubelet" - b.Env.InstallConfig.Ceph.NodesSubnet = "10.10.0.0/20" - b.Env.InstallConfig.Ceph.Hosts = []files.CephHost{ + dc.InstallConfig.Ceph.CsiKubeletDir = "/var/lib/k0s/kubelet" + // All data centers share the project's subnet; their Ceph clusters stay separate because + // each has its own hosts, monitors and FSID. + dc.InstallConfig.Ceph.NodesSubnet = vpcSubnetCIDR + dc.InstallConfig.Ceph.Hosts = []files.CephHost{ { - Hostname: b.Env.CephNodes[0].GetName(), + Hostname: dc.CephNodes[0].GetName(), IsMaster: true, - IPAddress: b.Env.CephNodes[0].GetInternalIP(), + IPAddress: dc.CephNodes[0].GetInternalIP(), }, { - Hostname: b.Env.CephNodes[1].GetName(), - IPAddress: b.Env.CephNodes[1].GetInternalIP(), + Hostname: dc.CephNodes[1].GetName(), + IPAddress: dc.CephNodes[1].GetInternalIP(), }, { - Hostname: b.Env.CephNodes[2].GetName(), - IPAddress: b.Env.CephNodes[2].GetInternalIP(), + Hostname: dc.CephNodes[2].GetName(), + IPAddress: dc.CephNodes[2].GetInternalIP(), }, } - b.Env.InstallConfig.Ceph.OSDs = []files.CephOSD{ + dc.InstallConfig.Ceph.OSDs = []files.CephOSD{ { SpecID: "default", Placement: files.CephPlacement{ @@ -175,48 +219,48 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { }, } - b.Env.InstallConfig.Kubernetes = files.KubernetesConfig{ + dc.InstallConfig.Kubernetes = files.KubernetesConfig{ ManagedByCodesphere: true, - APIServerHost: b.Env.ControlPlaneNodes[0].GetInternalIP(), + APIServerHost: dc.ControlPlaneNodes[0].GetInternalIP(), ControlPlanes: []files.K8sNode{ { - IPAddress: b.Env.ControlPlaneNodes[0].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[0].GetInternalIP(), }, }, Workers: []files.K8sNode{ { - IPAddress: b.Env.ControlPlaneNodes[0].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[0].GetInternalIP(), }, { - IPAddress: b.Env.ControlPlaneNodes[1].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[1].GetInternalIP(), }, { - IPAddress: b.Env.ControlPlaneNodes[2].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[2].GetInternalIP(), }, }, } - b.Env.InstallConfig.Cluster.Kyverno = &files.KyvernoConfig{ + dc.InstallConfig.Cluster.Kyverno = &files.KyvernoConfig{ Enabled: false, } - b.Env.InstallConfig.Cluster.Gateway.ServiceType = "LoadBalancer" - b.Env.InstallConfig.Cluster.Gateway.Annotations = map[string]string{ - "cloud.google.com/load-balancer-ipv4": b.Env.GatewayIP, + dc.InstallConfig.Cluster.Gateway.ServiceType = "LoadBalancer" + dc.InstallConfig.Cluster.Gateway.Annotations = map[string]string{ + "cloud.google.com/load-balancer-ipv4": dc.GatewayIP, } - b.Env.InstallConfig.Cluster.PublicGateway.ServiceType = "LoadBalancer" - b.Env.InstallConfig.Cluster.PublicGateway.Annotations = map[string]string{ - "cloud.google.com/load-balancer-ipv4": b.Env.PublicGatewayIP, + dc.InstallConfig.Cluster.PublicGateway.ServiceType = "LoadBalancer" + dc.InstallConfig.Cluster.PublicGateway.Annotations = map[string]string{ + "cloud.google.com/load-balancer-ipv4": dc.PublicGatewayIP, } - b.applySshProxyConfig() + b.applySshProxyConfig(dc) dnsProject := b.Env.DNSProjectID if b.Env.DNSProjectID == "" { dnsProject = b.Env.ProjectID } - b.Env.InstallConfig.Cluster.Certificates.Override = map[string]interface{}{ + dc.InstallConfig.Cluster.Certificates.Override = map[string]interface{}{ "issuers": map[string]interface{}{ "letsEncryptHttp": map[string]interface{}{ "enabled": !b.Env.GoogleACMEIssuer, @@ -248,26 +292,32 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { } acmeConfig.Server = "https://dv.acme-v02.api.pki.goog/directory" acmeConfig.EABKeyID = keyID - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAcmeEabMacKey, Fields: &files.SecretFields{Password: b64MacKey}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAcmeEabMacKey, Fields: &files.SecretFields{Password: b64MacKey}}) } - b.Env.InstallConfig.Codesphere.CertIssuer = files.CertIssuerConfig{ + dc.InstallConfig.Codesphere.CertIssuer = files.CertIssuerConfig{ Type: "acme", Acme: acmeConfig, } - b.Env.InstallConfig.Codesphere.Domain = "cs." + b.Env.BaseDomain - b.Env.InstallConfig.Codesphere.WorkspaceHostingBaseDomain = "ws." + b.Env.BaseDomain - b.Env.InstallConfig.Codesphere.PublicIP = b.Env.ControlPlaneNodes[1].GetExternalIP() - b.Env.InstallConfig.Codesphere.CustomDomains = files.CustomDomainsConfig{ - CNameBaseDomain: "ws." + b.Env.BaseDomain, + // The platform is served from one domain shared by all data centers, while workspaces and + // custom domains resolve to the data center hosting them. + dc.InstallConfig.Codesphere.Domain = "cs." + b.Env.BaseDomain + dc.InstallConfig.Codesphere.WorkspaceHostingBaseDomain = dc.WorkspaceHostingBaseDomain + dc.InstallConfig.Codesphere.CustomDomains = files.CustomDomainsConfig{ + CNameBaseDomain: dc.WorkspaceHostingBaseDomain, } - b.Env.InstallConfig.Codesphere.DNSServers = []string{"8.8.8.8"} - b.Env.InstallConfig.Codesphere.DeployConfig = bootstrap.DefaultCodesphereDeployConfig() - b.Env.InstallConfig.Codesphere.Plans = bootstrap.DefaultCodespherePlans() + if b.Env.MultiDC { + dc.InstallConfig.Codesphere.PublicIP = dc.PublicGatewayIP + } else { + dc.InstallConfig.Codesphere.PublicIP = dc.ControlPlaneNodes[1].GetExternalIP() + } + dc.InstallConfig.Codesphere.DNSServers = []string{"8.8.8.8"} + dc.InstallConfig.Codesphere.DeployConfig = bootstrap.DefaultCodesphereDeployConfig() + dc.InstallConfig.Codesphere.Plans = bootstrap.DefaultCodespherePlans() - b.Env.InstallConfig.Codesphere.GitProviders = &files.GitProvidersConfig{} + dc.InstallConfig.Codesphere.GitProviders = &files.GitProvidersConfig{} if b.Env.GitHubAppName != "" && b.Env.GitHubAppClientID != "" && b.Env.GitHubAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.GitHub = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.GitHub = &files.GitProviderConfig{ Enabled: true, URL: "https://github.com", API: files.APIConfig{ @@ -282,11 +332,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { InstallationURI: "https://github.com/apps/" + b.Env.GitHubAppName + "/installations/new", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientId, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientId, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientSecret}}) } if b.Env.GitLabAppClientID != "" && b.Env.GitLabAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.GitLab = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.GitLab = &files.GitProviderConfig{ Enabled: true, URL: "https://gitlab.com", API: files.APIConfig{ @@ -300,11 +350,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { RedirectURI: "https://cs." + b.Env.BaseDomain + "/ide/auth/gitlab/callback", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientId, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientSecret, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientId, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientSecret, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientSecret}}) } if b.Env.BitbucketAppClientID != "" && b.Env.BitbucketAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.Bitbucket = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.Bitbucket = &files.GitProviderConfig{ Enabled: true, URL: "https://bitbucket.org", API: files.APIConfig{ @@ -318,11 +368,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { RedirectURI: "https://cs." + b.Env.BaseDomain + "/ide/auth/bitbucket/callback", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientId, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientId, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientSecret}}) } if b.Env.AzureDevOpsAppClientID != "" && b.Env.AzureDevOpsAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.AzureDevOps = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.AzureDevOps = &files.GitProviderConfig{ Enabled: true, URL: "https://dev.azure.com", API: files.APIConfig{ @@ -337,15 +387,15 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { Scope: "openid offline_access https://app.vssps.visualstudio.com/vso.code_full", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientId, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientSecret, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientId, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientSecret, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientSecret}}) } if b.Env.OidcIssuerURL != "" && b.Env.OidcClientID != "" && b.Env.OidcClientSecret != "" { name := b.Env.OidcProviderName if name == "" { name = "OIDC" } - b.Env.InstallConfig.Codesphere.OAuth = &files.OAuthProvidersConfig{ + dc.InstallConfig.Codesphere.OAuth = &files.OAuthProvidersConfig{ Oidc: &files.OidcOAuthProvider{ Type: "oidc", Enabled: true, @@ -354,12 +404,12 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { Scopes: []string{"openid", "profile", "email"}, }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientId, Fields: &files.SecretFields{Password: b.Env.OidcClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientSecret, Fields: &files.SecretFields{Password: b.Env.OidcClientSecret}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientId, Fields: &files.SecretFields{Password: b.Env.OidcClientID}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientSecret, Fields: &files.SecretFields{Password: b.Env.OidcClientSecret}}) } if b.Env.CentralOtelPassword != "" || b.Env.LocalTraceEndpoint != "" { - b.Env.InstallConfig.Codesphere.TelemetryExport = &files.TelemetryExport{ + dc.InstallConfig.Codesphere.TelemetryExport = &files.TelemetryExport{ RemoteEndpoint: b.Env.CentralOtelEndpoint, RemoteExport: b.Env.CentralOtelPassword != "", Traces: b.Env.LocalTraceEndpoint != "", @@ -368,71 +418,185 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { } } - b.Env.InstallConfig.Codesphere.Internal = b.Env.InternalFlags - b.Env.InstallConfig.Codesphere.Preview = util.StringSliceToBoolMap(b.Env.PreviewFlags) - b.Env.InstallConfig.Codesphere.Features = util.StringSliceToBoolMap(b.Env.FeatureFlags) + dc.InstallConfig.Codesphere.Internal = b.Env.InternalFlags + dc.InstallConfig.Codesphere.Preview = util.StringSliceToBoolMap(b.Env.PreviewFlags) + dc.InstallConfig.Codesphere.Features = util.StringSliceToBoolMap(b.Env.FeatureFlags) // Only set when the flag is provided so a recovered config keeps its value on re-runs. if b.Env.ClusterAdminEmail != "" { - b.Env.InstallConfig.Codesphere.ClusterAdminEmail = b.Env.ClusterAdminEmail + dc.InstallConfig.Codesphere.ClusterAdminEmail = b.Env.ClusterAdminEmail } - b.applyExternalLokiConfig() - b.applyPrometheusRemoteWriteConfig() + b.applyExternalLokiConfig(dc) + b.applyPrometheusRemoteWriteConfig(dc) - if !b.Env.ExistingConfigUsed { - err := b.icg.GenerateSecrets() + // A secondary data center always generates: its vault was seeded from the primary's, so the + // sentinels stop everything except its own ingress CA and cephadm key from being regenerated. + if !dc.ExistingConfigUsed || !dc.IsPrimary() { + err := dc.icg.GenerateSecrets() if err != nil { return fmt.Errorf("failed to generate secrets: %w", err) } } else { - if err := b.regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname); err != nil { + if err := b.regeneratePostgresCerts(dc, previousPrimaryIP, previousPrimaryHostname); err != nil { + return err + } + } + + if !dc.IsPrimary() { + if err := b.verifySecondaryDataCenterSecrets(b.primaryDC(), dc); err != nil { return err } } if b.Env.CentralOtelUsername != "" && b.Env.CentralOtelPassword != "" { - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - b.Env.InstallConfig.Cluster.Monitoring.CentralOtelExport = &files.CentralOtelConfig{ + dc.InstallConfig.Cluster.Monitoring.CentralOtelExport = &files.CentralOtelConfig{ Enabled: true, Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretCentralOtelCreds, Fields: &files.SecretFields{Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretCentralOtelCreds, Fields: &files.SecretFields{Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword}}) } if b.Env.OpenBaoURI != "" { - b.Env.InstallConfig.Codesphere.OpenBao = &files.OpenBaoConfig{ + dc.InstallConfig.Codesphere.OpenBao = &files.OpenBaoConfig{ Engine: b.Env.OpenBaoEngine, URI: b.Env.OpenBaoURI, User: b.Env.OpenBaoUser, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOpenBaoPassword, Fields: &files.SecretFields{Password: b.Env.OpenBaoPassword}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOpenBaoPassword, Fields: &files.SecretFields{Password: b.Env.OpenBaoPassword}}) } - if err := b.icg.WriteInstallConfig(b.Env.InstallConfigPath, true); err != nil { + if err := dc.icg.WriteInstallConfig(dc.InstallConfigPath, true); err != nil { return fmt.Errorf("failed to write config file: %w", err) } - if err := b.icg.WriteVault(b.Env.SecretsFilePath, true); err != nil { + if err := dc.icg.WriteVault(dc.SecretsFilePath, true); err != nil { return fmt.Errorf("failed to write vault file: %w", err) } - err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallConfigPath, remoteInstallConfigPath) + // CopyFile creates the destination directory, so a secondary data center's secrets + // directory does not need to exist yet. + err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, dc.InstallConfigPath, dc.RemoteConfigPath) if err != nil { return fmt.Errorf("failed to copy install config to jumpbox: %w", err) } - err = b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.SecretsFilePath, b.Env.SecretsDir+"/prod.vault.yaml") + err = b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, dc.SecretsFilePath, dc.RemoteVaultPath()) if err != nil { return fmt.Errorf("failed to copy secrets file to jumpbox: %w", err) } + b.mirrorPrimaryDataCenter() + return nil } -func (b *GCPBootstrapper) applySshProxyConfig() { - b.Env.InstallConfig.PcApps = util.DeepMergeMaps(b.Env.InstallConfig.PcApps, files.ChartValues{ +// datacenterConfig describes a data center the way the install config does. All data centers of a +// bootstrapped instance live in the same GCP region, so city and country code are the same for all +// of them. +func datacenterConfig(dc *DataCenter) files.DatacenterConfig { + return files.DatacenterConfig{ + ID: dc.ID, + Name: dc.Name, + City: "Karlsruhe", + CountryCode: "DE", + } +} + +// applyDataCenterTopology tells the platform about every data center of the installation. Without +// it, the installer defaults the list to the local data center, so each data center of a multi-DC +// instance would render as a single-data-center one. The list is identical in every data center's +// config; dataCenter stays the local one, so the platform still knows which data center it runs in. +// +// A single data center keeps the list unset and relies on the installer's default, so its config is +// unchanged from what OMS has always written. +func (b *GCPBootstrapper) applyDataCenterTopology(dc *DataCenter) { + if len(b.Env.DataCenters) < 2 { + return + } + + all := make([]files.DatacenterConfig, 0, len(b.Env.DataCenters)) + for _, other := range b.Env.DataCenters { + all = append(all, datacenterConfig(other)) + } + dc.InstallConfig.DataCenters = all + // Clients land in the primary data center, which is also the one cs. resolves to. + dc.InstallConfig.DefaultDataCenterID = b.primaryDC().ID + + b.dropAvailableDataCentersOverride(dc) +} + +// dropAvailableDataCentersOverride removes the chart override OMS used to write before the +// installer supported dataCenters in the config. A recovered config may still carry it, where it +// would shadow the list above with a bare ID list. Any other override content is left alone. +func (b *GCPBootstrapper) dropAvailableDataCentersOverride(dc *DataCenter) { + global, ok := dc.InstallConfig.Codesphere.Override["global"].(map[string]interface{}) + if !ok { + return + } + + delete(global, "availableDataCenters") + if len(global) == 0 { + delete(dc.InstallConfig.Codesphere.Override, "global") + } + if len(dc.InstallConfig.Codesphere.Override) == 0 { + dc.InstallConfig.Codesphere.Override = nil + } +} + +// applyPostgresConfig points the data center at its PostgreSQL server. The primary data center +// installs the server on its own node; every other data center connects to that same server as +// an external one and skips the postgres install step. +// +// Returns the primary IP and hostname the config held before, so regeneratePostgresCerts can +// tell whether the server's identity changed. +func (b *GCPBootstrapper) applyPostgresConfig(dc *DataCenter) (previousIP, previousHostname string) { + if dc.ExternalPostgres { + dc.InstallConfig.Postgres = files.PostgresConfig{ + Mode: "external", + // The server certificate carries only an IP SAN, so the address must be the IP. + ServerAddress: b.Env.PostgreSQLNode.GetInternalIP(), + Port: sharedPostgresPort, + // The CA certificate lives in the config, not the vault, and cannot be re-derived + // from the CA key — so it has to be copied from the primary data center. + CACertPem: b.primaryDC().InstallConfig.Postgres.CACertPem, + Primary: nil, + Replica: nil, + } + b.skipInstallerStep(dc, "postgres") + + return "", "" + } + + if dc.InstallConfig.Postgres.Primary == nil { + dc.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ + Hostname: b.Env.PostgreSQLNode.GetName(), + } + } + + previousIP = dc.InstallConfig.Postgres.Primary.IP + previousHostname = dc.InstallConfig.Postgres.Primary.Hostname + dc.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() + dc.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() + + return previousIP, previousHostname +} + +// skipInstallerStep persists a skipped installer step in the data center's config, so manual +// `oms install codesphere` re-runs on the jumpbox skip it too. +func (b *GCPBootstrapper) skipInstallerStep(dc *DataCenter, step string) { + if dc.InstallConfig.Operations == nil { + dc.InstallConfig.Operations = &files.OperationsConfig{} + } + if !slices.Contains(dc.InstallConfig.Operations.Skip, step) { + dc.InstallConfig.Operations.Skip = append(dc.InstallConfig.Operations.Skip, step) + } +} + +func (b *GCPBootstrapper) applySshProxyConfig(dc *DataCenter) { + dc.InstallConfig.PcApps = util.DeepMergeMaps(dc.InstallConfig.PcApps, files.ChartValues{ "applications": map[string]any{ "ssh-workspace-proxy": map[string]any{ "enabled": true, @@ -440,9 +604,9 @@ func (b *GCPBootstrapper) applySshProxyConfig() { "service": map[string]any{ "enabled": true, "type": "LoadBalancer", - "loadBalancerIP": b.Env.SshProxyIP, + "loadBalancerIP": dc.SshProxyIP, "annotations": map[string]any{ - "cloud.google.com/load-balancer-ipv4": b.Env.SshProxyIP, + "cloud.google.com/load-balancer-ipv4": dc.SshProxyIP, }, }, }, @@ -451,16 +615,16 @@ func (b *GCPBootstrapper) applySshProxyConfig() { }) } -func (b *GCPBootstrapper) applyExternalLokiConfig() { +func (b *GCPBootstrapper) applyExternalLokiConfig(dc *DataCenter) { if b.Env.ExternalLokiEndpoint == "" { return } - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy == nil { - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy = &files.GrafanaAlloyConfig{} + if dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy == nil { + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy = &files.GrafanaAlloyConfig{} } loki := &files.LokiConnectionConfig{ @@ -469,42 +633,47 @@ func (b *GCPBootstrapper) applyExternalLokiConfig() { Password: b.Env.ExternalLokiSecret, } - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Enabled = true - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Loki = loki - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretLokiGatewayBasicAuthPassword, Fields: &files.SecretFields{Password: b.Env.ExternalLokiSecret}}) + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Enabled = true + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Loki = loki + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretLokiGatewayBasicAuthPassword, Fields: &files.SecretFields{Password: b.Env.ExternalLokiSecret}}) } -func (b *GCPBootstrapper) applyPrometheusRemoteWriteConfig() { +func (b *GCPBootstrapper) applyPrometheusRemoteWriteConfig(dc *DataCenter) { if b.Env.PrometheusRemoteWriteURL == "" { return } - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.Prometheus == nil { - b.Env.InstallConfig.Cluster.Monitoring.Prometheus = &files.PrometheusConfig{} + if dc.InstallConfig.Cluster.Monitoring.Prometheus == nil { + dc.InstallConfig.Cluster.Monitoring.Prometheus = &files.PrometheusConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite == nil { - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite = &files.RemoteWriteConfig{} + if dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite == nil { + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite = &files.RemoteWriteConfig{} } - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Enabled = true - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Url = b.Env.PrometheusRemoteWriteURL - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.ClusterName = b.Env.DatacenterName - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Username = b.Env.PrometheusRemoteWriteUser - b.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWritePassword", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWritePassword}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWriteUser", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWriteUser}}) + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Enabled = true + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Url = b.Env.PrometheusRemoteWriteURL + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.ClusterName = dc.Name + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Username = b.Env.PrometheusRemoteWriteUser + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWritePassword", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWritePassword}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWriteUser", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWriteUser}}) } // regeneratePostgresCerts regenerates PostgreSQL TLS certificates when the IP/hostname -// changed or no private key was loaded from the vault. -func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname string) error { - vault := b.icg.GetVault() +// changed or no private key was loaded from the vault. It is a no-op for a data center that +// connects to an external server, since that server owns its own certificates. +func (b *GCPBootstrapper) regeneratePostgresCerts(dc *DataCenter, previousPrimaryIP, previousPrimaryHostname string) error { + if dc.InstallConfig.Postgres.Primary == nil { + return nil + } + + vault := dc.icg.GetVault() primaryKeySecret := vault.GetSecret(files.SecretPostgresPrimaryServerKeyPem) primaryNeedsRegen := primaryKeySecret == nil || primaryKeySecret.File == nil || - previousPrimaryIP != b.Env.InstallConfig.Postgres.Primary.IP || - previousPrimaryHostname != b.Env.InstallConfig.Postgres.Primary.Hostname + previousPrimaryIP != dc.InstallConfig.Postgres.Primary.IP || + previousPrimaryHostname != dc.InstallConfig.Postgres.Primary.Hostname if primaryNeedsRegen { caSecret := vault.GetSecret(files.SecretPostgresCaKeyPem) @@ -513,9 +682,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri } primaryKeyPEM, primaryCertPEM, err := secrets.GenerateServerCertificate( caSecret.File.Content, - b.Env.InstallConfig.Postgres.CACertPem, - b.Env.InstallConfig.Postgres.Primary.Hostname, - []string{b.Env.InstallConfig.Postgres.Primary.IP}) + dc.InstallConfig.Postgres.CACertPem, + dc.InstallConfig.Postgres.Primary.Hostname, + []string{dc.InstallConfig.Postgres.Primary.IP}) if err != nil { return fmt.Errorf("failed to generate primary server certificate: %w", err) } @@ -523,9 +692,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri return fmt.Errorf("primary PostgreSQL cert/key validation failed: %w", err) } vault.SetSecret(files.SecretEntry{Name: files.SecretPostgresPrimaryServerKeyPem, File: &files.SecretFile{Name: "primary.key", Content: primaryKeyPEM}}) - b.Env.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem = primaryCertPEM + dc.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem = primaryCertPEM } - if b.Env.InstallConfig.Postgres.Replica != nil { + if dc.InstallConfig.Postgres.Replica != nil { replicaKeySecret := vault.GetSecret(files.SecretPostgresReplicaServerKeyPem) if replicaKeySecret == nil || replicaKeySecret.File == nil { caSecret := vault.GetSecret(files.SecretPostgresCaKeyPem) @@ -534,9 +703,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri } replicaKeyPEM, replicaCertPEM, err := secrets.GenerateServerCertificate( caSecret.File.Content, - b.Env.InstallConfig.Postgres.CACertPem, - b.Env.InstallConfig.Postgres.Replica.Name, - []string{b.Env.InstallConfig.Postgres.Replica.IP}) + dc.InstallConfig.Postgres.CACertPem, + dc.InstallConfig.Postgres.Replica.Name, + []string{dc.InstallConfig.Postgres.Replica.IP}) if err != nil { return fmt.Errorf("failed to generate replica server certificate: %w", err) } @@ -544,19 +713,25 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri return fmt.Errorf("replica PostgreSQL cert/key validation failed: %w", err) } vault.SetSecret(files.SecretEntry{Name: files.SecretPostgresReplicaServerKeyPem, File: &files.SecretFile{Name: "replica.key", Content: replicaKeyPEM}}) - b.Env.InstallConfig.Postgres.Replica.SSLConfig.ServerCertPem = replicaCertPEM + dc.InstallConfig.Postgres.Replica.SSLConfig.ServerCertPem = replicaCertPEM } } return nil } +// EnsureAgeKey generates the primary data center's age identity on the jumpbox. func (b *GCPBootstrapper) EnsureAgeKey() error { - hasKey := b.Env.Jumpbox.NodeClient.HasFile(b.Env.Jumpbox, b.Env.SecretsDir+"/age_key.txt") - if hasKey { + b.ensureDataCenters() + + return b.ensureAgeKey(b.primaryDC()) +} + +func (b *GCPBootstrapper) ensureAgeKey(dc *DataCenter) error { + if b.Env.Jumpbox.NodeClient.HasFile(b.Env.Jumpbox, dc.RemoteAgeKeyPath()) { return nil } - err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("mkdir -p %s; age-keygen -o %s/age_key.txt", b.Env.SecretsDir, b.Env.SecretsDir)) + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("mkdir -p %s; age-keygen -o %s", dc.SecretsDir, dc.RemoteAgeKeyPath())) if err != nil { return fmt.Errorf("failed to generate age key on jumpbox: %w", err) } @@ -564,24 +739,41 @@ func (b *GCPBootstrapper) EnsureAgeKey() error { return nil } +// EnsureSecrets loads the primary data center's vault if it already exists locally. func (b *GCPBootstrapper) EnsureSecrets() error { - if b.fw.Exists(b.Env.SecretsFilePath) { - err := b.icg.LoadVaultFromUnecryptedFile(b.Env.SecretsFilePath) + b.ensureDataCenters() + + return b.ensureSecrets(b.primaryDC()) +} + +func (b *GCPBootstrapper) ensureSecrets(dc *DataCenter) error { + if b.fw.Exists(dc.SecretsFilePath) { + err := dc.icg.LoadVaultFromUnecryptedFile(dc.SecretsFilePath) if err != nil { return fmt.Errorf("failed to load vault file: %w", err) } } - b.Env.Secrets = b.icg.GetVault() + if dc.IsPrimary() { + b.Env.Secrets = dc.icg.GetVault() + } + b.mirrorPrimaryDataCenter() return nil } +// EncryptVault encrypts the primary data center's vault on the jumpbox. func (b *GCPBootstrapper) EncryptVault() error { - err := b.Env.Jumpbox.RunSSHCommand("root", "cp "+b.Env.SecretsDir+"/prod.vault.yaml{,.bak}") + b.ensureDataCenters() + + return b.encryptVault(b.primaryDC()) +} + +func (b *GCPBootstrapper) encryptVault(dc *DataCenter) error { + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("cp %s{,.bak}", dc.RemoteVaultPath())) if err != nil { return fmt.Errorf("failed backup vault on jumpbox: %w", err) } - err = b.Env.Jumpbox.RunSSHCommand("root", "sops --encrypt --in-place --age $(age-keygen -y "+b.Env.SecretsDir+"/age_key.txt) "+b.Env.SecretsDir+"/prod.vault.yaml") + err = b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("sops --encrypt --in-place --age $(age-keygen -y %s) %s", dc.RemoteAgeKeyPath(), dc.RemoteVaultPath())) if err != nil { return fmt.Errorf("failed to encrypt vault on jumpbox: %w", err) } @@ -589,10 +781,10 @@ func (b *GCPBootstrapper) EncryptVault() error { return nil } -// decryptVault creates an unencrypted copy of the vault in dst on the jumpbox +// decryptVault creates an unencrypted copy of the data center's vault in dst on the jumpbox. // Make sure to delete the unencrypted file when not needed anymore. -func (b *GCPBootstrapper) decryptVault(dst string) error { - err := b.Env.Jumpbox.RunSSHCommand("root", "cp "+b.Env.SecretsDir+"/prod.vault.yaml "+dst) +func (b *GCPBootstrapper) decryptVault(dc *DataCenter, dst string) error { + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("cp %s %s", dc.RemoteVaultPath(), dst)) if err != nil { return fmt.Errorf("failed to create tmp vault on jumpbox: %w", err) } @@ -602,10 +794,102 @@ func (b *GCPBootstrapper) decryptVault(dst string) error { return fmt.Errorf("failed to make vault file readable only for root on jumpbox: %w", err) } - err = b.Env.Jumpbox.RunSSHCommand("root", "SOPS_AGE_KEY_FILE="+b.Env.SecretsDir+"/age_key.txt sops --decrypt --in-place "+dst) + err = b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("SOPS_AGE_KEY_FILE=%s sops --decrypt --in-place %s", dc.RemoteAgeKeyPath(), dst)) if err != nil { return fmt.Errorf("failed to decrypt vault on jumpbox: %w", err) } return nil } + +// writeDataCenterConfig writes the data center's install config and vault, then places the +// encrypted vault and its age identity on the jumpbox. +func (b *GCPBootstrapper) writeDataCenterConfig(dc *DataCenter) error { + err := b.stlog.Step(dc.StepName("Update install config"), func() error { + return b.updateInstallConfig(dc) + }) + if err != nil { + return fmt.Errorf("failed to update install config of data center %d: %w", dc.ID, err) + } + + err = b.stlog.Step(dc.StepName("Ensure age key"), func() error { + return b.ensureAgeKey(dc) + }) + if err != nil { + return fmt.Errorf("failed to ensure age key of data center %d: %w", dc.ID, err) + } + + err = b.stlog.Step(dc.StepName("Encrypt vault"), func() error { + return b.encryptVault(dc) + }) + if err != nil { + return fmt.Errorf("failed to encrypt vault of data center %d: %w", dc.ID, err) + } + + return nil +} + +// seedSecondaryDataCenter derives a secondary data center's config and vault from the primary +// one's, for the parts it does not already have. Secrets tied to the shared database and to +// cross-data-center authentication are copied verbatim; the per-cluster ones are dropped so +// GenerateSecrets regenerates them for this data center. +// +// Anything the data center already loaded from its own files is kept, so re-runs do not rotate a +// live installation's secrets. +func (b *GCPBootstrapper) seedSecondaryDataCenter(primary, dc *DataCenter) error { + if dc.InstallConfig == nil { + config, err := secrets.DeriveDataCenterConfig(primary.InstallConfig) + if err != nil { + return fmt.Errorf("failed to derive config from data center %d: %w", primary.ID, err) + } + dc.icg.SetInstallConfig(config) + dc.InstallConfig = dc.icg.GetInstallConfig() + } + + if len(dc.icg.GetVault().Secrets) == 0 { + dc.icg.SetVault(secrets.DeriveDataCenterVault(primary.icg.GetVault())) + } + + return nil +} + +// verifySecondaryDataCenterSecrets fails the bootstrap if a secondary data center's secrets +// diverged from the primary's where they must match. Divergent PostgreSQL roles would let one +// data center's install rotate credentials the other one is using, and a divergent token key +// would break cross-data-center authentication — both only visible long after the fact. +func (b *GCPBootstrapper) verifySecondaryDataCenterSecrets(primary, dc *DataCenter) error { + primaryVault := primary.icg.GetVault() + vault := dc.icg.GetVault() + + // openFgaPresharedKey is in here because a secondary data center that already has a vault is + // not re-derived, but GenerateSecrets still runs for it — so a vault predating the key gets a + // freshly generated one that the shared OpenFGA instance rejects. + shared := []string{files.SecretTokenPrivateKey, files.SecretPostgresPassword, files.SecretOpenFgaPresharedKey} + for _, svc := range codesphere.PostgresServices { + shared = append(shared, files.PostgresUserSecretName(svc.Name), files.PostgresPasswordSecretName(svc.Name)) + } + for _, name := range shared { + expected := primaryVault.GetSecret(name) + if expected == nil { + continue + } + if !reflect.DeepEqual(vault.GetSecret(name), expected) { + return fmt.Errorf("secret %q of data center %d differs from the primary data center, but both data centers share it", name, dc.ID) + } + } + + if dc.InstallConfig.Postgres.CACertPem == "" { + return fmt.Errorf("data center %d has no postgres CA certificate and could not verify the shared server", dc.ID) + } + + for _, name := range []string{files.SecretKubeConfig, files.SecretCephSshPrivateKey} { + if vault.GetSecret(name) == nil { + continue + } + if reflect.DeepEqual(vault.GetSecret(name), primaryVault.GetSecret(name)) { + return fmt.Errorf("secret %q of data center %d is the primary data center's, but they run separate clusters", name, dc.ID) + } + } + + return nil +} diff --git a/internal/bootstrap/gcp/install_config_test.go b/internal/bootstrap/gcp/install_config_test.go index 6ccd50d0..3e4b0c1d 100644 --- a/internal/bootstrap/gcp/install_config_test.go +++ b/internal/bootstrap/gcp/install_config_test.go @@ -344,6 +344,10 @@ var _ = Describe("Installconfig & Secrets", func() { Expect(bs.Env.InstallConfig.Datacenter.ID).To(Equal(1)) Expect(bs.Env.InstallConfig.Datacenter.Name).To(Equal("dev")) + // A single data center relies on the installer defaulting the topology to the + // local data center, so the config stays as it was before multi-DC support. + Expect(bs.Env.InstallConfig.DataCenters).To(BeEmpty()) + Expect(bs.Env.InstallConfig.DefaultDataCenterID).To(BeZero()) Expect(bs.Env.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) Expect(bs.Env.InstallConfig.Codesphere.Features).To(Equal(map[string]bool{})) Expect(bs.Env.InstallConfig.Codesphere.Internal).To(Equal(gcp.DefaultInternalFlags)) diff --git a/internal/bootstrap/gcp/mocks.go b/internal/bootstrap/gcp/mocks.go index 5b371325..74b64038 100644 --- a/internal/bootstrap/gcp/mocks.go +++ b/internal/bootstrap/gcp/mocks.go @@ -796,16 +796,16 @@ func (_c *MockGCPClientManager_CreateVPC_Call) RunAndReturn(run func(projectID s } // DeleteDNSRecordSets provides a mock function for the type MockGCPClientManager -func (_mock *MockGCPClientManager) DeleteDNSRecordSets(projectID string, zoneName string, baseDomain string) error { - ret := _mock.Called(projectID, zoneName, baseDomain) +func (_mock *MockGCPClientManager) DeleteDNSRecordSets(projectID string, zoneName string, records []DNSRecordName) error { + ret := _mock.Called(projectID, zoneName, records) if len(ret) == 0 { panic("no return value specified for DeleteDNSRecordSets") } var r0 error - if returnFunc, ok := ret.Get(0).(func(string, string, string) error); ok { - r0 = returnFunc(projectID, zoneName, baseDomain) + if returnFunc, ok := ret.Get(0).(func(string, string, []DNSRecordName) error); ok { + r0 = returnFunc(projectID, zoneName, records) } else { r0 = ret.Error(0) } @@ -820,12 +820,12 @@ type MockGCPClientManager_DeleteDNSRecordSets_Call struct { // DeleteDNSRecordSets is a helper method to define mock.On call // - projectID string // - zoneName string -// - baseDomain string -func (_e *MockGCPClientManager_Expecter) DeleteDNSRecordSets(projectID any, zoneName any, baseDomain any) *MockGCPClientManager_DeleteDNSRecordSets_Call { - return &MockGCPClientManager_DeleteDNSRecordSets_Call{Call: _e.mock.On("DeleteDNSRecordSets", projectID, zoneName, baseDomain)} +// - records []DNSRecordName +func (_e *MockGCPClientManager_Expecter) DeleteDNSRecordSets(projectID any, zoneName any, records any) *MockGCPClientManager_DeleteDNSRecordSets_Call { + return &MockGCPClientManager_DeleteDNSRecordSets_Call{Call: _e.mock.On("DeleteDNSRecordSets", projectID, zoneName, records)} } -func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID string, zoneName string, baseDomain string)) *MockGCPClientManager_DeleteDNSRecordSets_Call { +func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID string, zoneName string, records []DNSRecordName)) *MockGCPClientManager_DeleteDNSRecordSets_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { @@ -835,9 +835,9 @@ func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID if args[1] != nil { arg1 = args[1].(string) } - var arg2 string + var arg2 []DNSRecordName if args[2] != nil { - arg2 = args[2].(string) + arg2 = args[2].([]DNSRecordName) } run( arg0, @@ -853,7 +853,7 @@ func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Return(err error) *Mock return _c } -func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) RunAndReturn(run func(projectID string, zoneName string, baseDomain string) error) *MockGCPClientManager_DeleteDNSRecordSets_Call { +func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) RunAndReturn(run func(projectID string, zoneName string, records []DNSRecordName) error) *MockGCPClientManager_DeleteDNSRecordSets_Call { _c.Call.Return(run) return _c } diff --git a/internal/bootstrap/gcp/multi_dc_test.go b/internal/bootstrap/gcp/multi_dc_test.go new file mode 100644 index 00000000..3f62f3fc --- /dev/null +++ b/internal/bootstrap/gcp/multi_dc_test.go @@ -0,0 +1,327 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp_test + +import ( + "context" + "fmt" + "os" + + "cloud.google.com/go/compute/apiv1/computepb" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + "google.golang.org/api/cloudbilling/v1" + + "github.com/codesphere-cloud/oms/internal/bootstrap" + "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" + "github.com/codesphere-cloud/oms/internal/codesphere" + "github.com/codesphere-cloud/oms/internal/env" + "github.com/codesphere-cloud/oms/internal/github" + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" + "github.com/codesphere-cloud/oms/internal/portal" + "github.com/codesphere-cloud/oms/internal/util" +) + +// realConfigManager returns an install config manager that keeps its files in memory, so tests +// can inspect the config and vault a data center would actually be installed with. +func realConfigManager(writes map[string][]byte) installer.InstallConfigManager { + icm := installer.NewInstallConfigManager() + icm.(*installer.InstallConfig).SetFileIO(&recordingFileIO{writes: writes}) + return icm +} + +// recordingFileIO records written files and reports every other path as missing, so a bootstrap +// run behaves as if nothing existed locally beforehand. +type recordingFileIO struct { + util.FilesystemWriter + writes map[string][]byte +} + +func (f *recordingFileIO) CreateAndWrite(path string, content []byte, _ string) error { + f.writes[path] = content + return nil +} + +var _ = Describe("Multi-DC bootstrap", func() { + var ( + nodeClient *node.MockNodeClient + csEnv *gcp.CodesphereEnvironment + gc *gcp.MockGCPClientManager + fw *util.MockFileIO + writes map[string][]byte + primaryICG installer.InstallConfigManager + + bs *gcp.GCPBootstrapper + ) + + BeforeEach(func() { + nodeClient = node.NewMockNodeClient(GinkgoT()) + gc = gcp.NewMockGCPClientManager(GinkgoT()) + fw = util.NewMockFileIO(GinkgoT()) + writes = map[string][]byte{} + primaryICG = realConfigManager(writes) + + csEnv = &gcp.CodesphereEnvironment{ + MultiDC: true, + ProjectName: "test-project", + BillingAccount: "test-billing-account", + BaseDomain: "example.com", + Region: "us-central1", + Zone: "us-central1-a", + DNSProjectID: "dns-project", + DNSZoneName: "test-zone", + ProjectTTL: "1h", + SecretsDir: "/etc/codesphere/secrets", + DatacenterName: "multidc", + InstallConfigPath: "config.yaml", + SecretsFilePath: "prod.vault.yaml", + WriteConfig: true, + RegistryType: gcp.RegistryTypeGitHub, + RegistryUser: "registry-user", + GitHubPAT: "fake-pat", + SSHPublicKeyPath: "key.pub", + RootDiskSize: 50, + InternalFlags: gcp.DefaultInternalFlags, + PreviewFlags: gcp.DefaultPreviewFlags, + FeatureFlags: gcp.DefaultFeatureFlags, + } + }) + + JustBeforeEach(func() { + var err error + bs, err = gcp.NewGCPBootstrapper( + context.Background(), + env.NewEnv(), + bootstrap.NewStepLogger(false), + csEnv, + primaryICG, + gc, + fw, + nodeClient, + portal.NewMockPortal(GinkgoT()), + util.NewFakeTime(), + github.NewMockGitHubClient(GinkgoT()), + ) + Expect(err).NotTo(HaveOccurred()) + bs.NewConfigManager = func() installer.InstallConfigManager { return realConfigManager(writes) } + }) + + // expectBootstrapMocks sets up the GCP, SSH and file mocks for a full two-data-center run. + expectBootstrapMocks := func(projectID string) { + const vmCount = 14 + + fw.EXPECT().Exists(mock.Anything).Return(false) + fw.EXPECT().MkdirAll(mock.Anything, os.FileMode(0755)).Return(nil) + fw.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + fw.EXPECT().ReadFile(mock.Anything).Return([]byte("ssh-rsa AAA..."), nil).Times(vmCount) + + // EnsureProject only creates a project when the lookup fails with this exact message. + gc.EXPECT().GetProjectByName(mock.Anything, "test-project").Return(nil, fmt.Errorf("project not found: test-project")) + gc.EXPECT().CreateProjectID("test-project").Return(projectID) + gc.EXPECT().CreateProject(mock.Anything, mock.Anything, "test-project", mock.Anything).Return(mock.Anything, nil) + gc.EXPECT().GetBillingInfo(projectID).Return(&cloudbilling.ProjectBillingInfo{BillingEnabled: false}, nil) + gc.EXPECT().EnableBilling(projectID, "test-billing-account").Return(nil) + gc.EXPECT().EnableAPIs(projectID, mock.Anything).Return(nil) + gc.EXPECT().CreateServiceAccount(projectID, "cloud-controller", "cloud-controller").Return("cc@p.iam.gserviceaccount.com", false, nil) + gc.EXPECT().AssignIAMRole(projectID, "cloud-controller", projectID, []string{"roles/compute.admin"}).Return(nil) + gc.EXPECT().AssignIAMRole("dns-project", "cloud-controller", projectID, []string{"roles/dns.admin"}).Return(nil) + gc.EXPECT().CreateVPC(projectID, "us-central1", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + gc.EXPECT().CreateFirewallRule(projectID, mock.Anything).Return(nil).Times(5) + + mockGetInstanceNotFoundThenRunning(gc, projectID, "us-central1-a", makeRunningInstance("10.10.0.2", "1.2.3.4"), vmCount) + gc.EXPECT().CreateInstance(projectID, "us-central1-a", mock.Anything).Return(nil).Times(vmCount) + + // Two data centers reserve three addresses each, suffixed for the second one. + for _, name := range []string{ + "gateway", "public-gateway", "ssh-proxy", + "gateway-dc2", "public-gateway-dc2", "ssh-proxy-dc2", + } { + ip := fmt.Sprintf("203.0.113.%d", len(name)) + gc.EXPECT().GetAddress(projectID, "us-central1", name).Return(nil, fmt.Errorf("not found")).Once() + gc.EXPECT().CreateAddress(projectID, "us-central1", mock.MatchedBy(func(addr *computepb.Address) bool { + return addr.GetName() == name + })).Return(ip, nil).Once() + } + + gc.EXPECT().EnsureDNSManagedZone("dns-project", "test-zone", "example.com.", mock.Anything).Return(nil) + gc.EXPECT().EnsureDNSRecordSets("dns-project", "test-zone", mock.Anything).Return(nil) + + nodeClient.EXPECT().WaitReady(mock.Anything, mock.Anything).Return(nil) + nodeClient.EXPECT().HasFile(mock.Anything, mock.Anything).Return(false) + nodeClient.EXPECT().RunCommand(mock.Anything, mock.Anything, mock.Anything).Return(nil) + nodeClient.EXPECT().CopyFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + } + + It("bootstraps two data centers sharing one database", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + Expect(bs.Env.DataCenters).To(HaveLen(2)) + primary, secondary := bs.Env.DataCenters[0], bs.Env.DataCenters[1] + + By("giving each data center its own ceph and k0s nodes") + Expect(vmNamesOfNodes(primary.CephNodes)).To(Equal([]string{"ceph-1", "ceph-2", "ceph-3"})) + Expect(vmNamesOfNodes(primary.ControlPlaneNodes)).To(Equal([]string{"k0s-1", "k0s-2", "k0s-3"})) + Expect(vmNamesOfNodes(secondary.CephNodes)).To(Equal([]string{"ceph-1-dc2", "ceph-2-dc2", "ceph-3-dc2"})) + Expect(vmNamesOfNodes(secondary.ControlPlaneNodes)).To(Equal([]string{"k0s-1-dc2", "k0s-2-dc2", "k0s-3-dc2"})) + + By("sharing the platform domain and scoping the workspace domain") + Expect(primary.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) + Expect(secondary.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) + Expect(primary.InstallConfig.Codesphere.WorkspaceHostingBaseDomain).To(Equal("1.ws.example.com")) + Expect(secondary.InstallConfig.Codesphere.WorkspaceHostingBaseDomain).To(Equal("2.ws.example.com")) + Expect(secondary.InstallConfig.Codesphere.CustomDomains.CNameBaseDomain).To(Equal("2.ws.example.com")) + + By("giving each data center a distinct ID and name") + Expect(primary.InstallConfig.Datacenter.ID).To(Equal(1)) + Expect(secondary.InstallConfig.Datacenter.ID).To(Equal(2)) + // The k0s cluster is named codesphere-, so the names must differ. + Expect(primary.InstallConfig.Datacenter.Name).To(Equal("multidc")) + Expect(secondary.InstallConfig.Datacenter.Name).To(Equal("multidc-dc2")) + + By("installing postgres in the primary data center and reusing it in the secondary") + Expect(primary.InstallConfig.Postgres.Mode).To(Equal("install")) + Expect(primary.InstallConfig.Postgres.Primary.IP).To(Equal("10.10.0.2")) + Expect(secondary.InstallConfig.Postgres.Mode).To(Equal("external")) + // The server certificate carries only an IP SAN, so the address must be the IP. + Expect(secondary.InstallConfig.Postgres.ServerAddress).To(Equal("10.10.0.2")) + Expect(secondary.InstallConfig.Postgres.Primary).To(BeNil()) + Expect(secondary.InstallConfig.Postgres.Replica).To(BeNil()) + Expect(secondary.InstallConfig.Postgres.CACertPem).To(Equal(primary.InstallConfig.Postgres.CACertPem)) + Expect(secondary.InstallConfig.Postgres.CACertPem).NotTo(BeEmpty()) + + By("skipping the postgres install step in the secondary data center only") + Expect(secondary.InstallConfig.Operations.Skip).To(ContainElement("postgres")) + // Both data centers install their own ceph and kubernetes. + Expect(secondary.InstallConfig.Operations.Skip).NotTo(ContainElement("ceph")) + Expect(secondary.InstallConfig.Operations.Skip).NotTo(ContainElement("kubernetes")) + if primary.InstallConfig.Operations != nil { + Expect(primary.InstallConfig.Operations.Skip).NotTo(ContainElement("postgres")) + } + + By("telling both data centers about every data center of the installation") + // The installer defaults dataCenters to the local one, so without the list each data + // center renders as a single-data-center instance. + topology := []files.DatacenterConfig{ + {ID: 1, Name: "multidc", City: "Karlsruhe", CountryCode: "DE"}, + {ID: 2, Name: "multidc-dc2", City: "Karlsruhe", CountryCode: "DE"}, + } + for _, dc := range bs.Env.DataCenters { + Expect(dc.InstallConfig.DataCenters).To(Equal(topology), "data center %d", dc.ID) + Expect(dc.InstallConfig.DefaultDataCenterID).To(Equal(1), "data center %d", dc.ID) + // The local data center is still the one this config installs. + Expect(dc.InstallConfig.Datacenter.ID).To(Equal(dc.ID)) + } + + By("giving each data center its own vault directory on the shared jumpbox") + Expect(primary.InstallConfig.Secrets.BaseDir).To(Equal("/etc/codesphere/secrets")) + Expect(secondary.InstallConfig.Secrets.BaseDir).To(Equal("/etc/codesphere/secrets-dc2")) + Expect(writes).To(HaveKey("config.yaml")) + Expect(writes).To(HaveKey("config-dc2.yaml")) + Expect(writes).To(HaveKey("prod.vault.yaml")) + Expect(writes).To(HaveKey("prod-dc2.vault.yaml")) + + primaryVault := primary.ConfigManager().GetVault() + secondaryVault := secondary.ConfigManager().GetVault() + + By("sharing every secret both data centers need to agree on") + shared := []string{ + files.SecretPostgresPassword, + files.SecretPostgresReplicaPassword, + files.SecretPostgresCaKeyPem, + files.SecretTokenPrivateKey, + files.SecretTokenPublicKey, + files.SecretDomainAuthPrivateKey, + files.SecretMounterHmacSecret, + files.SecretMongoDbPasswordEncryptionKey, + } + for _, svc := range codesphere.PostgresServices { + shared = append(shared, files.PostgresUserSecretName(svc.Name), files.PostgresPasswordSecretName(svc.Name)) + } + for _, name := range shared { + Expect(primaryVault.GetSecret(name)).NotTo(BeNil(), "primary should have %s", name) + Expect(secondaryVault.GetSecret(name)).To(Equal(primaryVault.GetSecret(name)), "%s must be shared", name) + } + + By("regenerating the per-cluster secrets for the secondary data center") + for _, name := range []string{files.SecretSelfSignedCaKeyPem, files.SecretCephSshPrivateKey} { + Expect(secondaryVault.GetSecret(name)).NotTo(BeNil(), "secondary should have %s", name) + Expect(secondaryVault.GetSecret(name).File.Content). + NotTo(Equal(primaryVault.GetSecret(name).File.Content), "%s must be per data center", name) + } + Expect(secondary.InstallConfig.Cluster.Certificates.CA.CertPem). + NotTo(Equal(primary.InstallConfig.Cluster.Certificates.CA.CertPem)) + Expect(secondary.InstallConfig.Ceph.CephAdmSSHKey.PublicKey). + NotTo(Equal(primary.InstallConfig.Ceph.CephAdmSSHKey.PublicKey)) + + By("recording the DNS records it created so cleanup can delete them") + Expect(bs.Env.DNSRecords).To(Equal(gcp.DataCenterDNSRecordNames("example.com", bs.Env.DataCenters))) + + By("pointing each data center's install command at its own config and vault") + Expect(bs.InstallCommand(primary, "pkg.tar.gz")).To(ContainSubstring("-c /etc/codesphere/config.yaml")) + Expect(bs.InstallCommand(primary, "pkg.tar.gz")).To(ContainSubstring("--vault /etc/codesphere/secrets/prod.vault.yaml")) + Expect(bs.InstallCommand(secondary, "pkg.tar.gz")).To(ContainSubstring("-c /etc/codesphere/config-dc2.yaml")) + Expect(bs.InstallCommand(secondary, "pkg.tar.gz")).To(ContainSubstring("--vault /etc/codesphere/secrets-dc2/prod.vault.yaml")) + + By("producing postgres blocks the installer's validation accepts") + for _, dc := range bs.Env.DataCenters { + for _, problem := range dc.ConfigManager().ValidateInstallConfig() { + Expect(problem).NotTo(ContainSubstring("postgres"), "data center %d", dc.ID) + } + } + }) + + It("mirrors the primary data center onto the legacy infra file fields", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + primary := bs.Env.DataCenters[0] + Expect(bs.Env.ControlPlaneNodes).To(Equal(primary.ControlPlaneNodes)) + Expect(bs.Env.CephNodes).To(Equal(primary.CephNodes)) + Expect(bs.Env.GatewayIP).To(Equal(primary.GatewayIP)) + Expect(bs.Env.PublicGatewayIP).To(Equal(primary.PublicGatewayIP)) + Expect(bs.Env.SshProxyIP).To(Equal(primary.SshProxyIP)) + }) + + Describe("validateMultiDC", func() { + DescribeTable("rejects flag combinations it cannot satisfy", + func(mutate func(), wantErr string) { + mutate() + Expect(bs.ValidateInput()).To(MatchError(ContainSubstring(wantErr))) + }, + Entry("without write-config", func() { csEnv.WriteConfig = false }, "multi-dc requires write-config"), + Entry("with an explicit datacenter ID", func() { csEnv.DatacenterIDExplicit = true }, "datacenter-id cannot be combined with multi-dc"), + Entry("without a datacenter name", func() { csEnv.DatacenterName = "" }, "datacenter-name is required with multi-dc"), + Entry("without a config path", func() { csEnv.InstallConfigPath = "" }, "cannot derive a per-data-center path"), + Entry("without a secrets path", func() { csEnv.SecretsFilePath = "" }, "cannot derive a per-data-center path"), + ) + + It("accepts the default multi-dc flags", func() { + Expect(bs.ValidateInput()).To(Succeed()) + }) + + It("does not constrain a single-data-center bootstrap", func() { + csEnv.MultiDC = false + csEnv.WriteConfig = false + csEnv.DatacenterName = "" + csEnv.DatacenterIDExplicit = true + + Expect(bs.ValidateInput()).To(Succeed()) + }) + }) +}) + +// vmNamesOfNodes returns the names of the given nodes, in order. +func vmNamesOfNodes(nodes []*node.Node) []string { + names := make([]string, len(nodes)) + for i, n := range nodes { + names[i] = n.GetName() + } + return names +} diff --git a/internal/bootstrap/gcp/test_helpers_test.go b/internal/bootstrap/gcp/test_helpers_test.go index e7131a4b..2d9629c6 100644 --- a/internal/bootstrap/gcp/test_helpers_test.go +++ b/internal/bootstrap/gcp/test_helpers_test.go @@ -24,6 +24,15 @@ import ( func protoString(s string) *string { return &s } +// vmNames returns the names of the given VM definitions, in order. +func vmNames(defs []gcp.VMDef) []string { + names := make([]string, len(defs)) + for i, def := range defs { + names[i] = def.Name + } + return names +} + // makeInstance creates a computepb.Instance with the given status and IPs. func makeInstance(status, internalIP, externalIP string) *computepb.Instance { inst := &computepb.Instance{ diff --git a/internal/installer/argocd/install_and_apps.go b/internal/installer/argocd/install_and_apps.go index 625b2a9b..1b9f8236 100644 --- a/internal/installer/argocd/install_and_apps.go +++ b/internal/installer/argocd/install_and_apps.go @@ -11,6 +11,7 @@ import ( "github.com/codesphere-cloud/oms/internal/installer/files" "github.com/codesphere-cloud/oms/internal/installer/secrets" "github.com/codesphere-cloud/oms/internal/installer/vault" + "github.com/codesphere-cloud/oms/internal/util" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -69,13 +70,17 @@ func (i *AppInstaller) SyncVaultSecret(ctx context.Context) error { // InstallPCApps installs or upgrades pc-applications using the version from // the supplied installer BOM. func (i *AppInstaller) InstallPCApps(ctx context.Context, bomPath string) error { + // Values derived from the install config form the base; an explicit pcApps block in + // config.yaml wins over them, and the --pc-apps-values files win over both. + values := util.DeepMergeMaps(installer.OpenFgaPcAppsValues(&i.cfg.Config), i.cfg.Config.PcApps) + pcApps, err := installer.NewPcAppsFromBom( i.cfg.KubeClient, i.cfg.RESTConfig, bomPath, DefaultNamespace, i.cfg.PCAppsValues, - i.cfg.Config.PcApps, + values, ) if err != nil { return fmt.Errorf("failed to initialize pc-apps installer: %w", err) diff --git a/internal/installer/config_manager.go b/internal/installer/config_manager.go index d55f65fc..cf0e41d6 100644 --- a/internal/installer/config_manager.go +++ b/internal/installer/config_manager.go @@ -192,6 +192,36 @@ func (g *InstallConfig) ValidateInstallConfig() []string { } } + errors = append(errors, validateOpenFga(g.Config.Codesphere.OpenFga)...) + + return errors +} + +// validateOpenFga checks the codesphere.openFga block. A data center that does not deploy +// OpenFGA has nowhere to fall back to, so it must name the instance it uses; a data center that +// exposes one must say under which host. +func validateOpenFga(config *files.OpenFgaConfig) []string { + if config == nil { + return nil + } + + errors := []string{} + if !config.DeploysOpenFga() && config.APIURL == "" { + errors = append(errors, "OpenFGA apiUrl is required when codesphere.openFga.deploy is false") + } + if config.APIURL != "" { + if _, err := url.ParseRequestURI(config.APIURL); err != nil { + errors = append(errors, "OpenFGA apiUrl must be a valid URL") + } + } + if config.ExposesOpenFga() { + if config.Expose.Host == "" { + errors = append(errors, "OpenFGA expose host is required when codesphere.openFga.expose.enabled is true") + } + if !config.DeploysOpenFga() { + errors = append(errors, "OpenFGA cannot be exposed by a data center that does not deploy it") + } + } return errors } @@ -232,6 +262,16 @@ func (g *InstallConfig) ValidateVault() []string { } } + // Generated by EnsureSecrets for a fresh installation, but a data center joining an + // existing installation must carry over the primary data center's key — the services + // and the OpenFGA instance they share authenticate with the same one. + if !foundSecrets[files.SecretOpenFgaPresharedKey] { + errors = append(errors, fmt.Sprintf( + "required secret missing: %s (in a multi-data-center installation, copy it from the data center that deploys OpenFGA)", + files.SecretOpenFgaPresharedKey, + )) + } + return errors } diff --git a/internal/installer/config_manager_test.go b/internal/installer/config_manager_test.go index 1fd403d4..e2a00f2a 100644 --- a/internal/installer/config_manager_test.go +++ b/internal/installer/config_manager_test.go @@ -298,6 +298,55 @@ var _ = Describe("ConfigManager", func() { }) }) + Context("openFga validation", func() { + It("should accept an absent openFga block", func() { + configManager.Config.Codesphere.OpenFga = nil + errors := configManager.ValidateInstallConfig() + Expect(errors).NotTo(ContainElement(ContainSubstring("OpenFGA"))) + }) + + It("should require an apiUrl when the data center does not deploy OpenFGA", func() { + deploy := false + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{Deploy: &deploy} + errors := configManager.ValidateInstallConfig() + Expect(errors).To(ContainElement(ContainSubstring("OpenFGA apiUrl is required"))) + }) + + It("should validate the apiUrl format", func() { + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{APIURL: "not-a-valid-url"} + errors := configManager.ValidateInstallConfig() + Expect(errors).To(ContainElement(ContainSubstring("OpenFGA apiUrl must be a valid URL"))) + }) + + It("should require a host when exposing OpenFGA", func() { + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true}, + } + errors := configManager.ValidateInstallConfig() + Expect(errors).To(ContainElement(ContainSubstring("OpenFGA expose host is required"))) + }) + + It("should reject exposing an OpenFGA the data center does not deploy", func() { + deploy := false + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{ + Deploy: &deploy, + APIURL: "https://openfga.1.cs.example.com", + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.2.cs.example.com"}, + } + errors := configManager.ValidateInstallConfig() + Expect(errors).To(ContainElement(ContainSubstring("cannot be exposed by a data center that does not deploy it"))) + }) + + It("should accept a data center that deploys and exposes OpenFGA", func() { + configManager.Config.Codesphere.OpenFga = &files.OpenFgaConfig{ + APIURL: "https://openfga.1.cs.example.com", + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + } + errors := configManager.ValidateInstallConfig() + Expect(errors).NotTo(ContainElement(ContainSubstring("OpenFGA"))) + }) + }) + Context("ceph validation", func() { It("should require at least one Ceph host", func() { configManager.Config.Ceph.Hosts = []files.CephHost{} @@ -424,6 +473,7 @@ var _ = Describe("ConfigManager", func() { {Name: "selfSignedCaKeyPem"}, {Name: "domainAuthPrivateKey"}, {Name: "domainAuthPublicKey"}, + {Name: "openFgaPresharedKey"}, }, } }) @@ -443,9 +493,10 @@ var _ = Describe("ConfigManager", func() { }, } errors := configManager.ValidateVault() - Expect(errors).To(HaveLen(4)) + Expect(errors).To(HaveLen(5)) Expect(errors).To(ContainElement(ContainSubstring("domainAuthPrivateKey"))) Expect(errors).To(ContainElement(ContainSubstring("domainAuthPublicKey"))) + Expect(errors).To(ContainElement(ContainSubstring("openFgaPresharedKey"))) }) }) }) diff --git a/internal/installer/files/config_yaml.go b/internal/installer/files/config_yaml.go index b459920f..01b1db8e 100644 --- a/internal/installer/files/config_yaml.go +++ b/internal/installer/files/config_yaml.go @@ -360,6 +360,7 @@ type CodesphereConfig struct { OAuth *OAuthProvidersConfig `yaml:"oauth,omitempty"` ManagedServices []ManagedServiceConfig `yaml:"managedServices,omitempty"` OpenBao *OpenBaoConfig `yaml:"openBao,omitempty"` + OpenFga *OpenFgaConfig `yaml:"openFga,omitempty"` Migration *MigrationConfig `yaml:"migration,omitempty"` TelemetryExport *TelemetryExport `yaml:"telemetryExport,omitempty"` Override ChartOverride `yaml:"override,omitempty"` @@ -382,6 +383,38 @@ type OpenBaoConfig struct { User string `yaml:"user,omitempty"` } +// OpenFgaConfig configures the authorization store. One OpenFGA instance serves a whole +// installation, so in a multi-data-center setup exactly one data center deploys and exposes +// it (Deploy + Expose) and every other one only points at it (APIURL). +type OpenFgaConfig struct { + // Deploy controls whether pc-applications deploys OpenFGA in this data center. + // Defaults to true when unset, matching the pc-applications chart. + Deploy *bool `yaml:"deploy,omitempty"` + // APIURL is the URL the Codesphere services reach OpenFGA at. Defaults to the + // in-cluster service of a locally deployed OpenFGA; required when Deploy is false. + APIURL string `yaml:"apiUrl,omitempty"` + // Expose publishes the deployed OpenFGA through the Codesphere gateway so the other + // data centers can reach it. + Expose *OpenFgaExposeConfig `yaml:"expose,omitempty"` +} + +type OpenFgaExposeConfig struct { + Enabled bool `yaml:"enabled"` + // Host OpenFGA is served under. Must resolve to this data center's public IP and + // is what the other data centers put in their APIURL. + Host string `yaml:"host,omitempty"` +} + +// DeploysOpenFga reports whether pc-applications should deploy OpenFGA in this data center. +func (c *OpenFgaConfig) DeploysOpenFga() bool { + return c == nil || c.Deploy == nil || *c.Deploy +} + +// ExposesOpenFga reports whether the deployed OpenFGA is published through the gateway. +func (c *OpenFgaConfig) ExposesOpenFga() bool { + return c != nil && c.Expose != nil && c.Expose.Enabled +} + type OAuthProvidersConfig struct { Oidc *OidcOAuthProvider `yaml:"oidc,omitempty"` } diff --git a/internal/installer/files/secret_names.go b/internal/installer/files/secret_names.go index 41dfd305..c4791f43 100644 --- a/internal/installer/files/secret_names.go +++ b/internal/installer/files/secret_names.go @@ -44,6 +44,7 @@ const ( SecretOpenBaoPassword = "openBaoPassword" // OpenFGA + SecretOpenFgaPresharedKey = "openFgaPresharedKey" SecretOpenfgaDbBackupAccessKeyId = "openfgaDbBackupAccessKeyId" SecretOpenfgaDbBackupSecretAccessKey = "openfgaDbBackupSecretAccessKey" diff --git a/internal/installer/openfga_pc_apps.go b/internal/installer/openfga_pc_apps.go new file mode 100644 index 00000000..2841efa2 --- /dev/null +++ b/internal/installer/openfga_pc_apps.go @@ -0,0 +1,59 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +// OpenFgaPcAppsValues translates the customer-facing codesphere.openFga block of the install +// config into pc-applications values. +// +// OpenFGA is deployed by pc-applications, but whether a data center runs its own instance and +// whether that instance is published is an installation-level decision, not a chart detail — so +// operators configure it in config.yaml and this derives the chart values from it. The result is +// the *base* of the pc-apps values: an explicit `pcApps` block in config.yaml and any +// --pc-apps-values file still override it. +// +// Returns nil when the config says nothing about OpenFGA, leaving the pc-applications chart +// defaults untouched. +func OpenFgaPcAppsValues(config *files.RootConfig) files.ChartValues { + fga := config.Codesphere.OpenFga + if fga == nil { + return nil + } + + openfga := files.ChartValues{"enabled": fga.DeploysOpenFga()} + + if fga.Expose != nil { + gateway := files.ChartValues{"enabled": fga.Expose.Enabled} + if fga.Expose.Host != "" { + gateway["host"] = fga.Expose.Host + } + // The cert-manager ClusterIssuer the cluster step creates is named after the + // configured issuer type, so the gateway certificate follows the same issuer as + // the Codesphere frontend gateway. + gateway["tls"] = files.ChartValues{ + "certificate": files.ChartValues{ + "issuerRef": files.ChartValues{"name": certIssuerName(config)}, + }, + } + openfga["valuesObject"] = files.ChartValues{"gateway": gateway} + } + + return files.ChartValues{ + "applications": files.ChartValues{ + "openfga": openfga, + }, + } +} + +// certIssuerName returns the name of the ClusterIssuer for this installation, matching the +// naming the cluster step uses (the issuer type is the issuer name). +func certIssuerName(config *files.RootConfig) string { + if config.Codesphere.CertIssuer.Type != "" { + return string(config.Codesphere.CertIssuer.Type) + } + return string(files.CertIssuerTypeSelfSigned) +} diff --git a/internal/installer/openfga_pc_apps_test.go b/internal/installer/openfga_pc_apps_test.go new file mode 100644 index 00000000..9b704d59 --- /dev/null +++ b/internal/installer/openfga_pc_apps_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +var _ = Describe("OpenFgaPcAppsValues", func() { + configWith := func(fga *files.OpenFgaConfig, issuer files.CertIssuerType) *files.RootConfig { + config := &files.RootConfig{} + config.Codesphere.OpenFga = fga + config.Codesphere.CertIssuer.Type = issuer + return config + } + + // The application entry of the rendered values, or nil if there is none. + openfgaValues := func(values files.ChartValues) files.ChartValues { + apps, ok := values["applications"].(files.ChartValues) + Expect(ok).To(BeTrue(), "expected an applications map") + return apps["openfga"].(files.ChartValues) + } + + It("leaves the chart defaults alone when the config says nothing", func() { + Expect(installer.OpenFgaPcAppsValues(configWith(nil, ""))).To(BeNil()) + }) + + It("disables the application for a data center that uses a remote OpenFGA", func() { + deploy := false + values := installer.OpenFgaPcAppsValues(configWith(&files.OpenFgaConfig{ + Deploy: &deploy, + APIURL: "https://openfga.1.cs.example.com", + }, files.CertIssuerTypeACME)) + + fga := openfgaValues(values) + Expect(fga["enabled"]).To(BeFalse()) + // Nothing to expose, so the gateway is not configured at all. + Expect(fga).NotTo(HaveKey("valuesObject")) + }) + + It("defaults to deploying when only the exposure is configured", func() { + values := installer.OpenFgaPcAppsValues(configWith(&files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + }, files.CertIssuerTypeACME)) + + fga := openfgaValues(values) + Expect(fga["enabled"]).To(BeTrue()) + + gateway := fga["valuesObject"].(files.ChartValues)["gateway"].(files.ChartValues) + Expect(gateway["enabled"]).To(BeTrue()) + Expect(gateway["host"]).To(Equal("openfga.1.cs.example.com")) + }) + + It("issues the gateway certificate with the installation's cert issuer", func() { + values := installer.OpenFgaPcAppsValues(configWith(&files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + }, files.CertIssuerTypeACME)) + + gateway := openfgaValues(values)["valuesObject"].(files.ChartValues)["gateway"].(files.ChartValues) + issuerRef := gateway["tls"].(files.ChartValues)["certificate"].(files.ChartValues)["issuerRef"].(files.ChartValues) + Expect(issuerRef["name"]).To(Equal("acme")) + }) + + It("falls back to the self-signed issuer when no cert issuer is configured", func() { + values := installer.OpenFgaPcAppsValues(configWith(&files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + }, "")) + + gateway := openfgaValues(values)["valuesObject"].(files.ChartValues)["gateway"].(files.ChartValues) + issuerRef := gateway["tls"].(files.ChartValues)["certificate"].(files.ChartValues)["issuerRef"].(files.ChartValues) + Expect(issuerRef["name"]).To(Equal("self-signed")) + }) +}) diff --git a/internal/installer/secrets/datacenter.go b/internal/installer/secrets/datacenter.go index b60c48d1..194b6b3b 100644 --- a/internal/installer/secrets/datacenter.go +++ b/internal/installer/secrets/datacenter.go @@ -87,10 +87,31 @@ func DeriveDataCenterConfig(primary *files.RootConfig) (*files.RootConfig, error return nil, fmt.Errorf("clone primary data center config: %w", err) } clearDataCenterScopedConfig(derived) + deriveOpenFgaConfig(primary, derived) return derived, nil } +// deriveOpenFgaConfig points the derived data center at the primary's OpenFGA instead of letting +// it deploy one of its own. An installation has a single authorization store, so a second +// deployment would mean a second, empty set of permissions. +// +// Only possible when the primary exposes OpenFGA — that exposed host is the derived data center's +// only route to it. When it does not, the block is left as the primary wrote it, and completing it +// (exposing the primary, or pointing this data center at some other reachable instance) is up to +// whoever set up the config. +func deriveOpenFgaConfig(primary, derived *files.RootConfig) { + if !primary.Codesphere.OpenFga.ExposesOpenFga() { + return + } + + deploy := false + derived.Codesphere.OpenFga = &files.OpenFgaConfig{ + Deploy: &deploy, + APIURL: "https://" + primary.Codesphere.OpenFga.Expose.Host, + } +} + // clearDataCenterScopedConfig resets the config fields that are written by the Ensure* functions // alongside a data-center-scoped vault secret. Those pairs must always be mutated together: the // Ensure* functions are gated on the vault entry, so a config field left in place would keep a diff --git a/internal/installer/secrets/datacenter_test.go b/internal/installer/secrets/datacenter_test.go index 6c49b61f..281d2451 100644 --- a/internal/installer/secrets/datacenter_test.go +++ b/internal/installer/secrets/datacenter_test.go @@ -190,6 +190,44 @@ var _ = Describe("DeriveDataCenterConfig", func() { Expect(derived.Codesphere.CertIssuer.Acme).To(BeNil()) Expect(derived.Cluster.Certificates.CA.CertPem).To(BeEmpty()) }) + + It("points the derived data center at the primary's exposed OpenFGA", func() { + primary.Codesphere.OpenFga = &files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: true, Host: "openfga.1.cs.example.com"}, + } + + derived, err := secrets.DeriveDataCenterConfig(primary) + Expect(err).NotTo(HaveOccurred()) + + Expect(derived.Codesphere.OpenFga.DeploysOpenFga()).To(BeFalse()) + Expect(derived.Codesphere.OpenFga.APIURL).To(Equal("https://openfga.1.cs.example.com")) + // The derived data center has nothing of its own to expose. + Expect(derived.Codesphere.OpenFga.ExposesOpenFga()).To(BeFalse()) + // The primary keeps deploying and exposing it. + Expect(primary.Codesphere.OpenFga.DeploysOpenFga()).To(BeTrue()) + Expect(primary.Codesphere.OpenFga.ExposesOpenFga()).To(BeTrue()) + }) + + It("leaves the OpenFGA block alone when the primary does not expose it", func() { + primary.Codesphere.OpenFga = &files.OpenFgaConfig{ + Expose: &files.OpenFgaExposeConfig{Enabled: false}, + } + + derived, err := secrets.DeriveDataCenterConfig(primary) + Expect(err).NotTo(HaveOccurred()) + + Expect(derived.Codesphere.OpenFga.DeploysOpenFga()).To(BeTrue()) + Expect(derived.Codesphere.OpenFga.APIURL).To(BeEmpty()) + }) + + It("handles a config without an OpenFGA block", func() { + primary.Codesphere.OpenFga = nil + + derived, err := secrets.DeriveDataCenterConfig(primary) + Expect(err).NotTo(HaveOccurred()) + + Expect(derived.Codesphere.OpenFga).To(BeNil()) + }) }) var _ = Describe("secondary data center secret generation", func() { diff --git a/internal/installer/secrets/secrets.go b/internal/installer/secrets/secrets.go index d75d7e30..2224563b 100644 --- a/internal/installer/secrets/secrets.go +++ b/internal/installer/secrets/secrets.go @@ -49,6 +49,9 @@ func EnsureSecrets(vault *files.InstallVault, config *files.RootConfig) error { if err := EnsureMounterHmacSecret(vault); err != nil { return fmt.Errorf("ensure hmac secret: %w", err) } + if err := EnsureOpenFgaPresharedKey(vault); err != nil { + return fmt.Errorf("ensure openfga preshared key: %w", err) + } if err := EnsureDefaultSecrets(vault); err != nil { return fmt.Errorf("ensure default secrets: %w", err) } @@ -172,6 +175,28 @@ func EnsureMounterHmacSecret(vault *files.InstallVault) error { return nil } +// EnsureOpenFgaPresharedKey generates the preshared key that OpenFGA and the Codesphere +// services authenticate with, as 64 hex characters. Idempotent. +// +// Deliberately *not* a data-center-scoped secret (see datacenter.go): every data center's +// services talk to the same OpenFGA instance, so DeriveDataCenterVault must carry this key +// over to derived vaults unchanged. A data center whose vault is not derived from the +// primary one needs the key copied in by hand. +func EnsureOpenFgaPresharedKey(vault *files.InstallVault) error { + if vault.GetSecret(files.SecretOpenFgaPresharedKey) != nil { + return nil + } + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return fmt.Errorf("read random bytes: %w", err) + } + vault.SetSecret(files.SecretEntry{ + Name: files.SecretOpenFgaPresharedKey, + Fields: &files.SecretFields{Password: hex.EncodeToString(b)}, + }) + return nil +} + // EnsureNixSigningKeys generates an Ed25519 signing key pair for nix-cache in the // format "host:hexKey" if not already present. Idempotent. func EnsureNixSigningKeys(vault *files.InstallVault, host string) error { diff --git a/internal/installer/secrets/secrets_test.go b/internal/installer/secrets/secrets_test.go index 3d46c264..b11d8e7c 100644 --- a/internal/installer/secrets/secrets_test.go +++ b/internal/installer/secrets/secrets_test.go @@ -125,6 +125,39 @@ var _ = Describe("EnsureMounterHmacSecret", func() { }) }) +var _ = Describe("EnsureOpenFgaPresharedKey", func() { + It("creates a 64-character hex secret", func() { + vault := newVault() + Expect(secrets.EnsureOpenFgaPresharedKey(vault)).To(Succeed()) + + secret := vault.GetSecret("openFgaPresharedKey") + Expect(secret).NotTo(BeNil()) + Expect(secret.Fields).NotTo(BeNil()) + Expect(secret.Fields.Password).To(HaveLen(64)) + Expect(secret.Fields.Password).To(MatchRegexp("^[0-9a-f]+$")) + }) + + It("is idempotent", func() { + vault := newVault() + Expect(secrets.EnsureOpenFgaPresharedKey(vault)).To(Succeed()) + original := vault.GetSecret("openFgaPresharedKey").Fields.Password + + Expect(secrets.EnsureOpenFgaPresharedKey(vault)).To(Succeed()) + Expect(vault.GetSecret("openFgaPresharedKey").Fields.Password).To(Equal(original)) + }) + + It("is shared across data centers", func() { + vault := newVault() + Expect(secrets.EnsureOpenFgaPresharedKey(vault)).To(Succeed()) + primary := vault.GetSecret("openFgaPresharedKey").Fields.Password + + derived := secrets.DeriveDataCenterVault(vault) + Expect(secrets.EnsureOpenFgaPresharedKey(derived)).To(Succeed()) + + Expect(derived.GetSecret("openFgaPresharedKey").Fields.Password).To(Equal(primary)) + }) +}) + var _ = Describe("EnsureNixSigningKeys", func() { It("creates priv/pub keys in host:hexKey format", func() { vault := newVault()