fix(gcp): give every data center its own DNS records

cli/cmd/bootstrap_gcp_cleanup_test.go

@@ -322,7 +322,38 @@ var _ = Describe("BootstrapGcpCleanupCmd", func() {
 
 				mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true)
 				mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil)
-				mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", "example.com").Return(nil)
+				// An infra file without a recorded record list predates multi-DC support, so
+				// cleanup falls back to the single-data-center record names.
+				mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", gcp.GetDNSRecordNames("example.com")).Return(nil)
+				mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil)
+				mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil)
+
+				err := cleanupCmd.ExecuteCleanup(deps)
+				Expect(err).NotTo(HaveOccurred())
+			})
+		})
+
+		Context("when the infra file recorded the DNS records it created", func() {
+			It("should delete exactly those records", func() {
+				cleanupCmd.Opts.ProjectID = "test-project"
+				cleanupCmd.Opts.Force = true
+
+				recorded := []gcp.DNSRecordName{
+					{Name: "cs.example.com.", Rtype: "A"},
+					{Name: "2.ws.example.com.", Rtype: "A"},
+				}
+				validEnv := gcp.CodesphereEnvironment{
+					ProjectID:   "test-project",
+					BaseDomain:  "example.com",
+					DNSZoneName: "test-zone",
+					MultiDC:     true,
+					DNSRecords:  recorded,
+				}
+				envData, _ := json.Marshal(validEnv)
+
+				mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true)
+				mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil)
+				mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", recorded).Return(nil)
 				mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil)
 				mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil)
 

internal/bootstrap/gcp/cleanup.go

@@ -180,7 +180,20 @@ func (e *CleanupExecutor) CleanupDNSRecords() error {
 		log.Printf("Skipping DNS cleanup: missing base domain or DNS zone name (provide --base-domain/--dns-zone-name or use --skip-dns-cleanup)")
 		return nil
 	}
-	return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.BaseDomain)
+	return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.dnsRecords())
+}
+
+// dnsRecords returns the DNS records to delete. The bootstrap records what it created in the
+// infra file, which is authoritative. Older infra files predate that, and a cleanup driven only
+// by --project-id has no infra file at all, so both fall back to deriving the names.
+func (e *CleanupExecutor) dnsRecords() []DNSRecordName {
+	if len(e.InfraEnv.DNSRecords) > 0 {
+		return e.InfraEnv.DNSRecords
+	}
+	if len(e.InfraEnv.DataCenters) > 0 {
+		return DataCenterDNSRecordNames(e.BaseDomain, e.InfraEnv.DataCenters)
+	}
+	return GetDNSRecordNames(e.BaseDomain)
 }
 
 // RemoveDNSIAMBinding removes the cloud-controller service account's IAM binding

internal/bootstrap/gcp/datacenter.go

@@ -82,6 +82,16 @@ func (dc *DataCenter) RemoteAgeKeyPath() string {
 	return filepath.Join(dc.SecretsDir, "age_key.txt")
 }
 
+// PlatformDomain returns the host this data center serves its own platform services on. The
+// platform derives it from the data center ID and codesphere.domain, which OMS sets to
+// cs.<base-domain>, and the frontend calls it directly — the browser asks
+// <dc-id>.cs.<base-domain> for the config of the data center a workspace lives in. So it has to
+// resolve to this data center's platform gateway, not to the primary's, which is what
+// cs.<base-domain> and its wildcard point at.
+func (dc *DataCenter) PlatformDomain(baseDomain string) string {
+	return fmt.Sprintf("%d.cs.%s", dc.ID, baseDomain)
+}
+
 // K0sConfigScriptPath returns the local filename of this data center's k0s configuration script.
 func (dc *DataCenter) K0sConfigScriptPath() string {
 	return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix)

internal/bootstrap/gcp/datacenter_test.go

@@ -103,3 +103,30 @@ var _ = Describe("BuildDataCenters", func() {
 		Expect(gcp.BuildDataCenters(env, installer.NewInstallConfigManager)[0].Name).To(Equal("dev"))
 	})
 })
+
+var _ = Describe("DataCenterDNSRecordNames", func() {
+	It("returns the single-DC records for one data center", func() {
+		dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{BaseDomain: "example.com"}, nil)
+
+		Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(ConsistOf(gcp.GetDNSRecordNames("example.com")))
+	})
+
+	It("shares the platform names and scopes the workspace names per data center", func() {
+		dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{MultiDC: true, BaseDomain: "example.com"}, nil)
+
+		Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(Equal([]gcp.DNSRecordName{
+			{Name: "cs.example.com.", Rtype: "A"},
+			{Name: "*.cs.example.com.", Rtype: "A"},
+			{Name: "1.ws.example.com.", Rtype: "A"},
+			{Name: "*.1.ws.example.com.", Rtype: "A"},
+			{Name: "*.1.ssh.cs.example.com.", Rtype: "A"},
+			{Name: "1.cs.example.com.", Rtype: "A"},
+			{Name: "*.1.cs.example.com.", Rtype: "A"},
+			{Name: "2.ws.example.com.", Rtype: "A"},
+			{Name: "*.2.ws.example.com.", Rtype: "A"},
+			{Name: "*.2.ssh.cs.example.com.", Rtype: "A"},
+			{Name: "2.cs.example.com.", Rtype: "A"},
+			{Name: "*.2.cs.example.com.", Rtype: "A"},
+		}))
+	})
+})

internal/bootstrap/gcp/gcp.go

@@ -63,15 +63,17 @@ func CheckOMSManagedLabel(labels map[string]string) bool {
 	return exists && value == "true"
 }
 
-// GetDNSRecordNames returns the DNS record names that OMS creates for a given base domain.
-func GetDNSRecordNames(baseDomain string) []struct {
-	Name  string
-	Rtype string
-} {
-	return []struct {
-		Name  string
-		Rtype string
-	}{
+// DNSRecordName identifies a DNS record set that OMS manages.
+type DNSRecordName struct {
+	Name  string `json:"name"`
+	Rtype string `json:"rtype"`
+}
+
+// GetDNSRecordNames returns the DNS record names a single-data-center bootstrap creates for a
+// given base domain. It is the fallback for infra files written before multi-DC support, which
+// do not record the created records.
+func GetDNSRecordNames(baseDomain string) []DNSRecordName {
+	return []DNSRecordName{
 		{fmt.Sprintf("cs.%s.", baseDomain), "A"},
 		{fmt.Sprintf("*.cs.%s.", baseDomain), "A"},
 		{fmt.Sprintf("ws.%s.", baseDomain), "A"},
@@ -80,6 +82,30 @@ func GetDNSRecordNames(baseDomain string) []struct {
 	}
 }
 
+// DataCenterDNSRecordNames returns every DNS record OMS creates for the given data center
+// layout: the shared platform gateway names plus each data center's workspace and SSH names.
+func DataCenterDNSRecordNames(baseDomain string, dcs []*DataCenter) []DNSRecordName {
+	records := []DNSRecordName{
+		{fmt.Sprintf("cs.%s.", baseDomain), "A"},
+		{fmt.Sprintf("*.cs.%s.", baseDomain), "A"},
+	}
+	for _, dc := range dcs {
+		records = append(records,
+			DNSRecordName{fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), "A"},
+			DNSRecordName{fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), "A"},
+			DNSRecordName{fmt.Sprintf("*.%s.", dc.SshBaseDomain), "A"},
+		)
+		if len(dcs) > 1 {
+			records = append(records,
+				DNSRecordName{fmt.Sprintf("%s.", dc.PlatformDomain(baseDomain)), "A"},
+				DNSRecordName{fmt.Sprintf("*.%s.", dc.PlatformDomain(baseDomain)), "A"},
+			)
+		}
+	}
+
+	return records
+}
+
 // This should ALWAYS be empty. Internal flags are for internal feature
 // development and not intended for customer use.
 // Atm. it's not empty as the internal flags below are likely preview or
@@ -160,6 +186,9 @@ type CodesphereEnvironment struct {
 	MultiDC bool `json:"multi_dc"`
 	// DataCenters holds the per-data-center state. It always has at least one entry.
 	DataCenters []*DataCenter `json:"datacenters"`
+	// DNSRecords records the DNS records the bootstrap created, so cleanup deletes exactly
+	// those instead of recomputing the list.
+	DNSRecords []DNSRecordName `json:"dns_records,omitempty"`
 	// Mirrors of DataCenters[0], written for infra files consumed by cleanup and restart-vms.
 	ControlPlaneNodes []*node.Node `json:"control_plane_nodes"`
 	CephNodes         []*node.Node `json:"ceph_nodes"`
@@ -1104,6 +1133,8 @@ func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error {
 }
 
 func (b *GCPBootstrapper) EnsureDNSRecords() error {
+	b.ensureDataCenters()
+
 	gcpProject := b.Env.DNSProjectID
 	if b.Env.DNSProjectID == "" {
 		gcpProject = b.Env.ProjectID
@@ -1115,47 +1146,54 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error {
 		return fmt.Errorf("failed to ensure DNS managed zone: %w", err)
 	}
 
+	// The platform is served from one domain shared by all data centers, pointing at the
+	// primary data center's gateway.
 	records := []*dns.ResourceRecordSet{
-		{
-			Name:    fmt.Sprintf("cs.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.GatewayIP},
-		},
-		{
-			Name:    fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.GatewayIP},
-		},
-		{
-			Name:    fmt.Sprintf("*.ws.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.PublicGatewayIP},
-		},
-		{
-			Name:    fmt.Sprintf("ws.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.PublicGatewayIP},
-		},
-		{
-			Name:    fmt.Sprintf("*.ssh.cs.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.SshProxyIP},
-		},
+		dnsARecord(fmt.Sprintf("cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP),
+		dnsARecord(fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP),
+	}
+	// Workspaces and their SSH endpoints resolve per data center, so each one gets its own
+	// names pointing at its own public gateway and SSH proxy.
+	for _, dc := range b.Env.DataCenters {
+		records = append(records,
+			dnsARecord(fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP),
+			dnsARecord(fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP),
+			dnsARecord(fmt.Sprintf("*.%s.", dc.SshBaseDomain), dc.SshProxyIP),
+		)
+		// The platform calls each data center's own services at <dc-id>.cs.<base-domain>, which
+		// the wildcard above would send to the primary data center's gateway. A single data
+		// center is that primary, so it needs no record of its own.
+		if len(b.Env.DataCenters) > 1 {
+			platformDomain := dc.PlatformDomain(b.Env.BaseDomain)
+			records = append(records,
+				dnsARecord(fmt.Sprintf("%s.", platformDomain), dc.GatewayIP),
+				dnsARecord(fmt.Sprintf("*.%s.", platformDomain), dc.GatewayIP),
+			)
+		}
 	}
 
 	err = b.GCPClient.EnsureDNSRecordSets(gcpProject, zoneName, records)
 	if err != nil {
 		return fmt.Errorf("failed to ensure DNS record sets: %w", err)
 	}
 
+	// Record what was created so cleanup deletes exactly these records instead of recomputing
+	// the list from the base domain.
+	b.Env.DNSRecords = DataCenterDNSRecordNames(b.Env.BaseDomain, b.Env.DataCenters)
+
 	return nil
 }
 
+// dnsARecord builds a short-TTL A record set, as used during initial setup.
+func dnsARecord(name, ip string) *dns.ResourceRecordSet {
+	return &dns.ResourceRecordSet{
+		Name:    name,
+		Type:    "A",
+		Ttl:     300,
+		Rrdatas: []string{ip},
+	}
+}
+
 // InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in
 // ascending data center order. The order matters: the primary data center's install creates the
 // database, roles and schema that the secondary ones reuse.

internal/bootstrap/gcp/gcp_client.go

@@ -61,7 +61,7 @@ type GCPClientManager interface {
 	GetAddress(projectID, region, addressName string) (*computepb.Address, error)
 	EnsureDNSManagedZone(projectID, zoneName, dnsName, description string) error
 	EnsureDNSRecordSets(projectID, zoneName string, records []*dns.ResourceRecordSet) error
-	DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error
+	DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error
 	CreatePublicCAExternalAccountKey(projectID string) (keyID, b64MacKey string, err error)
 }
 
@@ -828,15 +828,15 @@ func (c *GCPClient) EnsureDNSRecordSets(projectID, zoneName string, records []*d
 	return nil
 }
 
-// DeleteDNSRecordSets deletes DNS record sets created by OMS for the given base domain.
-func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error {
+// DeleteDNSRecordSets deletes the given DNS record sets, ignoring those that no longer exist.
+func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error {
 	service, err := dns.NewService(c.ctx)
 	if err != nil {
 		return fmt.Errorf("failed to create DNS service: %w", err)
 	}
 
 	var deletions []*dns.ResourceRecordSet
-	for _, record := range GetDNSRecordNames(baseDomain) {
+	for _, record := range records {
 		existing, err := service.ResourceRecordSets.Get(projectID, zoneName, record.Name, record.Rtype).Context(c.ctx).Do()
 		if IsNotFoundError(err) {
 			continue

internal/bootstrap/gcp/gcp_test.go

@@ -1419,6 +1419,41 @@ var _ = Describe("GCP Bootstrapper", func() {
 				err := bs.EnsureDNSRecords()
 				Expect(err).NotTo(HaveOccurred())
 			})
+
+			It("points each data center's platform host at its own gateway", func() {
+				bs.Env.DataCenters = []*gcp.DataCenter{
+					{
+						ID: 1, GatewayIP: "1.1.1.1", PublicGatewayIP: "1.1.1.2", SshProxyIP: "1.1.1.3",
+						WorkspaceHostingBaseDomain: "1.ws.example.com", SshBaseDomain: "1.ssh.cs.example.com",
+					},
+					{
+						ID: 2, Suffix: "-dc2", GatewayIP: "2.2.2.1", PublicGatewayIP: "2.2.2.2", SshProxyIP: "2.2.2.3",
+						WorkspaceHostingBaseDomain: "2.ws.example.com", SshBaseDomain: "2.ssh.cs.example.com",
+					},
+				}
+
+				gc.EXPECT().EnsureDNSManagedZone(csEnv.DNSProjectID, csEnv.DNSZoneName, csEnv.BaseDomain+".", mock.Anything).Return(nil)
+				targets := map[string]string{}
+				gc.EXPECT().EnsureDNSRecordSets(csEnv.DNSProjectID, csEnv.DNSZoneName, mock.Anything).
+					RunAndReturn(func(_ string, _ string, records []*dns.ResourceRecordSet) error {
+						for _, r := range records {
+							targets[r.Name] = r.Rrdatas[0]
+						}
+						return nil
+					})
+
+				Expect(bs.EnsureDNSRecords()).To(Succeed())
+				// The shared platform name stays on the primary data center's gateway, but the
+				// per-data-center platform host the frontend calls resolves to its own gateway.
+				Expect(targets["cs.example.com."]).To(Equal("1.1.1.1"))
+				Expect(targets["*.cs.example.com."]).To(Equal("1.1.1.1"))
+				Expect(targets["1.cs.example.com."]).To(Equal("1.1.1.1"))
+				Expect(targets["2.cs.example.com."]).To(Equal("2.2.2.1"))
+				Expect(targets["*.2.cs.example.com."]).To(Equal("2.2.2.1"))
+				// Workspaces and SSH keep pointing at the public gateway and SSH proxy.
+				Expect(targets["2.ws.example.com."]).To(Equal("2.2.2.2"))
+				Expect(targets["*.2.ssh.cs.example.com."]).To(Equal("2.2.2.3"))
+			})
 		})
 
 		Describe("Invalid cases", func() {