diff --git a/cli/cmd/bootstrap_gcp_cleanup_test.go b/cli/cmd/bootstrap_gcp_cleanup_test.go index 3337fd4c..75c4da29 100644 --- a/cli/cmd/bootstrap_gcp_cleanup_test.go +++ b/cli/cmd/bootstrap_gcp_cleanup_test.go @@ -322,7 +322,38 @@ var _ = Describe("BootstrapGcpCleanupCmd", func() { mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true) mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil) - mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", "example.com").Return(nil) + // An infra file without a recorded record list predates multi-DC support, so + // cleanup falls back to the single-data-center record names. + mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", gcp.GetDNSRecordNames("example.com")).Return(nil) + mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil) + mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil) + + err := cleanupCmd.ExecuteCleanup(deps) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when the infra file recorded the DNS records it created", func() { + It("should delete exactly those records", func() { + cleanupCmd.Opts.ProjectID = "test-project" + cleanupCmd.Opts.Force = true + + recorded := []gcp.DNSRecordName{ + {Name: "cs.example.com.", Rtype: "A"}, + {Name: "2.ws.example.com.", Rtype: "A"}, + } + validEnv := gcp.CodesphereEnvironment{ + ProjectID: "test-project", + BaseDomain: "example.com", + DNSZoneName: "test-zone", + MultiDC: true, + DNSRecords: recorded, + } + envData, _ := json.Marshal(validEnv) + + mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true) + mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil) + mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", recorded).Return(nil) mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil) mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil) diff --git a/internal/bootstrap/gcp/cleanup.go b/internal/bootstrap/gcp/cleanup.go index f30e59a7..f16f3885 100644 --- a/internal/bootstrap/gcp/cleanup.go +++ b/internal/bootstrap/gcp/cleanup.go @@ -180,7 +180,20 @@ func (e *CleanupExecutor) CleanupDNSRecords() error { log.Printf("Skipping DNS cleanup: missing base domain or DNS zone name (provide --base-domain/--dns-zone-name or use --skip-dns-cleanup)") return nil } - return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.BaseDomain) + return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.dnsRecords()) +} + +// dnsRecords returns the DNS records to delete. The bootstrap records what it created in the +// infra file, which is authoritative. Older infra files predate that, and a cleanup driven only +// by --project-id has no infra file at all, so both fall back to deriving the names. +func (e *CleanupExecutor) dnsRecords() []DNSRecordName { + if len(e.InfraEnv.DNSRecords) > 0 { + return e.InfraEnv.DNSRecords + } + if len(e.InfraEnv.DataCenters) > 0 { + return DataCenterDNSRecordNames(e.BaseDomain, e.InfraEnv.DataCenters) + } + return GetDNSRecordNames(e.BaseDomain) } // RemoveDNSIAMBinding removes the cloud-controller service account's IAM binding diff --git a/internal/bootstrap/gcp/datacenter.go b/internal/bootstrap/gcp/datacenter.go index 7ef4fbcc..d263137f 100644 --- a/internal/bootstrap/gcp/datacenter.go +++ b/internal/bootstrap/gcp/datacenter.go @@ -82,6 +82,16 @@ func (dc *DataCenter) RemoteAgeKeyPath() string { return filepath.Join(dc.SecretsDir, "age_key.txt") } +// PlatformDomain returns the host this data center serves its own platform services on. The +// platform derives it from the data center ID and codesphere.domain, which OMS sets to +// cs., and the frontend calls it directly — the browser asks +// .cs. for the config of the data center a workspace lives in. So it has to +// resolve to this data center's platform gateway, not to the primary's, which is what +// cs. and its wildcard point at. +func (dc *DataCenter) PlatformDomain(baseDomain string) string { + return fmt.Sprintf("%d.cs.%s", dc.ID, baseDomain) +} + // K0sConfigScriptPath returns the local filename of this data center's k0s configuration script. func (dc *DataCenter) K0sConfigScriptPath() string { return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix) diff --git a/internal/bootstrap/gcp/datacenter_test.go b/internal/bootstrap/gcp/datacenter_test.go index 76522bbf..f822dbb4 100644 --- a/internal/bootstrap/gcp/datacenter_test.go +++ b/internal/bootstrap/gcp/datacenter_test.go @@ -103,3 +103,30 @@ var _ = Describe("BuildDataCenters", func() { Expect(gcp.BuildDataCenters(env, installer.NewInstallConfigManager)[0].Name).To(Equal("dev")) }) }) + +var _ = Describe("DataCenterDNSRecordNames", func() { + It("returns the single-DC records for one data center", func() { + dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{BaseDomain: "example.com"}, nil) + + Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(ConsistOf(gcp.GetDNSRecordNames("example.com"))) + }) + + It("shares the platform names and scopes the workspace names per data center", func() { + dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{MultiDC: true, BaseDomain: "example.com"}, nil) + + Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(Equal([]gcp.DNSRecordName{ + {Name: "cs.example.com.", Rtype: "A"}, + {Name: "*.cs.example.com.", Rtype: "A"}, + {Name: "1.ws.example.com.", Rtype: "A"}, + {Name: "*.1.ws.example.com.", Rtype: "A"}, + {Name: "*.1.ssh.cs.example.com.", Rtype: "A"}, + {Name: "1.cs.example.com.", Rtype: "A"}, + {Name: "*.1.cs.example.com.", Rtype: "A"}, + {Name: "2.ws.example.com.", Rtype: "A"}, + {Name: "*.2.ws.example.com.", Rtype: "A"}, + {Name: "*.2.ssh.cs.example.com.", Rtype: "A"}, + {Name: "2.cs.example.com.", Rtype: "A"}, + {Name: "*.2.cs.example.com.", Rtype: "A"}, + })) + }) +}) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 72aaa895..d6a2d19f 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -63,15 +63,17 @@ func CheckOMSManagedLabel(labels map[string]string) bool { return exists && value == "true" } -// GetDNSRecordNames returns the DNS record names that OMS creates for a given base domain. -func GetDNSRecordNames(baseDomain string) []struct { - Name string - Rtype string -} { - return []struct { - Name string - Rtype string - }{ +// DNSRecordName identifies a DNS record set that OMS manages. +type DNSRecordName struct { + Name string `json:"name"` + Rtype string `json:"rtype"` +} + +// GetDNSRecordNames returns the DNS record names a single-data-center bootstrap creates for a +// given base domain. It is the fallback for infra files written before multi-DC support, which +// do not record the created records. +func GetDNSRecordNames(baseDomain string) []DNSRecordName { + return []DNSRecordName{ {fmt.Sprintf("cs.%s.", baseDomain), "A"}, {fmt.Sprintf("*.cs.%s.", baseDomain), "A"}, {fmt.Sprintf("ws.%s.", baseDomain), "A"}, @@ -80,6 +82,30 @@ func GetDNSRecordNames(baseDomain string) []struct { } } +// DataCenterDNSRecordNames returns every DNS record OMS creates for the given data center +// layout: the shared platform gateway names plus each data center's workspace and SSH names. +func DataCenterDNSRecordNames(baseDomain string, dcs []*DataCenter) []DNSRecordName { + records := []DNSRecordName{ + {fmt.Sprintf("cs.%s.", baseDomain), "A"}, + {fmt.Sprintf("*.cs.%s.", baseDomain), "A"}, + } + for _, dc := range dcs { + records = append(records, + DNSRecordName{fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.SshBaseDomain), "A"}, + ) + if len(dcs) > 1 { + records = append(records, + DNSRecordName{fmt.Sprintf("%s.", dc.PlatformDomain(baseDomain)), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.PlatformDomain(baseDomain)), "A"}, + ) + } + } + + return records +} + // This should ALWAYS be empty. Internal flags are for internal feature // development and not intended for customer use. // Atm. it's not empty as the internal flags below are likely preview or @@ -160,6 +186,9 @@ type CodesphereEnvironment struct { MultiDC bool `json:"multi_dc"` // DataCenters holds the per-data-center state. It always has at least one entry. DataCenters []*DataCenter `json:"datacenters"` + // DNSRecords records the DNS records the bootstrap created, so cleanup deletes exactly + // those instead of recomputing the list. + DNSRecords []DNSRecordName `json:"dns_records,omitempty"` // Mirrors of DataCenters[0], written for infra files consumed by cleanup and restart-vms. ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` CephNodes []*node.Node `json:"ceph_nodes"` @@ -1104,6 +1133,8 @@ func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error { } func (b *GCPBootstrapper) EnsureDNSRecords() error { + b.ensureDataCenters() + gcpProject := b.Env.DNSProjectID if b.Env.DNSProjectID == "" { gcpProject = b.Env.ProjectID @@ -1115,37 +1146,30 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return fmt.Errorf("failed to ensure DNS managed zone: %w", err) } + // The platform is served from one domain shared by all data centers, pointing at the + // primary data center's gateway. records := []*dns.ResourceRecordSet{ - { - Name: fmt.Sprintf("cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.GatewayIP}, - }, - { - Name: fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.GatewayIP}, - }, - { - Name: fmt.Sprintf("*.ws.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.PublicGatewayIP}, - }, - { - Name: fmt.Sprintf("ws.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.PublicGatewayIP}, - }, - { - Name: fmt.Sprintf("*.ssh.cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.SshProxyIP}, - }, + dnsARecord(fmt.Sprintf("cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP), + dnsARecord(fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP), + } + // Workspaces and their SSH endpoints resolve per data center, so each one gets its own + // names pointing at its own public gateway and SSH proxy. + for _, dc := range b.Env.DataCenters { + records = append(records, + dnsARecord(fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", dc.SshBaseDomain), dc.SshProxyIP), + ) + // The platform calls each data center's own services at .cs., which + // the wildcard above would send to the primary data center's gateway. A single data + // center is that primary, so it needs no record of its own. + if len(b.Env.DataCenters) > 1 { + platformDomain := dc.PlatformDomain(b.Env.BaseDomain) + records = append(records, + dnsARecord(fmt.Sprintf("%s.", platformDomain), dc.GatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", platformDomain), dc.GatewayIP), + ) + } } err = b.GCPClient.EnsureDNSRecordSets(gcpProject, zoneName, records) @@ -1153,9 +1177,23 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return fmt.Errorf("failed to ensure DNS record sets: %w", err) } + // Record what was created so cleanup deletes exactly these records instead of recomputing + // the list from the base domain. + b.Env.DNSRecords = DataCenterDNSRecordNames(b.Env.BaseDomain, b.Env.DataCenters) + return nil } +// dnsARecord builds a short-TTL A record set, as used during initial setup. +func dnsARecord(name, ip string) *dns.ResourceRecordSet { + return &dns.ResourceRecordSet{ + Name: name, + Type: "A", + Ttl: 300, + Rrdatas: []string{ip}, + } +} + // InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in // ascending data center order. The order matters: the primary data center's install creates the // database, roles and schema that the secondary ones reuse. diff --git a/internal/bootstrap/gcp/gcp_client.go b/internal/bootstrap/gcp/gcp_client.go index 7b764de3..cb7a9c94 100644 --- a/internal/bootstrap/gcp/gcp_client.go +++ b/internal/bootstrap/gcp/gcp_client.go @@ -61,7 +61,7 @@ type GCPClientManager interface { GetAddress(projectID, region, addressName string) (*computepb.Address, error) EnsureDNSManagedZone(projectID, zoneName, dnsName, description string) error EnsureDNSRecordSets(projectID, zoneName string, records []*dns.ResourceRecordSet) error - DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error + DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error CreatePublicCAExternalAccountKey(projectID string) (keyID, b64MacKey string, err error) } @@ -828,15 +828,15 @@ func (c *GCPClient) EnsureDNSRecordSets(projectID, zoneName string, records []*d return nil } -// DeleteDNSRecordSets deletes DNS record sets created by OMS for the given base domain. -func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error { +// DeleteDNSRecordSets deletes the given DNS record sets, ignoring those that no longer exist. +func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error { service, err := dns.NewService(c.ctx) if err != nil { return fmt.Errorf("failed to create DNS service: %w", err) } var deletions []*dns.ResourceRecordSet - for _, record := range GetDNSRecordNames(baseDomain) { + for _, record := range records { existing, err := service.ResourceRecordSets.Get(projectID, zoneName, record.Name, record.Rtype).Context(c.ctx).Do() if IsNotFoundError(err) { continue diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index 073ef283..a47e8754 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1419,6 +1419,41 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureDNSRecords() Expect(err).NotTo(HaveOccurred()) }) + + It("points each data center's platform host at its own gateway", func() { + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, GatewayIP: "1.1.1.1", PublicGatewayIP: "1.1.1.2", SshProxyIP: "1.1.1.3", + WorkspaceHostingBaseDomain: "1.ws.example.com", SshBaseDomain: "1.ssh.cs.example.com", + }, + { + ID: 2, Suffix: "-dc2", GatewayIP: "2.2.2.1", PublicGatewayIP: "2.2.2.2", SshProxyIP: "2.2.2.3", + WorkspaceHostingBaseDomain: "2.ws.example.com", SshBaseDomain: "2.ssh.cs.example.com", + }, + } + + gc.EXPECT().EnsureDNSManagedZone(csEnv.DNSProjectID, csEnv.DNSZoneName, csEnv.BaseDomain+".", mock.Anything).Return(nil) + targets := map[string]string{} + gc.EXPECT().EnsureDNSRecordSets(csEnv.DNSProjectID, csEnv.DNSZoneName, mock.Anything). + RunAndReturn(func(_ string, _ string, records []*dns.ResourceRecordSet) error { + for _, r := range records { + targets[r.Name] = r.Rrdatas[0] + } + return nil + }) + + Expect(bs.EnsureDNSRecords()).To(Succeed()) + // The shared platform name stays on the primary data center's gateway, but the + // per-data-center platform host the frontend calls resolves to its own gateway. + Expect(targets["cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["*.cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["1.cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["2.cs.example.com."]).To(Equal("2.2.2.1")) + Expect(targets["*.2.cs.example.com."]).To(Equal("2.2.2.1")) + // Workspaces and SSH keep pointing at the public gateway and SSH proxy. + Expect(targets["2.ws.example.com."]).To(Equal("2.2.2.2")) + Expect(targets["*.2.ssh.cs.example.com."]).To(Equal("2.2.2.3")) + }) }) Describe("Invalid cases", func() { diff --git a/internal/bootstrap/gcp/mocks.go b/internal/bootstrap/gcp/mocks.go index 5b371325..74b64038 100644 --- a/internal/bootstrap/gcp/mocks.go +++ b/internal/bootstrap/gcp/mocks.go @@ -796,16 +796,16 @@ func (_c *MockGCPClientManager_CreateVPC_Call) RunAndReturn(run func(projectID s } // DeleteDNSRecordSets provides a mock function for the type MockGCPClientManager -func (_mock *MockGCPClientManager) DeleteDNSRecordSets(projectID string, zoneName string, baseDomain string) error { - ret := _mock.Called(projectID, zoneName, baseDomain) +func (_mock *MockGCPClientManager) DeleteDNSRecordSets(projectID string, zoneName string, records []DNSRecordName) error { + ret := _mock.Called(projectID, zoneName, records) if len(ret) == 0 { panic("no return value specified for DeleteDNSRecordSets") } var r0 error - if returnFunc, ok := ret.Get(0).(func(string, string, string) error); ok { - r0 = returnFunc(projectID, zoneName, baseDomain) + if returnFunc, ok := ret.Get(0).(func(string, string, []DNSRecordName) error); ok { + r0 = returnFunc(projectID, zoneName, records) } else { r0 = ret.Error(0) } @@ -820,12 +820,12 @@ type MockGCPClientManager_DeleteDNSRecordSets_Call struct { // DeleteDNSRecordSets is a helper method to define mock.On call // - projectID string // - zoneName string -// - baseDomain string -func (_e *MockGCPClientManager_Expecter) DeleteDNSRecordSets(projectID any, zoneName any, baseDomain any) *MockGCPClientManager_DeleteDNSRecordSets_Call { - return &MockGCPClientManager_DeleteDNSRecordSets_Call{Call: _e.mock.On("DeleteDNSRecordSets", projectID, zoneName, baseDomain)} +// - records []DNSRecordName +func (_e *MockGCPClientManager_Expecter) DeleteDNSRecordSets(projectID any, zoneName any, records any) *MockGCPClientManager_DeleteDNSRecordSets_Call { + return &MockGCPClientManager_DeleteDNSRecordSets_Call{Call: _e.mock.On("DeleteDNSRecordSets", projectID, zoneName, records)} } -func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID string, zoneName string, baseDomain string)) *MockGCPClientManager_DeleteDNSRecordSets_Call { +func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID string, zoneName string, records []DNSRecordName)) *MockGCPClientManager_DeleteDNSRecordSets_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { @@ -835,9 +835,9 @@ func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID if args[1] != nil { arg1 = args[1].(string) } - var arg2 string + var arg2 []DNSRecordName if args[2] != nil { - arg2 = args[2].(string) + arg2 = args[2].([]DNSRecordName) } run( arg0, @@ -853,7 +853,7 @@ func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Return(err error) *Mock return _c } -func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) RunAndReturn(run func(projectID string, zoneName string, baseDomain string) error) *MockGCPClientManager_DeleteDNSRecordSets_Call { +func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) RunAndReturn(run func(projectID string, zoneName string, records []DNSRecordName) error) *MockGCPClientManager_DeleteDNSRecordSets_Call { _c.Call.Return(run) return _c }