Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,10 @@ python -m venv .venv
|------|-------------|
| `-create-db` | Create the complete City2TABULA database (CityDB infrastructure + schemas + data import) |
| `-reset-db` | Reset everything: drop all schemas and recreate the complete database |
| `-reset-citydb` | Reset only CityDB infrastructure (drop CityDB schemas, recreate, and re-import data) |
| `-reset-city2tabula` | Reset only City2TABULA schemas (preserves CityDB) |
| `-extract-features` | Run the feature extraction pipeline |
| `-import-data` | Import new 3D city data into an existing database, skipping files already imported |
| `-extract-features` | Run the feature extraction pipeline over buildings not yet processed |
| `-link-pylovo` | Link 3D buildings to PyLovo res/oth via IoU spatial join (after `-extract-features`) |
| `-version` / `-v` | Print version and exit |

---
Expand Down
18 changes: 3 additions & 15 deletions cmd/c2t/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,11 @@ func main() {
utils.Info.Println(flagMessages.CreateDB.Progress)
if err := db.CreateCompleteDatabase(&config, pool, "", ""); err != nil {
if strings.Contains(err.Error(), "already exists") {
if strings.Contains(config.DB.Host, "docker") {
utils.Error.Println(flagMessages.CreateDB.Error)
utils.Info.Println(flagMessages.CreateDB.Custom)
os.Exit(1)
}
utils.Error.Println(flagMessages.CreateDB.Error)
utils.Info.Println(flagMessages.CreateDB.Custom)
os.Exit(1)
}
utils.Info.Println("Consider changing the database name in .env file or reset the existing database using the -reset-db flag.")
utils.Error.Fatalf(flagMessages.CreateDB.Error+": %v", err)

}
utils.Info.Println(flagMessages.CreateDB.Success)
}
Expand All @@ -76,14 +72,6 @@ func main() {
return
}

if f.ResetCityDB {
utils.Info.Println(flagMessages.ResetCityDB.Progress)
if err := db.ResetCityDBOnly(&config, pool); err != nil {
utils.Error.Fatalf(flagMessages.ResetCityDB.Error+": %v", err)
}
utils.Info.Println(flagMessages.ResetCityDB.Success)
}

if f.ImportData {
utils.Info.Println(flagMessages.ImportData.Progress)
if err := db.ImportAllData(&config, pool, f.Bbox, f.BboxMode); err != nil {
Expand Down
60 changes: 0 additions & 60 deletions internal/db/composite_setup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,66 +138,6 @@ func TestResetCompleteDatabase_CreateCompleteDatabaseFailure(t *testing.T) {
}
}

// --- ResetCityDBOnly ---

func TestResetCityDBOnly_CreateCityDBFailure(t *testing.T) {
cfg := fullCfg("citytabula_dbtest")
cfg.CityDB.SQLScripts.CreateDB = writeSQLFixture(t, `THIS IS NOT VALID SQL;`)
cfg.CityDB.SQLScripts.CreateSchema = writeSQLFixture(t, `CREATE SCHEMA :"schema_name";`)

err := db.ResetCityDBOnly(cfg, testPool)
if err == nil {
t.Fatal("expected an error when CreateCityDB fails, got nil")
}
if !strings.Contains(err.Error(), "failed to recreate CityDB") {
t.Errorf("expected ResetCityDBOnly's own error wrap, got: %v", err)
}
}

func TestResetCityDBOnly_ImportCityDBDataFailure(t *testing.T) {
ctx := context.Background()
cfg := fullCfg("citytabula_dbtest")
cfg.DB.Schemas.Lod2 = "rcdo_import_fail_lod2"
cfg.DB.Schemas.Lod3 = "rcdo_import_fail_lod3"
dropSchemasOnCleanup(t, ctx, cfg.DB.Schemas.Lod2, cfg.DB.Schemas.Lod3)
cfg.CityDB.SQLScripts.CreateDB = writeSQLFixture(t, `SELECT 1;`)
cfg.CityDB.SQLScripts.CreateSchema = writeSQLFixture(t, `CREATE SCHEMA :"schema_name";`)
cfg.CityDB.ToolPath = writeFakeCityDBExecutable(t, 1) // fails -help

err := db.ResetCityDBOnly(cfg, testPool)
if err == nil {
t.Fatal("expected an error when ImportCityDBData fails, got nil")
}
if !strings.Contains(err.Error(), "failed to import CityDB data") {
t.Errorf("expected ResetCityDBOnly's own error wrap, got: %v", err)
}
}

// TestResetCityDBOnly_Success is the one full end-to-end success case reachable
// without a real TABULA CSV or the real CityDB tool: ResetCityDBOnly never
// calls ImportSupplementaryData, and importCityDBFiles treats a missing LOD
// data directory as an optional skip (warn, not fail) rather than an error -
// so pointing Data.Lod2/Lod3 at paths that don't exist lets the fake citydb
// executable's -help check succeed and the whole call return nil.
func TestResetCityDBOnly_Success(t *testing.T) {
ctx := context.Background()
cfg := fullCfg("citytabula_dbtest")
cfg.DB.Schemas.Lod2 = "rcdo_success_lod2"
cfg.DB.Schemas.Lod3 = "rcdo_success_lod3"
dropSchemasOnCleanup(t, ctx, cfg.DB.Schemas.Lod2, cfg.DB.Schemas.Lod3)
cfg.CityDB.SQLScripts.CreateDB = writeSQLFixture(t, `SELECT 1;`)
cfg.CityDB.SQLScripts.CreateSchema = writeSQLFixture(t, `CREATE SCHEMA :"schema_name";`)
cfg.CityDB.ToolPath = writeFakeCityDBExecutable(t, 0) // succeeds -help
cfg.Data = &config.DataPaths{Lod2: "/nonexistent/lod2", Lod3: "/nonexistent/lod3"}

if err := db.ResetCityDBOnly(cfg, testPool); err != nil {
t.Fatalf("ResetCityDBOnly: %v", err)
}
if !schemaExists(t, ctx, cfg.DB.Schemas.Lod2) {
t.Errorf("expected schema %q to exist after ResetCityDBOnly", cfg.DB.Schemas.Lod2)
}
}

// --- ImportAllData ---

// TestImportAllData_ImportSupplementaryDataFailure covers ImportAllData's
Expand Down
23 changes: 0 additions & 23 deletions internal/db/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,29 +60,6 @@ func ResetCompleteDatabase(config *config.Config, conn *pgxpool.Pool) error {
return nil
}

// ResetCityDBOnly resets only the CityDB infrastructure (preserves City2TABULA schemas)
func ResetCityDBOnly(config *config.Config, conn *pgxpool.Pool) error {
utils.Info.Println("Resetting CityDB infrastructure only...")

// Step 1: Drop CityDB schemas
if err := DropCityDBSchemas(config, conn); err != nil {
utils.Warn.Printf("Warning during CityDB cleanup: %v", err)
}

// Step 2: Recreate CityDB
if err := CreateCityDB(config); err != nil {
return fmt.Errorf("failed to recreate CityDB: %w", err)
}

// Step 3: Re-import CityDB data only
if err := importer.ImportCityDBData(conn, config, "", ""); err != nil {
return fmt.Errorf("failed to import CityDB data: %w", err)
}

utils.Info.Println("CityDB reset completed successfully")
return nil
}

// RunCity2TabulaDBSetup creates the city2tabula and tabula schemas and runs all setup job queues.
func RunCity2TabulaDBSetup(config *config.Config, conn *pgxpool.Pool) error {
schemas := []string{config.DB.Schemas.City2Tabula, config.DB.Schemas.Tabula}
Expand Down
49 changes: 23 additions & 26 deletions internal/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import "flag"
type Flags struct {
CreateDB bool
ResetDB bool
ResetCityDB bool
ImportData bool
ResetC2T bool
ExtractFeatures bool
Expand All @@ -18,12 +17,11 @@ type Flags struct {

func ParseFlags() *Flags {
f := &Flags{}
flag.BoolVar(&f.CreateDB, "create-db", false, "Create the complete City2TABULA database (CityDB infrastructure + schemas + data import)")
flag.BoolVar(&f.ResetDB, "reset-db", false, "Reset everything: drop all schemas and recreate the complete database")
flag.BoolVar(&f.ResetCityDB, "reset-citydb", false, "Reset only CityDB infrastructure (drop CityDB schemas, recreate them, and re-import CityDB data)")
flag.BoolVar(&f.ImportData, "import-data", false, "Import data into existing CityDB schemas (useful if you want to keep existing City2TABULA schemas and import new 3D city data)")
flag.BoolVar(&f.CreateDB, "create-db", false, "Create the complete City2TABULA database (CityDB infrastructure + schemas + first data import). Refuses to run if the database already exists; use -import-data to add data to one")
flag.BoolVar(&f.ResetDB, "reset-db", false, "Destructive. Drop all schemas, including every extracted feature and hand correction, and recreate the database from scratch")
flag.BoolVar(&f.ImportData, "import-data", false, "Import new 3D city data into an existing database, skipping files already imported. Follow with -extract-features to process the new buildings")
flag.BoolVar(&f.ResetC2T, "reset-city2tabula", false, "Reset only City2TABULA schemas (preserve CityDB)")
flag.BoolVar(&f.ExtractFeatures, "extract-features", false, "Run the feature extraction pipeline")
flag.BoolVar(&f.ExtractFeatures, "extract-features", false, "Run the feature extraction pipeline over buildings not yet processed. Safe to re-run; already-processed buildings are skipped")
flag.BoolVar(&f.LinkPylovo, "link-pylovo", false, "Link 3D buildings to PyLovo res/oth via IoU spatial join (requires -extract-features to have run first)")
flag.BoolVar(&f.ShowVersion, "version", false, "print version and exit")
flag.BoolVar(&f.ShowV, "v", false, "print version and exit (shorthand)")
Expand All @@ -42,7 +40,6 @@ type Msg struct {

type CreateDBMsg Msg
type ResetDBMsg Msg
type ResetCityDBMsg Msg
type ResetC2TMsg Msg
type ExtractFeaturesMsg Msg
type LinkPylovoMsg Msg
Expand All @@ -51,31 +48,38 @@ type ImportDataMsg Msg
// Define messages for each flag
var (
CreateDBMessages = CreateDBMsg{
Custom: `Database already exists!
Custom: `Database already exists.

Please consider changing the database name using if you are processing different data, or reset the database if you want to overwrite existing data.

# To change the database name, use ONE of the following commands based on your operating system:
Pick the command that matches what you are trying to do:

----------------------------

1) For Linux: make configure
Adding new 3D city data to this database

2) For Windows: setup.bat configure
c2t -import-data
c2t -extract-features

3) For PowerShell: .\setup.ps1 configure
Files already imported are skipped and buildings already processed are
not reprocessed, so both are safe to re-run. This keeps everything
already extracted, including any hand-corrected buildings.

----------------------------

# To reset the database, use ONE of the following commands based on your operating system:
Building a second database alongside this one

----------------------------
Change DB_NAME, then re-run -create-db:

1) For Linux: make reset-db
Linux: make configure
Windows: setup.bat configure
PowerShell: .\setup.ps1 configure

2) For Windows: setup.bat reset-db
----------------------------

3) For PowerShell: .\setup.ps1 reset-db
Starting over and discarding everything in this database

Linux: make reset-db
Windows: setup.bat reset-db
PowerShell: .\setup.ps1 reset-db

----------------------------
`,
Expand All @@ -88,11 +92,6 @@ var (
Success: "Database reset successfully",
Error: "Failed to reset database",
}
ResetCityDBMessages = ResetCityDBMsg{
Progress: "Resetting CityDB...",
Success: "CityDB reset successfully",
Error: "Failed to reset CityDB",
}
ResetC2TMessages = ResetC2TMsg{
Progress: "Resetting City2TABULA schemas...",
Success: "City2TABULA schemas reset successfully",
Expand All @@ -119,7 +118,6 @@ var (
type Messages struct {
CreateDB CreateDBMsg
ResetDB ResetDBMsg
ResetCityDB ResetCityDBMsg
ResetC2T ResetC2TMsg
ExtractFeatures ExtractFeaturesMsg
LinkPylovo LinkPylovoMsg
Expand All @@ -129,7 +127,6 @@ type Messages struct {
var AllMessages = Messages{
CreateDB: CreateDBMessages,
ResetDB: ResetDBMessages,
ResetCityDB: ResetCityDBMessages,
ResetC2T: ResetC2TMessages,
ExtractFeatures: ExtractFeaturesMessages,
LinkPylovo: LinkPylovoMessages,
Expand Down
25 changes: 23 additions & 2 deletions internal/flags/flags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package flags
import (
"flag"
"os"
"strings"
"testing"
)

Expand All @@ -22,8 +23,8 @@ func TestParseFlags_EachFlagSetsItsOwnField(t *testing.T) {
get func(f *Flags) bool
}{
{"create-db", func(f *Flags) bool { return f.CreateDB }},
{"import-data", func(f *Flags) bool { return f.ImportData }},
{"reset-db", func(f *Flags) bool { return f.ResetDB }},
{"reset-citydb", func(f *Flags) bool { return f.ResetCityDB }},
{"reset-city2tabula", func(f *Flags) bool { return f.ResetC2T }},
{"extract-features", func(f *Flags) bool { return f.ExtractFeatures }},
{"link-pylovo", func(f *Flags) bool { return f.LinkPylovo }},
Expand Down Expand Up @@ -62,7 +63,7 @@ func TestParseFlags_NoFlagsAllDefaultFalse(t *testing.T) {

f := ParseFlags()

if f.CreateDB || f.ResetDB || f.ResetCityDB || f.ResetC2T ||
if f.CreateDB || f.ResetDB || f.ResetC2T || f.ImportData ||
f.ExtractFeatures || f.LinkPylovo || f.ShowVersion || f.ShowV {
t.Errorf("expected every flag to default to false with no args, got %+v", f)
}
Expand All @@ -83,3 +84,23 @@ func TestParseFlags_MultipleFlagsCombine(t *testing.T) {
t.Errorf("expected unrelated flags to stay false, got %+v", f)
}
}

// The guidance printed when -create-db meets an existing database is the only
// place a user holding new source files is told what to run instead. Offering
// reset-db ahead of -import-data there discards every extracted feature and
// hand correction in the database.
func TestCreateDBGuidanceOffersImportBeforeReset(t *testing.T) {
msg := AllMessages.CreateDB.Custom

importIdx := strings.Index(msg, "-import-data")
if importIdx == -1 {
t.Fatal("expected the already-exists guidance to name -import-data")
}
resetIdx := strings.Index(msg, "reset-db")
if resetIdx == -1 {
t.Fatal("expected the already-exists guidance to still name reset-db")
}
if resetIdx < importIdx {
t.Error("expected -import-data to be offered before reset-db")
}
}
Loading