diff --git a/README.md b/README.md index 2c30972..91c65a9 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,10 @@ python -m venv .venv |------|-------------| | `-create-db` | Create the complete City2TABULA database (CityDB infrastructure + schemas + data import) | | `-reset-db` | Reset everything: drop all schemas and recreate the complete database | -| `-reset-citydb` | Reset only CityDB infrastructure (drop CityDB schemas, recreate, and re-import data) | | `-reset-city2tabula` | Reset only City2TABULA schemas (preserves CityDB) | -| `-extract-features` | Run the feature extraction pipeline | +| `-import-data` | Import new 3D city data into an existing database, skipping files already imported | +| `-extract-features` | Run the feature extraction pipeline over buildings not yet processed | +| `-link-pylovo` | Link 3D buildings to PyLovo res/oth via IoU spatial join (after `-extract-features`) | | `-version` / `-v` | Print version and exit | --- diff --git a/cmd/c2t/main.go b/cmd/c2t/main.go index d0d3a77..618812c 100644 --- a/cmd/c2t/main.go +++ b/cmd/c2t/main.go @@ -54,15 +54,11 @@ func main() { utils.Info.Println(flagMessages.CreateDB.Progress) if err := db.CreateCompleteDatabase(&config, pool, "", ""); err != nil { if strings.Contains(err.Error(), "already exists") { - if strings.Contains(config.DB.Host, "docker") { - utils.Error.Println(flagMessages.CreateDB.Error) - utils.Info.Println(flagMessages.CreateDB.Custom) - os.Exit(1) - } + utils.Error.Println(flagMessages.CreateDB.Error) + utils.Info.Println(flagMessages.CreateDB.Custom) + os.Exit(1) } - utils.Info.Println("Consider changing the database name in .env file or reset the existing database using the -reset-db flag.") utils.Error.Fatalf(flagMessages.CreateDB.Error+": %v", err) - } utils.Info.Println(flagMessages.CreateDB.Success) } @@ -76,14 +72,6 @@ func main() { return } - if f.ResetCityDB { - utils.Info.Println(flagMessages.ResetCityDB.Progress) - if err := db.ResetCityDBOnly(&config, pool); err != nil { - utils.Error.Fatalf(flagMessages.ResetCityDB.Error+": %v", err) - } - utils.Info.Println(flagMessages.ResetCityDB.Success) - } - if f.ImportData { utils.Info.Println(flagMessages.ImportData.Progress) if err := db.ImportAllData(&config, pool, f.Bbox, f.BboxMode); err != nil { diff --git a/internal/db/composite_setup_test.go b/internal/db/composite_setup_test.go index 1afe8f5..a55cbc1 100644 --- a/internal/db/composite_setup_test.go +++ b/internal/db/composite_setup_test.go @@ -138,66 +138,6 @@ func TestResetCompleteDatabase_CreateCompleteDatabaseFailure(t *testing.T) { } } -// --- ResetCityDBOnly --- - -func TestResetCityDBOnly_CreateCityDBFailure(t *testing.T) { - cfg := fullCfg("citytabula_dbtest") - cfg.CityDB.SQLScripts.CreateDB = writeSQLFixture(t, `THIS IS NOT VALID SQL;`) - cfg.CityDB.SQLScripts.CreateSchema = writeSQLFixture(t, `CREATE SCHEMA :"schema_name";`) - - err := db.ResetCityDBOnly(cfg, testPool) - if err == nil { - t.Fatal("expected an error when CreateCityDB fails, got nil") - } - if !strings.Contains(err.Error(), "failed to recreate CityDB") { - t.Errorf("expected ResetCityDBOnly's own error wrap, got: %v", err) - } -} - -func TestResetCityDBOnly_ImportCityDBDataFailure(t *testing.T) { - ctx := context.Background() - cfg := fullCfg("citytabula_dbtest") - cfg.DB.Schemas.Lod2 = "rcdo_import_fail_lod2" - cfg.DB.Schemas.Lod3 = "rcdo_import_fail_lod3" - dropSchemasOnCleanup(t, ctx, cfg.DB.Schemas.Lod2, cfg.DB.Schemas.Lod3) - cfg.CityDB.SQLScripts.CreateDB = writeSQLFixture(t, `SELECT 1;`) - cfg.CityDB.SQLScripts.CreateSchema = writeSQLFixture(t, `CREATE SCHEMA :"schema_name";`) - cfg.CityDB.ToolPath = writeFakeCityDBExecutable(t, 1) // fails -help - - err := db.ResetCityDBOnly(cfg, testPool) - if err == nil { - t.Fatal("expected an error when ImportCityDBData fails, got nil") - } - if !strings.Contains(err.Error(), "failed to import CityDB data") { - t.Errorf("expected ResetCityDBOnly's own error wrap, got: %v", err) - } -} - -// TestResetCityDBOnly_Success is the one full end-to-end success case reachable -// without a real TABULA CSV or the real CityDB tool: ResetCityDBOnly never -// calls ImportSupplementaryData, and importCityDBFiles treats a missing LOD -// data directory as an optional skip (warn, not fail) rather than an error - -// so pointing Data.Lod2/Lod3 at paths that don't exist lets the fake citydb -// executable's -help check succeed and the whole call return nil. -func TestResetCityDBOnly_Success(t *testing.T) { - ctx := context.Background() - cfg := fullCfg("citytabula_dbtest") - cfg.DB.Schemas.Lod2 = "rcdo_success_lod2" - cfg.DB.Schemas.Lod3 = "rcdo_success_lod3" - dropSchemasOnCleanup(t, ctx, cfg.DB.Schemas.Lod2, cfg.DB.Schemas.Lod3) - cfg.CityDB.SQLScripts.CreateDB = writeSQLFixture(t, `SELECT 1;`) - cfg.CityDB.SQLScripts.CreateSchema = writeSQLFixture(t, `CREATE SCHEMA :"schema_name";`) - cfg.CityDB.ToolPath = writeFakeCityDBExecutable(t, 0) // succeeds -help - cfg.Data = &config.DataPaths{Lod2: "/nonexistent/lod2", Lod3: "/nonexistent/lod3"} - - if err := db.ResetCityDBOnly(cfg, testPool); err != nil { - t.Fatalf("ResetCityDBOnly: %v", err) - } - if !schemaExists(t, ctx, cfg.DB.Schemas.Lod2) { - t.Errorf("expected schema %q to exist after ResetCityDBOnly", cfg.DB.Schemas.Lod2) - } -} - // --- ImportAllData --- // TestImportAllData_ImportSupplementaryDataFailure covers ImportAllData's diff --git a/internal/db/setup.go b/internal/db/setup.go index 6dcb3a4..8de1c12 100644 --- a/internal/db/setup.go +++ b/internal/db/setup.go @@ -60,29 +60,6 @@ func ResetCompleteDatabase(config *config.Config, conn *pgxpool.Pool) error { return nil } -// ResetCityDBOnly resets only the CityDB infrastructure (preserves City2TABULA schemas) -func ResetCityDBOnly(config *config.Config, conn *pgxpool.Pool) error { - utils.Info.Println("Resetting CityDB infrastructure only...") - - // Step 1: Drop CityDB schemas - if err := DropCityDBSchemas(config, conn); err != nil { - utils.Warn.Printf("Warning during CityDB cleanup: %v", err) - } - - // Step 2: Recreate CityDB - if err := CreateCityDB(config); err != nil { - return fmt.Errorf("failed to recreate CityDB: %w", err) - } - - // Step 3: Re-import CityDB data only - if err := importer.ImportCityDBData(conn, config, "", ""); err != nil { - return fmt.Errorf("failed to import CityDB data: %w", err) - } - - utils.Info.Println("CityDB reset completed successfully") - return nil -} - // RunCity2TabulaDBSetup creates the city2tabula and tabula schemas and runs all setup job queues. func RunCity2TabulaDBSetup(config *config.Config, conn *pgxpool.Pool) error { schemas := []string{config.DB.Schemas.City2Tabula, config.DB.Schemas.Tabula} diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 898f0fd..1262dec 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -5,7 +5,6 @@ import "flag" type Flags struct { CreateDB bool ResetDB bool - ResetCityDB bool ImportData bool ResetC2T bool ExtractFeatures bool @@ -18,12 +17,11 @@ type Flags struct { func ParseFlags() *Flags { f := &Flags{} - flag.BoolVar(&f.CreateDB, "create-db", false, "Create the complete City2TABULA database (CityDB infrastructure + schemas + data import)") - flag.BoolVar(&f.ResetDB, "reset-db", false, "Reset everything: drop all schemas and recreate the complete database") - flag.BoolVar(&f.ResetCityDB, "reset-citydb", false, "Reset only CityDB infrastructure (drop CityDB schemas, recreate them, and re-import CityDB data)") - flag.BoolVar(&f.ImportData, "import-data", false, "Import data into existing CityDB schemas (useful if you want to keep existing City2TABULA schemas and import new 3D city data)") + flag.BoolVar(&f.CreateDB, "create-db", false, "Create the complete City2TABULA database (CityDB infrastructure + schemas + first data import). Refuses to run if the database already exists; use -import-data to add data to one") + flag.BoolVar(&f.ResetDB, "reset-db", false, "Destructive. Drop all schemas, including every extracted feature and hand correction, and recreate the database from scratch") + flag.BoolVar(&f.ImportData, "import-data", false, "Import new 3D city data into an existing database, skipping files already imported. Follow with -extract-features to process the new buildings") flag.BoolVar(&f.ResetC2T, "reset-city2tabula", false, "Reset only City2TABULA schemas (preserve CityDB)") - flag.BoolVar(&f.ExtractFeatures, "extract-features", false, "Run the feature extraction pipeline") + flag.BoolVar(&f.ExtractFeatures, "extract-features", false, "Run the feature extraction pipeline over buildings not yet processed. Safe to re-run; already-processed buildings are skipped") flag.BoolVar(&f.LinkPylovo, "link-pylovo", false, "Link 3D buildings to PyLovo res/oth via IoU spatial join (requires -extract-features to have run first)") flag.BoolVar(&f.ShowVersion, "version", false, "print version and exit") flag.BoolVar(&f.ShowV, "v", false, "print version and exit (shorthand)") @@ -42,7 +40,6 @@ type Msg struct { type CreateDBMsg Msg type ResetDBMsg Msg -type ResetCityDBMsg Msg type ResetC2TMsg Msg type ExtractFeaturesMsg Msg type LinkPylovoMsg Msg @@ -51,31 +48,38 @@ type ImportDataMsg Msg // Define messages for each flag var ( CreateDBMessages = CreateDBMsg{ - Custom: `Database already exists! + Custom: `Database already exists. - Please consider changing the database name using if you are processing different data, or reset the database if you want to overwrite existing data. - - # To change the database name, use ONE of the following commands based on your operating system: + Pick the command that matches what you are trying to do: ---------------------------- - 1) For Linux: make configure + Adding new 3D city data to this database - 2) For Windows: setup.bat configure + c2t -import-data + c2t -extract-features - 3) For PowerShell: .\setup.ps1 configure + Files already imported are skipped and buildings already processed are + not reprocessed, so both are safe to re-run. This keeps everything + already extracted, including any hand-corrected buildings. ---------------------------- - # To reset the database, use ONE of the following commands based on your operating system: + Building a second database alongside this one - ---------------------------- + Change DB_NAME, then re-run -create-db: - 1) For Linux: make reset-db + Linux: make configure + Windows: setup.bat configure + PowerShell: .\setup.ps1 configure - 2) For Windows: setup.bat reset-db + ---------------------------- - 3) For PowerShell: .\setup.ps1 reset-db + Starting over and discarding everything in this database + + Linux: make reset-db + Windows: setup.bat reset-db + PowerShell: .\setup.ps1 reset-db ---------------------------- `, @@ -88,11 +92,6 @@ var ( Success: "Database reset successfully", Error: "Failed to reset database", } - ResetCityDBMessages = ResetCityDBMsg{ - Progress: "Resetting CityDB...", - Success: "CityDB reset successfully", - Error: "Failed to reset CityDB", - } ResetC2TMessages = ResetC2TMsg{ Progress: "Resetting City2TABULA schemas...", Success: "City2TABULA schemas reset successfully", @@ -119,7 +118,6 @@ var ( type Messages struct { CreateDB CreateDBMsg ResetDB ResetDBMsg - ResetCityDB ResetCityDBMsg ResetC2T ResetC2TMsg ExtractFeatures ExtractFeaturesMsg LinkPylovo LinkPylovoMsg @@ -129,7 +127,6 @@ type Messages struct { var AllMessages = Messages{ CreateDB: CreateDBMessages, ResetDB: ResetDBMessages, - ResetCityDB: ResetCityDBMessages, ResetC2T: ResetC2TMessages, ExtractFeatures: ExtractFeaturesMessages, LinkPylovo: LinkPylovoMessages, diff --git a/internal/flags/flags_test.go b/internal/flags/flags_test.go index cf25878..35442f8 100644 --- a/internal/flags/flags_test.go +++ b/internal/flags/flags_test.go @@ -3,6 +3,7 @@ package flags import ( "flag" "os" + "strings" "testing" ) @@ -22,8 +23,8 @@ func TestParseFlags_EachFlagSetsItsOwnField(t *testing.T) { get func(f *Flags) bool }{ {"create-db", func(f *Flags) bool { return f.CreateDB }}, + {"import-data", func(f *Flags) bool { return f.ImportData }}, {"reset-db", func(f *Flags) bool { return f.ResetDB }}, - {"reset-citydb", func(f *Flags) bool { return f.ResetCityDB }}, {"reset-city2tabula", func(f *Flags) bool { return f.ResetC2T }}, {"extract-features", func(f *Flags) bool { return f.ExtractFeatures }}, {"link-pylovo", func(f *Flags) bool { return f.LinkPylovo }}, @@ -62,7 +63,7 @@ func TestParseFlags_NoFlagsAllDefaultFalse(t *testing.T) { f := ParseFlags() - if f.CreateDB || f.ResetDB || f.ResetCityDB || f.ResetC2T || + if f.CreateDB || f.ResetDB || f.ResetC2T || f.ImportData || f.ExtractFeatures || f.LinkPylovo || f.ShowVersion || f.ShowV { t.Errorf("expected every flag to default to false with no args, got %+v", f) } @@ -83,3 +84,23 @@ func TestParseFlags_MultipleFlagsCombine(t *testing.T) { t.Errorf("expected unrelated flags to stay false, got %+v", f) } } + +// The guidance printed when -create-db meets an existing database is the only +// place a user holding new source files is told what to run instead. Offering +// reset-db ahead of -import-data there discards every extracted feature and +// hand correction in the database. +func TestCreateDBGuidanceOffersImportBeforeReset(t *testing.T) { + msg := AllMessages.CreateDB.Custom + + importIdx := strings.Index(msg, "-import-data") + if importIdx == -1 { + t.Fatal("expected the already-exists guidance to name -import-data") + } + resetIdx := strings.Index(msg, "reset-db") + if resetIdx == -1 { + t.Fatal("expected the already-exists guidance to still name reset-db") + } + if resetIdx < importIdx { + t.Error("expected -import-data to be offered before reset-db") + } +}