From d19c3340a4426cb723514047660050c098e1d249 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Mon, 15 Dec 2025 09:22:28 +0530 Subject: [PATCH 1/2] chore: update parseable_server_logs schema add fields `workspace_id` and `org_id` --- resources/formats.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resources/formats.json b/resources/formats.json index ad5efae39..39ce494a1 100644 --- a/resources/formats.json +++ b/resources/formats.json @@ -1473,6 +1473,8 @@ "fields": [ "customer_id", "deployment_id", + "workspace_id", + "org_id", "timestamp", "level", "logger_context", From 857b032da1dc20575b21bece6c519b9a1aff93d8 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Tue, 16 Dec 2025 13:58:28 +0530 Subject: [PATCH 2/2] update test and regex --- resources/formats.json | 2 +- src/event/format/known_schema.rs | 30 +++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/resources/formats.json b/resources/formats.json index 39ce494a1..cbe2473ea 100644 --- a/resources/formats.json +++ b/resources/formats.json @@ -1469,7 +1469,7 @@ "name": "parseable_server_logs", "regex": [ { - "pattern": "^(?P\\S+)\\s+(?P\\S+)\\s+(?P\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z?)\\s+(?P\\w+)\\s+(?P\\S+)\\s+(?PThreadId\\(\\d+\\))\\s+(?P.*?):(?P\\d+):\\s+(?P.*)", + "pattern": "^(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z?)\\s+(?P\\w+)\\s+(?P\\S+)\\s+(?PThreadId\\(\\d+\\))\\s+(?P.*?):(?P\\d+):\\s+(?P.*)", "fields": [ "customer_id", "deployment_id", diff --git a/src/event/format/known_schema.rs b/src/event/format/known_schema.rs index b5a155972..ca32ebf4d 100644 --- a/src/event/format/known_schema.rs +++ b/src/event/format/known_schema.rs @@ -524,10 +524,10 @@ mod tests { let test_logs = vec![ // Current parseable format with ThreadId - "01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 2025-09-06T10:43:01.628980875Z WARN main ThreadId(01) parseable::handlers::http::cluster:919: node http://0.0.0.0:8010/ is not live", - "01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 2025-09-06T10:44:12.62276265Z ERROR actix-rt|system:0|arbiter:17 ThreadId(163) parseable_enterprise::http::handlers::query:43: JsonParse(\"Datafusion Error: Schema error: No field named a. Valid fields are serverlogs.log\")", - "01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 2025-09-06T05:16:46.092071318Z ERROR actix-rt|system:0|arbiter:21 ThreadId(167) parseable_enterprise::http::handlers::query:43: JsonParse(\"Datafusion Error: Schema error: No field named ansible.host.ip\")", - "01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 2025-09-06T11:22:07.500864363Z WARN main ThreadId(01) parseable_enterprise:70: Received shutdown signal, notifying server to shut down...", + "01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 2025-09-06T10:43:01.628980875Z WARN main ThreadId(01) parseable::handlers::http::cluster:919: node http://0.0.0.0:8010/ is not live", + "01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 2025-09-06T10:44:12.62276265Z ERROR actix-rt|system:0|arbiter:17 ThreadId(163) parseable_enterprise::http::handlers::query:43: JsonParse(\"Datafusion Error: Schema error: No field named a. Valid fields are serverlogs.log\")", + "01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 2025-09-06T05:16:46.092071318Z ERROR actix-rt|system:0|arbiter:21 ThreadId(167) parseable_enterprise::http::handlers::query:43: JsonParse(\"Datafusion Error: Schema error: No field named ansible.host.ip\")", + "01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 01K4SHM6VQASBJ7G8V0STZN6N1 2025-09-06T11:22:07.500864363Z WARN main ThreadId(01) parseable_enterprise:70: Received shutdown signal, notifying server to shut down...", ]; for (i, log_text) in test_logs.iter().enumerate() { @@ -543,7 +543,27 @@ mod tests { log_text ); - // Verify basic fields that should be present in all formats + // Verify fields that are always present + assert!( + obj.contains_key("customer_id"), + "Missing customer_id field for log {}", + i + 1 + ); + assert!( + obj.contains_key("deployment_id"), + "Missing deployment_id field for log {}", + i + 1 + ); + assert!( + obj.contains_key("workspace_id"), + "Missing workspace_id field for log {}", + i + 1 + ); + assert!( + obj.contains_key("org_id"), + "Missing org_id field for log {}", + i + 1 + ); assert!( obj.contains_key("timestamp"), "Missing timestamp field for log {}",