diff --git a/.gitignore b/.gitignore index 870bca4a26..863400b1b5 100644 --- a/.gitignore +++ b/.gitignore @@ -178,7 +178,9 @@ dataset-private/* appendonly.aof /p2p_db.sqlite old-pipeline/ - - +databases/regex_store/ +utils/immune_web_sim/runs/ +config/redis.conf +.vscode/launch.json # permanen p2p dbs and encryption keys permanent/ diff --git a/.secrets.baseline b/.secrets.baseline index 8609229b8b..eace8db17c 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -90,10 +90,6 @@ { "path": "detect_secrets.filters.allowlist.is_line_allowlisted" }, - { - "path": "detect_secrets.filters.common.is_baseline_file", - "filename": ".secrets.baseline" - }, { "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", "min_level": 2 @@ -127,7054 +123,29 @@ } ], "results": { - "config/local_ti_files/own_malicious_JA3.csv": [ - { - "type": "Hex High Entropy String", - "filename": "config/local_ti_files/own_malicious_JA3.csv", - "hashed_secret": "2e621bc4ae7af0e821c2a7f45b1e9ff83780ff3e", - "is_verified": false, - "line_number": 6 - }, - { - "type": "Hex High Entropy String", - "filename": "config/local_ti_files/own_malicious_JA3.csv", - "hashed_secret": "b6aa435b77ffc6bb2f5c3a1647eb4a5e45a316ee", - "is_verified": false, - "line_number": 7 - } - ], "config/slips.yaml": [ { "type": "Secret Keyword", "filename": "config/slips.yaml", - "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016", - "is_verified": false, - "line_number": 322 - } - ], - "dataset/test14-malicious-zeek-dir/http.log": [ - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "87a3ca87807af60f25513beda99b356b9fbdf09e", - "is_verified": false, - "line_number": 257 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "024dfe3e6bbc130f57727192eacc67342970aab4", - "is_verified": false, - "line_number": 266 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "600ab340e9ed99ff504f83d39d37a61acdd0b8ef", - "is_verified": false, - "line_number": 278 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "932842a6825242d1f77003ed74c00077205f2fd7", - "is_verified": false, - "line_number": 283 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "d2d51781277a43f75bbff43e5ae8e69faff9d32d", - "is_verified": false, - "line_number": 286 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "951d86e04d039b6d023a2f270977cf6aef1859c4", - "is_verified": false, - "line_number": 289 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "240285b3b9f0aae0b9817d23989a9104bed39680", - "is_verified": false, - "line_number": 297 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "186304d742baf2443142cca7cc9d08181fd0192a", - "is_verified": false, - "line_number": 316 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "e650b48c2cbc188fdc0be3116e4c6fe199c15a53", - "is_verified": false, - "line_number": 318 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "cc4b5287594e20e9c0dd65fd9e24cd2ac8e760ea", - "is_verified": false, - "line_number": 323 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "ed218f34534055322820bedfaf71eba498dd6aad", - "is_verified": false, - "line_number": 326 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "2c7e85c60afe34d2c6ebcdb7019f3362f3f96bd0", - "is_verified": false, - "line_number": 373 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "c1e4a39b43fec8745515f680279cb682c963fe78", - "is_verified": false, - "line_number": 376 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "613e385d1057b2e26e3e1cc6863f25c14a24b39a", - "is_verified": false, - "line_number": 381 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "a570d3f59aa3f1a19759a0c931b5beb934ae2d1a", - "is_verified": false, - "line_number": 382 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "684271cb5d0f473a0658698a40c4eb5303dcf4e1", - "is_verified": false, - "line_number": 386 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "23fb0f3a04797101b8887273f8c2b7b5f912d313", - "is_verified": false, - "line_number": 391 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "79038c7c41e365ab069071437e51fc6e9fbdc364", - "is_verified": false, - "line_number": 392 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "b32cec134c9c479109288445162de23dc4469f01", - "is_verified": false, - "line_number": 397 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "fd0b60739ef05510194b3edfc56d6e534642216b", - "is_verified": false, - "line_number": 401 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "fe14c850fe1b74a41355c5b707c1d12031710460", - "is_verified": false, - "line_number": 404 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "00574f4283401ef320e8fd9b373d51efaf31b37d", - "is_verified": false, - "line_number": 420 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "2b007cb159fd1cb99fbce87f37e82cd7d43c77a4", - "is_verified": false, - "line_number": 423 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "69e26a6bb775ad38a2cf62d64a0c3ea8eab319fc", - "is_verified": false, - "line_number": 433 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "d095634b4fe212ef09a90c31fa1927e5f1310604", - "is_verified": false, - "line_number": 446 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "7d79913f3af49356a53df4725935bb3651c08421", - "is_verified": false, - "line_number": 453 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "15d5bd1db4d42c7eeb464a1bf0dc4b799e678cd3", - "is_verified": false, - "line_number": 456 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "631d18d0259084c9784b44eedaf1667a3470fea6", - "is_verified": false, - "line_number": 511 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test14-malicious-zeek-dir/http.log", - "hashed_secret": "37522b5b749c6ceaa3820e74a0924758772a8793", - "is_verified": false, - "line_number": 514 - } - ], - "dataset/test15-malicious-zeek-dir/ssl.log": [ - { - "type": "Hex High Entropy String", - "filename": "dataset/test15-malicious-zeek-dir/ssl.log", - "hashed_secret": "b6aa435b77ffc6bb2f5c3a1647eb4a5e45a316ee", - "is_verified": false, - "line_number": 4 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test15-malicious-zeek-dir/ssl.log", - "hashed_secret": "3e1fd256a51b5ae9951f24ab5d2ff387894c3e67", - "is_verified": false, - "line_number": 13 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test15-malicious-zeek-dir/ssl.log", - "hashed_secret": "2ceebe1f14715c0c71f7f48929918d4ca749b388", - "is_verified": false, - "line_number": 22 - } - ], - "dataset/test9-mixed-zeek-dir/files.log": [ - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "037f99aa1702e6b14b9d79995108d92f21b61dd1", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "db8a6dc0071f8133f8cb259857c96eec29928d33", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6f6215a7a80f68bb53f3a4b23eeb8c591d05349d", - "is_verified": false, - "line_number": 2 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a1ef755c605fe818df95b8282fa259713096e0bf", - "is_verified": false, - "line_number": 2 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5a01d29d1a4c754c28b08b4c91bcac23108b8013", - "is_verified": false, - "line_number": 3 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b71ac1afbe41f4bcc69f992e27d8b1d286201c31", - "is_verified": false, - "line_number": 3 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cd3326488f11485e6a23636b9d3c754f27782e03", - "is_verified": false, - "line_number": 6 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ff8a9de63de3992e9dca09dd96b0e5a4bef03bd3", - "is_verified": false, - "line_number": 6 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "37616e4d9e2f970cff67e31ac7f2b0036cf42e2d", - "is_verified": false, - "line_number": 7 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7b7b2a81f03ebd6643d0984b0d85a2d6f23fbeff", - "is_verified": false, - "line_number": 7 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7bb5ba477014899b3ce789bbd6cbd92bfbb2f4d5", - "is_verified": false, - "line_number": 8 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d6495632b3cabb956c2ea272bf977f85df639b46", - "is_verified": false, - "line_number": 8 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2f95fe81231f3a51021c6113907b74de2850a2c1", - "is_verified": false, - "line_number": 9 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f79ac4a3b539b0f9be594c3ec9235a46e04bafdd", - "is_verified": false, - "line_number": 9 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "148a4091d7aff32cfdd968bb6cabd7f9394079b5", - "is_verified": false, - "line_number": 10 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3a19cbd7d7ad1d557325d326f2e26811b61a1a7e", - "is_verified": false, - "line_number": 10 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "203af34e409fb2faefc62345708828de94e9078e", - "is_verified": false, - "line_number": 11 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3926fc8a3c44fad136453e723b2d4f3041367531", - "is_verified": false, - "line_number": 11 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e70954c61d633510d8816b3bf6d6c8a3a6d55f84", - "is_verified": false, - "line_number": 12 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f32ca81169eb1a32bd4c01f0f6ebfb88686676bc", - "is_verified": false, - "line_number": 12 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "12143b3f6120fc0afb759035b6f199fcd8eeb316", - "is_verified": false, - "line_number": 13 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6e31a1fe005ca1c218948f90757f2f5616e40492", - "is_verified": false, - "line_number": 13 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "093955387d63592c48b9b4d2e8badd72e52139c4", - "is_verified": false, - "line_number": 14 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4466090b52c585c2acc9907952f75c1b71bc2d5c", - "is_verified": false, - "line_number": 14 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1fd1a74808d4a00ce86ccc71da670d9143dd9f25", - "is_verified": false, - "line_number": 15 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "866e6b0a310c2715b117799e22d096368b341666", - "is_verified": false, - "line_number": 15 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dce463daacb1af8aff1f796d20a5cf56decfde53", - "is_verified": false, - "line_number": 16 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e55c2de753ab1583e3699daa1419a34bd86cb9eb", - "is_verified": false, - "line_number": 16 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "673eee61775d1c9d3ef72939c7baba7180680054", + "hashed_secret": "02ecb94373bfb3dfe827ca18409f50b016e8302a", "is_verified": false, - "line_number": 17 + "line_number": 310 }, { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ce2125137fa7a089f29a0762b0f497baba66899f", - "is_verified": false, - "line_number": 17 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0557637edcfbb6cb54e170ea6ab99d5ec0ed21ac", - "is_verified": false, - "line_number": 18 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "55cadc4ce2cfd91d62849aaedefd8eee9fc63de4", - "is_verified": false, - "line_number": 18 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "450ac938c05f48dc477cd681abfbff16e19d51d1", - "is_verified": false, - "line_number": 19 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ea8efd0c1bc9a3dadc557a58e3f98612a2826ace", - "is_verified": false, - "line_number": 19 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "373e9918cd573e35ddc034930b5cabeb09f678c7", - "is_verified": false, - "line_number": 20 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "91af07438cf97de1f90e966d6d282bb2098e3450", - "is_verified": false, - "line_number": 20 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "05f79de456b2e3a19df5135bce83b025df8750cf", - "is_verified": false, - "line_number": 21 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5b60bddf3097131557acbdc30a1524aa6aca7604", - "is_verified": false, - "line_number": 21 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7fda2d09dbc12810818d8bc113cb06c81cc73ee3", - "is_verified": false, - "line_number": 22 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8421bc55cc3e08b26214568e1315d7a33e2c5bc8", - "is_verified": false, - "line_number": 22 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "128a0614682cfd322c366cf8137f35d31f5219bd", - "is_verified": false, - "line_number": 23 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c7632218baf03644a11fb88260f19b4c0399ebe4", - "is_verified": false, - "line_number": 23 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d40e837078ce3aaf5f096f54408eae9ec3ed453d", - "is_verified": false, - "line_number": 24 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d9f9fc9e75a7f9f104ca371309aff7a42c7b95e6", - "is_verified": false, - "line_number": 24 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "024cbc52eb544ea9151e47016728fb601b37801e", - "is_verified": false, - "line_number": 25 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "48136b8707533a805db2f1cb9e6c5ed6aeca9a55", - "is_verified": false, - "line_number": 25 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "17f0598f64574dc4e0bd020b7bae87068707efab", - "is_verified": false, - "line_number": 26 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "60a6787e9ea9e17b1249ed04594c96bed6c9634c", - "is_verified": false, - "line_number": 26 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0e36f12dd9ada1e455c04eba1fda4b3ecbb1f0ea", - "is_verified": false, - "line_number": 27 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dddf27f3f1b6eb4b5098d0412ba487c8947c82fa", + "type": "Secret Keyword", + "filename": "config/slips.yaml", + "hashed_secret": "f8ca0d7266886f4b5be9adddc9b66017b3bf1a4b", "is_verified": false, - "line_number": 27 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a4c5a8a28bc1a77a06fe947e8bb999f9784aa75d", - "is_verified": false, - "line_number": 28 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dc5e9a1b3d9e64b5963c8cc231669df18ae3de63", - "is_verified": false, - "line_number": 28 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "44cfb649380e50ddeed157409fb1802fd60fc21b", - "is_verified": false, - "line_number": 29 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "98afbbcadc9543a2e956e49ee0d8228c0fee95c5", - "is_verified": false, - "line_number": 29 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ccc785d4b4277356c359a6450e95aff8cb0fe9f5", - "is_verified": false, - "line_number": 30 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dbb3104578a9f4ea9f59cb7cbfd70b148ad64a32", - "is_verified": false, - "line_number": 30 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1a1456511ff0127d47caf44ddc465dd89353faf8", - "is_verified": false, - "line_number": 31 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fd9c5b6ff43513ff67fd083cbe63e27225e26133", - "is_verified": false, - "line_number": 31 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dbf1c801375b03899abae6b44f957dfc06f5d08f", - "is_verified": false, - "line_number": 32 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fb7d1303960e73e13088645e1817fb2be9617e42", - "is_verified": false, - "line_number": 32 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0e28d7b43e5e8ed63bfc5df8993f843a4cb66ecd", - "is_verified": false, - "line_number": 33 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e8f473ed18ebb003860ed3e93c60ef11f581de10", - "is_verified": false, - "line_number": 33 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4a948fab3663ec813f208bc14c7bc68e890226a8", - "is_verified": false, - "line_number": 34 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bd765df764a7efb8a6132c0a5a18211534f77df0", - "is_verified": false, - "line_number": 34 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1efed16f0f1aa3354252bdaaf75a9cc9ebb7fd1d", - "is_verified": false, - "line_number": 35 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9417c668f317719ff4f3075a3fa617e5cd14c2ed", - "is_verified": false, - "line_number": 35 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0d9a2b31d9f93a3468ed03eb5810bb4b176da034", - "is_verified": false, - "line_number": 36 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3c0ba5d33ba3a33f4191e9d9ac373422b3e947d4", - "is_verified": false, - "line_number": 36 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "01b3e88f40ecb35005ed29e9f72e685a7857aa05", - "is_verified": false, - "line_number": 37 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "17e11e1e28cc3732837590efced61f1b2142a6d4", - "is_verified": false, - "line_number": 37 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "93e0646086364b05e631aa651b4c5b126e9b7498", - "is_verified": false, - "line_number": 38 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a8e0802face524a2a7f43692bc231206a518a18c", - "is_verified": false, - "line_number": 38 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "87bce89bf38190ca071f0eaf267059d8f463338e", - "is_verified": false, - "line_number": 39 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d1440a3b58980feb37511a34eec351446adb51df", - "is_verified": false, - "line_number": 39 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "28fb380b420f198a56b6c399609a67d0a45d01fd", - "is_verified": false, - "line_number": 40 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "35861bb69f028e80ae41094fb0464452989b37a1", - "is_verified": false, - "line_number": 40 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "621585ccb591e65a95edcae93a9ec38f12cd5fea", - "is_verified": false, - "line_number": 41 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dc4bba45d6687bb97f5f636d0c8eeea6f8a40bdc", - "is_verified": false, - "line_number": 41 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "20e993fd0cc773bb5baf917cb1be917b2759bbd2", - "is_verified": false, - "line_number": 42 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f5d98f9c55c6ca39cba4a6b14f51ec3b94ab13fb", - "is_verified": false, - "line_number": 42 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0ceaa05a071af86a386b4d7c014daa2ce8ad3f0b", - "is_verified": false, - "line_number": 43 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6ae599522834d0dd168ccd571183f9f9c07cd5e2", - "is_verified": false, - "line_number": 43 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3c6cfb6ec24e359ac6a30245b06314f6f884af8e", - "is_verified": false, - "line_number": 44 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c2cb05807c549a46c2cff2b293a631d468109e8f", - "is_verified": false, - "line_number": 44 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4494ce95a374af85356fe69abef72958a6be4007", - "is_verified": false, - "line_number": 45 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "48a14568c051c218422417780dcc539f25c7d6df", - "is_verified": false, - "line_number": 45 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "205b671e7caf74d708978ce0b881f85774cefc9c", - "is_verified": false, - "line_number": 46 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dbc6727f3ec44db6f7e86cee7286899dfdbb0fbf", - "is_verified": false, - "line_number": 46 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "92d95af89c5d923c549f61a01c5fb2bf519c8062", - "is_verified": false, - "line_number": 47 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a9ee7c57f3b4c0f496eb717b4c4514fb65778aa3", - "is_verified": false, - "line_number": 47 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aaa9cff89176a79ddd20afe9d0980f8f504752b6", - "is_verified": false, - "line_number": 48 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c22b7ab36a3004cc721c66b95ea70cb2afb5b9bf", - "is_verified": false, - "line_number": 48 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "14716369c64e7d0453134ed95e8c5a5a6d957ac0", - "is_verified": false, - "line_number": 49 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c25aaf9991d3aa2afdb2e76e4b3fe3217a242a31", - "is_verified": false, - "line_number": 49 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aa29f21f2a34d816d4bb7cc89fffffaf32888805", - "is_verified": false, - "line_number": 50 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f68e50a417fa459c765a2673e29a6e94e82ecf63", - "is_verified": false, - "line_number": 50 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2e28848badd31bd9e5c3b95dcee12782fcd74139", - "is_verified": false, - "line_number": 51 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9d9adb7acd5d1685671afdc862c5dcaef1c59245", - "is_verified": false, - "line_number": 51 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7a69f9c906eae6820829c4b668b1947c39c80a51", - "is_verified": false, - "line_number": 52 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aac4a3c9efdc50fa200394307b59f53774983677", - "is_verified": false, - "line_number": 52 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "46b99a918844abd2ae185a98a5498e1e0a0d1b55", - "is_verified": false, - "line_number": 53 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cda935582cc84d80dccac1c3db873a3667346c98", - "is_verified": false, - "line_number": 53 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b5eebe73d290975f7befb86b617693190ee9c189", - "is_verified": false, - "line_number": 54 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f9581f9898c5692c41b6519875e8963c373c5f02", - "is_verified": false, - "line_number": 54 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "58848017f7ef1ed89a895f2c35bb0b9aba694044", - "is_verified": false, - "line_number": 55 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7f0bd33d5ace46a6911b1049329a20a632dd8b23", - "is_verified": false, - "line_number": 55 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0a7d38314da55106b78482f651ebd85203a13baa", - "is_verified": false, - "line_number": 56 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "12c1a3ccbeba560d70519806d82ae3850d58eccd", - "is_verified": false, - "line_number": 56 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2adae5a969dc3e96e2a618f204afb7c2b3055029", - "is_verified": false, - "line_number": 57 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "71e314c40baef8c688f019d1520c3b65238fbbe9", - "is_verified": false, - "line_number": 57 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ca7828394540c474178e88583a123b7af0e81890", - "is_verified": false, - "line_number": 58 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fcc7b21546b5f602615151fa1db8603fd9b000db", - "is_verified": false, - "line_number": 58 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9d4bdce874bbd78ac31652b28e8bbd5966571c5c", - "is_verified": false, - "line_number": 59 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e721aaf1bb5a12b0ec924ba95540984506c50210", - "is_verified": false, - "line_number": 59 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c1f5abbe0fd0f4cc5c007f9937765a13410cd94d", - "is_verified": false, - "line_number": 60 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c46cb4514af5c98b44ae20d34b06a3543daa88d7", - "is_verified": false, - "line_number": 60 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2f8bf50afee718b2ed6a33418307725895e9de03", - "is_verified": false, - "line_number": 61 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e511cbe859fcb2d0e6d9224875df8cd4f29fa2c5", - "is_verified": false, - "line_number": 61 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c9fb494343e1e03b56163aa405f13c718c985da2", - "is_verified": false, - "line_number": 62 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e28e52dd6897f13f1821d202e86e840be6e437be", - "is_verified": false, - "line_number": 62 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "38d1ef40c0048ee2929664f6c241ca8a206b87a0", - "is_verified": false, - "line_number": 63 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e0333d9a94f0aa4ce03a6563c15f4eb394546720", - "is_verified": false, - "line_number": 63 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "59a0d13594eb7c9c28f36ad1b4fd92b6e805af83", - "is_verified": false, - "line_number": 64 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5e32c9735c2879b74d6d77a5c5eb2d5154a770e7", - "is_verified": false, - "line_number": 64 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a0fef201c9f6c317ff96f12df3abfefca88d46a2", - "is_verified": false, - "line_number": 65 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a9728348f0dbad8c586da63597e1392f71837548", - "is_verified": false, - "line_number": 65 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "64dda110f44aba77b472389bf93fad43e34ff96d", - "is_verified": false, - "line_number": 66 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bb7082ba0599de9369919ebebe10116bd0d4f2b9", - "is_verified": false, - "line_number": 66 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9cb17beca955c2bf0e5d6d3bb643ce169dbc2ecd", - "is_verified": false, - "line_number": 67 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "deb9e07c2310d2c5428e983153cf81a44e10ff54", - "is_verified": false, - "line_number": 67 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9d04bac4c7f274e0011a47a433879c9fa4513632", - "is_verified": false, - "line_number": 68 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ca1894542d67240d9b86d9f05dc5facb210dcbca", - "is_verified": false, - "line_number": 68 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3af31e6308b63711fa5804d42b7b1a66bf34d1a4", - "is_verified": false, - "line_number": 69 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "80c446d21c4c4571861ae704650faf3c3d5f9a18", - "is_verified": false, - "line_number": 69 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7a47154876bc7b561df90d32e8b1036e0f33bae9", - "is_verified": false, - "line_number": 70 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e19c8b2ecedbbfeb2bad47ff8cdaa3cbc4c54f49", - "is_verified": false, - "line_number": 70 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "259b99395a2a5112d67ad102185561a5277b0d4a", - "is_verified": false, - "line_number": 71 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7a6fefd606af5897cf9872dd428c8f3d4f04eff3", - "is_verified": false, - "line_number": 71 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "39c20acb634a62bfa034c0c9f777472c80de81be", - "is_verified": false, - "line_number": 72 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "74d0628424f55dd771da1444c6da9f78547f1ede", - "is_verified": false, - "line_number": 72 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a36b2ab4d02611693fb0f5bc6b3fe4e04ada1c6b", - "is_verified": false, - "line_number": 73 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e69922bcfe0d936646c90d1d8e7c1edcec259855", - "is_verified": false, - "line_number": 73 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "38fbb263c29d992211a6cd0e1b23adbe5d43490b", - "is_verified": false, - "line_number": 74 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b97143f0aa9ef8e6d9d2901947d4f61fbab85183", - "is_verified": false, - "line_number": 74 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "85de1f177d69b5c0ea6a802be8e2ec3062b53c5f", - "is_verified": false, - "line_number": 75 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ce2b01110f522794b3319cb0e61a0a3a7ee403e5", - "is_verified": false, - "line_number": 75 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5a54c5325107b782a956f62964512aa163bc9cbf", - "is_verified": false, - "line_number": 76 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d8dd406692029cc124740bfec45cbf4e34c4af9c", - "is_verified": false, - "line_number": 76 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "27d72661a89271b5d139b657e10cf3edfc2b74e6", - "is_verified": false, - "line_number": 77 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8622650ee27983d3fe16c1e520e71b35a4b4bf87", - "is_verified": false, - "line_number": 77 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7f58e5ffefa03593022f2712fb906e631bf43f7a", - "is_verified": false, - "line_number": 78 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ddb602977603d4a3e5b0eeada46b1f0f0d0de0dc", - "is_verified": false, - "line_number": 78 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5706393efcf10b5885ab1fe217952f41b870dd65", - "is_verified": false, - "line_number": 79 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c753f364f06df5da5319ad00eb7ab6e7c176a1fb", - "is_verified": false, - "line_number": 79 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a849e2e162b51d7fecad11a6f6b2eeaee0d3d7ac", - "is_verified": false, - "line_number": 80 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cf1788c0fc4a3959fb08ed6c56906ed7b6bd151a", - "is_verified": false, - "line_number": 80 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "af9fc2a488c999f100a9991a2aa51095839ee961", - "is_verified": false, - "line_number": 81 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dc0e6791bb47c415d1640982e913943ba3312e20", - "is_verified": false, - "line_number": 81 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "40c415ec0fa8442cc20c4ee502a3d64e3c64ef13", - "is_verified": false, - "line_number": 82 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "720a1a02264cc46047bf6f88bae83e122684bf94", - "is_verified": false, - "line_number": 82 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6528296ccb84d9f952e9c2bba3a1824b888e7ae5", - "is_verified": false, - "line_number": 83 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9409c779d382a60a545500d53960d3b082a7bd54", - "is_verified": false, - "line_number": 83 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2c97cd2d98562c8554840c2fe213dce3667089d2", - "is_verified": false, - "line_number": 84 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4f4aeac8b8abb894e66928fa6f02cc0a5340be5b", - "is_verified": false, - "line_number": 84 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2ad23d559807909901c3145f7f1fb9b5230726dd", - "is_verified": false, - "line_number": 85 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "36d62133117c62c80722dfd0f32942fd7716caac", - "is_verified": false, - "line_number": 85 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4a81e91cffdf884abe2fe31ec8c9aaa8fa003ae0", - "is_verified": false, - "line_number": 86 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a8a592eee0b4caa9aea04c8bf8872e1901141645", - "is_verified": false, - "line_number": 86 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2eb97c88ff9abe23dd55bfc77942243f2b30e766", - "is_verified": false, - "line_number": 87 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ed7a7d44bd86e6cba725b23f7549dd903026738a", - "is_verified": false, - "line_number": 87 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "19abc53b95df98766518345c20121b30a596be55", - "is_verified": false, - "line_number": 88 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2eab73935e3c46fbd555afe79e24415f9620d664", - "is_verified": false, - "line_number": 88 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "914fdbb402d269007e4d07d17ce6b33b15641b5d", - "is_verified": false, - "line_number": 89 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d7bc22c451151bceaa9f5e3f3902e10c9efd330c", - "is_verified": false, - "line_number": 89 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5f7328b037b4c2e4143a9adb7c132dcedb013757", - "is_verified": false, - "line_number": 90 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a5f95a3301c512483968080ed4c436e479e323b6", - "is_verified": false, - "line_number": 90 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "443c44bfaed7d24e43ec35e40c2d28c82da8345a", - "is_verified": false, - "line_number": 91 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e6ae2277928e6b397bdc2b9b5ee563cb9d1b8d25", - "is_verified": false, - "line_number": 91 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "79f34cbb1083f7b145fa8cecc6744f9e58f43cca", - "is_verified": false, - "line_number": 92 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b582a621ebfea1822c2f11cd2900d0602cf49d99", - "is_verified": false, - "line_number": 92 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7085dbc7dd8edd788d1f62b9ac410ba741c38273", - "is_verified": false, - "line_number": 93 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7ba9042f36b6722fe135aef5160332e6b641ee99", - "is_verified": false, - "line_number": 93 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "532e736ea13db6169d0206c5f0f63cb99a383857", - "is_verified": false, - "line_number": 94 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "91dccf159f34c11f85f91bcca6e66d742e7dfa2a", - "is_verified": false, - "line_number": 94 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7cb63e5c2f2be36097655a863dbe0c0237110c40", - "is_verified": false, - "line_number": 95 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ba0c102507398ee2f4b3b0a2ee02aaf65a7568aa", - "is_verified": false, - "line_number": 95 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "807ff60f7264a67e5ba5470b82cad52229541111", - "is_verified": false, - "line_number": 96 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f40f4eb4c3fd3ba881c474fdd5636d6e5e0115f5", - "is_verified": false, - "line_number": 96 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e0150203ac88c3d94d63a4cfe77273327889d22b", - "is_verified": false, - "line_number": 97 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f61046f2b86e9370a1c1358d390db790d6579056", - "is_verified": false, - "line_number": 97 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5729aa1cdc64e129965cae0c19a565723cf0fbf4", - "is_verified": false, - "line_number": 98 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a15232e26e04ac6a8f58b91d1c5c3a11949eef11", - "is_verified": false, - "line_number": 98 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6600d5946f9ac141c302b123a10eae0ba60d72f2", - "is_verified": false, - "line_number": 99 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b7c23ee05b6b01bd44acdca8fa1ed09b7d826ed4", - "is_verified": false, - "line_number": 99 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3bb00c2936259180ab9b396a9f53aa0619a66983", - "is_verified": false, - "line_number": 100 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "daa077128c11a4919c8954df5332d6ecf65bbbf7", - "is_verified": false, - "line_number": 100 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7a3d79415d9b679b592a855e745dd65db3c70d31", - "is_verified": false, - "line_number": 101 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ecc8b3b3607cc2a7dcee97d5576d96ac9c02ce7d", - "is_verified": false, - "line_number": 101 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "30cca4cf546a93c75495c88a15f4909cdee7d2db", - "is_verified": false, - "line_number": 102 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "45b33529aab6eeb7d5721b6e3a60598d9857c9bc", - "is_verified": false, - "line_number": 102 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "306bd8312d8c9bc418b0c292e17c51af5ad123dc", - "is_verified": false, - "line_number": 103 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4040289d2d025088564328948daf15b99ef84fca", - "is_verified": false, - "line_number": 103 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6f29f1b486379a1fb0f62603cd5653178734709f", - "is_verified": false, - "line_number": 104 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e934c9aedd17b44d6fac26461864c4b2b06eae96", - "is_verified": false, - "line_number": 104 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "eaaf015dc8731e4c36d0750d3227df991984c277", - "is_verified": false, - "line_number": 105 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f518cb1e3d2e438415edf2b6564d86e02bd2fe41", - "is_verified": false, - "line_number": 105 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c12db0e808bed864845c2b83186ee7e48cb26c7e", - "is_verified": false, - "line_number": 106 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ead0b759096f09cba2f6208fa2d3fa7846438ef0", - "is_verified": false, - "line_number": 106 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "32d8a4faecbc7f0bd2f60c2df74b0d6a09f9d267", - "is_verified": false, - "line_number": 107 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cb9e066e06d73d9e72d92f9974d328e88f0097ab", - "is_verified": false, - "line_number": 107 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bf31bfc309e962e24b95c9a77a931c1d2db6b65f", - "is_verified": false, - "line_number": 108 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cebfb9657feb8d503ac5fd54401bce5376ca6623", - "is_verified": false, - "line_number": 108 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5bda1378e9c5b088b4a5a41a6eb380b5d11361ce", - "is_verified": false, - "line_number": 109 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fc8cd300c9e1c3dd836a7e492f4d0c39c601dad0", - "is_verified": false, - "line_number": 109 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e69536d5ab2c923f6fc82f4e846ac48d4aed67c8", - "is_verified": false, - "line_number": 110 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ea03cc8e9ff94b8bab062ae425afe0f2d8a7ef0e", - "is_verified": false, - "line_number": 110 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c118cbb2cdf9d5ceba4dc327475fc35dba3a4bec", - "is_verified": false, - "line_number": 111 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f3c57c527857504a049978e1afe48fcd91a82521", - "is_verified": false, - "line_number": 111 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6470dc0ff278be241ec1de1309d8e1a1c53646ac", - "is_verified": false, - "line_number": 112 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c1216d68a6ae3d786010ae283748c848846a54bd", - "is_verified": false, - "line_number": 112 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9160b3ffccad442eadc720cd54da21dba515c35e", - "is_verified": false, - "line_number": 113 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d3c1c47b976d0654adb7f58b312c9d934e2ee93e", - "is_verified": false, - "line_number": 113 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b4ba97bfc2b10909bbc1fac4113458872879e762", - "is_verified": false, - "line_number": 114 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d0c671cf30921b4bc4f87959ecec0f3b556f4fb3", - "is_verified": false, - "line_number": 114 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a9c3a490ec89f842856aaa6b1a029646e42f507a", - "is_verified": false, - "line_number": 115 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b399cad6315199a3c46e8e241b14bb95319f0d02", - "is_verified": false, - "line_number": 115 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2849cc1e2695edff5118d4578fab416401003bf7", - "is_verified": false, - "line_number": 116 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fe359df5923debae9bf9e1351a75e04937626033", - "is_verified": false, - "line_number": 116 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3615cc3137bf9b31a257599d8048975d11fff794", - "is_verified": false, - "line_number": 117 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b29441bbe2db611c079b98f69ec11ef720fd1a34", - "is_verified": false, - "line_number": 117 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aa80d8d59a250be68c19924d3c103da79b189447", - "is_verified": false, - "line_number": 119 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dc0d91194b28633e886a4a85872d6fb412dc42ec", - "is_verified": false, - "line_number": 119 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4075887f12b7c0926527f484d1f6c29baa42ec87", - "is_verified": false, - "line_number": 120 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6aa6b14e3ff94fc42e1dc4945c6c152a73efd6e2", - "is_verified": false, - "line_number": 120 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2ce098be0f0fe3d3172091d3bc098d0f42f92ef6", - "is_verified": false, - "line_number": 121 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ca221d58cd2cdbab6be3828acbd27bd466c2231a", - "is_verified": false, - "line_number": 121 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a1a28a6de83a094ed0808e9ac41dae2e6fa149b2", - "is_verified": false, - "line_number": 122 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ec79fb331ee6b35dad103dd21e29c015c4398555", - "is_verified": false, - "line_number": 122 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5ec163854965a21e8c1bcd965ae15e6de49f8f55", - "is_verified": false, - "line_number": 123 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "876603d3dec1f6e8d74afd0d6918a10391678b82", - "is_verified": false, - "line_number": 123 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "36949f38679815b1b23fdc0ac1a701e6fcdb0570", - "is_verified": false, - "line_number": 124 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8f67c9b0527458d335d7a13a7a873cf4c60a9e1a", - "is_verified": false, - "line_number": 124 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "87c637e804ee870548ed69d100bec716baf882d4", - "is_verified": false, - "line_number": 125 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "af42e2da2e9e3772b52a29fe2b5defdca6f43464", - "is_verified": false, - "line_number": 125 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7f0e6ef625a12c06bcd570857fc7c210f84cafe0", - "is_verified": false, - "line_number": 126 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8bbff2d7ca23f047d082f8d0921e337561ec0907", - "is_verified": false, - "line_number": 126 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e278874c2a76905d835df23ba3bb6c416f1c28cc", - "is_verified": false, - "line_number": 127 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e9427c142541757f0dea4825bd52e734e4f2ddcf", - "is_verified": false, - "line_number": 127 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2212141e69165175f9afb09456a4a7f3f2a51d48", - "is_verified": false, - "line_number": 128 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "296ca5dfe2f5f87b56a7c98398feff71396892b1", - "is_verified": false, - "line_number": 128 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5abe82536527e5e803efed4a8a21ba565cee790a", - "is_verified": false, - "line_number": 129 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c9f2e7306d641181c2bd2173c65e264af194f0c2", - "is_verified": false, - "line_number": 129 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0b1029137f43a18ad7c59e4dd0befb84a50d87ea", - "is_verified": false, - "line_number": 130 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6c2fd1d29844b5ac6c1ef752bb0af7d3f12f5690", - "is_verified": false, - "line_number": 130 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2919598fdb5bcfc9d0fe745b3784729b161e490c", - "is_verified": false, - "line_number": 131 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "80baaac32159e84fabf086d2162abbbed06def96", - "is_verified": false, - "line_number": 131 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "09ffa2bd5f9cd927a2e3f4a7c0cdf8a95188f81a", - "is_verified": false, - "line_number": 132 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cbc8ba7fc7dcf55bbbbce47c4a2e9c59338f0606", - "is_verified": false, - "line_number": 132 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9006cbb0400b1dc343fb7cd06f855a631fa109fd", - "is_verified": false, - "line_number": 133 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d7ef704c3053ca297790c60e7017971fde154e3b", - "is_verified": false, - "line_number": 133 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e962cf7f3b15cd09021f6334d6d07906fa779a4e", - "is_verified": false, - "line_number": 134 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ee8a0480a8f9a0752d5c6e0ad8d07fa97fb60e9b", - "is_verified": false, - "line_number": 134 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9c10ac3d14103725c33b6ea6fb274f393a28c394", - "is_verified": false, - "line_number": 135 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f7c46eb71e2178f5b28f98c43629f0bfa0203ee0", - "is_verified": false, - "line_number": 135 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "13aa189261ad630be29f13dd419f827f8bcbec5d", - "is_verified": false, - "line_number": 136 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6b22f49fa1dcb2a31d323068a69ba55c1f5a0b3c", - "is_verified": false, - "line_number": 136 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3fc7f4bf758f33fd60f59926560e2a5ace0ff8b2", - "is_verified": false, - "line_number": 137 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8c5c89f0fce895fd7e5a3d4ec97dc1754fc4a532", - "is_verified": false, - "line_number": 137 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "24af6df06adf146a8e72355e5fb76863c1c6fe36", - "is_verified": false, - "line_number": 138 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a3c4af143728ed6649b2d5d054a5f2e3340915eb", - "is_verified": false, - "line_number": 138 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "43dd4a0c63b53a27f6e83f17fc344401a9eae917", - "is_verified": false, - "line_number": 139 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4b9533044d3d191e1c337dbc6fc7d1c605b5ed97", - "is_verified": false, - "line_number": 139 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c5cd9da76f9ae3906e19e78f45b0d4b86c44917e", - "is_verified": false, - "line_number": 140 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "feac9d1577a8d2b4068fc4cb1d1ad6268bdf1ca0", - "is_verified": false, - "line_number": 140 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2cb98c3d74c2a2dfa13efcfec5e40d7c72eae353", - "is_verified": false, - "line_number": 141 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bb0633ea206c5fb214d4cd0cc26f4e5382b2e0e5", - "is_verified": false, - "line_number": 141 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b89dec3ff54ee7c9b6acc0865271cfba7cbb7939", - "is_verified": false, - "line_number": 142 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fc7310caddcd2d764371efe1e566e363a8a93cbc", - "is_verified": false, - "line_number": 142 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "035e2238c3ad2dffe879481e148e1d7d6b033324", - "is_verified": false, - "line_number": 143 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "792581e67c8809bb0e42930cea251271e45a3d99", - "is_verified": false, - "line_number": 143 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "733184d60ae66e2f356ff240644e9b59c3bccc03", - "is_verified": false, - "line_number": 144 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9905ab27f1bbd7343dd5f3039334f49296b83f0e", - "is_verified": false, - "line_number": 144 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6a59cfda48573838a97154904300bc2c822b8081", - "is_verified": false, - "line_number": 145 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f925806d053c468c0605c61eb44fbd375acd66ad", - "is_verified": false, - "line_number": 145 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "03f5559ae6b93131bbfcc8e6d2ec6e767c630b3e", - "is_verified": false, - "line_number": 146 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c9ac7a0776407f0a34dfc967146bd8f1c828bf9a", - "is_verified": false, - "line_number": 146 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "46f90fa2716f055d2e5af17ff7225c2208cb9874", - "is_verified": false, - "line_number": 147 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a33133d561bf32724b5901604789458032527069", - "is_verified": false, - "line_number": 147 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6e266cc65f886c19840d2d6b400be6e85c51bf55", - "is_verified": false, - "line_number": 148 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b6d98d1f0ca349ed761b735481a2dda895f29bca", - "is_verified": false, - "line_number": 148 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "11f30aa2db598090f26d3c6d5a4d27229d8a8374", - "is_verified": false, - "line_number": 149 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "91d456862a28c209d1b6cf6e628715555fff1e64", - "is_verified": false, - "line_number": 149 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7f48cb4fea8055120031fe8da18d54b4d19dd40c", - "is_verified": false, - "line_number": 150 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e8f9f0b142581d30e2dabc5c77929c860f1d5216", - "is_verified": false, - "line_number": 150 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2fcc671a4e7850c9be6ab6a2f1dbaa50b1243652", - "is_verified": false, - "line_number": 151 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "81c42a742059a9e8ec4cd4ed85f8dfacc5287b71", - "is_verified": false, - "line_number": 151 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8f55d068fd232092cf97e4f8a0505413f9c6349c", - "is_verified": false, - "line_number": 152 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dfdee1ae12c120aa25fa48774db1f2bddc680448", - "is_verified": false, - "line_number": 152 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d8d1ac855caba26db06bd4ba20162eea66ab355b", - "is_verified": false, - "line_number": 153 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f33187b400aee38959187a46d157d7bbcd461e26", - "is_verified": false, - "line_number": 153 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "16e0116c9c932679d68bdd8874afc8cbe3dc915f", - "is_verified": false, - "line_number": 154 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bd342979b2d35648dbd63b045a05c050d7bac508", - "is_verified": false, - "line_number": 154 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0858b986c99dd38ab7d287d249bce2c5a80f3939", - "is_verified": false, - "line_number": 155 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "67c75eff8fb72ff5f57fb487536f6cb57fc11338", - "is_verified": false, - "line_number": 155 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2aba6defc8670e3951c2f4e62bf901a80ed85b24", - "is_verified": false, - "line_number": 156 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b0257d866b1d172aa9abbcf1698a760550d6ecf1", - "is_verified": false, - "line_number": 156 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "01a11a34e0d8d6dc8f2e7ed4f7c6234d19c89ac0", - "is_verified": false, - "line_number": 157 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4619bb3323078a7ae811967d056218b7201378ac", - "is_verified": false, - "line_number": 157 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "08efa9681831133a6b54cd50e529b3e2abee3882", - "is_verified": false, - "line_number": 158 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1e3a4434d49dd244413c6b4c0c725b8fc484b150", - "is_verified": false, - "line_number": 158 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "10d531c7be3110dd561fc071ac91b105c3e97778", - "is_verified": false, - "line_number": 159 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2ddb9795aeac1e085b4fcaaa17c6540899fe5e21", - "is_verified": false, - "line_number": 159 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3149dcbf2c1078c005f8819f7513e3e599dcf356", - "is_verified": false, - "line_number": 160 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6efc34bcc6b8056413592e0f4fe0a5134947f6a6", - "is_verified": false, - "line_number": 160 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1607a39799f542cc24b66741bbef1175b19577a8", - "is_verified": false, - "line_number": 161 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5d6af6b30bc405bb256cb326c3dc626a0db1adf0", - "is_verified": false, - "line_number": 161 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aa88fc8eccbde98b44415c41be15aa6d83e196eb", - "is_verified": false, - "line_number": 162 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bdab846c867ae5181b6919e1df68798af27d051d", - "is_verified": false, - "line_number": 162 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4aaff86e991396368639d83e635b28528a385b8a", - "is_verified": false, - "line_number": 163 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5b4f40442b106049054501decfafaaae6684e068", - "is_verified": false, - "line_number": 163 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3f8d79b46c31d2400ba332d882105188939c0d84", - "is_verified": false, - "line_number": 164 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b04897764142130a2aedc95b620491c28510c86e", - "is_verified": false, - "line_number": 164 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3565d60dc4a43aec783eb5a5927ff74e3839e1df", - "is_verified": false, - "line_number": 165 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "877a89ba60d4a41b08e12a34325cff73422487b7", - "is_verified": false, - "line_number": 165 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8454599e213e243b796c1f0d30fddaf496eeb34b", - "is_verified": false, - "line_number": 166 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a0e743272ebb27dc306a7394e7fa04cd73fc2fa4", - "is_verified": false, - "line_number": 166 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4374a229210fd28e68c54449cd5d5cc5442d39b2", - "is_verified": false, - "line_number": 167 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4d17315d467a4d78d138ffc71f64305eb203b00d", - "is_verified": false, - "line_number": 167 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2ec605a3f80edbc147ec37e891742c97c077e5ad", - "is_verified": false, - "line_number": 168 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9ff793db2f96f0e4934c7e4cac100b2841086b39", - "is_verified": false, - "line_number": 168 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7eb0fdfcd3126e04678ed48034865547312dc1bc", - "is_verified": false, - "line_number": 169 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "adbeca08e3d9e374a8b7ea7cde05fdd3573e1106", - "is_verified": false, - "line_number": 169 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4c4bee1bc85c3d6c3a60847b26de257ef523ab6e", - "is_verified": false, - "line_number": 170 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f3cb88dd8612244747c5531c5deb10a91dc9879c", - "is_verified": false, - "line_number": 170 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "076886bac88c84c67090146f1dfbb52e2779d2a4", - "is_verified": false, - "line_number": 171 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f86b69a8e5200a18c1f4927a9a0dd1f3f7448e76", - "is_verified": false, - "line_number": 171 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7e52867432531bc8d0a156799ec715a6a73182e5", - "is_verified": false, - "line_number": 172 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d514b01c3c0381f49044d49a1b57bc12feddd3df", - "is_verified": false, - "line_number": 172 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "543a8c351868d359144f8c13763167cfbe953c88", - "is_verified": false, - "line_number": 173 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fd6d1527b2cc418c8489843ac5815a5455025355", - "is_verified": false, - "line_number": 173 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "30f7fae10c90ed0c1277f6df3293fc000d39cb2c", - "is_verified": false, - "line_number": 174 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5b0cc46b7ebbca1b817f72811013ed9879c88abc", - "is_verified": false, - "line_number": 174 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "02b608c3c42e60d514d500d8580198125756db6f", - "is_verified": false, - "line_number": 175 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4b383d4c6de8c689f2907b71643b6184a3f2d7a0", - "is_verified": false, - "line_number": 175 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2f35209929fc129ab819d4d58f55e179d80b8c54", - "is_verified": false, - "line_number": 177 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "85c4c4e08f0e7665ad28a87ee385e582cc14bf77", - "is_verified": false, - "line_number": 177 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "16d9d1c3e7d2d7bc3f433d2a2402cebb3a2ca5fa", - "is_verified": false, - "line_number": 178 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e4fd0fb865964669aead61d5b0f110c02fee107e", - "is_verified": false, - "line_number": 178 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "065e64bc498b9a08ff651390bd642455462ca23f", - "is_verified": false, - "line_number": 179 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1ac9bf495d97947dd4b9257b4033ccd73fcd54c8", - "is_verified": false, - "line_number": 179 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "001f9af0bfb6a207e229dd0791d26b33e76eec5e", - "is_verified": false, - "line_number": 181 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "344469c8c542d560da3a97d7cb8394e6c1c29d29", - "is_verified": false, - "line_number": 181 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "590a3665555769742a4905586e14f5bbddcbfcf2", - "is_verified": false, - "line_number": 182 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7a1e6d7ebf16ed40a317f755afff1ca78302cf1c", - "is_verified": false, - "line_number": 182 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8100b77934e7fb54d4b16196b6415876f34f51c0", - "is_verified": false, - "line_number": 183 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a4b711bf52cadbd43a39bb0756f7db87272796ed", - "is_verified": false, - "line_number": 183 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "00cac3bf1292a90bd03e4d818065c04b8e1edfa5", - "is_verified": false, - "line_number": 184 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8b273d480d5a397803c647a9748bdabe40c16314", - "is_verified": false, - "line_number": 184 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6d010656b1201b17773d2b614c148f491df0a626", - "is_verified": false, - "line_number": 185 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c424ca12ff9944303fe0748dcc865a051b7861bd", - "is_verified": false, - "line_number": 185 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "93203c118b8f0049b131e53461206d7c81524dae", - "is_verified": false, - "line_number": 186 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dc183210c3ff0ce4386e97e131195b1858b18701", - "is_verified": false, - "line_number": 186 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bab2c5a57ec5a6a967dc872346aa37a2191f259d", - "is_verified": false, - "line_number": 187 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d29c3d1487be1a2338ad6ed63dc985a0ed09caf6", - "is_verified": false, - "line_number": 187 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0c4bcd7af14be1a9608bef87f1b68192549778b3", - "is_verified": false, - "line_number": 189 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c298c297f1fa1a0a52ea758dce8f1f974e2a86c4", - "is_verified": false, - "line_number": 189 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d4d5d795de4ef3d556687c9fbbcd60a0a91c0598", - "is_verified": false, - "line_number": 190 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e59d48000d6be5fe0c3033f63fb21af04b00ad43", - "is_verified": false, - "line_number": 190 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5468f108e8611f72799711808fc082cabfbe06bc", - "is_verified": false, - "line_number": 191 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fee7a1852415087beca5a5c86ed3176255d7d836", - "is_verified": false, - "line_number": 191 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b875193d57fe6b72294c71898e83ea27b8297e2d", - "is_verified": false, - "line_number": 192 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "efede22e984c1ee56060ea70868daac13854774c", - "is_verified": false, - "line_number": 192 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6bdc346066b3f504e9af686d03c077ebcbc48ee9", - "is_verified": false, - "line_number": 193 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f62616ac060fe7005a4723b4d383b5daec5fb135", - "is_verified": false, - "line_number": 193 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "481c37cf988af93549a4233b4099f1f1c4ee3f28", - "is_verified": false, - "line_number": 197 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cac7e7243d2d7fa9a1f9b155d2daddd63898944c", - "is_verified": false, - "line_number": 197 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ec2646a4e62c055bc8871237f967ad83e3a2ee1a", - "is_verified": false, - "line_number": 202 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f49f6ccefbe938a3ea5c8eb77bab3ff2e2833486", - "is_verified": false, - "line_number": 202 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7c85ac02397a5da644ab05ff603fdcc360efca64", - "is_verified": false, - "line_number": 207 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e24818399cc55db96459817a84cd43144ac25113", - "is_verified": false, - "line_number": 207 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "68bb4b0e2bf90eb1091ca4c9b23eda607c2e0aa9", - "is_verified": false, - "line_number": 211 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ae2119a9b9e08b1948e4e2b821cc26cc8b7028b4", - "is_verified": false, - "line_number": 211 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "00ca85bb2018ba41a809c9e752e0f47b84520d8c", - "is_verified": false, - "line_number": 212 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a8edd3649779144880603e41be21339cba2ed4a6", - "is_verified": false, - "line_number": 212 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7914866336c781ca3e04c85174391d281afd21d9", - "is_verified": false, - "line_number": 213 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b1ec47a2a1192f80aca3619c259bea6b94452239", - "is_verified": false, - "line_number": 213 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "39eb7656db5d93aa9666596fdbd4929bdbdce7bb", - "is_verified": false, - "line_number": 214 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7ac612273867a0fb8d4a05ce3be976faa839051f", - "is_verified": false, - "line_number": 214 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b226baf1faf562c2f490934584a02f2e23aacefc", - "is_verified": false, - "line_number": 216 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fda5ce46fc2f82dad8c4d14fb639691310d093b4", - "is_verified": false, - "line_number": 216 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0ec00ee98fa24da5f771b784f0fd694a73c3db74", - "is_verified": false, - "line_number": 220 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8d9dd9ad8ac461328e35b246866a8624cfce0f75", - "is_verified": false, - "line_number": 220 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "16796a99ef5b45bb2a11d44929d628dbd0dd2bab", - "is_verified": false, - "line_number": 235 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e9555e31d929fddae3ad7c38a4ed939eaafeb78d", - "is_verified": false, - "line_number": 235 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7fdecdfad1513692754db95dcdc5e181ec7ca674", - "is_verified": false, - "line_number": 236 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "faac37e145afe8853bc13aef72c753f592bce5f2", - "is_verified": false, - "line_number": 236 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "12f5876fd08ee0d6993d4495678dc26e08efcead", - "is_verified": false, - "line_number": 237 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b6b70e0b1d4c5e959cf2f59c0b8d3506b571856f", - "is_verified": false, - "line_number": 237 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "878ab1593b6a9cd026223e6430d36496f965f773", - "is_verified": false, - "line_number": 241 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e0facaaa720d85f59e98dfd4f60d6fc794e66d19", - "is_verified": false, - "line_number": 241 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "61fa03bc68dd394f1d7aac0445bcb6261d26bf29", - "is_verified": false, - "line_number": 246 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "89d3e930e9fa1b94fd2101e06184642937dab8b0", - "is_verified": false, - "line_number": 246 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6d63bf74c84a32b2eddbef55b9dbc9780ce99cdb", - "is_verified": false, - "line_number": 250 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fb296cf612e00780b2ad026dabcfa041ec1dec31", - "is_verified": false, - "line_number": 250 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bcf200ae55540cddc396842b75593b625f105d04", - "is_verified": false, - "line_number": 255 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fa3e8aed173e674d4becee281d587a819f2f4279", - "is_verified": false, - "line_number": 255 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7fe4e68723284f93ad0083b895f661b826c00343", - "is_verified": false, - "line_number": 257 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b14e20c5c80edb2827713d15880841470947de6e", - "is_verified": false, - "line_number": 257 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "64a4c1bf493f575c9f60ce367fff5e6c5cd90998", - "is_verified": false, - "line_number": 258 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "85cae3fdf761086fc3123023cdd78ff75f65a005", - "is_verified": false, - "line_number": 258 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1e05e6b9d480f2a3abc3772047aa78f3543180c7", - "is_verified": false, - "line_number": 259 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3c8bf5bce896909a5c3d651621031b0d491c52d2", - "is_verified": false, - "line_number": 259 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "10f3cfcecd9d7f888927d598e384c6657ed5fed0", - "is_verified": false, - "line_number": 265 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dd173f9a5d41daf44cc2e0fc3f2668d6d8d7687d", - "is_verified": false, - "line_number": 265 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "746327e68a89d4a6333ad8fd4b273ce86c1396f1", - "is_verified": false, - "line_number": 266 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7df76ca3f95806281b44b04cf6b2707c3dbbedf3", - "is_verified": false, - "line_number": 266 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6c731be66ca6a42269501267a3deb1f33a112176", - "is_verified": false, - "line_number": 270 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b5890f7074f1d22a34b364181e8d5466c4592692", - "is_verified": false, - "line_number": 270 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0c02df6c26058bc5dffdffbf5eb06585cb5396f4", - "is_verified": false, - "line_number": 271 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "760e30c1ce29c7a5a0bd1db7520aebbe89acf35f", - "is_verified": false, - "line_number": 271 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "820f415fb3ee5709fce928bb3b6063722f0b8e33", - "is_verified": false, - "line_number": 273 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ff2c89505aebb7f9359ca62634028af69f5b114a", - "is_verified": false, - "line_number": 273 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "791749b6747b839943bb02208b9858f1073e2353", - "is_verified": false, - "line_number": 277 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "96a018050f209ac7212a89ec7a1c0d2b301e365b", - "is_verified": false, - "line_number": 277 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "93c6deef2a460a167201603885b6dbe9c7b2636e", - "is_verified": false, - "line_number": 278 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "eabd8b995cc3cde5f13d864138a7a761057efdb5", - "is_verified": false, - "line_number": 278 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "db18aaa8c7a5e5ae08e42bddc33cf07e59268355", - "is_verified": false, - "line_number": 279 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f77eeda6e777399117ed067ae0a1c78475d97cc4", - "is_verified": false, - "line_number": 279 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0f3e22c9d2236ee3ac24744fd074abadc17b23a9", - "is_verified": false, - "line_number": 281 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "918ba5a3146ede01ddb3e5c27353c9bc29eacd51", - "is_verified": false, - "line_number": 281 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0235c8942f4927df8009f4f5c64a1ca302237b4d", - "is_verified": false, - "line_number": 283 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aeadbf70f96cba43d6c3f137633c3215f571b3cf", - "is_verified": false, - "line_number": 283 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9f359f34dac4853e0a099d6c5c9adeb265796e43", - "is_verified": false, - "line_number": 287 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e8a44ac967d4ddc3728cc49a44a1bc00009dc3bb", - "is_verified": false, - "line_number": 287 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1d0c6043c5eceadd52e1f62a31a91a1a9f3a4fe9", - "is_verified": false, - "line_number": 288 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1fcadc54721436d58af56057c7baa51e50f626de", - "is_verified": false, - "line_number": 288 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a6c9aaa74edd9d7cfb26217785518cf96dfc0618", - "is_verified": false, - "line_number": 289 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c2e089b8a52846771aed8b5d267d19899e15d9bd", - "is_verified": false, - "line_number": 289 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "afcb0b37d3406bf3eda66f21faf4c77e40296149", - "is_verified": false, - "line_number": 290 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e96deddd62063d8cee786fc49d7b3b6ef5ff7535", - "is_verified": false, - "line_number": 290 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0b6d301ef03f5ebc8af553b5c63ccd370f0e3cef", - "is_verified": false, - "line_number": 291 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "517ee7509f19944c16c9fafba01a8ea8a52ed7d2", - "is_verified": false, - "line_number": 291 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0d554068d3090ad64999d6986d799ef0b28fa5b3", - "is_verified": false, - "line_number": 292 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fb9f2a9cec5cad2bb1bdfb8034cc114a1776dc3b", - "is_verified": false, - "line_number": 292 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2246be55e20e8bf2525972626a3b369899815b22", - "is_verified": false, - "line_number": 296 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "45c1d71be972c985821fc38e32cb47b71e94e853", - "is_verified": false, - "line_number": 296 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "29d0b7834ed39e0ccd2162b6ec3ba42564edd0c4", - "is_verified": false, - "line_number": 297 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9c1cc093d52bb022729554c1be0793d26224d530", - "is_verified": false, - "line_number": 297 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "747bd537361a2dc342deecb34d2e7f2f5980e202", - "is_verified": false, - "line_number": 298 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "da0998fbb4b1e0630b67583768f180fefca6c755", - "is_verified": false, - "line_number": 298 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "01fc8614152bb131f54839fc4e9d8d8d0a337090", - "is_verified": false, - "line_number": 299 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9407f02e8977f4cf8f0e8938c77ea0b9f3f6653e", - "is_verified": false, - "line_number": 299 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0c0148ae0d3eae4ad7c1158bcbbee1e81cea3b49", - "is_verified": false, - "line_number": 300 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "61b5d154f22973ecb9b5d10081acb17216ff9769", - "is_verified": false, - "line_number": 300 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2ef39d5da7cd9e6364d6abccd4f9ac96e6f05c0c", - "is_verified": false, - "line_number": 301 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4694ff09472459c2962a6f452cb1e278a99e11ad", - "is_verified": false, - "line_number": 301 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "64ab4aefb3c0ee85bc2fa52052c9700d385d22d9", - "is_verified": false, - "line_number": 303 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9d1f5f7dd99eec22ad1e2f549567d19b9af51356", - "is_verified": false, - "line_number": 303 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "444da910a02204dbbd70f9fb18ec9bf1a8be8d50", - "is_verified": false, - "line_number": 304 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a2bd608b43f65ef77fa3ef6bf7de018dc056f0b1", - "is_verified": false, - "line_number": 304 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4b278a108a9d9412f096016750c46287008c9134", - "is_verified": false, - "line_number": 307 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "805832529eb6bc6c54d6fe1e35a3f62aeae14874", - "is_verified": false, - "line_number": 307 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4e65dd37a0bd0b9bc27fb7f91de63e2746892061", - "is_verified": false, - "line_number": 312 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "935bc601508e511beeadfa5e7f2695614cbbc967", - "is_verified": false, - "line_number": 312 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1bca3f86fe30b4a736e6170c03f5ba2111a59848", - "is_verified": false, - "line_number": 314 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c7bca3808444467f254d253b022f989f11784a93", - "is_verified": false, - "line_number": 314 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "603e2336059fcd1530d52dbdf38de5a3d2a3e544", - "is_verified": false, - "line_number": 325 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "be1ac3530a1db91ceaa30035755248dd6422eb84", - "is_verified": false, - "line_number": 325 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7e105a8ad86d9d95640e06abc4aa99f05840df57", - "is_verified": false, - "line_number": 327 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ed686ee51822ef75f6e3e8a44b26f309f1898893", - "is_verified": false, - "line_number": 327 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6c711b42df1bb016c54fc56c840eb004d640f62f", - "is_verified": false, - "line_number": 330 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bb9ea2ffc605363652352ede59a32f19687243d0", - "is_verified": false, - "line_number": 330 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "726ff7d3765631a443731e1aeb9957d5e285b0dd", - "is_verified": false, - "line_number": 336 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8e98da4aef56bc393690327c716e729c8f4338b4", - "is_verified": false, - "line_number": 336 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cc77bea4847409e02f410e3f63a3d2661bca06b0", - "is_verified": false, - "line_number": 342 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e53fa86661b9e8ffa3819f75ad609cd0717ad4b6", - "is_verified": false, - "line_number": 342 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "31e3b9d565ea374cf99e1ae3ed68be6a0548b8a2", - "is_verified": false, - "line_number": 346 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c56ebb10fd71dedc5c0bd6e7c1221f497b8e01af", - "is_verified": false, - "line_number": 346 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0b79fd0f0d1ceb1b0d9af96d1cc3e5fad940fb98", - "is_verified": false, - "line_number": 351 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "12550631dc2b8e6fc452f2e8522eec85e6282a15", - "is_verified": false, - "line_number": 351 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d4af480158e2df2f1c4801c76e56c0365d9b71fd", - "is_verified": false, - "line_number": 353 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ebb571094ce833ef63223bd427a807a643bb9b81", - "is_verified": false, - "line_number": 353 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "15de5fa328d02f8cf8cc553747abcdaca722d25a", - "is_verified": false, - "line_number": 354 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "975b331ce39202c899492eb6397c7231d826bda5", - "is_verified": false, - "line_number": 354 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0089684dde0d13ce3e5123c56fd3a5e041318a8e", - "is_verified": false, - "line_number": 360 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5ed0a7b57cb657cbc0c3362fa6c4810b9caf31ca", - "is_verified": false, - "line_number": 360 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9e015014d6de908154cfeddb7d3fe3247791dd24", - "is_verified": false, - "line_number": 361 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a819170c76c8839e6f21c346fde6cd79f5d03706", - "is_verified": false, - "line_number": 361 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c0cfb61aa8c800db1185c2606136c60418a33cee", - "is_verified": false, - "line_number": 362 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "de137c016badd32e96c77ded0d165984283ee435", - "is_verified": false, - "line_number": 362 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "07bd79d9ac01c8f829dac476732ae2c4fda42252", - "is_verified": false, - "line_number": 363 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5a9e66ff59e6d3e5c60fdb1f9af1cc98958c9c9b", - "is_verified": false, - "line_number": 363 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "10626ac2219cce1b2c960591ad4d210f7e2e6af4", - "is_verified": false, - "line_number": 364 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6174fc0b38cf830007f0c014a4ab5474559fe2cd", - "is_verified": false, - "line_number": 364 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c8487f58ab9e612fd22ea6012fefcc36e2308eba", - "is_verified": false, - "line_number": 366 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e402326cdb35aa451fee0bd17b26bedb0e26cae3", - "is_verified": false, - "line_number": 366 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b19c1c77d05d9023601d6a4887d807389945c921", - "is_verified": false, - "line_number": 369 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d21fa2e3f0b8e9cf42780d686a9cb8bd0641c17d", - "is_verified": false, - "line_number": 369 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1665de1e0fb41349bf5ce4ee3b4a7c89f6809b44", - "is_verified": false, - "line_number": 375 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "df96b348280a86c15f38ee6759e39bbc4c469c6f", - "is_verified": false, - "line_number": 375 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "61ea21c752e153c8379bcba3bb269f901c400c68", - "is_verified": false, - "line_number": 377 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e31b729d066533f332bed03d9c326e06e80dff70", - "is_verified": false, - "line_number": 377 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "723e1f2b6208b596db3942461ce7527f154152bc", - "is_verified": false, - "line_number": 378 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c3c9b33ab219ba18c38579b99d94868a781411ab", - "is_verified": false, - "line_number": 378 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "06540955ce719f73230e1f718f896524fee0e64c", - "is_verified": false, - "line_number": 379 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "530cf670ccb950ea1bacad235e9901015aed3299", - "is_verified": false, - "line_number": 379 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0b80fa65c99d5edc86b5ff5321f002e128dc3b92", - "is_verified": false, - "line_number": 380 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fb84fdb13286ae5def6154cb65d7d64c31d87646", - "is_verified": false, - "line_number": 380 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1968665ba366a0ae4a05c14a68cbf12bbe02b5ca", - "is_verified": false, - "line_number": 381 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f7baaf2b2d272e8a51dfa35415b0af47f7f84d6b", - "is_verified": false, - "line_number": 381 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c2e6ba82f3384296519847ed2f7925b858cb237c", - "is_verified": false, - "line_number": 382 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c4c6df0579ccfb18d0a6d7ef53a3650c5f377cdb", - "is_verified": false, - "line_number": 382 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1fb6e0c58f94a6f22b89bee3a197e486b604db63", - "is_verified": false, - "line_number": 383 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4cbb2ad6ae9e18d1995b6891a93207ea56bc543d", - "is_verified": false, - "line_number": 383 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4ec3f04871ca2568b6894478b709902490c0d4ad", - "is_verified": false, - "line_number": 384 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "57093a7dcac3ceb408d86f111fec71550721a6ca", - "is_verified": false, - "line_number": 384 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "04053ceecc10b969bb49a80135f085f81c0b0b9a", - "is_verified": false, - "line_number": 385 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "88ca9ea762a00cb263211007eb78f39b416054a8", - "is_verified": false, - "line_number": 385 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5116f3ce0230a4f5aa1ea349d529a8c5adea42b2", - "is_verified": false, - "line_number": 386 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a47ab14a1f98de2e4697dc85eb21374c245bea6e", - "is_verified": false, - "line_number": 386 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4a9c1eb5b2134031bac5ab7a83a9889fcad8437c", - "is_verified": false, - "line_number": 387 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f10b4b6045ad4ede7bd8e55726e3393238016a27", - "is_verified": false, - "line_number": 387 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4ce856e98be06bc139968c1cd1bb100a7cb62c9a", - "is_verified": false, - "line_number": 389 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d58ccbfcbf73fba08357cf292b77833f11795cac", - "is_verified": false, - "line_number": 389 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "160048cea3039816bf266d9134a94b6bf164fbc2", - "is_verified": false, - "line_number": 390 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c70d951883dff2337ca200db23d903a7259c50a6", - "is_verified": false, - "line_number": 390 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0dd6d1be28054c44bb456cf2a2100ab915f0bdee", - "is_verified": false, - "line_number": 391 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d20eb21e5eb98eeabf58b2db607e45c9ef10a05e", - "is_verified": false, - "line_number": 391 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4b330a5c11af6e53f73c539663e037ebc6eab32b", - "is_verified": false, - "line_number": 393 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "77a2bf50a1d4c84fb92a1673914244cd4332cfda", - "is_verified": false, - "line_number": 393 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "40c3500dde24d9232c66ab713123e8dd1eee729c", - "is_verified": false, - "line_number": 394 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9d2c8e6e7b9718b1a498379746aac2c38ec646f0", - "is_verified": false, - "line_number": 394 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3615a235e5e1180d8f4497acbec5dfcb8ba00e47", - "is_verified": false, - "line_number": 395 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "99e4c3ab3e2d1a0acf5eb89e0c826b0c51391d0c", - "is_verified": false, - "line_number": 395 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5787d18a318a5f6464ecc165eb584bc8e00fca12", - "is_verified": false, - "line_number": 396 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6e61a65d1ac2417205ce7fe787638134a9f3fb6c", - "is_verified": false, - "line_number": 396 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "54e5e9a62ad965052b24eabe800358ea2684a720", - "is_verified": false, - "line_number": 397 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e55095702779e068adfbd90c0a10365f58b1fefc", - "is_verified": false, - "line_number": 397 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a0574918b428c60e131b4fd0fcd05497c1fda2eb", - "is_verified": false, - "line_number": 399 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dc2124c9099fed447ef23d7a40e3b6b8e9478738", - "is_verified": false, - "line_number": 399 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "80ca06f1ac15f58147ff439dc213660e1c0a5c23", - "is_verified": false, - "line_number": 407 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a8310d532c6751b048ae765fbff956b5cfa471c7", - "is_verified": false, - "line_number": 407 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4d83718be60b7211613611b335cd255053df242d", - "is_verified": false, - "line_number": 408 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a4a9a273b8b80650f85bd3c626644fdd5c86dcc5", - "is_verified": false, - "line_number": 408 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "21f0ca9f31b15c924c6c9163fcb4a535f3cd03fd", - "is_verified": false, - "line_number": 409 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "59339704831756e99874a322e3321ac69e2997e5", - "is_verified": false, - "line_number": 409 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2928152ae94de2bd81ad9a004253b5150f636cdd", - "is_verified": false, - "line_number": 411 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6f260139b77fc5501d7d0da4ecda5bd9fa052920", - "is_verified": false, - "line_number": 411 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8ea025d76a8cca6404c4ff143cc2f9107c2ebbfa", - "is_verified": false, - "line_number": 415 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "960d3ff90f92300a7b257b01a59b15a43c67a550", - "is_verified": false, - "line_number": 415 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c138fda37d041e4847dae976f96785ba3d0746c3", - "is_verified": false, - "line_number": 416 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cdac69669a403cbab50757e6c1da9f38602d18d6", - "is_verified": false, - "line_number": 416 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "52f30efaf64df16e03a84e098a01293c2d0cde59", - "is_verified": false, - "line_number": 417 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bbc5bacaa1b62ae1ab38a58afef93b5af0f9c7c4", - "is_verified": false, - "line_number": 417 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d28a5e0ac52374c03ce8bdea42969baa87118164", - "is_verified": false, - "line_number": 418 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "eb9ec87db96c81d555e9c771ac2ffe645ab92bd7", - "is_verified": false, - "line_number": 418 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0db435d6492071b9458b4b0aca99fca1ad4210e8", - "is_verified": false, - "line_number": 420 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1f0349aced127f1dc486b425b041ec6d71ef95fa", - "is_verified": false, - "line_number": 420 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a5d50ddbfbbb92f192d7cc08e4ea696696c20248", - "is_verified": false, - "line_number": 421 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e7581147c98ca11e1f2bafadc9cd6c785cb234cb", - "is_verified": false, - "line_number": 421 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "484a3c1bc050e3730d977bf84a7008ff07e4eb44", - "is_verified": false, - "line_number": 422 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "eed6d135f1dc611b66faa6728c8aec2d7e52f2b2", - "is_verified": false, - "line_number": 422 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9fc21bcc6b27664ec919f38968b98ce68585bed1", - "is_verified": false, - "line_number": 423 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bb095aac04eb551aee2cff6c4b7ca065b2a79d4b", - "is_verified": false, - "line_number": 423 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "08ffd9e2d68fb46370b932f619eaa08e5a773071", - "is_verified": false, - "line_number": 424 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "37d9b3210c412f29a007624b995c9188cb14cd8b", - "is_verified": false, - "line_number": 424 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a3ae565156eda1a752d5c959617c811e60d27e2a", - "is_verified": false, - "line_number": 427 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e026336b7a7ca8370fdf8a5006d75c44e963e75a", - "is_verified": false, - "line_number": 427 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0134c130ebd1f3f3038b1115e17fabc664ead36c", - "is_verified": false, - "line_number": 431 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "034386ad15c37e5978ff71b03b062efdf0e6b6a5", - "is_verified": false, - "line_number": 431 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "28f3dbcb99f8f17dc0d38504c5eb71af9a2b4d5c", - "is_verified": false, - "line_number": 434 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e60430506f2e0765ff42dee1e6c9f85022f08a69", - "is_verified": false, - "line_number": 434 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "67176ceda42663a85fa1ebb104adac549dc3a148", - "is_verified": false, - "line_number": 435 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b81ea665fda5332f20a776a960cfdcca6e02ab60", - "is_verified": false, - "line_number": 435 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "241336fa79430003f523d3ecd70388e7cdbf7f28", - "is_verified": false, - "line_number": 438 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3351c9b3c45e1c50e510f533a7ff44ec303c1ce4", - "is_verified": false, - "line_number": 438 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5e5ded71f83c990bb02501943befb9e03acf8df2", - "is_verified": false, - "line_number": 445 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "603cb21d95f6ef58b3031f18237ff3e8f0c80349", - "is_verified": false, - "line_number": 445 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8ea9924ef39b55f31177f2166540f5b5f843648f", - "is_verified": false, - "line_number": 446 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f0f546b61165a022d4fdb30c26990bec500db71f", - "is_verified": false, - "line_number": 446 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3bc7baebeae674de46195c2e2eeaadeee3c981e6", - "is_verified": false, - "line_number": 451 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4ca523f682d90d36d5436b5e2cc65a71a132d795", - "is_verified": false, - "line_number": 451 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2d5c22895d86b8e20ca249da430e41a0243a6601", - "is_verified": false, - "line_number": 456 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b580d42a1f496655bcd96d12f3bc3968ee6c6a76", - "is_verified": false, - "line_number": 456 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "876b6064199c42946cbebcef81419097b5a20f45", - "is_verified": false, - "line_number": 457 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ed5e8f8d2429b6f59bc1528392e676db53855c81", - "is_verified": false, - "line_number": 457 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cc4524f283cce97da26332159dcd9b8cef0c52ed", - "is_verified": false, - "line_number": 460 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cee7c60399b6b1695d0edbd603f84d1dfbc036d8", - "is_verified": false, - "line_number": 460 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3277dd9180968501e33ef1c9e1d44775e03a0f86", - "is_verified": false, - "line_number": 467 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "82ffa1dcef55b4e9604eaa46905af65661949134", - "is_verified": false, - "line_number": 467 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5bfb315d9a38563531cee6e9129b24efb43e2309", - "is_verified": false, - "line_number": 468 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5ee6de921d6ca2088ab8c49abca05b19e6e81440", - "is_verified": false, - "line_number": 468 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "87b20b76c060d6bc33b751f81358bcf49b413be2", - "is_verified": false, - "line_number": 469 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f81920baf2c2ce6b61cceab7b533e70aca2a1a02", - "is_verified": false, - "line_number": 469 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b48dc8128b164303ef6b1a1d6eefbbbe15796d57", - "is_verified": false, - "line_number": 478 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e4ce5e47f8f1f45c2503b4ded58de5bbb02ba815", - "is_verified": false, - "line_number": 478 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1e14bbbf8a8499cd8e73647350b5ccc5040de0c9", - "is_verified": false, - "line_number": 494 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "25d4169a0831828a57bf2371e45c1f70af2b8d18", - "is_verified": false, - "line_number": 494 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "25854ae8f95263b6f4a1a8433ecb140634188213", - "is_verified": false, - "line_number": 498 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c6e779ea7b0d8aa669de96a0bd825a6f060323fd", - "is_verified": false, - "line_number": 498 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "744c1e761909d6312406a8ae23a40205ae6d1bd6", - "is_verified": false, - "line_number": 499 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "804e3480a8fa256f5ab590a9aa2cbcbf1d2a08d8", - "is_verified": false, - "line_number": 499 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "63a5e715ad080c032490269b390f1beaa668e265", - "is_verified": false, - "line_number": 504 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "657edca2ebda0750da9ad5964cbcae95fa7f19af", - "is_verified": false, - "line_number": 504 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cd846789d440edce459c028415ffbb38b173979f", - "is_verified": false, - "line_number": 509 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e0f09d75e6f0e8c6c42f461fe117d3af2584016a", - "is_verified": false, - "line_number": 509 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "18a333f59c7c8f89252f327063a3e69e7807b82f", - "is_verified": false, - "line_number": 510 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ac8d945911064fa6f5af1c6ef78e443e279a4649", - "is_verified": false, - "line_number": 510 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "12d24cfa2a77b84e1862ceec929363f46f646911", - "is_verified": false, - "line_number": 511 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "25c1f20145f80223e467ead28e966aca36e873fb", - "is_verified": false, - "line_number": 511 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "57665dad2eeed1d86ea60c5a8c4980a08f929955", - "is_verified": false, - "line_number": 512 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6b6c6965418255e3b27901ff8f0f91a4d18134e8", - "is_verified": false, - "line_number": 512 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8b72a67ff7fcf80f80c381addff9b7a21804c888", - "is_verified": false, - "line_number": 513 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ceae2b71965b980e6ae40316b08fb5d974942cdb", - "is_verified": false, - "line_number": 513 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0d553608771d7b2717610f6766e518fb09db8fc0", - "is_verified": false, - "line_number": 514 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dd5a32b212349118d4b76102b09822189bdb5fd6", - "is_verified": false, - "line_number": 514 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aa633011e24c42340257d6938733144d40f53189", - "is_verified": false, - "line_number": 519 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b9b181f5428370e37cc104b9cf5103a87b76d632", - "is_verified": false, - "line_number": 519 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0ec02cdf46d39d6eaee1c1b40be7e631e40f138f", - "is_verified": false, - "line_number": 520 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "822917c9179c49ffe5bbd7a81c468b1634794d12", - "is_verified": false, - "line_number": 520 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "668bc8e83392b07478e54765f7311b3b4a042e2e", - "is_verified": false, - "line_number": 521 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d736a1dc0aaa47da944e17ed61ad9808c67de7de", - "is_verified": false, - "line_number": 521 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "58b91f872323aa52c1c00cd907bf48a8e44072e1", - "is_verified": false, - "line_number": 522 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f66f815f15efcb52278d1d3f23b91100ad36f553", - "is_verified": false, - "line_number": 522 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2d58772a2ad8edd7abb1d649a059b768f37b2cef", - "is_verified": false, - "line_number": 529 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8a393e6013cadabddcf5740bfe251ec344fb2a70", - "is_verified": false, - "line_number": 529 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b2e8e4ce1f7a3ea1862f1106d7f2457ccd988849", - "is_verified": false, - "line_number": 532 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d11c2f3b2daa8ea137d7a2045641c6514e920de2", - "is_verified": false, - "line_number": 532 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "53c0b4592f6f3cf706202106a326b707e595389d", - "is_verified": false, - "line_number": 533 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5c854ae3a9ba0eded540ef09321c85f16eed865e", - "is_verified": false, - "line_number": 533 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "34250434eb77d3d7952c62d36aaeb457d62bbd15", - "is_verified": false, - "line_number": 534 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b98c948df77306372650dcd9dd49c98b340ff4d3", - "is_verified": false, - "line_number": 534 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "00952510084110d7d1e247299db269d230a27975", - "is_verified": false, - "line_number": 535 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f9ba99fa8981681fd8cfcafcac936a65851d0008", - "is_verified": false, - "line_number": 535 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "115588177967631dc3d47b903f79c1e033992211", - "is_verified": false, - "line_number": 563 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3e27e60eeab439623380fb96b9d08566902eae71", - "is_verified": false, - "line_number": 563 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7da0286c0783cf86526cd3b1288ba0d560e277de", - "is_verified": false, - "line_number": 567 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b1729e63d95a5873327ce55cfc5cb1741a14cfad", - "is_verified": false, - "line_number": 567 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8e987af5a4cc7f4d781d107d5a47554fa3171184", - "is_verified": false, - "line_number": 579 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f3a82f84c7eb837c15286edea0a5057c062868e7", - "is_verified": false, - "line_number": 579 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7d81d17722640af3d8f181cac4541dcb2b2a7e60", - "is_verified": false, - "line_number": 580 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e1cf9c746684e0e64102b1e90563018e96ce6c0a", - "is_verified": false, - "line_number": 580 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8cd7835afac940b8f3d6c67900c0a38d3e4259b9", - "is_verified": false, - "line_number": 581 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e0346c76fbad1459480c41284c5147ccf7e5ed04", - "is_verified": false, - "line_number": 581 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b951e097c53dbac20ea5986aa59b963978e6d75b", - "is_verified": false, - "line_number": 582 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "eec898302304ca8bc6590c3c3f50fd37c989f08c", - "is_verified": false, - "line_number": 582 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "83ef90182b9b802cc7cb7c9699b4edd1fecf291f", - "is_verified": false, - "line_number": 583 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bccfaea177a5a40ec19a91d0c6a9b816757cfbaf", - "is_verified": false, - "line_number": 583 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "49a73cde0f9423d5cd68e72b6b7e4afd4162ac72", - "is_verified": false, - "line_number": 586 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "68ff3d3f903b13a1ab721befeeab36453d7861c0", - "is_verified": false, - "line_number": 586 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2e637782b7c7860ae5a93f96042478b4f12e5742", - "is_verified": false, - "line_number": 589 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6006750b4b3387d14079045d63e5e66ad6bbdb14", - "is_verified": false, - "line_number": 589 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "203bf5010756923a1d8eb5198fff534029860eb7", - "is_verified": false, - "line_number": 590 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "adc376b7395e243598b579c6f2e9a73d44056a1d", - "is_verified": false, - "line_number": 590 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "30d22eacb7ffb59b99da905f499b4d3302b1b802", - "is_verified": false, - "line_number": 591 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d4eea56a1598265e24514784185b9312e366c0ad", - "is_verified": false, - "line_number": 591 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "08b6270783c3ba662583f4774b60367ac52e47a9", - "is_verified": false, - "line_number": 596 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ed04588170fa743f695321183717df5eeb3978b4", - "is_verified": false, - "line_number": 596 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5df0f61cc115ca8f2f2123c3f9d8c00b01f7638e", - "is_verified": false, - "line_number": 597 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e56d21cf701a66b16f9bdc871a09749044aad9c1", - "is_verified": false, - "line_number": 597 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "48b9f0492715834ffe5343daeb23c85c43e5ef0d", - "is_verified": false, - "line_number": 598 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "85593b38f7d4409d6d3eef370c48299af3145b26", - "is_verified": false, - "line_number": 598 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "840cf1d48cdd08209447a271e33c989200012a2e", - "is_verified": false, - "line_number": 606 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dbec54972257d22db18c9f352d5a8425d3664300", - "is_verified": false, - "line_number": 606 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3790ed45ce60f5291dd7036d5e6062b70949b4e8", - "is_verified": false, - "line_number": 607 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "eb1924896a2d0d758f94d50957f4dce6f6f0d042", - "is_verified": false, - "line_number": 607 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6d319b9426df28a0d54e3c42c633ad39cce01f46", - "is_verified": false, - "line_number": 608 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b09ca4281235113cc49a04078dd3d5905d764a86", - "is_verified": false, - "line_number": 608 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "420bf797d2c15ebdafd752e5d54ee26a6d0508e4", - "is_verified": false, - "line_number": 609 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c9b11b9dfd477008fb31b7964012ef4fb11d6b3f", - "is_verified": false, - "line_number": 609 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6f8363f9d7c45790d5ddac19617949672385773c", - "is_verified": false, - "line_number": 610 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f2dbcb84d73b626ecd5910b5f4c5ac62bfc84098", - "is_verified": false, - "line_number": 610 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "402f302f6d4d0063b2c8342a6a105017fc9ba83f", - "is_verified": false, - "line_number": 611 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "add9a5f66b0724fa3eb002718422597d02534ab9", - "is_verified": false, - "line_number": 611 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0e7a3ae5a5c2592cff5b241b54130f46400ca996", - "is_verified": false, - "line_number": 613 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3a586303e0403d2fa619479c9edfa7feda5b7071", - "is_verified": false, - "line_number": 613 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "974609fc2d7c61e124c289874fc0db80194310e1", - "is_verified": false, - "line_number": 614 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d68875f77ffb3fdc6cd6578ccd37f353fe07f062", - "is_verified": false, - "line_number": 614 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5b582f78eb97be5168ee0bd47a89f5b112640627", - "is_verified": false, - "line_number": 615 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9e742eb15ef51f7fa401d2c66cd2a2f57beb4b7f", - "is_verified": false, - "line_number": 615 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6ba0abad5b1eb888c2663516aaf46d98acd0efa0", - "is_verified": false, - "line_number": 616 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cba8a4eb98ed078a4f019248e5cd1c97966b6d05", - "is_verified": false, - "line_number": 616 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "85047897a641bba9c3f5122e6679d605815ee91d", - "is_verified": false, - "line_number": 617 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c902ccd66b2e0c10f1da0185f2512118861e3feb", - "is_verified": false, - "line_number": 617 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0f78daf9a085e580c555279b9707d0610a82d7f1", - "is_verified": false, - "line_number": 618 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9511b45d2dd57b1686923615ea7177b7b889dbbc", - "is_verified": false, - "line_number": 618 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3754d7213ad80bf72cb20f5714e7317f006e0db2", - "is_verified": false, - "line_number": 619 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3db9581e562901c5deb666d9735fd11dde36d5c1", - "is_verified": false, - "line_number": 619 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4d99bc0f12274dfcc54b17bc12c2621d58d3d62d", - "is_verified": false, - "line_number": 620 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aeeec43bf4c7b0f3ce9b9526292303d5242c193c", - "is_verified": false, - "line_number": 620 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cf8592876194cb8f43b2f9f099b18487c067659a", - "is_verified": false, - "line_number": 621 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "db43986a4f4b168182b1e53b4206347f5608bb64", - "is_verified": false, - "line_number": 621 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0adef81c03ae3ea39fcfb1c41083bbcb8a5be05a", - "is_verified": false, - "line_number": 622 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d907dde82198bb1c50072d5ab01843d2cb643440", - "is_verified": false, - "line_number": 622 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "320ee6b151fb6dcdfa8c45565c0ea0cd1bc3082c", - "is_verified": false, - "line_number": 627 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bfc8e396b036b20903c2cf331c0f15e9abab9c85", - "is_verified": false, - "line_number": 627 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0b9d1caf93acd21168a6834cd24b7bf5b2247058", - "is_verified": false, - "line_number": 629 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3508855c72c1c010877626f777d3fbeeb84df9a7", - "is_verified": false, - "line_number": 629 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ab261f4b800f8d381e7eb6b076901be4adb8bdbb", - "is_verified": false, - "line_number": 647 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e3bf6cb9a133654b38c7bc8ebb1523247d29bf48", - "is_verified": false, - "line_number": 647 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "282765f22421175dfbc985c92b685d3033f9a1eb", - "is_verified": false, - "line_number": 648 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "67af472da06999f74872f4689aa9885e71fd6399", - "is_verified": false, - "line_number": 648 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1176b15a39ddd1f5d197bdf25fcc40f1eb34d102", - "is_verified": false, - "line_number": 649 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "695ad1307bf14447ea7b31b9e5c05c508dd31d3c", - "is_verified": false, - "line_number": 649 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "26d3227d715f0a896a978e0f7e1164d69777ec27", - "is_verified": false, - "line_number": 650 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3945bc031d0bc612416ca22c85aad7fe659fe905", - "is_verified": false, - "line_number": 650 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3aaaebb97fa6cd400c5e23e511faa5007547cd80", - "is_verified": false, - "line_number": 651 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6bfdfc06179a174a6905ac86498ed5d60bccdc4f", - "is_verified": false, - "line_number": 651 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "619d5f2c8dec4c55467d798b3c63952516d513be", - "is_verified": false, - "line_number": 696 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8c5fef359f02b35814bf62ac410513cafea4591d", - "is_verified": false, - "line_number": 696 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "10bc66e24c5c49205455503a5e39a369be06941e", - "is_verified": false, - "line_number": 705 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fc9dcb7058ac3dea684c40b9a875dd21ca0401fd", - "is_verified": false, - "line_number": 705 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2e7dcd2ccf9d6d430fea6ac98ffc1b9d42f7f65d", - "is_verified": false, - "line_number": 711 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9e2f3fa6cb3139cf9bbfecddd2be60592f7491af", - "is_verified": false, - "line_number": 711 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0135de5ac17ed5236e1a82a8887bbf1e2e7ad181", - "is_verified": false, - "line_number": 712 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e48868449fa2b1eb09a81e3122a0e200ce455fcd", - "is_verified": false, - "line_number": 712 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "49f903dad4fd2fe36ff840593ebe5a5db9efda11", - "is_verified": false, - "line_number": 714 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "504342ba596fa724ee131daa699b1d91cccf31d2", - "is_verified": false, - "line_number": 714 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0122eae225e93a080e38cdfdd3b9383a940816f4", - "is_verified": false, - "line_number": 718 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c0b651b2d8f8b8fbaa692960471efd10c87f6eaa", - "is_verified": false, - "line_number": 718 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "016d3288d50e871ec5cfac882eb5f64eacd565da", - "is_verified": false, - "line_number": 724 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "88e10725567d7907c8d8637d4fc40a7c9f8d5738", - "is_verified": false, - "line_number": 724 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "be592e5f782f9d6768ff513fb4e888c323c95794", - "is_verified": false, - "line_number": 733 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d2ece94611876b26a3730ba39a3e8575bbd58b29", - "is_verified": false, - "line_number": 733 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ab50308b64efaa28005bf4377036d14c648fbb26", - "is_verified": false, - "line_number": 734 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f5dd2504ca1677707cb687397fe01f98a1c08fab", - "is_verified": false, - "line_number": 734 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "86360885827733cbc8053fa49759b7d8895cb5a0", - "is_verified": false, - "line_number": 735 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b6c25851e11397eb3fc81273997ac5717fdeed59", - "is_verified": false, - "line_number": 735 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "884bcfe83836a3f814af90419d6fa44ac0f73e4c", - "is_verified": false, - "line_number": 745 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "de68b580d56f9cb0cb45636915b9de91559f202c", - "is_verified": false, - "line_number": 745 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5b1edf8342ae3375fa1330ed75ec5e7797f53a3a", - "is_verified": false, - "line_number": 751 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a79bc0ed7ed14de358946cb1ce3dbcdff090777f", - "is_verified": false, - "line_number": 751 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "85a1e96fab0340c9f957471d497a84f618a2e444", - "is_verified": false, - "line_number": 752 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "efec807123b0681cb299c3693f2f6e8bec49326c", - "is_verified": false, - "line_number": 752 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "70fe648317e187f485af174df5c8dd386f0daa9a", - "is_verified": false, - "line_number": 753 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f7fc7e144342e9b8434806aabef7a2f23d762272", - "is_verified": false, - "line_number": 753 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "64ca326b8fcd5df5907e3c3b449ba562f27d252c", - "is_verified": false, - "line_number": 754 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f48a765b0acf94fc63293b47c9137a3de75ebcb6", - "is_verified": false, - "line_number": 754 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "346685a6018c2c491de868d0be7650a96aa0ae28", - "is_verified": false, - "line_number": 755 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e77e5537bfe212b745ae24419890deff5fece50d", - "is_verified": false, - "line_number": 755 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a12ac0eebc6ae1baf022dec2b3903ac069f86578", - "is_verified": false, - "line_number": 760 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ac81aa25967bd0dd82a6c092ca462d2a244a044b", - "is_verified": false, - "line_number": 760 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8a451db871c50764f3b8106eec5ce26961018958", - "is_verified": false, - "line_number": 761 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f654a748eca4bd1fc0756acd7e26db31b861e29d", - "is_verified": false, - "line_number": 761 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cf379622c9811fe604a0e89c921d6a85f469097b", - "is_verified": false, - "line_number": 762 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "df3708f6c0f441394e1831ac6f12aa3e807c1acf", - "is_verified": false, - "line_number": 762 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "43a706e4bf89b60a10fbff59218cb118a1270547", - "is_verified": false, - "line_number": 783 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4b70550293dbe88ea61438ffd70b0d8843afa1b1", - "is_verified": false, - "line_number": 783 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a8722fde111ee1b4416c2339ddb5204ab9cab779", - "is_verified": false, - "line_number": 784 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e458026d5fc0194eeabe65008d6a5a16b0732dff", - "is_verified": false, - "line_number": 784 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c811787dfe9223a372cfbb82e6b8226648d43fbf", - "is_verified": false, - "line_number": 785 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "da744e4f2477fc3b45a045fd57bd4af86d0cc3da", - "is_verified": false, - "line_number": 785 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "aa97f1be1c4e8c20b062d751347fea8cecf50f42", - "is_verified": false, - "line_number": 786 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b9fd5c57504715da2ee39bc1c22add54ccb54110", - "is_verified": false, - "line_number": 786 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0f96272328093b3a917faf219a133985888cbbda", - "is_verified": false, - "line_number": 787 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cb1819efe3b9255409b9eb92af5b371937e90f1b", - "is_verified": false, - "line_number": 787 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ca7f34c33d36658bb64e5c6e6c5ddce16a721349", - "is_verified": false, - "line_number": 803 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d9d8a5e72bc0b2674951fb15ebaf9381cb175f3f", - "is_verified": false, - "line_number": 803 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2cf8da58ededcd24e8d972f2ac091c913ef143db", - "is_verified": false, - "line_number": 805 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b7bd16890928c2af829e2b6ddec5a74bab246838", - "is_verified": false, - "line_number": 805 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2e1950f45fd6f1cd6e0eb90b7453f1f42f31b1f6", - "is_verified": false, - "line_number": 806 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e5cdf15a35f51762d1b0916260de05b520b8031d", - "is_verified": false, - "line_number": 806 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "1f7d497f701c66cc51f009d7d9ff7cd74cdfb29a", - "is_verified": false, - "line_number": 807 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d484216fe0269eee8eff27c0e3c7689b0c8322f1", - "is_verified": false, - "line_number": 807 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "04554029367a20bfc0f0a40df2e33ff430807e6a", - "is_verified": false, - "line_number": 808 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3e69cc53ccd14ed34521220dc0da95190983439f", - "is_verified": false, - "line_number": 808 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4e7a99f1495e023c5ab99e10e55606233f8cd976", - "is_verified": false, - "line_number": 813 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5a32b30681213ecdd57afa81a3c91c7e965523c1", - "is_verified": false, - "line_number": 813 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d88e692967c36f2b73a3b71312ec877692c1c05e", - "is_verified": false, - "line_number": 817 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fa0a362a98ac102a1f73c9281f731474ec599aab", - "is_verified": false, - "line_number": 817 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "05e4cd48e8f6443a3e97cb31a2ca5c6fa4b7b896", - "is_verified": false, - "line_number": 818 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cd9229a36451f2aa996811d77b56912c584bd474", - "is_verified": false, - "line_number": 818 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3b0aba910edbe716482c94591ce409f335ad8d5f", - "is_verified": false, - "line_number": 819 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6916c4db5296a464a6fc9308878869881951ca84", - "is_verified": false, - "line_number": 819 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3bd557233052edf9246be00b5d391eb0fd5a1caa", - "is_verified": false, - "line_number": 823 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ad1931614c872fe11eb3ec8c252337f83b8720ad", - "is_verified": false, - "line_number": 823 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "48e4199ad69e4d0224771556251b4bbb55edbfb3", - "is_verified": false, - "line_number": 824 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e04ab8db0c01dfea66946c9aaa91d22b375aff61", - "is_verified": false, - "line_number": 824 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "60fa3bd739ad68f1f5a381b6ee27f127c0ba32ad", - "is_verified": false, - "line_number": 827 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e96fce94c758e3a85900f76fc057ff3724071844", - "is_verified": false, - "line_number": 827 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "5b47541012e8f2755e345222478f2583dd8ac22f", - "is_verified": false, - "line_number": 829 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d958e6b8bb9d19f867db4ae63095b2dc336dfca2", - "is_verified": false, - "line_number": 829 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7801e816d029ff12c3e8c9be1a203001d3e72566", - "is_verified": false, - "line_number": 836 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e8aa939e8f12b33b7e02703cd4a1a720f6271d92", - "is_verified": false, - "line_number": 836 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0db71c6d3ca3091c1e0553d441bea452e860e373", - "is_verified": false, - "line_number": 840 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f0d20428987080d04fc01a28ce6230e9383e01d6", - "is_verified": false, - "line_number": 840 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ea6f7fb73d4b29787b64e646a5c24bc038974c24", - "is_verified": false, - "line_number": 842 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "fd6342ff46ffd226ab011dda7dfbabaa936a7454", - "is_verified": false, - "line_number": 842 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a941e691d5afb02087f212aafb96f0ff381013a9", - "is_verified": false, - "line_number": 846 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cdc017091b65aa223da7676a79499431ce22ca92", - "is_verified": false, - "line_number": 846 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "29f577087c91ce4cac58125b385e9b6b56bd3948", - "is_verified": false, - "line_number": 848 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3d7479c30d90d38511d5f5d5b5eb86477a4861c5", - "is_verified": false, - "line_number": 848 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "48f21e2e3cd237dc0fe7dc7fb479a6d8db35154e", - "is_verified": false, - "line_number": 854 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b66f6f577a43b3ba50aebb02d8fea8bf38767380", - "is_verified": false, - "line_number": 854 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ae6e7a1df413e4ff565bc1c8314d1ab51a8d42ef", - "is_verified": false, - "line_number": 856 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b024ebf502aca28c5cff54c6fa8abec3cc5bd53a", - "is_verified": false, - "line_number": 856 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6da56e48511e6fdfbf4519fbd5fabe75e7abfd8e", - "is_verified": false, - "line_number": 859 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a37fb57cac54e4bfd401b8f744b6d835f618565c", - "is_verified": false, - "line_number": 859 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "61e1aee596b0cfbb2a9c3ab7c9e1fea0356f8dfb", - "is_verified": false, - "line_number": 873 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a3b427fa51843a81806c9819cad6123117d64d45", - "is_verified": false, - "line_number": 873 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7f00582e3698cfde33cb176ec7ef90724460ec6d", - "is_verified": false, - "line_number": 878 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "af3820ff0a884c252eb6bf2d2eb9b207652a0951", - "is_verified": false, - "line_number": 878 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "46dbd04b9455a3d4c670ae9c430fe65532bce655", - "is_verified": false, - "line_number": 880 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bf7ab2fb8ce43fd3fd03ec6558fc376fcd25c573", - "is_verified": false, - "line_number": 880 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c637f4a40dc091e3dfeeb98127c5eb7c06feba4d", - "is_verified": false, - "line_number": 881 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "dc19d4073c3611195c7519b465b413fa901a9ae4", - "is_verified": false, - "line_number": 881 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "3c4d6702e0c7866383b386eca52bbe0fffbce0f3", - "is_verified": false, - "line_number": 882 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ede30abda8d269bb05c75c2286f5f12f66c3360f", - "is_verified": false, - "line_number": 882 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "7914b28a9505ee65fe5a50dc058f322a069781ed", - "is_verified": false, - "line_number": 893 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ffcd7655917ab1469744c4d79c3bf1270ce94a20", - "is_verified": false, - "line_number": 893 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "2d2c64b88205b797d38815c8f7ebda62059b8660", - "is_verified": false, - "line_number": 895 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "e533630a6c19676d53cd29301357edee1d943481", - "is_verified": false, - "line_number": 895 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ca5296128b3fa72795bd19f04427ea24526f0105", - "is_verified": false, - "line_number": 896 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "ec18126931229f89429bcf972094132ae2105422", - "is_verified": false, - "line_number": 896 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "63a999310c82808f4d3b8808f5957f207bea438d", - "is_verified": false, - "line_number": 898 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "bfc851fb4908464b9cded648831010c1d205c1f9", - "is_verified": false, - "line_number": 898 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "9ae9ab78e654fdfae0bd41e85cd55e51987acfc6", - "is_verified": false, - "line_number": 903 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "d3264a7cc9d82a156a6680125df87e806e73b8e3", - "is_verified": false, - "line_number": 903 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "6ec87d1b26bd1031be0985df77158ce86037a640", - "is_verified": false, - "line_number": 904 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "a6c09a4674feeedeb31bddde1ec843f6c605f3d7", - "is_verified": false, - "line_number": 904 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "0f8966c5b70f83947fb4a238ae475b6c6db5f0f2", - "is_verified": false, - "line_number": 913 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "73f0301c448e6e4efe86910313ba0f8e0a15bb4a", - "is_verified": false, - "line_number": 913 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4344ea2555a802a8b97ebd67e5ad36d7c6167a86", - "is_verified": false, - "line_number": 914 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "f4ff617bd75911c38fd7101293f979acead6400e", - "is_verified": false, - "line_number": 914 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "be3bef15bc7bed6bafcb9179169b4832fb564227", - "is_verified": false, - "line_number": 915 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "cca8cdd6f518b2b879b3197195dac4e3042a1ad3", - "is_verified": false, - "line_number": 915 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", - "is_verified": false, - "line_number": 916 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c28b4da54bc68775fa8e2e0ea98f852eb49d5870", - "is_verified": false, - "line_number": 916 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "4b7d6ec00d8e651549d3e939e4d3ebff394b7655", - "is_verified": false, - "line_number": 919 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "c95063c24bdf721f80f88bce7c52565e574c3de9", - "is_verified": false, - "line_number": 919 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "8a16496a48de4448d6a430af9f8d756b41690e19", - "is_verified": false, - "line_number": 920 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/files.log", - "hashed_secret": "b9c473ba6eac8ef97783fe05078383c757919085", - "is_verified": false, - "line_number": 920 - } - ], - "dataset/test9-mixed-zeek-dir/http.log": [ - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "87a3ca87807af60f25513beda99b356b9fbdf09e", - "is_verified": false, - "line_number": 257 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "024dfe3e6bbc130f57727192eacc67342970aab4", - "is_verified": false, - "line_number": 266 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "600ab340e9ed99ff504f83d39d37a61acdd0b8ef", - "is_verified": false, - "line_number": 278 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "932842a6825242d1f77003ed74c00077205f2fd7", - "is_verified": false, - "line_number": 283 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "d2d51781277a43f75bbff43e5ae8e69faff9d32d", - "is_verified": false, - "line_number": 286 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "951d86e04d039b6d023a2f270977cf6aef1859c4", - "is_verified": false, - "line_number": 289 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "240285b3b9f0aae0b9817d23989a9104bed39680", - "is_verified": false, - "line_number": 297 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "186304d742baf2443142cca7cc9d08181fd0192a", - "is_verified": false, - "line_number": 316 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "e650b48c2cbc188fdc0be3116e4c6fe199c15a53", - "is_verified": false, - "line_number": 318 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "cc4b5287594e20e9c0dd65fd9e24cd2ac8e760ea", - "is_verified": false, - "line_number": 323 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "ed218f34534055322820bedfaf71eba498dd6aad", - "is_verified": false, - "line_number": 326 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "2c7e85c60afe34d2c6ebcdb7019f3362f3f96bd0", - "is_verified": false, - "line_number": 373 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "c1e4a39b43fec8745515f680279cb682c963fe78", - "is_verified": false, - "line_number": 376 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "613e385d1057b2e26e3e1cc6863f25c14a24b39a", - "is_verified": false, - "line_number": 381 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "a570d3f59aa3f1a19759a0c931b5beb934ae2d1a", - "is_verified": false, - "line_number": 382 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "684271cb5d0f473a0658698a40c4eb5303dcf4e1", - "is_verified": false, - "line_number": 386 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "23fb0f3a04797101b8887273f8c2b7b5f912d313", - "is_verified": false, - "line_number": 391 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "79038c7c41e365ab069071437e51fc6e9fbdc364", - "is_verified": false, - "line_number": 392 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "b32cec134c9c479109288445162de23dc4469f01", - "is_verified": false, - "line_number": 397 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "fd0b60739ef05510194b3edfc56d6e534642216b", - "is_verified": false, - "line_number": 401 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "fe14c850fe1b74a41355c5b707c1d12031710460", - "is_verified": false, - "line_number": 404 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "00574f4283401ef320e8fd9b373d51efaf31b37d", - "is_verified": false, - "line_number": 420 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "2b007cb159fd1cb99fbce87f37e82cd7d43c77a4", - "is_verified": false, - "line_number": 423 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "69e26a6bb775ad38a2cf62d64a0c3ea8eab319fc", - "is_verified": false, - "line_number": 433 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "d095634b4fe212ef09a90c31fa1927e5f1310604", - "is_verified": false, - "line_number": 446 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "7d79913f3af49356a53df4725935bb3651c08421", - "is_verified": false, - "line_number": 453 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "15d5bd1db4d42c7eeb464a1bf0dc4b799e678cd3", - "is_verified": false, - "line_number": 456 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "631d18d0259084c9784b44eedaf1667a3470fea6", - "is_verified": false, - "line_number": 511 - }, - { - "type": "Base64 High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/http.log", - "hashed_secret": "37522b5b749c6ceaa3820e74a0924758772a8793", - "is_verified": false, - "line_number": 514 - } - ], - "dataset/test9-mixed-zeek-dir/ssl.log": [ - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/ssl.log", - "hashed_secret": "b6aa435b77ffc6bb2f5c3a1647eb4a5e45a316ee", - "is_verified": false, - "line_number": 3 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/ssl.log", - "hashed_secret": "3e1fd256a51b5ae9951f24ab5d2ff387894c3e67", - "is_verified": false, - "line_number": 12 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/ssl.log", - "hashed_secret": "2e621bc4ae7af0e821c2a7f45b1e9ff83780ff3e", - "is_verified": false, - "line_number": 21 - } - ], - "dataset/test9-mixed-zeek-dir/x509.log": [ - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "f34136b5a87c4cb95d5e487d4463833ed86454ad", - "is_verified": false, - "line_number": 2 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "46b5d2bd612f8dce52bc86ef3e465782eae8173c", - "is_verified": false, - "line_number": 3 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "bd4c2da3eeeeddc50e66133cc83374e5428bb883", - "is_verified": false, - "line_number": 4 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "7788161150e362c7061502b3b35af68ad64cca4c", - "is_verified": false, - "line_number": 5 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "b56e50a64d3a03e23569ba69d96c6864ca9f89f8", - "is_verified": false, - "line_number": 8 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "a00c54038c0792e334d861b15b8d576dac97ad19", - "is_verified": false, - "line_number": 9 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "19020555c49bb460d5dff18494d8894bfcd8445e", - "is_verified": false, - "line_number": 10 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "6f4f3d6218ca8d67825c75bc4b406f5adb15062d", - "is_verified": false, - "line_number": 12 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "7261fc0376613073c3cd26d1086c2f8c41a46700", - "is_verified": false, - "line_number": 13 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "5fcd7a54eedca48c1014e4fe59d357ee7f76dda7", - "is_verified": false, - "line_number": 17 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "e98cf8fe7835fa3fd4381b9905f3535f473f93f2", - "is_verified": false, - "line_number": 19 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "564c8ca56634e562408b5d21ecf425318f699e99", - "is_verified": false, - "line_number": 20 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "1d1020c0ed2078d092fa0460b2d8429ba5bc0183", - "is_verified": false, - "line_number": 22 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "9c29e868c008b4a69d58765e716a7bc95cda4179", - "is_verified": false, - "line_number": 23 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "4f0a5d97ee702e9983b4cee024785768cbb2ed2d", - "is_verified": false, - "line_number": 26 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "b228735f75406b16e7e6064fccf4499bc22bb4f9", - "is_verified": false, - "line_number": 28 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "9de370c6bcff259e30b91efd334b03a3e90a17bf", - "is_verified": false, - "line_number": 30 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "dc059acc514a62a4565aef2f7bf33c155a3d5076", - "is_verified": false, - "line_number": 31 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "cd1954e3f5046d21f057108032c55b17f2ef2c2c", - "is_verified": false, - "line_number": 32 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "238f921fd901f97c1a4f0b66d7dbdfa167347043", - "is_verified": false, - "line_number": 34 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "f7e9c021d8a79837bc3a19884c93579f4b0228a7", - "is_verified": false, - "line_number": 35 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "0e9f747bd7dd0b04cc17f30a59b9cbbbc4df74fc", - "is_verified": false, - "line_number": 36 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "965b9cef9a2079cc00cf761f76317ae94baf7e37", - "is_verified": false, - "line_number": 37 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "809c41d4584d0e4106c536207bebda47127b6829", - "is_verified": false, - "line_number": 40 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "7e3a586c5e92c4363c8c204d8d413b55f2b17f77", - "is_verified": false, - "line_number": 44 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "c7432f209ed2f15d66b7dc24531d19d890cf30d3", - "is_verified": false, - "line_number": 48 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "afdd5d551ac51dc4c41eb73fb32415b3deadb238", - "is_verified": false, - "line_number": 55 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "23df393583038210874f7d4cb738c23b27dfad6b", - "is_verified": false, - "line_number": 64 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "8180568237fcf502d6409cbb0bcef4cc1d60c3db", - "is_verified": false, - "line_number": 79 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "5965d4754cddaee1640f9d8f54bec4ee7d5d11ca", - "is_verified": false, - "line_number": 82 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "e2193ad3e89e08be7d1a954f59c9d215a96c65bd", - "is_verified": false, - "line_number": 84 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "b5f1c5da0069d32cfc9c9cad2f6e09a05e58bf0c", - "is_verified": false, - "line_number": 86 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "06c3d8e0dbcc85fff48eead096323f172a430db5", - "is_verified": false, - "line_number": 91 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "9b2f18e6b7bcbb8f03046e280c12bfb0b55fddcc", - "is_verified": false, - "line_number": 95 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "da15450fa7e05a97b21b8a04670ef9781a891d23", - "is_verified": false, - "line_number": 97 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "c61fa50ca431dc2fd83a53f3c6991b33b399fa56", - "is_verified": false, - "line_number": 100 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "787d4d382aa0e35c16412532424971f7d5a31f5d", - "is_verified": false, - "line_number": 107 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "ed8271f4f83be86408c7200b9fe5cc0364b0297f", - "is_verified": false, - "line_number": 113 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "64ca3ed304a175c99797a20a15e91b459e52685d", - "is_verified": false, - "line_number": 114 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "e75dcf7574460f71ccae0f3deaed29994e13a84d", - "is_verified": false, - "line_number": 115 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "44733ca3f1081f76ae4da02ea90250abf598aa06", - "is_verified": false, - "line_number": 116 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "c24b923da80ced80251768e7b4e838a0bcd9658c", - "is_verified": false, - "line_number": 117 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "2178d118c7c194f7b8f84f27cd65e55d8dfffa1d", - "is_verified": false, - "line_number": 121 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "9a6c8df77cd74f8d8226027bc4256417d0c31754", - "is_verified": false, - "line_number": 122 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "f5b128091d514c47f9beafb2e2271a2df6e583c8", - "is_verified": false, - "line_number": 124 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "4bf438eb8cb7e337b4b20bb49b4a81a0856e6d36", - "is_verified": false, - "line_number": 125 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "0c1397bd8f7fd878b7fda181dbe959b927369474", - "is_verified": false, - "line_number": 127 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "12d8eedfe40cf61c4eda4d4799a19b79c1be2811", - "is_verified": false, - "line_number": 134 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "9c5c9e2a4a8ee4f59a298b1f07497f05a35cde2d", - "is_verified": false, - "line_number": 135 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "113ac71d4f18787710e164ed7cc4755c6a3c00b2", - "is_verified": false, - "line_number": 136 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "7d1353a6ce6c72e2969ff12ef5c34023e25496c3", - "is_verified": false, - "line_number": 141 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "98a3e793dec2e0df207be33dd719e7d6f00026ac", - "is_verified": false, - "line_number": 142 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "9d584d5b4bc19c9bda6f1b149b765b6ad907fef0", - "is_verified": false, - "line_number": 144 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "1ff7dfab101732838ad27d10ad08dc73c5d36a6b", - "is_verified": false, - "line_number": 147 - }, - { - "type": "Hex High Entropy String", - "filename": "dataset/test9-mixed-zeek-dir/x509.log", - "hashed_secret": "37767078fc296efe6555f03f57583d9b54e60966", - "is_verified": false, - "line_number": 148 - } - ], - "docs/detection_modules.md": [ - { - "type": "Hex High Entropy String", - "filename": "docs/detection_modules.md", - "hashed_secret": "2e621bc4ae7af0e821c2a7f45b1e9ff83780ff3e", - "is_verified": false, - "line_number": 448 - }, - { - "type": "Hex High Entropy String", - "filename": "docs/detection_modules.md", - "hashed_secret": "b6aa435b77ffc6bb2f5c3a1647eb4a5e45a316ee", - "is_verified": false, - "line_number": 449 - } - ], - "docs/features.md": [ - { - "type": "Hex High Entropy String", - "filename": "docs/features.md", - "hashed_secret": "2e621bc4ae7af0e821c2a7f45b1e9ff83780ff3e", - "is_verified": false, - "line_number": 643 - }, - { - "type": "Hex High Entropy String", - "filename": "docs/features.md", - "hashed_secret": "b6aa435b77ffc6bb2f5c3a1647eb4a5e45a316ee", - "is_verified": false, - "line_number": 644 - } - ], - "docs/feel_project.md": [ - { - "type": "JSON Web Token", - "filename": "docs/feel_project.md", - "hashed_secret": "11616e43bc161521a1f92431e6e5a5aaf075d263", - "is_verified": false, - "line_number": 578 - }, - { - "type": "JSON Web Token", - "filename": "docs/feel_project.md", - "hashed_secret": "5edb46569721ec245fa12e05be5108d6b91b756d", - "is_verified": false, - "line_number": 624 - }, - { - "type": "JSON Web Token", - "filename": "docs/feel_project.md", - "hashed_secret": "0836e434425db8ed6ba5e0106c3abb05b7cb7197", - "is_verified": false, - "line_number": 676 - }, - { - "type": "JSON Web Token", - "filename": "docs/feel_project.md", - "hashed_secret": "d7f9abc8ff1f9fe8cac0335b92588f6ad0f5b33b", - "is_verified": false, - "line_number": 701 - } - ], - "docs/slips_in_action.md": [ - { - "type": "Secret Keyword", - "filename": "docs/slips_in_action.md", - "hashed_secret": "afc795ad602e173aaf56d4aed92bb865450bd38d", - "is_verified": false, - "line_number": 116 - } - ], - "modules/p2ptrust/data_format.md": [ - { - "type": "Base64 High Entropy String", - "filename": "modules/p2ptrust/data_format.md", - "hashed_secret": "0e8deb4c72059a235e960774c4b31f76a194e6f6", - "is_verified": false, - "line_number": 31 - } - ], - "modules/p2ptrust/testing/json_data.py": [ - { - "type": "Base64 High Entropy String", - "filename": "modules/p2ptrust/testing/json_data.py", - "hashed_secret": "4f6790425aa66507a5f531a6699f8c7603f7f6a5", - "is_verified": false, - "line_number": 17 - }, - { - "type": "Base64 High Entropy String", - "filename": "modules/p2ptrust/testing/json_data.py", - "hashed_secret": "e56c817fb6a6e6abfc5d993b2a1b0b019935237e", - "is_verified": false, - "line_number": 37 - } - ], - "modules/rnn_cc_detection/training_code/evaluation-cc-detection.ipynb": [ - { - "type": "Base64 High Entropy String", - "filename": "modules/rnn_cc_detection/training_code/evaluation-cc-detection.ipynb", - "hashed_secret": "9a8f66eb3c59916e0f00df60fd1f0d9d6cadf3e9", - "is_verified": false, - "line_number": 666 - }, - { - "type": "Base64 High Entropy String", - "filename": "modules/rnn_cc_detection/training_code/evaluation-cc-detection.ipynb", - "hashed_secret": "6043e3eda33f5914b5d060b95a1da8302fd9b220", - "is_verified": false, - "line_number": 676 - } - ], - "tests/integration_tests/fides_config.yaml": [ - { - "type": "Secret Keyword", - "filename": "tests/integration_tests/fides_config.yaml", - "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016", - "is_verified": false, - "line_number": 199 + "line_number": 317 }, { "type": "Secret Keyword", - "filename": "tests/integration_tests/fides_config.yaml", - "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997", - "is_verified": false, - "line_number": 323 - } - ], - "tests/integration_tests/test.yaml": [ - { - "type": "Secret Keyword", - "filename": "tests/integration_tests/test.yaml", - "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016", - "is_verified": false, - "line_number": 170 - }, - { - "type": "Secret Keyword", - "filename": "tests/integration_tests/test.yaml", - "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997", - "is_verified": false, - "line_number": 275 - } - ], - "tests/integration_tests/test2.yaml": [ - { - "type": "Secret Keyword", - "filename": "tests/integration_tests/test2.yaml", + "filename": "config/slips.yaml", "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016", "is_verified": false, - "line_number": 201 - }, - { - "type": "Secret Keyword", - "filename": "tests/integration_tests/test2.yaml", - "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997", - "is_verified": false, - "line_number": 325 - } - ], - "tests/test_circllu.py": [ - { - "type": "Hex High Entropy String", - "filename": "tests/test_circllu.py", - "hashed_secret": "125fbc14773f228e72f16d55be21bad750d30b19", - "is_verified": false, - "line_number": 78 - } - ], - "tests/test_go_director.py": [ - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "c7e0bc120ee16cfa95983e550301dd6555b23ea2", - "is_verified": false, - "line_number": 33 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "b586d0c2cdebc6defd0f81605610399ae8aaeed8", - "is_verified": false, - "line_number": 34 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "35f29e14d00c0fc6f35a1587c68889c112e24bcc", - "is_verified": false, - "line_number": 35 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "9afc0822c443ae9cecbb373900429e890da8e231", - "is_verified": false, - "line_number": 44 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "675c1e48dfa313a3cebe293cfc950c4ae2618d7f", - "is_verified": false, - "line_number": 45 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "a73d04a4137a59599d3ff20111762f1cddc8c470", - "is_verified": false, - "line_number": 46 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "5fe39eccf20874c5a842ed8bd7238d65a07bc053", - "is_verified": false, - "line_number": 96 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "165ace5923919eea8e15c4ad9279f68f35d4b5b8", - "is_verified": false, - "line_number": 97 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "9e039b5e2fb24f9d8feddf914c6dac07cb1eca66", - "is_verified": false, - "line_number": 98 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "dd31e02ba5fefdf6409c0cd23aaf360287626787", - "is_verified": false, - "line_number": 99 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "ef475528c602bfac33d4198ba750deb1e3fb904e", - "is_verified": false, - "line_number": 119 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "e486019fb66365e852c77158645768bb5038da87", - "is_verified": false, - "line_number": 120 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "032ab16f80631bb3df49b7b941bc95f187e893db", - "is_verified": false, - "line_number": 150 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "de261b4d0d37516dccffd67dcf0e216723ee3790", - "is_verified": false, - "line_number": 151 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/test_go_director.py", - "hashed_secret": "81a86724010e489b2063575bfc9439a2c9ba45b5", - "is_verified": false, - "line_number": 152 - } - ], - "tests/test_set_evidence.py": [ - { - "type": "Hex High Entropy String", - "filename": "tests/test_set_evidence.py", - "hashed_secret": "2e621bc4ae7af0e821c2a7f45b1e9ff83780ff3e", - "is_verified": false, - "line_number": 1394 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/test_set_evidence.py", - "hashed_secret": "b6aa435b77ffc6bb2f5c3a1647eb4a5e45a316ee", - "is_verified": false, - "line_number": 1401 - } - ], - "tests/test_slips_utils.py": [ - { - "type": "Hex High Entropy String", - "filename": "tests/test_slips_utils.py", - "hashed_secret": "0142af6be109425ab73fc66b36d8981ec919ba0b", - "is_verified": false, - "line_number": 20 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/test_slips_utils.py", - "hashed_secret": "10470c3b4b1fed12c3baac014be15fac67c6e815", - "is_verified": false, - "line_number": 66 - }, - { - "type": "Basic Auth Credentials", - "filename": "tests/test_slips_utils.py", - "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", - "is_verified": false, - "line_number": 510 - } - ], - "tests/test_threat_intelligence.py": [ - { - "type": "Hex High Entropy String", - "filename": "tests/test_threat_intelligence.py", - "hashed_secret": "125fbc14773f228e72f16d55be21bad750d30b19", - "is_verified": false, - "line_number": 638 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/test_threat_intelligence.py", - "hashed_secret": "47784580758b20256793a484ce89c74d6724936c", - "is_verified": false, - "line_number": 663 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/test_threat_intelligence.py", - "hashed_secret": "7d7c596baa46487dce0e2036e14982612f6b50da", - "is_verified": false, - "line_number": 1220 - } - ], - "tests/test_update_file_manager.py": [ - { - "type": "Hex High Entropy String", - "filename": "tests/test_update_file_manager.py", - "hashed_secret": "2431dcd348f1cc7e2d70c13eed1df1ee77452bfb", - "is_verified": false, - "line_number": 322 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/test_update_file_manager.py", - "hashed_secret": "13603b78502e7568249304e035f904029e4c81c6", - "is_verified": false, - "line_number": 791 + "line_number": 662 } ] }, - "generated_at": "2026-05-04T20:22:30Z" + "generated_at": "2026-06-23T22:22:57Z" } diff --git a/README.md b/README.md index 14a0a9dc51..e8361a096c 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ Slips aborts updating to new versions when there are changes to Slips local conf * You can also specify whether to ```train``` or ```test``` the ML models * You can enable [popup notifications](https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#popup-notifications) of evidence, enable [blocking](https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#slips-permissions), [plug in your own zeek script](https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#plug-in-a-zeek-script) and more. +* The `t_cell` section is enabled by default so Slips can consume centrally tagged `PAMP` and `DAMP` evidence, match extracted antigens against accepted regexes, and then carry a signal-specific priming profile into later co-stimulation and context decisions. `DAMP` can now create a weaker state-1 cell with stricter later thresholds and a shorter waiting window, while all responder state stays in its own SQLite DB. [More details about the config file options here]( https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#modifying-the-configuration-file) @@ -192,6 +193,10 @@ Slips key features are: * **HTTPS Anomaly Detection**: Adaptive TLS/HTTPS anomaly detection with drift handling and a local HTML report generator for deep dives. * **Integration with External Platforms**: Modules in Slips can look up IP addresses on external platforms such as VirusTotal and RiskIQ. * **Graphical User Interface**: Slips provides a web interface and an optional Kalipso terminal interface through the `modules/kalipso` submodule. +* **Shared LLM Access**: Slips can expose configured LLM backends such as Ollama, OpenAI, and Anthropic to other modules through Redis channels. +* **Hierarchical Alert Summaries**: Slips can turn correlated alert evidence into analyst-facing one-paragraph incident summaries, recursively reducing oversized evidence sets instead of truncating them. +* **Pseudo-Random Regex Generation**: Slips can generate and validate pseudo-random regexes for DNS domains, URIs, filenames, TLS SNI, and certificate CN fields for later Zeek-side use. +* **Immune-Style T Cell Response**: Slips can consume centrally tagged `PAMP` and `DAMP` evidence, correlate extracted antigens with accepted regexes, and use signal-specific T-cell priming plus mixed `PAMP`/`DAMP` danger pressure to decide whether to stay tolerant, activate, contain, or store long-term memory. * **Peer-to-Peer (P2P) Module**: Slips includes a complex automatic system to find other peers in the network and share IoC data automatically in a balanced, trusted manner. The P2P module can be enabled as needed. * **Docker Implementation**: Running Slips through Docker on Linux systems is simplified, allowing real-time traffic analysis. * **Detailed Documentation**: Slips provides detailed documentation guiding users through usage instructions for efficient utilization of its features. @@ -222,6 +227,11 @@ We appreciate your contributions and thank you for helping to improve Slips! # Documentation [User documentation](https://stratospherelinuxips.readthedocs.io/en/develop/) +T Cell design and configuration: [docs/t_cell_module.md](docs/t_cell_module.md) + +T Cell offline report generation and interpretation: +[docs/t_cell_module.md#offline-html-report](docs/t_cell_module.md#offline-html-report) + [Code docs](https://stratospherelinuxips.readthedocs.io/en/develop/code_documentation.html ) --- diff --git a/config/slips.yaml b/config/slips.yaml index 3cfa54227a..5115d4244a 100644 --- a/config/slips.yaml +++ b/config/slips.yaml @@ -35,7 +35,6 @@ output: logs: slips.log ############################# - parameters: # The verbosity is related to how much data you want to see about the # detections. @@ -242,7 +241,348 @@ modules: timeline_human_timestamp: true ############################# -flow_ml_detection: +llm_proxy: + # Enable the shared LLM service module. + enabled: true + + # If a request does not specify a backend alias, this one is used. + # The value must match one of the keys under backends: below. + default_backend: local_qwen + + # Number of worker threads processing LLM requests in parallel. + # All caller modules share the same request queue and the same response + # channel, so modules must correlate replies by request_id. + worker_threads: 2 + + # Maximum number of pending requests kept in memory. + queue_size: 100 + + # Each backend is a named connection that other modules select by alias + # through the request field backend. + backends: + slips-goldilock: + # slips-goldilock is the fine-tunned model in Slips for general summary and detection. It is fast enough and good enough + provider: ollama + # Default model for this backend alias. Caller modules may override the + # model in an individual request while still using this connection. + model: stratosphere/qwen2.5-1.5b-slips-immune-risk:q5_k_m + base_url: http://127.0.0.1:11434 + # Optional HTTP timeout in seconds. + timeout: 300 + + slips-fast: + # slips-fast is the fine-tunned model in Slips for fast summary and detection. It has lower quality + provider: ollama + # Default model for this backend alias. Caller modules may override the + # model in an individual request while still using this connection. + model: stratosphere/qwen2.5-1.5b-slips-immune-risk:q4_k_m + base_url: http://127.0.0.1:11434 + # Optional HTTP timeout in seconds. + timeout: 300 + + slips-high: + # slips-high is the fine-tunned model in Slips for high quality. It is slower + provider: ollama + # Default model for this backend alias. Caller modules may override the + # model in an individual request while still using this connection. + model: stratosphere/qwen2.5-1.5b-slips-immune-risk:q8_0 + base_url: http://127.0.0.1:11434 + # Optional HTTP timeout in seconds. + timeout: 300 + + + local_qwen: + # Supported providers: ollama, openai, anthropic + provider: ollama + # Default model for this backend alias. Caller modules may override the + # model in an individual request while still using this connection. + model: qwen2.5:3b + base_url: http://127.0.0.1:11434 + # Optional HTTP timeout in seconds. + timeout: 300 + + openai_default: + provider: openai + model: gpt-4o-mini + base_url: https://api.openai.com/v1 + # Provide one of api_key, api_key_env, or api_key_file for providers + # that require authentication. + api_key_env: OPENAI_API_KEY + timeout: 60 + + claude_default: + provider: anthropic + model: claude-sonnet-4-5 + base_url: https://api.anthropic.com + api_key_env: ANTHROPIC_API_KEY + # Optional Anthropic API version header. + anthropic_version: 2023-06-01 + timeout: 60 + +############################# +alert_summary: + # Enable the analyst-facing alert summary module. + enabled: true + + # Write operational logs for this module to: + # /llm_proxy-summary/alert_summary.log + # log_verbosity levels: + # 0 = only create the file + # 1 = startup, shutdown, and failures + # 2 = per-alert queueing, requests, writes, and reduction progress + # 3 = debug details such as prompt-budget decisions + log_verbosity: 3 + + # Preferred LLM backend aliases for alert summaries. If empty, the module + # falls back to the runtime-ready LLM default backend. + #allowed_backends: [local_qwen] + allowed_backends: [slips-goldilock] + + # Keep the sampling conservative because this module writes one analyst + # summary paragraph per alert. + llm_temperature: 0.2 + + # Token budget for the final analyst summary paragraph. + llm_max_tokens: 220 + + # Hard timeout in seconds for one in-flight shared-LLM request. + # Large alerts may require several internal reduction requests before the + # final summary request. If a request times out, the module writes a + # heuristic fallback summary for that alert. Set to 0 to wait indefinitely. + llm_response_timeout_seconds: 300 + + # Include recent alert history for the same source/profile as extra context + # in the final analyst-summary prompt. + history_enabled: true + + # Number of prior summarized alerts kept in memory per profile. + history_max_alerts: 3 + + # Approximate token budget reserved for recent alert history in the final + # analyst-summary prompt. + history_max_tokens: 700 + + # Number of dominant grouped patterns stored from each prior alert. + history_patterns_per_alert: 2 + +############################# +regex_generator: + # Enable the shared regex generator module. + enabled: true + enabled: false + + # Create output/regex_generator.log with detailed internal progress logs. + # This file follows the global parameters.rotation / rotation_period policy. + create_log_file: true + + # Wait this many seconds between completed generation cycles. + # Set to 0 to start the next cycle immediately after the previous one + # finishes and the module is ready again. + generation_interval_seconds: 0 + + # Preferred LLM backend aliases for this module. If empty, the module falls + # back to the runtime-ready LLM default backend. + allowed_backends: [local_qwen] + + # Keep the temperature high enough to encourage variation over time. + llm_temperature: 1.2 + + # Token budget for the LLM response. The prompt asks for one regex line only, + # so this should stay very small. + llm_max_tokens: 80 + + # Soft warning threshold in seconds while waiting for the matching + # llm_response. The module keeps waiting after this and only logs that the + # LLM is slow. Set to 0 to disable the warning. + llm_response_timeout_seconds: 300 + + # Prompt history is not sent to the LLM. Repetition is checked locally with a + # bloom filter plus exact DB lookup, so keep this at 0. + recent_history_size: 0 + + # Reject generated regexes longer than this many characters. + max_regex_length: 180 + + # Hard wall-clock timeout for local regex validation and benign-corpus + # matching. This prevents one pathological regex from freezing the module. + # Set to 0 to disable the timeout. + regex_validation_timeout_seconds: 2 + + # Benign match strength threshold from 0 to 100. A generated regex is + # rejected only when its strongest match against a benign string reaches + # this score. Higher values are more permissive. + benign_match_strength_threshold: 75 + + # Weighted random choice for the next regex type to generate. + type_weights: + dns_domain: 1 + uri: 1 + filename: 1 + tls_sni: 1 + certificate_cn: 1 + + # Directory that stores the benign corpus DB and the generated regex DB. + # Absolute paths are used as-is. Relative paths are resolved inside the + # output directory of the current Slips run. + store_dir: output/regex_generator + + # Stable directory for the regex SQLite files. Relative paths are resolved + # inside parameters.permanent_dir. If set, it takes precedence over + # store_dir and lets the generator reuse the same DBs across many Slips + # restarts. + persistent_store_dir: databases/regex_store + + # Persist rejected regexes in generated_regexes.sqlite. + # Leave this disabled unless you need audit/debug history, otherwise disk + # usage will grow with low-value rejected rows. + store_rejected_regexes: false + + # If rejected regex persistence is enabled, keep at most this many rejected + # rows and prune older ones first. Set to 0 for unlimited retention. + max_stored_rejected_regexes: 10000 + + # Seed the benign corpus DB once with a small built-in sample for each type. + # The module also imports domain entries from the configured local whitelist + # file into the benign corpus on each run for dns_domain, tls_sni, and + # certificate_cn checks. + seed_benign_samples: true + +############################# +t_cell: + # Enable the immune-inspired T Cell responder module. + enabled: false + + # Create output/t_cell.log with human-readable transition logs. + create_log_file: true + + # Keep ANSI colors in the T Cell log file for quick scanning. + log_colors: true + + # File log verbosity: + # 1 = transitions and terminal actions only + # 2 = add decision summaries such as waiting for co-stimulation/context + # 3 = add per-evidence debug details such as extracted antigens + log_verbosity: 3 + + # Optional evaluation trace for auditing why thresholds passed or failed. + # off = disabled + # transitions = write detailed traces only when a state transition happens + # all = also trace waiting evaluations + decision_trace_mode: on + + # Separate trace file used only when decision_trace_mode is not off. + # This path is always resolved inside the selected output directory for the + # current Slips run. Absolute or outside paths are not honored. + decision_trace_file: t_cell_trace.jsonl + + # Maximum number of contributing evidence rows stored per contributor list + # inside the trace file. + decision_trace_max_evidence: 10 + + # Directory that stores the isolated T Cell SQLite database. + # Absolute paths are used as-is. Relative paths are resolved inside the + # output directory of the current Slips run. + store_dir: output/t_cell/data + + # Optional stable directory for the T Cell SQLite database. + # Leave this empty for normal runs because T Cell state is run-specific. + # If set, relative paths are resolved inside parameters.permanent_dir. + persistent_store_dir: "" + + # Keep processed evidence observations for this many seconds. + observation_retention_seconds: 604800 + + # How long a cell remains anergic after a PAMP does not match any accepted + # regex for its antigen. + anergy_ttl_seconds: 21600 + + # Time window used to count related PAMP observations for the same + # responsible IP. + related_lookback_seconds: 3600 + + # Saturation point for related PAMP scoring. A value of 5 means that 5 or + # more related observations contribute the full score. + related_pamps_saturation: 5 + + # Saturation point for weighted profile danger: + # sum(threat_level_value * confidence) / danger_saturation + danger_saturation: 2.5 + + # DAMP observations also contribute to danger pressure for the same + # responsible IP. That mixed pressure is used by co-stimulation and + # context: + # combined_raw_danger = pamp_raw_danger + damp_danger_weight * damp_raw_danger + damp_danger_weight: 1.5 + + # Base activation threshold for co-stimulation. Signal-specific priming + # profiles can raise this later for weaker priming sources such as DAMP. + co_stimulation_threshold: 0.65 + + # Co-stimulation weights are normalized internally. + co_stimulation_weights: + confidence: 0.35 + related_pamps: 0.25 + danger: 0.40 + + # Both PAMP and DAMP can move a mature T Cell into state 1 when an + # extracted antigen matches an accepted regex. The priming profile + # determines how strong that state-1 cell is by shifting later + # thresholds, required supporting evidence counts, and how long the + # cell can wait before timing out. + priming_profiles: + PAMP: + strength: 1.0 + co_stimulation_threshold_offset: 0.0 + effector_threshold_offset: 0.0 + memory_threshold_offset: 0.0 + state_wait_timeout_factor: 1.0 + effector_min_related_count_offset: 0 + memory_min_related_count_offset: 0 + DAMP: + strength: 0.6 + co_stimulation_threshold_offset: 0.15 + effector_threshold_offset: 0.10 + memory_threshold_offset: 0.05 + state_wait_timeout_factor: 0.5 + effector_min_related_count_offset: 1 + memory_min_related_count_offset: 1 + + # Novelty lookback window used to decide whether a matched regex is new. + novelty_window_seconds: 86400 + + # Recent context window. The previous context window is the immediately + # preceding window of the same size. + context_recent_window_seconds: 1800 + + # Base context threshold for escalating to the effector state. The + # effective threshold may be increased by the priming profile. + effector_threshold: 0.70 + + # Base minimum number of related recent observations required before an + # effector response is allowed. The priming profile may add to this. + effector_min_related_count: 4 + + # Cooldown between effector responses for the same T Cell. + effector_cooldown_seconds: 1800 + + # Base context threshold for moving into the memory state. The effective + # threshold may be increased by the priming profile. + memory_threshold: 0.60 + + # Maximum recent/previous pressure ratio to consider the threat to be + # decreasing enough for memory. + memory_trend_ratio_max: 0.60 + + # Base minimum number of related recent observations required before + # memory is stored. The priming profile may add to this. + memory_min_related_count: 3 + + # If blocking or ARP-poisoning modules are not running, log a simulated + # effector action instead of publishing new_blocking. + simulate_effector_without_blocking: true + +############################# +flowmldetection: # This is a module that uses machine learning for detection. # It can be used in train mode or test mode. # The mode 'train' should be used to tell the flow_ml_detection module @@ -405,6 +745,9 @@ whitelists: # 2 weeks = 1209600 seconds online_whitelist_update_period: 86400 online_whitelist: https://tranco-list.eu/download/X5QNN/10000 + # Only store the first N Tranco domains in the database. + # RegexGenerator and the offline regex coverage report use this limit. + tranco_top_benign_limit: 1000 # if this parameter is set to false, Slips runs with no whitelists at all. # May cause a lot of false positives @@ -549,7 +892,7 @@ DisabledAlerts: # CONNECTION_TO_MULTIPLE_PORTS, HIGH_ENTROPY_DNS_ANSWER, # INVALID_DNS_RESOLUTION, PORT_0_CONNECTION, MALICIOUS_JA3, MALICIOUS_JA3S, # DATA_UPLOAD, BAD_SMTP_LOGIN, SMTP_LOGIN_BRUTEFORCE, MALICIOUS_SSL_CERT, - # MALICIOUS_FLOW, SUSPICIOUS_USER_AGENT, EMPTY_CONNECTIONS, + # ANOMALOUS_FLOW, MALICIOUS_FLOW, SUSPICIOUS_USER_AGENT, EMPTY_CONNECTIONS, # INCOMPATIBLE_USER_AGENT, EXECUTABLE_MIME_TYPE, MULTIPLE_USER_AGENT, # HTTP_TRAFFIC, MALICIOUS_JARM, NETWORK_GPS_LOCATION_LEAKED, # ICMP_TIMESTAMP_SCAN, ICMP_ADDRESS_SCAN, ICMP_ADDRESS_MASK_SCAN, @@ -560,6 +903,44 @@ DisabledAlerts: # disabled_detections = [THREAT_INTELLIGENCE_BLACKLISTED_IP] disabled_detections: [] +############################# +EvidenceSignals: + + # Slips adds an evidence_signal field to every evidence centrally when it + # reaches the shared evidence pipeline. Modules do not need to set it. + # + # Allowed values are: + # - PAMP + # - DAMP + # + # If an evidence type is not listed under overrides, Slips uses + # default_signal. Unknown or invalid entries also fall back to PAMP. + default_signal: PAMP + + # Override the evidence signal per evidence type. + # By default ANOMALOUS_FLOW and MALICIOUS_FLOW are marked as DAMP because + # they are emitted by anomaly-detection modules. You can also override + # additional evidence types here. + overrides: + ANOMALOUS_FLOW: DAMP + MALICIOUS_FLOW: DAMP + UNSOLICITED_ARP: DAMP + CONNECTION_TO_MULTIPLE_PORTS: DAMP + CONNECTION_TO_PRIVATE_IP: DAMP + CONNECTION_WITHOUT_DNS: DAMP + DNS_WITHOUT_CONNECTION: DAMP + HIGH_ENTROPY_DNS_ANSWER: DAMP + LONG_CONNECTION: DAMP + MULTIPLE_RECONNECTION_ATTEMPTS: DAMP + MULTIPLE_SSH_VERSIONS: DAMP + NON_SSL_PORT_443_CONNECTION: DAMP + UNKNOWN_PORT: DAMP + YOUNG_DOMAIN: DAMP + HTTP_TRAFFIC: DAMP + MULTIPLE_USER_AGENT: DAMP + NON_HTTP_PORT_80_CONNECTION: DAMP + SSH_SUCCESSFUL: DAMP + ############################# Docker: # ID and group id of the user who started to docker container diff --git a/docs/alert_summary_module.md b/docs/alert_summary_module.md new file mode 100644 index 0000000000..a5a55942a8 --- /dev/null +++ b/docs/alert_summary_module.md @@ -0,0 +1,153 @@ +# Alert Summary Module + +The `AlertSummary` module creates one analyst-facing summary paragraph for every Slips alert. + +It listens for generated alerts, gathers the evidence that caused each alert, queries the shared `LLM` module, and writes one summary line per alert to `output/alerts/alerts-summary.log`. + +## What it does + +For each alert, the module: + +1. Receives the alert on `new_alert`. +2. Loads all evidence records referenced by `alert.correl_id`. +3. Groups similar evidence descriptions into an incident-style digest. +4. Estimates whether the digest fits the final prompt budget. +5. Sends either the final summary prompt or one reduction prompt on + `llm_request`. +6. Waits for the matching `llm_response`. +7. Repeats reduction layers as needed until the final prompt fits. +8. Normalizes the last reply into one plain-text paragraph and appends it to + the alert summary log. + +If the LLM pipeline fails, the module writes a local heuristic fallback summary +for the alert instead of leaving the run without output. + +## Recursive summary hierarchy + +This module follows the prompt style from `https://github.com/stratosphereips/Slips-tools/alert_summary/inference.py` as per the original specifgication, but it is completely implemented and adapted to live Slips alerts in this module. + +When the grouped evidence is too large for the final prompt, the module does +not truncate it. It performs a hierarchy of summaries instead: + +1. Split the grouped digest into prompt-sized chunks. +2. Ask the shared LLM for one intermediate digest per chunk. +3. If the combined chunk digests are still too large, split and summarize + them again. +4. Send the reduced digest to the final analyst-summary prompt. + +If one grouped line is too large by itself, it is split on sentence or word +boundaries so the content is preserved without clipping. + +## Configuration + +Example section in `config/slips.yaml`: + +```yaml +alert_summary: + enabled: false + log_verbosity: 2 + allowed_backends: [] + llm_temperature: 0.2 + llm_max_tokens: 220 + llm_response_timeout_seconds: 120 +``` + +Configuration reference: + +- `enabled`: enables or disables the module. +- `log_verbosity`: controls how much operational detail is written to + `/llm-summary/alert_summary.log`. Use `0` for an empty file, + `1` for startup, shutdown, and failures, `2` for per-alert queueing and + request flow, and `3` for prompt-budget and reduction-layer details. +- `allowed_backends`: preferred runtime-ready LLM backend aliases for this + module. If empty, the module falls back to the shared LLM default backend. +- `llm_temperature`: low-temperature setting used to keep summaries stable and + analyst-oriented. +- `llm_max_tokens`: output budget for the final analyst paragraph. +- `llm_response_timeout_seconds`: hard timeout for one in-flight shared-LLM + request. If set to `0`, the module waits indefinitely. + +## Prompt design + +The final prompt contains: + +- alert metadata +- grouped evidence patterns with time ranges, counts, severities, and sample + IPs or ports +- explicit guidance that evidence threat levels must be weighed differently, + and that `info` evidence is context only rather than a security finding by + itself +- explicit separation between current-alert evidence and historical context, + so history can inform recurrence and risk without being restated as if it + happened in the current alert +- instructions to explain the suspicious behavior, strongest supporting or + weakening evidence, likely alert validity, and operational risk + +Reduction prompts reuse the same alert metadata but ask the model to compress +one chunk into a shorter intermediate digest for the next reduction layer +while preserving threat-level distinctions, including the rule that `info` +evidence remains contextual. + +Recent-history prompt context is now intentionally structured from historical +patterns and alert-level metrics, not prior free-text LLM summaries. That +avoids recursively feeding one generated explanation into later prompts, which +can otherwise cause old details to be repeated as if they were part of the +current alert. + +## Shared LLM integration + +The module uses the existing shared LLM contract: + +- request channel: `llm_request` +- response channel: `llm_response` + +Each request contains: + +- `request_id` +- `requester = "alert_summary"` +- `backend` +- `messages` +- `temperature` +- `max_tokens` +- `metadata.alert_id` +- `metadata.profileid` +- `metadata.timewindow` +- `metadata.evidence_count` +- `metadata.grouped_item_count` +- `metadata.reduction_layer` +- `metadata.prompt_version` + +Reduction requests also include `metadata.chunk_index` and +`metadata.chunk_count`. + +The module accepts only the response whose `request_id` matches the active +in-flight request. + +## Output files + +Analyst summaries are written to: + +```text +output/alerts/alerts-summary.log +``` + +Operational logs for this module are written to: + +```text +/llm-summary/alert_summary.log +``` + +Operational logs for the shared LLM backend are written to: + +```text +/LLM/llm.log +``` + +The alert-summary log records queueing, prompt-budget checks, reduction-layer +progress, request publication, replies, failures, and shutdown handling. + +## Shutdown behavior + +The module keeps waiting during shutdown while a shared LLM request is still +in flight. That prevents the old race where the shared `LLM` module finished +later and published a reply after `alert_summary` had already exited. diff --git a/docs/create_new_module.md b/docs/create_new_module.md index b8bbda9e4e..9cdf43ddf3 100644 --- a/docs/create_new_module.md +++ b/docs/create_new_module.md @@ -62,6 +62,7 @@ cp -a modules/template modules/local_connection_detector Each module in Slips should have a name, author and description. Use a snake_case module package and main file name in the `x_y_doer` style already used in the repository, for example `http_analyzer` or `local_connection_detector`. +The class-level `name` attribute must also be snake_case; Slips raises a `RuntimeError` when a module class defines a non-snake-case name. We should change the name inside the python file by finding the lines with the name and description in the class 'Module' and changing them: diff --git a/docs/detection_modules.md b/docs/detection_modules.md index 0a8cacec9d..97da31220e 100644 --- a/docs/detection_modules.md +++ b/docs/detection_modules.md @@ -128,6 +128,26 @@ tr:nth-child(even) { module to detect malicious flows using machine learning ✅ + + LLM + shared service module that sends prompts to configured OpenAI, Anthropic, or Ollama backends and publishes the replies for other modules + ✅ + + + Alert Summary + analyst-facing module that summarizes each alert and its correlated evidence into one human-readable paragraph using the shared LLM service + ✅ + + + RegexGenerator + shared service module that continuously generates pseudo-random regexes, rejects those matching benign corpora, and stores accepted regexes for later modules + ✅ + + + T Cell + immune-style responder that consumes PAMP evidence, matches accepted regexes, and escalates to blocking or memory using a per-antigen state machine + ✅ + @@ -146,6 +166,60 @@ It correlates repeated SSH sessions by source IP, destination IP, destination po For the full design and configuration details, see: - [Bruteforcing Module](bruteforcing.md) +## LLM Module + +The LLM module is a shared service for other Slips modules. + +It listens on the Redis channel `llm_request`, sends the request to the selected +configured backend, and publishes the result on `llm_response`. + +For the full request and response format, backend configuration, and examples, +see [LLM Module](llm_module.md). + +## Alert Summary Module + +The Alert Summary module is an analyst-facing consumer of the shared LLM +service. + +It listens on `new_alert`, loads the evidence records referenced by the alert, +sends that structured context to the shared `LLM` module, and writes one +paragraph per alert to `output/alerts/alerts-summary.log`. + +The generated paragraph is meant to help a human analyst quickly judge whether +the alert looks real, how strong the evidence is, and what operational risk it +implies. + +For the full configuration, prompt behavior, and output format, see +[Alert Summary Module](alert_summary_module.md). + +## Regex Generator Module + +The RegexGenerator module is a shared service for other Slips modules. + +It uses the shared LLM module to generate one regex at a time for DNS domains, +URIs, filenames, TLS SNI, and certificate CN fields, tests that regex against +a benign corpus, and stores accepted regexes in a local SQLite database. + +For the full configuration, acceptance pipeline, and DB helper usage, see +[Regex Generator Module](regex_generator_module.md). + +## T Cell Module + +The T Cell module is a second-stage immune responder for Slips. + +It listens on `evidence_added`, uses the central `evidence_signal` field, +extracts structured antigens from evidence and linked altflows, and checks +those values against accepted regexes already stored by `RegexGenerator`. +Depending on co-stimulation and context signals, it becomes tolerant, +activates, requests containment over `new_blocking`, or stores memory in its +own SQLite DB. Both `PAMP` and `DAMP` can create a recognized cell when a +regex matches an extracted antigen, but `DAMP` primes a weaker cell with +stricter later thresholds and a shorter wait window. Stored `DAMP` +observations also raise the danger pressure used by co-stimulation and +context. + +For the full state machine, formulas, DB schema, and configuration, see +[T Cell Module](t_cell_module.md). ## HTTPS Anomaly Detection Module diff --git a/docs/evidence_signals.md b/docs/evidence_signals.md new file mode 100644 index 0000000000..c9331cb72d --- /dev/null +++ b/docs/evidence_signals.md @@ -0,0 +1,160 @@ +# Evidence Signals + +Slips now adds an `evidence_signal` field to every evidence when the evidence reaches the shared evidence pipeline. Detection modules do not need to set this field themselves. + +The `T Cell` module consumes this same central field and uses it to decide how +strongly a new cell should be primed. Both `PAMP` and `DAMP` evidence can +create `0 -> 1` when an extracted antigen matches an accepted regex. +`PAMP` keeps the base downstream thresholds, while `DAMP` stores a weaker +priming profile with stricter later thresholds, higher corroboration counts, +and a shorter waiting window. `DAMP` evidence still contributes to the mixed +danger pressure used in co-stimulation and context calculations for the same +responsible IP, and each new `DAMP` also reevaluates cells that are already +waiting on that responsible IP. See +[T Cell Module](t_cell_module.md) for the responder details. + +The supported values are: + +- `PAMP` +- `DAMP` + +Unknown evidence types default to `PAMP`. + +## Configuration + +Configure the default signal and per-evidence overrides in `config/slips.yaml`: + +```yaml +EvidenceSignals: + default_signal: PAMP + overrides: + ANOMALOUS_FLOW: DAMP + MALICIOUS_FLOW: DAMP + UNSOLICITED_ARP: DAMP + CONNECTION_TO_MULTIPLE_PORTS: DAMP + CONNECTION_TO_PRIVATE_IP: DAMP + CONNECTION_WITHOUT_DNS: DAMP + DNS_WITHOUT_CONNECTION: DAMP + HIGH_ENTROPY_DNS_ANSWER: DAMP + LONG_CONNECTION: DAMP + MULTIPLE_RECONNECTION_ATTEMPTS: DAMP + MULTIPLE_SSH_VERSIONS: DAMP + NON_SSL_PORT_443_CONNECTION: DAMP + UNKNOWN_PORT: DAMP + YOUNG_DOMAIN: DAMP + HTTP_TRAFFIC: DAMP + MULTIPLE_USER_AGENT: DAMP + NON_HTTP_PORT_80_CONNECTION: DAMP + SSH_SUCCESSFUL: DAMP +``` + +Rules: + +- `default_signal` is applied to every evidence type that is not listed in `overrides`. +- `overrides` keys are evidence type names from `EvidenceType`. +- Invalid values fall back to `PAMP`. +- The default shipped mapping marks the following evidence types as `DAMP`: + `ANOMALOUS_FLOW`, `MALICIOUS_FLOW`, `UNSOLICITED_ARP`, + `CONNECTION_TO_MULTIPLE_PORTS`, `CONNECTION_TO_PRIVATE_IP`, + `CONNECTION_WITHOUT_DNS`, `DNS_WITHOUT_CONNECTION`, + `HIGH_ENTROPY_DNS_ANSWER`, `LONG_CONNECTION`, + `MULTIPLE_RECONNECTION_ATTEMPTS`, `MULTIPLE_SSH_VERSIONS`, + `NON_SSL_PORT_443_CONNECTION`, `UNKNOWN_PORT`, `YOUNG_DOMAIN`, + `HTTP_TRAFFIC`, `MULTIPLE_USER_AGENT`, `NON_HTTP_PORT_80_CONNECTION`, + and `SSH_SUCCESSFUL`. + +## Propagation + +The field is added centrally before the evidence is stored or published, so it is available consistently in: + +- Redis-stored evidence +- `alerts.json` +- STIX/TAXII export +- SlipsWeb dashboard payloads + +## Current Evidence Inventory + +The table below lists the evidence types currently emitted by Slips modules and their default signal classification. + +| Module | Evidence type | Default signal | +| --- | --- | --- | +| `anomaly_detection_https` | `ANOMALOUS_FLOW` | `DAMP` | +| `arp` | `ARP_SCAN` | `PAMP` | +| `arp` | `ARP_OUTSIDE_LOCALNET` | `PAMP` | +| `arp` | `UNSOLICITED_ARP` | `DAMP` | +| `arp` | `MITM_ARP_ATTACK` | `PAMP` | +| `flowalerts` | `BAD_SMTP_LOGIN` | `PAMP` | +| `flowalerts` | `CN_URL_MISMATCH` | `PAMP` | +| `flowalerts` | `CONNECTION_TO_MULTIPLE_PORTS` | `DAMP` | +| `flowalerts` | `CONNECTION_TO_PRIVATE_IP` | `DAMP` | +| `flowalerts` | `CONNECTION_WITHOUT_DNS` | `DAMP` | +| `flowalerts` | `DATA_UPLOAD` | `PAMP` | +| `flowalerts` | `DEVICE_CHANGING_IP` | `PAMP` | +| `flowalerts` | `DGA_NXDOMAINS` | `PAMP` | +| `flowalerts` | `DIFFERENT_LOCALNET` | `PAMP` | +| `flowalerts` | `DNS_ARPA_SCAN` | `PAMP` | +| `flowalerts` | `DNS_WITHOUT_CONNECTION` | `DAMP` | +| `flowalerts` | `GRE_SCAN` | `PAMP` | +| `flowalerts` | `GRE_TUNNEL` | `PAMP` | +| `flowalerts` | `HIGH_ENTROPY_DNS_ANSWER` | `DAMP` | +| `flowalerts` | `HORIZONTAL_PORT_SCAN` | `PAMP` | +| `flowalerts` | `INCOMPATIBLE_CN` | `PAMP` | +| `flowalerts` | `INVALID_DNS_RESOLUTION` | `PAMP` | +| `flowalerts` | `LONG_CONNECTION` | `DAMP` | +| `flowalerts` | `MALICIOUS_JA3` | `PAMP` | +| `flowalerts` | `MALICIOUS_JA3S` | `PAMP` | +| `flowalerts` | `MALICIOUS_SSL_CERT` | `PAMP` | +| `flowalerts` | `MULTIPLE_RECONNECTION_ATTEMPTS` | `DAMP` | +| `flowalerts` | `MULTIPLE_SSH_VERSIONS` | `DAMP` | +| `flowalerts` | `NON_SSL_PORT_443_CONNECTION` | `DAMP` | +| `flowalerts` | `PASSWORD_GUESSING` | `PAMP` | +| `flowalerts` | `PASTEBIN_DOWNLOAD` | `PAMP` | +| `flowalerts` | `PORT_0_CONNECTION` | `PAMP` | +| `flowalerts` | `SELF_SIGNED_CERTIFICATE` | `PAMP` | +| `flowalerts` | `SMTP_LOGIN_BRUTEFORCE` | `PAMP` | +| `flowalerts` | `SSH_SUCCESSFUL` | `DAMP` | +| `flowalerts` | `UNKNOWN_PORT` | `DAMP` | +| `flowalerts` | `VERTICAL_PORT_SCAN` | `PAMP` | +| `flowalerts` | `YOUNG_DOMAIN` | `DAMP` | +| `flowmldetection` | `MALICIOUS_FLOW` | `DAMP` | +| `http_analyzer` | `EMPTY_CONNECTIONS` | `PAMP` | +| `http_analyzer` | `EXECUTABLE_MIME_TYPE` | `PAMP` | +| `http_analyzer` | `HTTP_TRAFFIC` | `DAMP` | +| `http_analyzer` | `INCOMPATIBLE_USER_AGENT` | `PAMP` | +| `http_analyzer` | `MULTIPLE_USER_AGENT` | `DAMP` | +| `http_analyzer` | `NON_HTTP_PORT_80_CONNECTION` | `DAMP` | +| `http_analyzer` | `PASTEBIN_DOWNLOAD` | `PAMP` | +| `http_analyzer` | `SUSPICIOUS_USER_AGENT` | `PAMP` | +| `http_analyzer` | `WEIRD_HTTP_METHOD` | `PAMP` | +| `ip_info` | `MALICIOUS_JARM` | `PAMP` | +| `leak_detector` | `NETWORK_GPS_LOCATION_LEAKED` | `PAMP` | +| `network_discovery` | `DHCP_SCAN` | `PAMP` | +| `network_discovery` | `ICMP_ADDRESS_MASK_SCAN` | `PAMP` | +| `network_discovery` | `ICMP_ADDRESS_SCAN` | `PAMP` | +| `network_discovery` | `ICMP_TIMESTAMP_SCAN` | `PAMP` | +| `network_discovery.horizontal_portscan` | `HORIZONTAL_PORT_SCAN` | `PAMP` | +| `network_discovery.vertical_portscan` | `VERTICAL_PORT_SCAN` | `PAMP` | +| `p2ptrust` | `MALICIOUS_IP_FROM_P2P_NETWORK` | `PAMP` | +| `p2ptrust` | `P2P_REPORT` | `PAMP` | +| `p2ptrust.go_director` | `P2P_REPORT` | `PAMP` | +| `rnn_cc_detection` | `COMMAND_AND_CONTROL_CHANNEL` | `PAMP` | +| `threat_intelligence` | `MALICIOUS_DOWNLOADED_FILE` | `PAMP` | +| `threat_intelligence` | `THREAT_INTELLIGENCE_ANSWER_OF_BLACKLISTED_QUERY` | `PAMP` | +| `threat_intelligence` | `THREAT_INTELLIGENCE_BLACKLISTED_ASN` | `PAMP` | +| `threat_intelligence` | `THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER` | `PAMP` | +| `threat_intelligence` | `THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN` | `PAMP` | +| `threat_intelligence` | `THREAT_INTELLIGENCE_TO_BLACKLISTED_IP` | `PAMP` | +| `threat_intelligence.urlhaus` | `MALICIOUS_DOWNLOADED_FILE` | `PAMP` | +| `threat_intelligence.urlhaus` | `THREAT_INTELLIGENCE_MALICIOUS_URL` | `PAMP` | + +`ANOMALOUS_FLOW` is emitted by `anomaly_detection_https`, while `MALICIOUS_FLOW` +is emitted by `flowmldetection`. Both are marked as `DAMP` by default in the +central signal configuration. The additional shipped DAMP overrides in +`config/slips.yaml` are `UNSOLICITED_ARP`, +`CONNECTION_TO_MULTIPLE_PORTS`, `CONNECTION_TO_PRIVATE_IP`, +`CONNECTION_WITHOUT_DNS`, `DNS_WITHOUT_CONNECTION`, +`HIGH_ENTROPY_DNS_ANSWER`, `LONG_CONNECTION`, +`MULTIPLE_RECONNECTION_ATTEMPTS`, `MULTIPLE_SSH_VERSIONS`, +`NON_SSL_PORT_443_CONNECTION`, `UNKNOWN_PORT`, `YOUNG_DOMAIN`, +`HTTP_TRAFFIC`, `MULTIPLE_USER_AGENT`, `NON_HTTP_PORT_80_CONNECTION`, +and `SSH_SUCCESSFUL`. diff --git a/docs/images/regex_generator/coverage_report_overview.png b/docs/images/regex_generator/coverage_report_overview.png new file mode 100644 index 0000000000..1fa7a03cc9 Binary files /dev/null and b/docs/images/regex_generator/coverage_report_overview.png differ diff --git a/docs/images/t_cell/t_cell_report_overview.png b/docs/images/t_cell/t_cell_report_overview.png new file mode 100644 index 0000000000..82fcbf4649 Binary files /dev/null and b/docs/images/t_cell/t_cell_report_overview.png differ diff --git a/docs/index.rst b/docs/index.rst index 58e4378770..5d0d11f132 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,12 @@ This documentation gives an overview how Slips works, how to use it and how to h - **Detection modules**. Explanation of detection modules in Slips, types of input and output. See :doc:`Detection modules `. - **brute_force_detector**. Dedicated documentation for the SSH brute force detector module. See :doc:`brute_force_detector `. +- **LLM module**. Shared access to configured LLM backends from other Slips modules. See :doc:`LLM module `. +- **Alert Summary module**. Analyst-facing one-paragraph summaries for alerts using the shared LLM service. See :doc:`Alert Summary module `. + +- **Regex Generator module**. Shared service that generates and validates pseudo-random regexes for later Zeek-side use. See :doc:`Regex Generator module `. + +- **T Cell module**. Immune-style responder that consumes PAMP evidence, regex matches, and context to decide blocking or memory. See :doc:`T Cell module `. - **HTTPS anomaly detection**. Detailed design and behavior of the HTTPS anomaly detector. See :doc:`HTTPS anomaly detection `. @@ -29,6 +35,8 @@ This documentation gives an overview how Slips works, how to use it and how to h - **Exporting**. The exporting module allows Slips to export to Slack and STIX servers. See :doc:`Exporting `. +- **Evidence signals**. Central PAMP/DAMP classification for evidence, configuration overrides, and the current evidence inventory. See :doc:`Evidence signals `. + - **Slips in Action**. Example of using slips to analyze different PCAPs See :doc:`Slips in action `. - **Contributing**. Explanation how to contribute to Slips, and instructions how to implement new detection module in Slips. See :doc:`Contributing `. @@ -54,11 +62,16 @@ This documentation gives an overview how Slips works, how to use it and how to h architecture detection_modules brute_force_detector + llm_module + alert_summary_module + regex_generator_module + t_cell_module https_anomaly_detection flow_alerts features training exporting + evidence_signals P2P fides create_new_module diff --git a/docs/llm_module.md b/docs/llm_module.md new file mode 100644 index 0000000000..deaa02950d --- /dev/null +++ b/docs/llm_module.md @@ -0,0 +1,298 @@ +# LLM Module + +The `LLM` module provides shared access to configured language model backends +for the rest of Slips. + +Instead of each module managing its own API keys, URLs, and HTTP logic, they +can publish a request to Redis and read the answer from a shared response +channel. + +## What It Does + +The module: + +1. Reads LLM backend configuration from `config/slips.yaml` +2. Connects to one or more configured providers +3. Subscribes to the Redis channel `llm_request` +4. Sends each request to the selected backend +5. Publishes the result to `llm_response` + +Supported providers: + +- `ollama` +- `openai` +- `anthropic` + +## Implementation Layout + +The `LLM` class in `modules/llm/llm.py` is the only class in this module that +implements `IModule`. Backend support is split into smaller files: + +- `llm_backend_config.py`: validates one backend alias configuration. +- `llm_backend.py`: owns shared HTTP request, URL, usage, and text helpers. +- `openai_backend_mixin.py`: contains `MixinOpenAIBackend`. +- `anthropic_backend_mixin.py`: contains `MixinAnthropicBackend`. +- `ollama_backend_mixin.py`: contains `MixinOllamaBackend`. +- `llm_errors.py`: contains shared configuration and request exceptions. + +New provider-specific backend behavior should be added as a mixin file under +`modules/llm/`, with a class name prefixed by `Mixin` and a filename ending in +`_mixin.py`. + +## Configuration + +Example section in `config/slips.yaml`: + +```yaml +llm: + enabled: true + default_backend: local_qwen + worker_threads: 2 + queue_size: 100 + backends: + local_qwen: + provider: ollama + model: qwen2.5:3b + base_url: http://127.0.0.1:11434 + timeout: 120 + openai_default: + provider: openai + model: gpt-4o-mini + base_url: https://api.openai.com/v1 + api_key_env: OPENAI_API_KEY + timeout: 60 + claude_default: + provider: anthropic + model: claude-sonnet-4-5 + base_url: https://api.anthropic.com + api_key_env: ANTHROPIC_API_KEY + timeout: 60 +``` + +Configuration reference: + +- `enabled`: enables or disables the LLM service module. +- `default_backend`: backend alias used when a request omits `backend`. +- `worker_threads`: number of requests processed in parallel. +- `queue_size`: maximum number of queued requests in memory. +- `backends`: mapping of backend alias to backend configuration. + +Per-backend options: + +- `provider`: one of `ollama`, `openai`, or `anthropic`. +- `model`: default model for that backend alias. +- `base_url`: provider endpoint. If omitted, the provider default is used. +- `timeout`: HTTP timeout in seconds. +- `api_key`: optional inline API key for `openai` or `anthropic`. +- `api_key_env`: optional environment variable containing the API key. +- `api_key_file`: optional file path containing the API key. +- `anthropic_version`: optional Anthropic API version header. Default is + `2023-06-01`. + +Each backend is a named connection. Other modules select it by name using the +`backend` field in the request. + +## Discovery Helper + +Other modules should discover available backends with: + +```python +available = self.db.get_available_llm_backends() +``` + +This returns only runtime-ready backends: + +```json +{ + "default_backend": "local_qwen", + "backends": { + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b" + }, + "openai_default": { + "provider": "openai", + "model": "gpt-4o-mini" + } + } +} +``` + +During startup the helper may temporarily return: + +```json +{ + "default_backend": "", + "backends": {} +} +``` + +Caller modules should retry later if they need LLM access and the registry is +still empty. + +## How Caller Modules Must Correlate Responses + +The current design uses: + +- one shared request channel: `llm_request` +- one shared response channel: `llm_response` + +This means caller modules must correlate replies themselves. + +Required caller pattern: + +1. Subscribe to `llm_response` during module initialization. +2. Discover runtime-ready backends with + `self.db.get_available_llm_backends()`. +3. Choose a backend alias from the returned registry. +4. Generate a unique `request_id` before publishing. +5. Keep local pending state keyed by `request_id`. +6. Publish the request to `llm_request`. +7. When reading `llm_response`, ignore any response whose `request_id` is not + one of yours. + +If multiple caller modules send requests at the same time, `request_id` is what +separates the replies. `requester` is only a human-readable label and should +not be treated as the primary routing key. + +## Redis Contract + +### Request channel + +Channel: `llm_request` + +Minimal request: + +```json +{ + "request_id": "req-123", + "backend": "local_qwen", + "prompt": "Summarize this alert" +} +``` + +Structured request: + +```json +{ + "request_id": "req-456", + "requester": "HTTP Analyzer", + "backend": "openai_default", + "messages": [ + {"role": "system", "content": "You are a concise security analyst."}, + {"role": "user", "content": "Analyze this flow."} + ], + "temperature": 0.2, + "max_tokens": 300, + "metadata": {"uid": "C1abc"} +} +``` + +Fields: + +- `request_id`: technically optional, but caller modules should always set it. + This is the main correlation key on the shared response channel. +- `requester`: optional caller name. +- `backend`: optional if `default_backend` is set. +- `prompt`: shortcut for one user message. +- `messages`: list of text messages with roles `system`, `user`, or `assistant`. +- `model`: optional model override for the selected backend. +- `temperature`: optional sampling control. +- `max_tokens`: optional response length limit. +- `metadata`: optional passthrough object echoed back in the response. + +### Response channel + +Channel: `llm_response` + +Success: + +```json +{ + "request_id": "req-456", + "requester": "HTTP Analyzer", + "backend": "openai_default", + "provider": "openai", + "model": "gpt-4o-mini", + "success": true, + "text": "The flow looks like repeated beaconing with stable timing.", + "usage": { + "input_tokens": 120, + "output_tokens": 40, + "total_tokens": 160 + }, + "metadata": {"uid": "C1abc"}, + "ts": 1760000000.0 +} +``` + +Failure: + +```json +{ + "request_id": "req-999", + "backend": "missing_backend", + "success": false, + "error": "Unknown LLM backend requested: missing_backend", + "text": "", + "metadata": {}, + "ts": 1760000000.0 +} +``` + +## Example Integration from Another Module + +Publish: + +```python +import json +import uuid + +available = self.db.get_available_llm_backends() +backend = available["default_backend"] +if not backend: + return + +request_id = f"{self.name}-{uuid.uuid4()}" +pending_requests[request_id] = {"profileid": profileid} + +request = { + "request_id": request_id, + "requester": self.name, + "backend": backend, + "prompt": "Summarize this alert in 2 lines.", + "metadata": {"profileid": profileid}, +} +self.db.publish("llm_request", json.dumps(request)) +``` + +Subscribe: + +```python +self.c_llm = self.db.subscribe("llm_response") +self.channels["llm_response"] = self.c_llm +``` + +Read: + +```python +if msg := self.get_msg("llm_response"): + response = json.loads(msg["data"]) + request_id = response["request_id"] + if request_id not in pending_requests: + return + + context = pending_requests.pop(request_id) + text = response["text"] +``` + +## Operational Notes + +- The module uses one shared response channel, so requesters must match on + `request_id`. +- Caller modules should always generate `request_id` themselves instead of + relying on the service to create one. +- The first version is text-only. +- If the module is disabled or no valid backends are configured, it will stop + cleanly and no request processing will occur. +- Backend selection is by runtime-ready backend alias, not only by model name. diff --git a/docs/regex_generator_module.md b/docs/regex_generator_module.md new file mode 100644 index 0000000000..470bf0838d --- /dev/null +++ b/docs/regex_generator_module.md @@ -0,0 +1,434 @@ +# Regex Generator Module + +The `RegexGenerator` module continuously creates one pseudo-random regex at a +time for later Zeek-side use. + +It uses the shared `LLM` module over Redis, validates the generated regex +against a benign corpus, and stores accepted regexes in a dedicated local +SQLite database that later Slips modules can read through `DBManager`. + +## What it does + +The module: + +1. Reads its configuration from `config/slips.yaml` +2. Discovers runtime-ready LLM backends using the shared LLM registry +3. Chooses the next regex type with weighted random selection +4. Sends one generation request over `llm_request` +5. Waits for the matching `llm_response` +6. Validates the regex and tests it against a benign corpus +7. Stores accepted results in local SQLite. Rejected results are only persisted + if explicitly enabled. + +Supported regex types: + +- `dns_domain` +- `uri` +- `filename` +- `tls_sni` +- `certificate_cn` + +## Configuration + +Example section in `config/slips.yaml`: + +```yaml +regex_generator: + enabled: false + create_log_file: false + generation_interval_seconds: 5 + allowed_backends: [] + llm_temperature: 1.2 + llm_max_tokens: 80 + llm_response_timeout_seconds: 90 + recent_history_size: 0 + max_regex_length: 180 + regex_validation_timeout_seconds: 2 + benign_match_strength_threshold: 75 + type_weights: + dns_domain: 1 + uri: 1 + filename: 1 + tls_sni: 1 + certificate_cn: 1 + store_dir: output/regex_generator + persistent_store_dir: databases/regex_store + store_rejected_regexes: false + max_stored_rejected_regexes: 10000 + seed_benign_samples: true + +whitelists: + tranco_top_benign_limit: 1000 +``` + +Configuration reference: + +- `enabled`: enables or disables the module. +- `create_log_file`: creates `output/regex_generator.log` with detailed module + progress messages. This file rotates on the same global + `parameters.rotation` / `parameters.rotation_period` schedule used by the + current Slips run. +- `generation_interval_seconds`: delay between completed generation cycles. + Set `0` to start the next cycle immediately after the previous one finishes. +- `allowed_backends`: preferred backend aliases for this module. +- `llm_temperature`: generation temperature. Kept high to keep outputs varied. +- `llm_max_tokens`: max tokens for the LLM reply. The module asks for one regex + line only, so keep this small. +- `llm_response_timeout_seconds`: soft warning threshold while waiting for the + matching `llm_response`. The module keeps waiting after this. Set `0` to + disable the warning. +- `recent_history_size`: compatibility knob kept at `0`. Prompt history is not + sent to the LLM; repetition is checked locally. +- `max_regex_length`: hard reject longer regexes. +- `regex_validation_timeout_seconds`: hard wall-clock timeout for local regex + validation and benign-corpus matching. This prevents one pathological regex + from freezing the module. Set `0` to disable it. +- `benign_match_strength_threshold`: score from `0` to `100` used during the + benign scan. A regex is rejected only if its strongest benign match reaches + or exceeds this threshold. Higher values are more permissive. +- `type_weights`: weighted random choice among the supported regex types. +- `store_dir`: directory containing `benign_corpus.sqlite` and + `generated_regexes.sqlite`. Absolute paths are used as-is. Relative paths are + resolved inside the current Slips run output directory. The default + `output/regex_generator` therefore becomes `/regex_generator`. +- `persistent_store_dir`: stable directory for the regex SQLite files. Relative + paths are resolved inside `parameters.permanent_dir`; the default + `databases/regex_store` therefore becomes + `/databases/regex_store`. If set, it takes precedence over + `store_dir` and lets the generator reuse the same DBs across many Slips + restarts. +- `store_rejected_regexes`: stores rejected regexes in SQLite for audit/debug + purposes. Default `false` so discarded candidates do not fill the disk. +- `max_stored_rejected_regexes`: retention cap for rejected rows when + `store_rejected_regexes` is enabled. Set `0` for unlimited retention. +- `seed_benign_samples`: seed the benign DB once with a small built-in sample. +- `whitelists.tranco_top_benign_limit`: number of ordered Tranco whitelist + domains reused as benign data by `RegexGenerator` and the offline coverage + report. + +## LLM request and response usage + +The module uses the existing shared LLM channels only: + +- request channel: `llm_request` +- response channel: `llm_response` + +Each generation request includes: + +- `request_id` +- `requester = "RegexGenerator"` +- `backend` +- `messages` +- `temperature` +- `max_tokens` +- `metadata.regex_type` +- `metadata.prompt_version` +- `metadata.generation_nonce` + +The prompt requires the model to return exactly one regex line. No JSON, +explanation, or code fences. The parser still accepts JSON-shaped replies as a +fallback for compatibility, but the active prompt is raw-regex only. + +After the reply arrives, the module does not reject on any benign hit. It +streams the benign corpus for the selected type, computes a benign +match-strength score for each regex/string match, and rejects only if some +benign string reaches or exceeds `benign_match_strength_threshold`. + +V1 keeps one request in flight at a time, so response correlation is simple: +only the matching `request_id` is accepted. +If the local LLM is slow, the module keeps waiting and only logs a warning +after `llm_response_timeout_seconds`. + +If `create_log_file` is enabled, the module also writes detailed progress logs +to: + +```text +output/regex_generator.log +``` + +This file records: + +- selected regex type +- selected backend +- published `llm_request` `request_id` +- slow-wait warnings while the LLM is still working +- accepted regexes +- rejected regexes and rejection reasons + +Accepted regexes are stored in the configured persistent store by default: + +```text +/databases/regex_store/generated_regexes.sqlite +``` + +If `persistent_store_dir` is empty, the fallback location is +`/regex_generator/generated_regexes.sqlite`. + +Rejected regexes are tracked in memory during the current run to reduce cheap +repeats, but they are not stored on disk unless `store_rejected_regexes` is +enabled. + +## Acceptance pipeline + +After the matching `llm_response` arrives, the module: + +1. Extracts one regex line from the LLM reply +3. Rejects empty or malformed results +4. Applies static safety validation +5. Checks local duplicates with a bloom filter and exact SQLite lookup +6. Streams the benign corpus for the selected type +7. Computes a benign match-strength score for each regex/string match +8. Rejects only if some benign string reaches the configured threshold +9. Stores accepted regexes for later use + +Static validation rejects: + +- non-ASCII regexes +- regexes longer than `max_regex_length` +- lookbehind +- backreferences +- unbounded `.*`-style prefix/suffix patterns +- obviously broad patterns such as `.*` and `.+` +- nested wildcard structures that risk catastrophic backtracking +- invalid syntax + +The benign match-strength score is an estimate from `0` to `100`. It is +computed per regex and per benign string using the strongest match span found +by Python `re.finditer()`. + +For one matched span, the score is: + +```text +score = + 40 * span_ratio + + 12 * start_bonus + + 12 * end_bonus + + 16 * full_bonus + + 30 * specificity_ratio + - 18 * wildcard_penalty +``` + +The result is clipped to `0..100`. The regex keeps the highest score it +obtains against that benign string. If any benign string reaches or exceeds +`benign_match_strength_threshold`, the regex is rejected. + +The terms mean: + +- `span_ratio = matched_span_length / benign_string_length` +- `start_bonus = 1` if the match starts at offset `0`, else `0` +- `end_bonus = 1` if the match ends at the final character, else `0` +- `full_bonus = 1` if the match covers the entire benign string, else `0` +- `specificity_ratio = literal_chars / (literal_chars + meta_tokens)` +- `wildcard_penalty = min(1.0, wildcard_points / ((literal_chars + meta_tokens) / 2))` + +Regex-specific features are measured from the regex text itself: + +- `literal_chars` counts explicit alphanumeric and common structural literal + characters such as `-`, `_`, `/`, `:`, `,`, `@`, and `=` +- escaped literals such as `\.` count as literal characters +- `meta_tokens` counts regex syntax such as `.`, `[]`, `*`, `+`, `?`, groups, + anchors, and generic escapes +- `wildcard_points` penalize broad constructs: + - `.*` or `.+` adds `2.5` + - bare `.` adds `1.5` + - `[` character classes add `1.2` + - `*`, `+`, and `?` add `1.0` + - generic escapes such as `\w` also add penalty + +Examples: + +- Regex `^google\.com$` against benign string `google.com` + - full span match, starts at `0`, ends at the end, full-match bonus applies + - specificity is high because most of the pattern is literal + - wildcard penalty is low + - score is very high, so this benign match is rejected + +- Regex `google` against benign string `google.com` + - only part of the string is covered + - it starts at `0` but does not end at the final character + - no full-match bonus + - score is lower and may stay below the threshold + +- Regex `.*com` + - may match a long suffix, but it is penalized heavily by the wildcard term + - this keeps broad permissive patterns from automatically looking “strong” + +## Benign corpus and bloom filters + +The module creates a benign corpus DB once and can seed it with a small sample +for all five regex types. + +On each run, it also imports domain entries from the configured Slips local +whitelist file into the benign corpus for the matching domain-like regex +types: + +- `dns_domain` +- `tls_sni` +- `certificate_cn` + +If the daily Tranco whitelist has already been downloaded by Slips, the module +also imports the ordered configured Tranco top benign domains from Redis into +the same domain-like benign corpus. + +During runtime, the module also listens for `tw_closed`. When a finished time +window belongs to one of the host IPs of the machine running Slips, it checks +that host TW for alerts and evidence: + +- if the host TW has any alert or any evidence, it imports nothing from that TW +- if the host TW has zero alerts and zero evidence, it imports additional + benign strings from that clean local TW + +The runtime benign import currently uses: + +- DNS query names -> `dns_domain` +- HTTP hostnames -> `dns_domain` +- TLS `server_name` -> `tls_sni` +- certificate `subject` CN -> `certificate_cn` +- filenames derived from HTTP URIs -> `filename` + +The module logs the total alert count, total evidence count, and a separate +best-effort anomaly-evidence count for that finished host TW. The anomaly +count is informative only; the actual import gate is strict zero alerts and +zero evidence. + +Redis storage note: + +- Slips still stores the full downloaded Tranco whitelist in Redis under + `tranco_whitelisted_domains` while preserving download order. +- `RegexGenerator` reads the configured number of top-ranked entries from that + ordered whitelist cache when it needs benign high-reputation domains for + domain-like regex testing. +- The number of domains read is configured with + `whitelists.tranco_top_benign_limit`. + +It also builds one in-memory bloom filter per benign type and one bloom filter +for generated regex hashes, but these do not replace the benign corpus scan. +They help with exact membership checks and future scale improvements, while the +acceptance decision still requires computing the benign match-strength score +against the benign corpus and rejecting the regex only if some benign string +reaches or exceeds `benign_match_strength_threshold`. + +The current benign acceptance gate is: + +```sql +SELECT value FROM benign_strings WHERE regex_type = ? +``` + +streamed line by line while the module computes the benign match-strength score +for each string. The regex is rejected only if a score reaches the configured +threshold. + +## Reading accepted regexes from other modules + +Later modules should not open the SQLite files directly. + +Use the DB helpers: + +```python +self.db.get_generated_regexes(regex_type="dns_domain", limit=100) +self.db.get_generated_regexes_count(regex_type="dns_domain") +``` + +These helpers return accepted regexes by default. + +## Offline coverage report + +There is also a standalone offline report script for estimating how much the +accepted regexes cover several reference populations for a given Slips run. + +Example HTML report overview: + +![RegexGenerator coverage report overview](images/regex_generator/coverage_report_overview.png) + +This report is useful for explaining the module because it shows, in one page, +how many accepted regexes exist for each data type, how strongly they cover the +reference-union and malicious TI populations, and how much benign spillover +remains after negative selection. + +Example: + +```bash +./venv/bin/python scripts/regex_coverage_report.py \ + --run-output-dir output/eno1_2026-03-18_10:00:30 \ + --redis-port 6379 +``` + +By default, large populations are sampled so the script finishes in practical +time. It prints terminal progress while it runs, for example: + +```text +🧪 sampled estimate ███████░░░░░░░░░░░░ 31.62% | regex 247/781 | cmp 560,840/1,770,991 | type DNS Domain | ETA ⏳ 00:00:14 +``` + +In that progress line: + +- `regex 247/781` means 247 accepted regexes have been evaluated out of 781 total accepted regexes. +- `cmp 560,840/1,770,991` means regex-versus-string match operations, not raw TI entries. The number grows because many regexes are checked against many strings across the benign corpus, malicious TI, observed traffic, and reference-union populations. + +The report reuses the same `0..100` match-strength function as the live +generator, but it applies it to every regex/string comparison in the selected +populations: + +- non-match: score `0` +- match: the same span/anchor/specificity/wildcard formula used by + `RegexGenerator` + +For each regex and each population, the report computes: + +- `match_count`: number of strings matched at all +- `avg_all ± std_all`: average and standard deviation over all tested strings, + with non-matches counted as `0` +- `avg_match ± std_match`: average and standard deviation over only the strings + that matched + +The top-regex ranking uses: + +```text +strength_gap = malicious_avg_all - benign_avg_all +``` + +So the “best” regexes in the report are the ones that are stronger and/or +broader on malicious strings while staying weak on benign strings. + +The HTML output also adds a `Strength Scatter` plot per regex type: + +- X axis: benign `avg_all` +- Y axis: malicious `avg_all` +- ideal area: upper-left + +This gives a faster view of many regexes than a table alone. + +If you want the exhaustive run for research, use: + +```bash +./venv/bin/python scripts/regex_coverage_report.py \ + --run-output-dir /path/to/regex_store \ + --redis-port 23456 \ + --ti-cache-port 6379 \ + --ti-cache-db 1 \ + --full-scan +``` + +Useful knobs: + +- `--sampling-ratio`: fraction of strings to evaluate from each regex-type population in estimate mode. This is applied separately to the benign corpus values, malicious TI values, observed traffic values, and reference-union values. Default: `0.1`. +- `--max-population-size`: hard cap on the number of strings evaluated for each regex type inside each population, after `--sampling-ratio` is applied. +- `--full-scan`: disable both `--sampling-ratio` and `--max-population-size`, and scan all strings in all populations for every regex type. +- `--match-timeout-seconds`: timeout for one regex tested against one regex-type population of strings. + +The script writes: + +- `regex_generator_coverage_report.html` +- `regex_generator_coverage_report.json` + +inside the selected run output directory. + +The estimate is based on: + +- the RegexGenerator benign corpus DB, grouped by regex type +- the configured Tranco top benign domains from `whitelists.tranco_top_benign_limit` as extra benign data for domain-like types, when available in the Slips cache +- TI-derived malicious reference strings from Redis and TI cache files, grouped by regex type +- observed traffic strings from Zeek logs or `flows.sqlite`, grouped by regex type +- the per-type reference union, which is `malicious TI ∪ observed traffic` + +This is an offline report only. It does not run continuously inside Slips. diff --git a/docs/t_cell_module.md b/docs/t_cell_module.md new file mode 100644 index 0000000000..11f8897392 --- /dev/null +++ b/docs/t_cell_module.md @@ -0,0 +1,726 @@ +# T Cell Module + +The `T Cell` module is an immune-inspired responder that consumes centrally +classified Slips evidence, looks for extracted antigens that match the +accepted RegexGenerator regex corpus, and then escalates through a small state +machine until it either becomes tolerant, publishes a containment request, or +stores a memory snapshot for later reuse. Both `PAMP` and `DAMP` evidence can +prime a new cell when an extracted antigen matches an accepted regex, but the +signal decides how strong that new cell is. `PAMP` keeps the base downstream +thresholds and waiting window, while `DAMP` stores a weaker priming profile +with stricter later thresholds, higher corroboration counts, and a shorter +waiting window. `DAMP` observations also raise the danger pressure used later +in co-stimulation and context decisions and trigger reevaluation of already +waiting cells for the same responsible IP. + +The module is started by the normal Slips module loader and is enabled by +default through `t_cell.enabled: true`. + +## Goals + +The module adds a second-stage decision layer without changing detector +modules: + +1. It listens to the shared `evidence_added` channel. +2. It creates or advances cells from `0 - mature` by using `PAMP` or `DAMP` + evidence with extractable antigens and accepted regex matches. +3. It extracts structured antigen values from evidence and linked altflows. +4. It matches those values against accepted regexes already stored by + `RegexGenerator`. +5. It stores `DAMP` observations as responsible-IP danger signals, folds them + into co-stimulation and context pressure, and uses them to create weaker + DAMP-primed cells when they match an accepted regex. +6. It computes co-stimulation and context scores using the per-cell priming + profile that was stored at recognition time. +7. It either becomes tolerant, activates, requests blocking, or stores memory. + +The target of any effector response is the IP that T Cell identifies as the +responsible source for the attack. This is not always the same as +`evidence.profile.ip`. + +## Profile, Source, and Target + +The evidence object carries three different notions that must not be mixed: + +- `evidence.profile.ip`: the Slips profile bucket that the evidence belongs to. + It is the host related to the evidence in the current time window. +- `evidence.attacker`: the attacking or responsible entity. In IDMEF export, + this becomes `Source`. +- `evidence.victim`: the attacked entity. In IDMEF export, this becomes + `Target`. + +The `direction` field on `attacker` or `victim` says whether that entity was +seen on the network flow source side (`SRC`) or destination side (`DST`). That +flow-side direction is separate from the attack role: + +- `attacker` maps to IDMEF `Source` +- `victim` maps to IDMEF `Target` +- `direction=SRC/DST` maps to the network flow side + +T Cell uses a separate derived value called the responsible IP: + +1. If `evidence.attacker` is an IP, T Cell uses `evidence.attacker.value`. +2. Otherwise, if either evidence entity is an IP on the network `SRC` side, + T Cell uses that IP. +3. Otherwise, it falls back to `evidence.profile.ip`. + +This responsible IP is the IP that T Cell: + +- keys the cell on +- aggregates co-stimulation and context observations on +- sends to `new_blocking` when effector action is approved + +The original `evidence.profile.ip` is still logged, because it tells you which +host/time-window context produced the evidence. + +Example: + +- `profile.ip = 147.32.80.37` +- `Source.IP = 138.68.100.107` +- `Target.IP = 147.32.80.37` + +In that case, T Cell keeps `147.32.80.37` as the related profile context, but +the responsible IP for analysis and blocking is `138.68.100.107`. + +## State Machine + +One T Cell is tracked per: + +- responsible IP +- regex type +- normalized antigen value + +The persisted states are: + +- `0 - mature` +- `1 - antigen-recognized` +- `2 - anergic` +- `3 - activated` +- `4 - effector` +- `5 - memory` + +States `1 - antigen-recognized` and `3 - activated` can also carry an +explicit waiting substatus in the stored cell context: + +- `1 - antigen-recognized (waiting for co-stimulation)` +- `3 - activated (waiting for context)` + +This does not create new state numbers. It is an explicit runtime marker that +the cell is still in state `1` or `3`, but is currently waiting for the next +reevaluation. + +Mermaid state diagram: + +```mermaid +stateDiagram-v2 + [*] --> S0 : new cell + + state "0 - mature" as S0 + state "1 - antigen-recognized" as S1 + state "2 - anergic" as S2 + state "3 - activated" as S3 + state "4 - effector" as S4 + state "5 - memory" as S5 + + S0 --> S1 : PAMP or DAMP + antigen extracted\n+ accepted regex match + S0 --> S2 : PAMP or DAMP + antigen extracted\n+ no regex match + S0 --> S0 : no antigen extracted + + S2 --> S0 : anergy TTL expired + + S1 --> S3 : co-stimulation >= threshold\nwithin 1 Slips TW + S1 --> S1 : re-evaluate on later PAMP or DAMP\nwhile below threshold + S1 --> S2 : co-stimulation timeout\nafter 1 Slips TW + + S3 --> S4 : context says novel + intense + S3 --> S5 : context says familiar + cooling down + S3 --> S3 : re-evaluate on later PAMP or DAMP\nwhile undecided + S3 --> S0 : context timeout\nafter 1 Slips TW + + S5 --> S5 : later matching evidence retained + S4 --> S4 : repeated hits gated by\neffector cooldown + + note right of S0 + PAMP keeps the base priming profile. + DAMP can also create state 1, + but stores a weaker priming profile + with stricter later gates. + end note + + note right of S1 + Co-stimulation combines: + current PAMP confidence + related PAMP count + weighted PAMP+DAMP danger + for the same responsible IP. + end note + + note right of S3 + Context uses the same mixed pressure model + to decide whether to contain now + or store memory for later. + end note +``` + +The runtime flow is: + +1. Slips publishes an evidence on `evidence_added`. +2. The module stores one observation row in its own SQLite DB. +3. If the evidence signal is `DAMP`, the module stores the observation, + reevaluates any waiting cells for the same responsible IP, logs + `damp_reverification`, and still continues to antigen recognition if + extractable antigens are present. +4. If the evidence signal is neither `PAMP` nor `DAMP`, the module logs + `ignored_non_pamp` and stops for that evidence after storing the + observation. +5. If no structured antigen can be extracted, the module logs + `no_antigen_extracted` and stops for that evidence. +6. For each antigen candidate, the module loads or creates the cell in + `0 - mature`. +7. If the cell is still under `anergic_until`, the module logs suppression and + does nothing else. +8. If the cell is `2 - anergic` and the TTL expired, it transitions back to + `0 - mature`. +9. If no accepted regex matches the antigen, the mature cell goes + `0 -> 2 - anergic` and stores a new `anergic_until`, regardless of whether + the evidence was `PAMP` or `DAMP`. +10. If a regex matches, the cell goes `0 -> 1`, stores the chosen regex + metadata, and stores a priming profile snapshot derived from the signal + (`PAMP` or `DAMP`). +11. The recognition observation is marked as consumed for that cell, so it + cannot also count toward the next transition. +12. The module computes co-stimulation from the current evidence confidence, + related `PAMP`s, and stored mixed `PAMP` + weighted `DAMP` danger for the + same responsible IP. +13. If co-stimulation crosses the cell's effective threshold from its priming + profile, the cell goes `1 -> 3`. +14. That activating observation is also marked as consumed for that cell, so + it cannot also count toward the next context transition. +15. If co-stimulation stays below threshold, the cell can wait in + `1 - antigen-recognized` for at most the effective wait window from its + priming profile, with the cell explicitly marked as waiting for + co-stimulation. +16. If that wait expires without enough co-stimulation, the + cell goes `1 -> 2 - anergic`. +17. In state `3`, the module computes context signals from the same mixed + pressure model: related `PAMP`s plus weighted `DAMP` danger. +18. If the situation is novel and intense enough for the cell's effective + effector gate, the cell goes to `4 - effector`. +19. If the situation is familiar and clearly cooling down enough for the + cell's effective memory gate, the cell goes to `5 - memory`. +20. If state `3` cannot decide effector or memory within the cell's effective + wait window, the cell goes `3 -> 0 - mature`. + +Both waiting states are reevaluated on later matching `PAMP`s and on later +`DAMP` observations for the same responsible IP. Because transition-causing +observations are remembered and excluded from later counts, pressure, and +novelty checks, the same evidence cannot drive a full chain of activations for +one cell. + +State `4` publishes the existing `new_blocking` payload for the responsible IP +when blocking support is present. If blocking or ARP poisoning modules are not +running, the module can simulate the effector decision and log the exact +payload instead. + +State `5` stores the matched regex and the full context snapshot in the T Cell +SQLite DB when the cell first enters memory. It does not emit a new Slips +evidence. Later matching evidence keeps the cell in `5 - memory`, but it does +not create repeated `memory_stored` actions for the same cell. + +## Antigen Extraction + +The module reuses the same field semantics already used by RegexGenerator. + +Supported antigen types: + +- `dns_domain` +- `uri` +- `filename` +- `tls_sni` +- `certificate_cn` + +Extraction sources: + +- evidence attacker or victim domain values -> `dns_domain` +- evidence attacker URL values -> hostname as `dns_domain`, path as `uri`, + basename as `filename` +- evidence attacker or victim `SNI` -> `tls_sni` +- DNS altflow `query` -> `dns_domain` +- HTTP altflow `host` -> `dns_domain` +- HTTP altflow `uri` -> `uri` +- HTTP altflow URI basename -> `filename` +- SSL altflow `server_name` -> `tls_sni` +- SSL altflow `subject` `CN=` -> `certificate_cn` + +If a `PAMP` has no structured antigen, the module logs and skips it. It does +not create an anergic cell for that case. + +## Regex Matching + +Matching only uses accepted regexes already stored by `RegexGenerator`. + +For one antigen candidate: + +- the module loads accepted regexes of the same `regex_type` +- it keeps only those that actually match the antigen value +- it ranks them by strongest match strength against the antigen +- it uses regex specificity and then newest `created_at` as tie-breakers + +The chosen regex metadata is stored in the cell, transitions table, and any +memory row. + +## Co-Stimulation + +Co-stimulation measures how dangerous the current situation looks for the +matched antigen: + +```text +co_stimulation = + wc * confidence + + wr * related_pamp_score + + wd * profile_danger_score +``` + +Where: + +- `confidence = current evidence.confidence` +- `related_pamp_score = min(1, related_pamp_count / related_pamps_saturation)` +- `profile_danger_score = min(1, combined_danger_raw / danger_saturation)` +- `combined_danger_raw = pamp_danger_raw + damp_danger_weight * damp_danger_raw` +- `pamp_danger_raw = sum(threat_level_value * confidence)` over recent `PAMP` + observations for the same responsible IP +- `damp_danger_raw = sum(threat_level_value * confidence)` over recent `DAMP` + observations for the same responsible IP + +Related PAMPs are recent `PAMP` observations for the same responsible IP that +share either: + +- the same antigen value, or +- the same matched regex hash + +Default weights are normalized from configuration: + +- `confidence = 0.35` +- `related_pamps = 0.25` +- `danger = 0.40` +- `damp_danger_weight = 1.5` + +Default activation threshold: + +- `co_stimulation_threshold = 0.65` + +Effective priming profiles: + +- `PAMP`: keeps the base threshold, base related-count requirements, and the + full wait window +- `DAMP`: raises the later co-stimulation threshold, raises later effector and + memory thresholds, adds one more required related `PAMP` for effector and + memory, and shortens the wait window by a factor + +Interpretation: + +- `PAMP`s still provide antigen identity and the related-antigen correlation. +- `DAMP`s also increase the danger term, so the same recognized antigen is + treated as riskier when the responsible IP is also showing damage or anomaly + signals. +- If a `DAMP` itself matched an accepted regex and created the cell, the + effective co-stimulation threshold is higher and the wait is shorter because + the cell is marked as DAMP-primed. +- The observation that caused `0 -> 1` is excluded from later co-stimulation + calculations for that cell. + +Wait limit: + +- state `1 - antigen-recognized` can wait for co-stimulation for at most the + effective wait limit from its priming profile +- by default, `PAMP` uses one configured Slips time window + (`parameters.time_window_width`) and `DAMP` uses half of that +- if that wait expires, the cell goes `1 -> 2 - anergic` + +## Context Signals + +Context signals decide how to respond once a cell is activated. + +Definitions: + +- `novelty_score = 1` when the matched regex has no stored memory row and no + recent prior regex activity in `novelty_window_seconds`; otherwise `0` +- `recent_pressure` is the normalized combined danger score over + `context_recent_window_seconds` +- `previous_pressure` is the same combined danger score over the previous + adjacent context window +- each pressure window uses + `combined_danger_raw = pamp_danger_raw + damp_danger_weight * damp_danger_raw` +- `trend_ratio = recent_pressure / max(previous_pressure, 0.01)` +- `recent_related_score = min(1, recent_related_count / related_pamps_saturation)` +- `decrease_score = clamp(1 - trend_ratio, 0, 1)` +- `familiarity_score = 1 - novelty_score` +- `stability_score = min(1, recent_related_count / memory_min_related_count)` + +Effector score: + +```text +effector_score = + 0.45 * recent_pressure + + 0.25 * recent_related_score + + 0.30 * novelty_score +``` + +Memory score: + +```text +memory_score = + 0.60 * decrease_score + + 0.25 * familiarity_score + + 0.15 * stability_score +``` + +Base decisions: + +- `effector` requires: + - `effector_score >= 0.70` + - `recent_related_count >= 4` + - novelty still present +- `memory` requires: + - `memory_score >= 0.60` + - `trend_ratio <= 0.60` + - `recent_related_count >= 3` + - familiarity already present + +If both would pass, `effector` wins. + +Effective decisions: + +- `PAMP`-primed cells use the base effector and memory gates. +- `DAMP`-primed cells raise the effector threshold, raise the memory + threshold, and require one more related recent `PAMP` for both decisions. +- The observation that caused the previous state transition is excluded from + later context pressure, recent-related counts, and novelty checks for that + cell. + +Wait limit: + +- state `3 - activated` can wait for context for at most the effective wait + limit from its priming profile +- if that wait expires without effector or memory, the cell goes + `3 -> 0 - mature` + +## Containment Behavior + +When the cell reaches `4 - effector`, the module publishes the same payload +shape used by the existing Slips blocking path: + +```json +{ + "ip": "", + "block": true, + "tw": 1, + "interface": null +} +``` + +Notes: + +- `ip` is the derived responsible IP, not necessarily `evidence.profile.ip` +- `tw` is `evidence.timewindow.number` +- `interface` uses the same `utils.get_interface_of_ip()` lookup as the rest + of Slips +- `from` and `to` are omitted, so the existing blocking module falls back to + blocking both directions +- the same cell is rate-limited with `effector_cooldown_seconds` + +If no blocking-capable module is running: + +- with `simulate_effector_without_blocking: true`, the module logs a simulated + effector decision and the exact would-be payload +- with `false`, it keeps the state but only logs that the effector path is not + available + +## SQLite Storage + +The T Cell module uses its own isolated SQLite DB and does not change the core +Slips evidence schema, Redis evidence payloads, `alerts.json`, STIX/TAXII +export, or SlipsWeb payloads. + +Default DB location: + +```text +/t_cell/t_cell.sqlite +``` + +Tables: + +- `observations`: one processed evidence row with confidence, threat level, + extracted antigens, matched regexes, the tracked responsible IP, and the raw + evidence JSON +- `cells`: current state for each `responsible_ip + regex_type + antigen_value` +- `transitions`: auditable state transitions with reasons and score snapshots +- `memories`: stored state-5 regex/context snapshots + +The DB is accessed through `DBManager.get_t_cell_storage()`. + +## Logging + +If `create_log_file` is enabled, the module writes: + +```text +/t_cell/t_cell.log +``` + +The log is intentionally short and human-readable. It writes one line per +decision or transition, with: + +- timestamp +- action +- resulting state +- evidence type and ID +- related profile IP +- responsible IP +- target IP when the evidence victim is an IP +- cell key +- matched regex hash and value when relevant +- main scores + +`log_verbosity` controls how much decision detail is written: + +- `1`: transitions and terminal actions only +- `2`: also log why a cell is waiting, for example + `waiting_for_co_stimulation` with the current score, threshold, elapsed + wait time, wait limit, and the split between `PAMP` and `DAMP` danger +- `3`: also log per-evidence debug details such as extracted antigens + +### Decision Trace + +For verification runs, the module also supports a separate audit trace file: + +```text +/t_cell/t_cell_trace.jsonl +``` + +This trace is disabled by default. When enabled, each JSON line explains one +co-stimulation or context evaluation and includes: + +- the action being decided, for example `co_stimulation_threshold_met`, + `context_memory`, or `waiting_for_context` +- the related profile IP, responsible IP, and target IP +- the candidate antigen and matched regex +- the exact score, threshold, and weighted formula terms +- the evidence IDs that contributed to the related-PAMP count +- the evidence IDs that contributed to `PAMP` and `DAMP` danger totals +- omitted-contributor counts when the trace limit is reached + +The trace path is always resolved under the t_cell module output directory +inside the selected run output. If the config contains an absolute path or a +path that tries to escape the output directory, the module collapses it back +under `/t_cell/` before writing the file. + +Recommended usage: + +- keep `decision_trace_mode: off` during normal runs +- use `decision_trace_mode: transitions` when you only want threshold-passing + and state-change explanations +- use `decision_trace_mode: all` only for focused evaluation runs where you + also want waiting decisions + +Performance note: + +- with `decision_trace_mode: off`, there is effectively no extra trace cost +- trace mode performs extra observation lookups and extra file writes, so it + should be treated as a verification feature, not the normal default path + +### Offline HTML Report + +The module includes a separate offline report generator: + +```bash +./venv/bin/python modules/t_cell/analyze_t_cell.py \ + --run-output-dir output/ +``` + +By default it writes: + +```text +/t_cell_report.html +``` + +You can then open that HTML file directly in any browser. If you want a +different output filename, pass `--out `. + +The report is static and self-contained. It reads the T Cell SQLite DB as the +primary source, then enriches the page with `t_cell/t_cell.log` and +`t_cell/t_cell_trace.jsonl` when those files exist. This means: + +- it still explains the run when `log_verbosity` is `1` +- it gains richer per-evidence detail when `log_verbosity` is `2` or `3` +- it gains threshold-by-threshold explanations when decision tracing is enabled + +Example report screenshot from a real run: + +![T Cell HTML report overview](images/t_cell/t_cell_report_overview.png) + +The page focuses on the run itself, including: + +- total `PAMP` and `DAMP` observations +- evidence type mix +- a rendered T-cell state-machine graph with per-state and per-transition counts +- extracted antigens and matched regexes +- current cells and their states +- transition reasons and state-path counts +- memories stored so far +- observation, transition, and trace timelines +- a sortable Recent Observations table at the bottom of the page +- a sortable Transitions table that defaults to grouping rows by T cell +- a compact, collapsed configuration snapshot at the very end + +How to read the report: + +- **Quick Summary** and **Run Findings** tell you first whether the module saw + mostly `PAMP` or `DAMP`, whether cells were created at all, and whether the + run stalled because no supported antigen could be extracted. +- **Observation / Transition timelines** show when pressure and state changes + happened over time. This is the fastest way to see whether the module was + mostly idle, mostly collecting danger, or actively moving cells. +- **T Cell State Machine** overlays the abstract state machine with run data: + each node shows how many cells are currently in that state, and each arrow + shows how many times that transition happened in the run. +- **Signals**, **Evidence Types**, and the top-* panels show what fed the + danger model: which evidence classes dominated, which responsible IPs or + targets were involved most often, and which antigens or unmatched `PAMP` + values kept appearing. +- **Transitions** is the per-cell transition history. It is sortable and + defaults to grouping rows by T cell, so you can read one cell's path from + `0 - mature` onward without manually regrouping the table. +- **Current Cells** shows the cells that still exist now, their current state, + any explicit waiting substatus such as `waiting for co-stimulation` or + `waiting for context`, and the latest co-stimulation / effector / memory + scores that were stored on the cell. +- **Stored Memories** shows which cells have already reached + `5 - memory`, along with the saved context snapshot that will be reused + later. +- **Decision Trace** is the threshold-audit section. When enabled, it is where + you verify why a threshold passed by checking the weighted formula terms and + contributing evidence IDs. +- **Recent Observations** stays at the bottom as the raw sortable evidence + audit table. It is the best section to correlate what Slips generated with + what T Cell actually received and stored. + +Color mapping: + +- `0 - mature` -> cyan +- `1 - antigen-recognized` -> yellow +- `2 - anergic` -> blue +- `3 - activated` -> magenta +- `4 - effector` -> red +- `5 - memory` -> green + +## Configuration + +Example section from `config/slips.yaml`: + +```yaml +t_cell: + enabled: true + create_log_file: true + log_colors: true + log_verbosity: 1 + decision_trace_mode: off + decision_trace_file: t_cell_trace.jsonl + decision_trace_max_evidence: 10 + store_dir: output/t_cell + persistent_store_dir: "" + observation_retention_seconds: 604800 + anergy_ttl_seconds: 21600 + related_lookback_seconds: 3600 + related_pamps_saturation: 5 + danger_saturation: 2.5 + damp_danger_weight: 1.5 + co_stimulation_threshold: 0.65 + co_stimulation_weights: + confidence: 0.35 + related_pamps: 0.25 + danger: 0.40 + priming_profiles: + PAMP: + strength: 1.0 + co_stimulation_threshold_offset: 0.0 + effector_threshold_offset: 0.0 + memory_threshold_offset: 0.0 + state_wait_timeout_factor: 1.0 + effector_min_related_count_offset: 0 + memory_min_related_count_offset: 0 + DAMP: + strength: 0.6 + co_stimulation_threshold_offset: 0.15 + effector_threshold_offset: 0.10 + memory_threshold_offset: 0.05 + state_wait_timeout_factor: 0.5 + effector_min_related_count_offset: 1 + memory_min_related_count_offset: 1 + novelty_window_seconds: 86400 + context_recent_window_seconds: 1800 + effector_threshold: 0.70 + effector_min_related_count: 4 + effector_cooldown_seconds: 1800 + memory_threshold: 0.60 + memory_trend_ratio_max: 0.60 + memory_min_related_count: 3 + simulate_effector_without_blocking: true +``` + +Reference: + +- `enabled`: enable or disable the module +- `create_log_file`: create `/t_cell/t_cell.log` +- `log_colors`: keep ANSI colors in the module log +- `log_verbosity`: `1` logs transitions/actions only, `2` adds decision + summaries, `3` adds per-evidence debug details +- `decision_trace_mode`: `off`, `transitions`, or `all` +- `decision_trace_file`: JSONL audit file for threshold explanations, always + created under the t_cell module output directory +- `decision_trace_max_evidence`: contributor cap per trace list +- `store_dir`: run-local directory for the SQLite DB +- `persistent_store_dir`: optional stable directory for the SQLite DB. Leave + it empty for normal runs because T Cell state is tied to one Slips run. + If set, relative paths are resolved inside `parameters.permanent_dir`. +- `observation_retention_seconds`: retention for observation rows +- `anergy_ttl_seconds`: how long a non-matching cell remains tolerant +- `related_lookback_seconds`: lookback for co-stimulation correlation +- `related_pamps_saturation`: saturation point for related PAMP score +- `danger_saturation`: saturation point for weighted combined profile danger +- `damp_danger_weight`: multiplier applied to raw `DAMP` danger before it is + added to the `PAMP` danger term +- `co_stimulation_threshold`: threshold for `1 -> 3` +- `co_stimulation_weights`: normalized internally +- `priming_profiles`: signal-specific offsets and wait factors applied after + `0 -> 1` to mark how strongly a cell was primed +- `priming_profiles.PAMP`: base-strength profile, usually zero offsets +- `priming_profiles.DAMP`: weaker profile that raises later thresholds, + raises related-count requirements, and shortens waiting +- `novelty_window_seconds`: window for novelty suppression +- `context_recent_window_seconds`: context window size +- `effector_threshold`: minimum effector score +- `effector_min_related_count`: minimum related count before effector +- `effector_cooldown_seconds`: per-cell effector cooldown +- `memory_threshold`: minimum memory score +- `memory_trend_ratio_max`: maximum recent/previous pressure ratio for memory +- `memory_min_related_count`: minimum related count before memory +- `simulate_effector_without_blocking`: log a simulated effector action when + blocking modules are absent + +## Evidence Signal Dependency + +The module relies on the central `evidence_signal` field that Slips adds before +evidence is stored or published. + +See [Evidence Signals](evidence_signals.md) for: + +- the global `PAMP` / `DAMP` configuration +- the current evidence inventory by module +- the default shipped signal mapping + +T Cell antigen recognition and state creation now support both `PAMP` and +`DAMP` when an extracted antigen matches an accepted regex. The signal is +stored as a priming profile inside the cell and changes the later effective +thresholds, required related-counts, and wait limits. `DAMP` observations are +still stored in the T Cell observation table and still contribute weighted +danger to co-stimulation and context calculations for the same responsible IP. diff --git a/docs/usage.md b/docs/usage.md index 1b91cfa92d..3a1800533a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -243,6 +243,9 @@ The output process collects output from the modules and handles the display of i analysis and detected malicious behaviour can be analyzed as following: - **alerts.json and alerts.txt in the output folder** - collects all evidences and detections generated by Slips in a .txt and .json formats. +- **alerts-summary.log in `output/alerts/`** - when the `alert_summary` module is enabled, stores one analyst-facing paragraph per alert based on grouped correlated evidence and, when needed, recursive reduction summaries from the shared LLM reply path. +- **`llm-summary/alert_summary.log` in the output folder** - when the `alert_summary` module is enabled, stores operational logs for alert queueing, prompt-budget checks, recursive reduction layers, LLM requests, responses, failures, and shutdown handling according to `alert_summary.log_verbosity`. +- **`LLM/llm.log` in the output folder** - when the shared `LLM` module is enabled, stores backend readiness, per-request queueing, dispatch, success or failure, and response publication details for the run. - **log files in a folder _current-date-time_** - separates the traffic into files according to a profile and timewindow and summarize the traffic according to each profile and timewindow. - **Web interface** - browser based GUI for viewing slips detections, incoming and outgoing traffic, and an organized timeline of flows. @@ -409,6 +412,13 @@ Slips download the top 10k domains from this list and by default and whitelists all evidence and alerts from and to these domains. Slips still shows the flows to and from these IoC. +Redis storage detail: + +- Slips stores the full downloaded Tranco whitelist in Redis under + `tranco_whitelisted_domains` while preserving download order. +- Other modules, such as `RegexGenerator`, read the first + `whitelists.tranco_top_benign_limit` entries from that ordered whitelist + cache as benign data without losing ranking order. The tranco list is updated daily by default in Slips, but you can change how often to update it using the ```online_whitelist_update_period``` key in config/slips.yaml. @@ -516,7 +526,9 @@ This parameter allows you to tailor SLIPS's analysis focus based on your specifi Use ```permanent_dir``` to choose where Slips stores databases and runtime-generated files that must persist across different Slips runs and should not be overwritten. -This includes persistent artifacts such as ```p2p_trust_runtime/``` and shared module databases like the Fides cache. +This includes persistent artifacts such as ```p2p_trust_runtime/```, +```databases/regex_store/```, and shared module databases like the Fides cache +under ```databases/```. **Live Slips auto update** diff --git a/managers/process_manager.py b/managers/process_manager.py index 3ec48991a2..7b0699a4c8 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -161,48 +161,66 @@ def get_disabled_modules(self) -> Tuple[List[str], List[str]]: A tuple containing user-disabled modules and modules disabled by Slips runtime rules. """ + return ( + self.get_user_disabled_modules(), + self.get_runtime_disabled_modules(), + ) + + def get_user_disabled_modules(self) -> List[str]: + """ + Get modules disabled by the user configuration. + + Returns: + User-disabled module names stripped of surrounding whitespace. + """ user_disabled_modules: List[str] = self.main.conf.read_configuration( "modules", "disable", ["template"] ) - user_disabled_modules = [ - module.strip() for module in user_disabled_modules - ] + return [module.strip() for module in user_disabled_modules] + + def get_runtime_disabled_modules(self) -> List[str]: + """ + Get modules disabled by Slips runtime rules. + Returns: + Module names disabled by Slips runtime conditions. + """ is_running_non_stop = self.main.db.is_running_non_stop() - slips_disabled_modules: List[str] = [] + disabled_modules: List[str] = [] if not self._is_exporting_module_enabled(): - slips_disabled_modules.append("exporting_alerts") + disabled_modules.append("exporting_alerts") use_p2p = self.main.conf.use_local_p2p() if not (use_p2p and is_running_non_stop): - slips_disabled_modules.append("p2p_trust") + disabled_modules.append("p2p_trust") use_global_p2p = self.main.conf.use_global_p2p() if not (use_global_p2p and is_running_non_stop): - slips_disabled_modules.extend(("fides", "iris")) + disabled_modules.extend(("fides", "iris")) if not ( self.main.conf.send_to_warden() or self.main.conf.receive_from_warden() ): - slips_disabled_modules.append("cesnet") + disabled_modules.append("cesnet") if not (self.main.args.clearblocking or self.main.args.blocking): - slips_disabled_modules.extend(("blocking", "arp_poisoner")) + disabled_modules.extend(("blocking", "arp_poisoner")) if self.main.input_type != InputType.PCAP: - slips_disabled_modules.append("leak_detector") + disabled_modules.append("leak_detector") if not self._reading_flows_from_cyst(): - slips_disabled_modules.append("cyst") + disabled_modules.append("cyst") - for module in self.slips_disabled_modules: - if module not in slips_disabled_modules: - slips_disabled_modules.append(module) + if not self.main.conf.llm_enabled(): + # the last 2 depend on the disabled llm_proxy module + for module in ("llm_proxy", "regex_generator", "t_cell"): + disabled_modules.append(module) - return user_disabled_modules, slips_disabled_modules + return disabled_modules def _is_exporting_module_enabled(self) -> bool: """ @@ -221,7 +239,9 @@ def get_all_disabled_modules(self) -> List[str]: Returns: User-disabled modules followed by Slips-disabled modules. """ - return self.user_disabled_modules + self.slips_disabled_modules + return list( + set(self.user_disabled_modules + self.slips_disabled_modules) + ) def declare_that_slips_done_starting_all_children(self): self.all_children_started = True diff --git a/managers/redis_manager.py b/managers/redis_manager.py index f8061f0297..53a8b94c36 100644 --- a/managers/redis_manager.py +++ b/managers/redis_manager.py @@ -6,6 +6,7 @@ import os import psutil import socket +import sys import time import subprocess from typing import Dict, Union @@ -166,6 +167,10 @@ def log_redis_server_pid(self, redis_port: int, redis_pid: int): "Save the DB\n" ) + db = getattr(self.main, "db", None) + zeek_dir = '""' + if db: + zeek_dir = db.get_zeek_output_dir() zeek_dir = self.main.args.output f.write( @@ -598,9 +603,13 @@ def ask_user_to_confirm_altering_a_currently_used_server( f"being used.\nAre you sure you want to {alter} it? [" f"y/n]\n> " ) + if not sys.stdin.isatty(): + return True answer = input(msg) if answer.lower() == "y": return True + except EOFError: + return True except KeyboardInterrupt: pass @@ -624,8 +633,14 @@ def confirm_server_altering( return False def _get_dbmanager_without_starting_a_new_server(self, port): + logger = getattr(self.main, "logger", None) + if logger is None: + logger = Output( + create_logfiles=False, + slips_args=getattr(self.main, "args", None), + ) return DBManager( - Output(), + logger, self.main.args.output, port, self.main.conf, @@ -757,6 +772,24 @@ def remove_server_from_log(self, redis_port): os.remove(tmpfile) raise e + def _get_redis_server_selection(self) -> int: + """ + Read the selected Redis server from stdin. + + Returns: + The selected server number, or 0 when stdin is unavailable. + """ + if sys.stdin.isatty(): + try: + return int(input()) + except EOFError: + return 0 + except ValueError: + print("Invalid input.") + self.main.terminate_slips() + + return 0 + def flush_and_kill(self, pid: int, port): """ raises UserCancelledErr or AlreadyKilledErr if redis isnt killed, @@ -794,14 +827,11 @@ def close_open_redis_servers(self): if not open_servers: self.main.terminate_slips() - try: - server_to_close: int = int(input()) - except ValueError: - print("Invalid input.") - self.main.terminate_slips() + server_to_close: int = self._get_redis_server_selection() - # close all ports in running_slips_logs.txt and in our supported range if server_to_close == 0: + # close all ports in running_slips_logs.txt + # and in our supported range self.close_all_ports() self.main.terminate_slips() return diff --git a/modules/alert_summary/README.md b/modules/alert_summary/README.md new file mode 100644 index 0000000000..9bd5457864 --- /dev/null +++ b/modules/alert_summary/README.md @@ -0,0 +1,187 @@ +# Alert Summary Module + +The `AlertSummary` module creates one analyst-facing summary paragraph for each +Slips alert. + +It listens for `new_alert`, loads the correlated evidence that triggered the +alert, uses the shared `LLM` module to build an analyst summary, and writes the +result to `output/alerts/alerts-summary.log`. + +## What it does + +For each alert, the module: + +1. Receives the alert from `new_alert`. +2. Loads every evidence record referenced by `alert.correl_id`. +3. Groups similar evidence descriptions into an incident-style digest. +4. Estimates whether the grouped digest fits the prompt budget. +5. If it fits, sends the final analyst-summary request to `llm_request` channel. +6. If it does not fit, recursively reduces the digest through one or more + intermediate LLM summaries until the final prompt fits. +7. Waits for the matching `llm_response`. +8. Writes one plain-text paragraph per alert to `output/alerts/alerts-summary.log`. + +If the LLM pipeline fails, the module writes a local heuristic fallback summary +instead of leaving the alert without analyst context. + +The module can also keep a bounded recent-history memory per source/profile. +When enabled, that history is added to the final analyst-summary prompt as +extra context so the current alert can be explained relative to recent past +activity instead of being summarized in isolation. When the current alert +matches repeated prior alerts, that recurrence is intended to act as +cumulative supporting context that can raise confidence and urgency. + +## Recursive reduction + +The module now follows the incident-style prompt design used in +`Slips-tools/alert_summary/inference.py`, but adapts it to live Slips alerts. + +When an alert is too large for the final prompt budget, the module does not +truncate the evidence. Instead it: + +1. Groups similar evidence lines. +2. Splits the grouped digest into chunks that fit a reduction prompt. +3. Requests one intermediate summary per chunk. +4. Repeats that reduction on the summaries if the combined digest is still too + large. +5. Sends the reduced digest to the final analyst-summary prompt. + +If one grouped line is still too large, the module splits it into multiple +segments on sentence or word boundaries so the full content is preserved. + +## Configuration + +Example section in `config/slips.yaml`: + +```yaml +alert_summary: + enabled: false + log_verbosity: 2 + allowed_backends: [] + llm_temperature: 0.2 + llm_max_tokens: 220 + llm_response_timeout_seconds: 120 + history_enabled: false + history_max_alerts: 3 + history_max_tokens: 700 + history_patterns_per_alert: 2 +``` + +Configuration reference: + +- `enabled`: enables or disables the module. +- `log_verbosity`: controls how much operational detail is written to + `/llm-summary/alert_summary.log`. Use `0` for an empty file, + `1` for startup, shutdown, and failures, `2` for per-alert request flow, + and `3` for prompt-budget and reduction-layer details. +- `allowed_backends`: preferred runtime-ready LLM backend aliases for this + module. If empty, the module falls back to the shared LLM default backend. +- `llm_temperature`: low-temperature setting used to keep summaries stable and + analyst-oriented. +- `llm_max_tokens`: output budget for the final analyst paragraph. +- `llm_response_timeout_seconds`: hard timeout for one in-flight shared-LLM + request. If set to `0`, the module waits indefinitely. +- `history_enabled`: keeps recent prior alert summaries in memory and adds + them to the final prompt for the same source/profile. +- `history_max_alerts`: maximum number of prior summarized alerts kept per + source/profile. +- `history_max_tokens`: approximate token budget reserved for recent-history + context inside the final prompt. +- `history_patterns_per_alert`: number of dominant grouped evidence patterns + stored from each prior alert. + +## Prompt design + +The final prompt is built around: + +- incident metadata +- recent alert history for the same source/profile when enabled +- grouped evidence patterns with time ranges, counts, severities, and samples +- explicit weighting guidance for evidence threat levels, where `info` + evidence is treated as context only and not as a standalone security finding +- explicit separation between current-alert evidence and historical context, + so prior details are not restated as if they happened in the current alert +- instructions to explain the suspicious behavior, strongest supporting or + weakening evidence, likely true-positive or false-positive status, + operational risk, and whether the current alert looks like a continuation, + escalation, repetition, diversification, or a different pattern relative to + recent past activity + +Intermediate reduction prompts use the same incident metadata but ask the model +to compress one evidence chunk into a shorter digest for the next reduction +layer or the final summary while preserving those threat-level distinctions. + + +## Recent alert history + +When `history_enabled` is on, the module stores a small in-memory history of +completed alert summaries per source/profile. Each stored entry contains: + +- time window and compact time range +- accumulated threat level +- alert confidence +- a few dominant grouped evidence patterns +- the final summary text + +That history is added only to the final analyst-summary prompt, not to +intermediate reduction prompts. The current alert evidence remains the primary +source of truth, but repeated aligned alerts are meant to be treated as +cumulative supporting context. In other words, recurrence should not replace +the current alert evidence, but it can strengthen the assessment of risk, +urgency, and likely true-positive status when the current alert matches the +historical pattern. + +## Shared LLM integration + +The module uses the shared LLM contract: + +- request channel: `llm_request` +- response channel: `llm_response` + +Each request contains: + +- `request_id` +- `requester = "alert_summary"` +- `backend` +- `messages` +- `temperature` +- `max_tokens` +- `metadata.alert_id` +- `metadata.profileid` +- `metadata.timewindow` +- `metadata.evidence_count` +- `metadata.grouped_item_count` +- `metadata.reduction_layer` +- `metadata.prompt_version` + +Reduction requests also include `metadata.chunk_index` and +`metadata.chunk_count`. + +## Logs and output + +Analyst summaries are written to: + +```text +/alerts/alerts-summary.log +``` + +Alert-summary operational logs are written to: + +```text +/llm-summary/alert_summary.log +``` + +Shared LLM runtime logs are written to: + +```text +/llm_proxy/llm_proxy.log +``` + +The alert-summary log records queueing, prompt-budget decisions, reduction +layers, request publication, replies, failures, and shutdown handling. + +## Shutdown behavior + +The module keeps waiting during shutdown while a shared LLM request is still +in flight. This prevents the old race where the shared `LLM` module finished +later and published a reply after `alert_summary` had already exited. diff --git a/modules/alert_summary/__init__.py b/modules/alert_summary/__init__.py new file mode 100644 index 0000000000..f436f14183 --- /dev/null +++ b/modules/alert_summary/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only diff --git a/modules/alert_summary/alert_summary.py b/modules/alert_summary/alert_summary.py new file mode 100644 index 0000000000..92bb9cb802 --- /dev/null +++ b/modules/alert_summary/alert_summary.py @@ -0,0 +1,1870 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +import os +import re +import time +import uuid +from collections import defaultdict, deque +from datetime import datetime + +from slips_files.common.abstracts.imodule import IModule +from slips_files.common.output_paths import get_alerts_path_inside_output_dir +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import utils +from slips_files.core.structures.alerts import dict_to_alert +from slips_files.core.structures.evidence import dict_to_evidence + + +PROMPT_VERSION = "alert-summary-v4" +LOG_VERBOSITY_SUMMARY = 1 +LOG_VERBOSITY_REQUESTS = 2 +LOG_VERBOSITY_DEBUG = 3 +APPROX_CHARS_PER_TOKEN = 4 +FINAL_PROMPT_INPUT_TOKEN_BUDGET = 3200 +REDUCTION_PROMPT_INPUT_TOKEN_BUDGET = 2400 +REDUCTION_MAX_TOKENS = 180 +MAX_REDUCTION_DEPTH = 6 +MAX_SAMPLE_VALUES = 5 +DEFAULT_HISTORY_MAX_ALERTS = 3 +DEFAULT_HISTORY_MAX_TOKENS = 700 +DEFAULT_HISTORY_PATTERNS_PER_ALERT = 2 +SYSTEM_PROMPT = """ +You are a very professional and senior cybersecurity researcher and incident analyst. +Use only the provided alert and evidence data. +If recent alert history is provided, use it as cumulative context for the current alert. +If the current alert evidence aligns with repeated historical alerts, treat that recurrence as additional support that can increase confidence and urgency. +Do not treat historical activity as proof of the current alert when the current alert evidence conflicts with it. +Describe the current alert only from the current alert evidence digest. +Do not restate ports, IPs, destinations, or behaviors from historical context as if they happened in the current alert unless they also appear in the current alert evidence digest. +Evidence threat levels matter. Treat informational (`info`) evidence as context only, not as a security finding by itself. +Give analytical weight to low, medium, high, and critical evidence, with higher threat levels carrying more weight. +Do not let informational evidence inflate the verdict, confidence, urgency, or risk unless it is supported by non-info evidence. +Write exactly one paragraph of plain text for a human analyst. +Explain the main suspicious behavior, what evidence most strongly supports or weakens the alert, +whether it looks like a likely true positive, likely false positive, or uncertain, and how risky it appears. +If the evidence is weak, incomplete, or conflicting, say that clearly. +Do not use bullet points, markdown, headings, or JSON. +Do not invent missing facts. +""".strip() +REDUCTION_SYSTEM_PROMPT = """ +You are compressing raw security evidence into a compact intermediate digest for a later analyst summary. +Use only the provided evidence subset. +Write exactly one plain-text paragraph. +Preserve concrete behaviors, time ranges, counts, suspicious indicators, and false-positive clues when they matter. +Preserve threat-level distinctions. Informational (`info`) evidence is context only and should not be described as a threat indicator by itself. +Do not invent missing facts and do not add introductions or meta-commentary. +""".strip() + + +class AlertSummary(IModule): + name = "alert_summary" + description = "Summarizes alerts for analysts using the shared LLM module" + authors = ["Sebastian Garcia"] + + def init(self): + """Initialize channels, queues, and runtime configuration.""" + self.enabled = False + self.allowed_backends = [] + self.llm_temperature = 0.2 + self.llm_max_tokens = 220 + self.llm_response_timeout_seconds = 120 + self.log_verbosity = LOG_VERBOSITY_REQUESTS + self.history_enabled = False + self.history_max_alerts = DEFAULT_HISTORY_MAX_ALERTS + self.history_max_tokens = DEFAULT_HISTORY_MAX_TOKENS + self.history_patterns_per_alert = DEFAULT_HISTORY_PATTERNS_PER_ALERT + self.pending_alerts = deque() + self.active_job = None + self.pending_request = None + self.summary_log = None + self.operation_log = None + self.last_logged_pending_llm_requests = None + self.alert_history_by_profile = defaultdict(deque) + self.operation_log_path = os.path.join( + self.parent_output_dir, + "llm-summary", + "alert_summary.log", + ) + self.summary_log_path = os.path.join( + get_alerts_path_inside_output_dir(self.parent_output_dir), + "alerts-summary.log", + ) + self.read_configuration() + + def subscribe_to_channels(self): + """Subscribe to alert and shared LLM response channels.""" + self.c_alert = self.db.subscribe("new_alert") + self.c_llm = self.db.subscribe(self.db.channels.LLM_RESPONSE) + self.channels = { + "new_alert": self.c_alert, + self.db.channels.LLM_RESPONSE: self.c_llm, + } + + def read_configuration(self): + """Read alert summary settings from the active Slips configuration.""" + conf = ( + self.conf + if hasattr(self.conf, "alert_summary_enabled") + else ConfigParser() + ) + self.enabled = conf.alert_summary_enabled() + self.allowed_backends = conf.alert_summary_allowed_backends() + self.llm_temperature = conf.alert_summary_llm_temperature() + self.llm_max_tokens = conf.alert_summary_llm_max_tokens() + self.llm_response_timeout_seconds = ( + conf.alert_summary_llm_response_timeout_seconds() + ) + self.log_verbosity = conf.alert_summary_log_verbosity() + self.history_enabled = conf.alert_summary_history_enabled() + self.history_max_alerts = conf.alert_summary_history_max_alerts() + self.history_max_tokens = conf.alert_summary_history_max_tokens() + self.history_patterns_per_alert = ( + conf.alert_summary_history_patterns_per_alert() + ) + + def pre_main(self): + """Drop privileges and initialize the output files if enabled.""" + utils.drop_root_privs_permanently() + + if not self.enabled: + self.print("AlertSummary module disabled in config.", 2, 0) + return True + + self._init_operation_log_file() + self._init_summary_log_file() + self._log_operation( + "AlertSummary module ready. " + f"summary_log={self.summary_log_path} " + f"operation_log={self.operation_log_path} " + f"prompt_version={PROMPT_VERSION}", + verbosity=LOG_VERBOSITY_SUMMARY, + ) + + @staticmethod + def _is_process_alive(pid: int) -> bool: + """ + Check whether a process PID is still alive. + + :param pid: Process ID. + :return: True when the PID is alive. + """ + try: + os.kill(int(pid), 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + def shutdown_gracefully(self): + """Flush unresolved work to the summary file and close log handles.""" + if self.active_job: + alert = self.active_job["alert"] + self._write_summary_entry( + alert, + self._build_fallback_summary( + alert, + self.active_job["evidences"], + "Module stopped before the LLM reply was processed.", + ), + ) + self._log_operation( + f"Shutdown flushed active alert_id={alert.id}.", + verbosity=LOG_VERBOSITY_SUMMARY, + ) + self.active_job = None + self.pending_request = None + + if self.pending_alerts: + self._flush_queued_alerts_without_backend( + "Module stopped before pending alerts were summarized." + ) + + self._log_operation( + "AlertSummary module stopped.", + verbosity=LOG_VERBOSITY_SUMMARY, + ) + if self.summary_log is not None: + self.summary_log.close() + if self.operation_log is not None: + self.operation_log.close() + + def main(self): + """Queue alerts, process LLM replies, and advance the active job.""" + self._queue_new_alert() + + if self.pending_request: + self._handle_pending_response() + if self.pending_request: + return + + if self.active_job: + self._advance_active_job() + if self.pending_request: + return + + if not self.pending_alerts: + return + + available_backends = self.db.get_available_llm_backends() + backend = self._select_backend(available_backends) + if not backend: + if self.termination_event.is_set(): + self._flush_queued_alerts_without_backend( + "No runtime-ready LLM backend available." + ) + elif self.pending_alerts: + self._log_operation( + "No runtime-ready LLM backend available yet. " + f"queued_alerts={len(self.pending_alerts)}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + return + + self._start_next_alert_job(backend) + + def _init_operation_log_file(self): + """Create or clear the per-run alert summary operation log file.""" + os.makedirs(os.path.dirname(self.operation_log_path), exist_ok=True) + utils.initialize_logfile( + self.operation_log_path, + getattr(self.args, "is_slips_started_by_an_update", False), + ) + self.operation_log = open( + self.operation_log_path, "a", encoding="utf-8" + ) + + conf = ConfigParser() + utils.change_logfiles_ownership( + self.operation_log_path, + conf.get_UID(), + conf.get_GID(), + ) + + def _init_summary_log_file(self): + """Create or clear alerts-summary.log for the current Slips run.""" + os.makedirs(os.path.dirname(self.summary_log_path), exist_ok=True) + utils.initialize_logfile( + self.summary_log_path, + getattr(self.args, "is_slips_started_by_an_update", False), + ) + self.summary_log = open(self.summary_log_path, "a", encoding="utf-8") + + conf = ConfigParser() + utils.change_logfiles_ownership( + self.summary_log_path, + conf.get_UID(), + conf.get_GID(), + ) + + def _queue_new_alert(self): + """ + Parse and enqueue a new alert together with its evidence records. + queues alerts to the self.pending_alerts + """ + msg = self.get_msg("new_alert") + if not msg: + return + + try: + alert = dict_to_alert(json.loads(msg["data"])) + except (TypeError, ValueError, KeyError, json.JSONDecodeError) as exc: + self.print(f"Unable to parse new_alert payload: {exc}", 0, 1) + self._log_operation( + f"Unable to parse new_alert payload: {exc}", + verbosity=LOG_VERBOSITY_SUMMARY, + ) + return + + evidences = self._get_alert_evidence(alert) + self.pending_alerts.append( + { + "alert": alert, + "evidences": evidences, + } + ) + self._log_operation( + f"Queued alert_id={alert.id} " + f"profileid={alert.profile} " + f"timewindow={alert.timewindow} " + f"evidence_count={len(evidences)} " + f"queue_size={len(self.pending_alerts)}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + + def _get_alert_evidence(self, alert) -> list: + """Load and normalize all evidence records referenced by the alert.""" + profileid = str(alert.profile) + twid = str(alert.timewindow) + raw_evidence = self.db.get_twid_evidence(profileid, twid) or {} + + evidence_records = [] + for evidence_id in alert.correl_id: + payload = raw_evidence.get(evidence_id) + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError: + continue + + if not isinstance(payload, dict): + continue + + try: + evidence_records.append(dict_to_evidence(payload)) + except (KeyError, TypeError, ValueError): + continue + + if not evidence_records: + evidence_records = [alert.last_evidence] + + evidence_records.sort(key=self._get_evidence_sort_key) + return evidence_records + + def _get_evidence_sort_key(self, evidence) -> tuple: + """Build a stable sort key even when evidence timestamps vary in type.""" + timestamp = getattr(evidence, "timestamp", "") + + if isinstance(timestamp, (int, float)): + return (0, float(timestamp)) + + try: + return ( + 0, + float(utils.convert_ts_format(timestamp, "unixtimestamp")), + ) + except (TypeError, ValueError): + return (1, str(timestamp)) + + def _select_backend(self, available_backends: dict) -> str: + """Choose a runtime-ready backend using module preferences first.""" + available = available_backends.get("backends", {}) + if not available: + return "" + + for backend in self.allowed_backends: + if backend in available: + return backend + + default_backend = available_backends.get("default_backend", "") + if default_backend in available: + return default_backend + + if self.allowed_backends: + return "" + + return sorted(available)[0] + + def _start_next_alert_job(self, backend: str): + """Create a multi-step summary job for the next queued alert.""" + queued_alert = self.pending_alerts.popleft() + alert = queued_alert["alert"] + evidences = queued_alert["evidences"] + grouped_items = self._build_grouped_evidence_items(evidences) + self.active_job = { + "alert": alert, + "evidences": evidences, + "backend": backend, + "grouped_item_count": len(grouped_items), + "initial_grouped_items": list(grouped_items), + "current_items": grouped_items, + "reduction_layer": 0, + "current_chunks": [], + "completed_chunk_summaries": [], + } + self._log_operation( + f"Started alert summary job alert_id={alert.id} " + f"backend={backend} evidence_count={len(evidences)} " + f"grouped_items={len(grouped_items)}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + self._advance_active_job() + + def _advance_active_job(self): + """Dispatch the next LLM request for the active alert summary job.""" + if not self.active_job or self.pending_request: + return + + job = self.active_job + alert = job["alert"] + final_messages = self._build_prompt_messages( + alert, + job["current_items"], + len(job["evidences"]), + job["grouped_item_count"], + job["reduction_layer"], + ) + final_token_estimate = self._estimate_messages_tokens(final_messages) + self._log_operation( + f"Evaluated final summary prompt alert_id={alert.id} " + f"estimated_input_tokens={final_token_estimate} " + f"budget={FINAL_PROMPT_INPUT_TOKEN_BUDGET} " + f"reduction_layer={job['reduction_layer']} " + f"digest_items={len(job['current_items'])}", + verbosity=LOG_VERBOSITY_DEBUG, + ) + if self._messages_fit(final_messages, FINAL_PROMPT_INPUT_TOKEN_BUDGET): + self._dispatch_llm_request( + phase="final_summary", + messages=final_messages, + max_tokens=self.llm_max_tokens, + metadata={ + "alert_id": alert.id, + "profileid": str(alert.profile), + "timewindow": str(alert.timewindow), + "evidence_count": len(job["evidences"]), + "grouped_item_count": job["grouped_item_count"], + "digest_item_count": len(job["current_items"]), + "reduction_layer": job["reduction_layer"], + "prompt_version": PROMPT_VERSION, + }, + ) + return + + if job["reduction_layer"] >= MAX_REDUCTION_DEPTH: + self._fail_active_job( + "Prompt remained too large after recursive evidence reduction." + ) + return + + chunks = self._chunk_items_for_reduction( + alert, + job["current_items"], + job["reduction_layer"], + ) + if not chunks: + self._fail_active_job( + "Unable to build reduction chunks for alert summary." + ) + return + + job["current_chunks"] = chunks + job["completed_chunk_summaries"] = [] + self._log_operation( + f"Starting reduction layer={job['reduction_layer'] + 1} " + f"for alert_id={alert.id} chunks={len(chunks)} " + f"source_items={len(job['current_items'])}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + self._dispatch_reduction_chunk(0) + + def _dispatch_reduction_chunk(self, chunk_index: int): + """Send one evidence chunk for intermediate summarization.""" + job = self.active_job + if not job: + return + + chunk_items = job["current_chunks"][chunk_index] + messages = self._build_reduction_messages( + job["alert"], + chunk_items, + job["reduction_layer"] + 1, + chunk_index + 1, + len(job["current_chunks"]), + len(job["current_items"]), + ) + self._dispatch_llm_request( + phase="reduction", + messages=messages, + max_tokens=REDUCTION_MAX_TOKENS, + metadata={ + "alert_id": job["alert"].id, + "profileid": str(job["alert"].profile), + "timewindow": str(job["alert"].timewindow), + "evidence_count": len(job["evidences"]), + "grouped_item_count": job["grouped_item_count"], + "digest_item_count": len(job["current_items"]), + "reduction_layer": job["reduction_layer"] + 1, + "chunk_index": chunk_index + 1, + "chunk_count": len(job["current_chunks"]), + "prompt_version": PROMPT_VERSION, + }, + ) + + def _dispatch_llm_request( + self, + phase: str, + messages: list, + max_tokens: int, + metadata: dict, + ): + """Publish one LLM request for either reduction or final summarization.""" + if not self.active_job: + return + + request_id = f"{self.name}-{uuid.uuid4()}" + request = { + "request_id": request_id, + "requester": self.name, + "backend": self.active_job["backend"], + "messages": messages, + "temperature": self.llm_temperature, + "max_tokens": max_tokens, + "metadata": metadata, + } + self.pending_request = { + "request_id": request_id, + "backend": self.active_job["backend"], + "alert": self.active_job["alert"], + "evidences": self.active_job["evidences"], + "phase": phase, + "sent_at": time.time(), + "metadata": metadata, + } + + try: + self.db.publish(self.db.channels.LLM_REQUEST, json.dumps(request)) + except Exception: + self.pending_request = None + raise + + self._log_operation( + f"Published llm_request request_id={request_id} " + f"alert_id={self.active_job['alert'].id} " + f"phase={phase} " + f"backend={self.active_job['backend']} " + f"max_tokens={max_tokens} " + f"metadata={json.dumps(metadata, sort_keys=True)}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + + def _build_prompt_messages( + self, + alert, + evidence_items: list[str], + evidence_count: int, + grouped_item_count: int, + reduction_layer: int, + ) -> list: + """ + Create the final analyst-summary prompt for one alert. + + :param alert: Alert being summarized. + :param evidence_items: Grouped evidence lines or reduced digest items. + :param evidence_count: Total evidence records attached to the alert. + :param grouped_item_count: Count of grouped evidence patterns. + :param reduction_layer: Number of prior reduction layers applied. + :return: Chat messages for the shared LLM module. + """ + history_context = self._build_recent_history_text( + alert, evidence_items + ) + user_prompt = ( + "You are a security analyst. Translate this Slips alert into one " + "clear, concise paragraph for a human analyst.\n\n" + f"{self._build_alert_metadata_text(alert, evidence_count, grouped_item_count, reduction_layer)}\n\n" + f"{history_context}" + "CURRENT ALERT EVIDENCE DIGEST:\n" + f"{self._format_digest_items(evidence_items)}\n\n" + "YOUR TASK:\n" + "1. Explain the main suspicious behavior in plain language.\n" + "2. Identify the strongest evidence that supports or weakens the alert.\n" + "3. State whether it looks like a likely true positive, likely false positive, or uncertain.\n" + "4. State the likely operational risk or urgency.\n" + "5. If recent alert history is present, explicitly explain whether the current alert looks like a continuation, escalation, repetition, diversification, or a different pattern.\n" + "6. If recent alert history shows repeated aligned alerts, explicitly state whether that recurrence raises, lowers, or does not materially change confidence and urgency.\n\n" + "OUTPUT RULES:\n" + "- Write exactly one paragraph.\n" + "- Use plain text only.\n" + "- Base the assessment only on the provided data.\n" + "- Describe the current alert using only details from CURRENT ALERT EVIDENCE DIGEST.\n" + "- Do not present historical-only details as part of the current alert.\n" + "- If you mention history-specific details, mark them explicitly as historical or prior activity.\n" + "- Weigh evidence according to threat level.\n" + "- Treat informational (`info`) evidence as context only, not as suspicious evidence by itself.\n" + "- Use recent alert history as cumulative context when it aligns with the current alert evidence.\n" + "- Do not use recent alert history as replacement evidence when the current alert evidence is weak or conflicting.\n" + "- When recent alert history is present, include one explicit clause about how recurrence affects confidence or risk.\n" + "- If the evidence is repetitive, weak, incomplete, or contradictory, say so clearly.\n" + f"- Prompt version: {PROMPT_VERSION}" + ) + return [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ] + + def _build_reduction_messages( + self, + alert, + evidence_items: list[str], + reduction_layer: int, + chunk_index: int, + chunk_count: int, + source_item_count: int, + ) -> list: + """ + Create an intermediate reduction prompt for one evidence chunk. + + :param alert: Alert being summarized. + :param evidence_items: Chunk items to compress further. + :param reduction_layer: One-based reduction layer number. + :param chunk_index: One-based chunk position in this layer. + :param chunk_count: Total chunk count in this layer. + :param source_item_count: Number of digest items before chunking. + :return: Chat messages for the shared LLM module. + """ + user_prompt = ( + "Compress this alert evidence subset into a compact intermediate " + "digest for a later final analyst summary.\n\n" + f"{self._build_alert_metadata_text(alert, len(self.active_job['evidences']), self.active_job['grouped_item_count'], reduction_layer - 1)}\n" + f"Reduction layer: {reduction_layer}\n" + f"Chunk: {chunk_index}/{chunk_count}\n" + f"Source digest items in this layer: {source_item_count}\n\n" + "EVIDENCE SUBSET:\n" + f"{self._format_digest_items(evidence_items)}\n\n" + "OUTPUT RULES:\n" + "- Write exactly one paragraph.\n" + "- Keep it shorter than the source evidence subset.\n" + "- Preserve the most important behaviors, time ranges, counts, indicators, and false-positive clues.\n" + "- Preserve threat-level distinctions and keep informational (`info`) evidence as context only.\n" + "- Do not include introductions, bullet points, markdown, or JSON.\n" + "- Do not make a final analyst verdict for the whole alert.\n" + f"- Prompt version: {PROMPT_VERSION}" + ) + return [ + {"role": "system", "content": REDUCTION_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ] + + def _build_alert_metadata_text( + self, + alert, + evidence_count: int, + grouped_item_count: int, + reduction_layer: int, + ) -> str: + """ + Format the alert metadata section used by both prompt types. + + :param alert: Alert being summarized. + :param evidence_count: Total evidence records attached to the alert. + :param grouped_item_count: Count of grouped evidence patterns. + :param reduction_layer: Number of completed reduction layers. + :return: Multi-line metadata block. + """ + profileid = str(alert.profile) + hostname = self.db.get_hostname_from_profile(profileid) or "" + profile = alert.profile.ip + if hostname: + profile = f"{profile} ({hostname})" + + start_time = self._format_timestamp_for_prompt( + getattr(alert.timewindow, "start_time", "") + ) + end_time = self._format_timestamp_for_prompt( + getattr(alert.timewindow, "end_time", "") + ) + if start_time and end_time: + time_range = f"{start_time} to {end_time}" + else: + time_range = start_time or end_time or "Unknown" + + return ( + "INCIDENT METADATA:\n" + f"- Alert ID: {alert.id}\n" + f"- Source IP: {profile}\n" + f"- Timewindow: {alert.timewindow.number}\n" + f"- Time Range: {time_range}\n" + f"- Accumulated Threat Level: {alert.accumulated_threat_level}\n" + f"- Alert Confidence: {alert.confidence:.2f}\n" + f"- Correlated Evidence Records: {evidence_count}\n" + f"- Grouped Evidence Patterns: {grouped_item_count}\n" + f"- Completed Reduction Layers: {reduction_layer}" + ) + + def _format_digest_items(self, evidence_items: list[str]) -> str: + """ + Render grouped evidence or digest items for a prompt body. + + :param evidence_items: Items to format. + :return: Multi-line evidence block. + """ + if not evidence_items: + return "- No evidence details were available." + return "\n".join(f"- {item}" for item in evidence_items) + + def _build_recent_history_text( + self, alert, current_items: list[str] + ) -> str: + """ + Render bounded recent alert history for the same profile. + + :param alert: Alert being summarized. + :param current_items: Current grouped digest items. + :return: Prompt section or an empty string when history is unavailable. + """ + recent_history = self._get_recent_alert_history(alert) + if not recent_history: + return "" + + history_analysis = self._analyze_recent_history( + alert, recent_history, current_items + ) + lines = [] + remaining_tokens = self.history_max_tokens + header_lines = [ + "HISTORICAL PROGRESSION ONLY (same source/profile, context for the current alert):", + f"- Prior summarized alerts: {history_analysis['prior_alert_count']}", + f"- Prior alerts with overlapping dominant patterns: {history_analysis['matching_alert_count']}", + f"- Repeated dominant current patterns seen before: {history_analysis['repeated_pattern_count']}", + f"- Same-timewindow prior alerts: {history_analysis['same_timewindow_alert_count']}", + f"- Threat trend versus recent history: {history_analysis['threat_trend']}", + f"- Confidence trend versus recent history: {history_analysis['confidence_trend']}", + "- Guidance: if the current alert matches the repeated prior pattern, treat recurrence as cumulative supporting context that can raise confidence and urgency.", + "- Guidance: do not restate ports, IPs, destinations, or behaviors from this history as current-alert facts unless they also appear in CURRENT ALERT EVIDENCE DIGEST.", + ] + header_text = "\n".join(header_lines) + header_tokens = self._estimate_text_tokens(header_text) + if header_tokens >= remaining_tokens: + return "" + remaining_tokens -= header_tokens + + for entry in recent_history: + line = self._format_history_entry(entry) + line_tokens = self._estimate_text_tokens(line) + if line_tokens > remaining_tokens: + if lines: + break + line = self._truncate_text_to_budget(line, remaining_tokens) + line_tokens = self._estimate_text_tokens(line) + lines.append(f"- {line}") + remaining_tokens -= line_tokens + if remaining_tokens <= 0: + break + + if not lines: + return "" + + return ( + header_text + + "\nRECENT ALERT HISTORY (most recent first):\n" + + "\n".join(lines) + + "\n\n" + ) + + def _get_recent_alert_history(self, alert) -> list[dict]: + """ + Return recent stored summaries for the same profile, newest first. + + :param alert: Alert being summarized. + :return: List of history entries. + """ + if ( + not self.history_enabled + or self.history_max_alerts <= 0 + or self.history_max_tokens <= 0 + ): + return [] + + profileid = str(alert.profile) + history = list(self.alert_history_by_profile.get(profileid, [])) + if not history: + return [] + return list(reversed(history[-self.history_max_alerts :])) + + def _format_history_entry(self, entry: dict) -> str: + """ + Convert one stored history entry into prompt text. + + :param entry: Stored history entry. + :return: Single-line history description. + """ + top_patterns = entry.get("top_patterns") or [] + pattern_text = ( + "; ".join(top_patterns) + or "No dominant historical patterns stored." + ) + return ( + f"TW {entry.get('timewindow', '?')} | " + f"{entry.get('time_range', 'Unknown')} | " + f"threat={entry.get('accumulated_threat_level', 0.0):.2f} | " + f"conf={entry.get('confidence', 0.0):.2f} | " + f"historical patterns: {pattern_text}" + ) + + def _analyze_recent_history( + self, + alert, + recent_history: list[dict], + current_items: list[str], + ) -> dict: + """ + Summarize how recent history aligns with the current alert. + + :param alert: Alert being summarized. + :param recent_history: Stored recent alert history entries. + :param current_items: Current grouped digest items. + :return: History alignment metrics. + """ + current_signatures = set(self._build_pattern_signatures(current_items)) + matching_alert_count = 0 + repeated_signatures = set() + same_timewindow_alert_count = 0 + prior_threat_values = [] + prior_confidence_values = [] + current_timewindow = str(getattr(alert.timewindow, "number", "?")) + + for entry in recent_history: + prior_threat_values.append( + float(entry.get("accumulated_threat_level", 0.0) or 0.0) + ) + prior_confidence_values.append( + float(entry.get("confidence", 0.0) or 0.0) + ) + if str(entry.get("timewindow", "?")) == current_timewindow: + same_timewindow_alert_count += 1 + + entry_signatures = set(entry.get("pattern_signatures") or []) + overlap = current_signatures & entry_signatures + if overlap: + matching_alert_count += 1 + repeated_signatures.update(overlap) + + average_prior_threat = ( + sum(prior_threat_values) / len(prior_threat_values) + if prior_threat_values + else 0.0 + ) + average_prior_confidence = ( + sum(prior_confidence_values) / len(prior_confidence_values) + if prior_confidence_values + else 0.0 + ) + + return { + "prior_alert_count": len(recent_history), + "matching_alert_count": matching_alert_count, + "repeated_pattern_count": len(repeated_signatures), + "same_timewindow_alert_count": same_timewindow_alert_count, + "threat_trend": self._classify_history_trend( + float(getattr(alert, "accumulated_threat_level", 0.0) or 0.0), + average_prior_threat, + 0.75, + ), + "confidence_trend": self._classify_history_trend( + float(getattr(alert, "confidence", 0.0) or 0.0), + average_prior_confidence, + 0.05, + ), + } + + def _build_pattern_signatures(self, grouped_items: list[str]) -> list[str]: + """ + Derive normalized pattern signatures from grouped digest items. + + :param grouped_items: Grouped digest items. + :return: Deduplicated normalized signatures. + """ + signatures = [] + seen = set() + for item in grouped_items or []: + signature = self._extract_pattern_signature(item) + if not signature or signature in seen: + continue + seen.add(signature) + signatures.append(signature) + return signatures + + def _extract_pattern_signature(self, grouped_item: str) -> str: + """ + Strip timing and examples from one grouped item for overlap matching. + + :param grouped_item: One grouped digest item. + :return: Normalized signature. + """ + normalized = self._normalize_summary_text(grouped_item) + if "|" in normalized: + normalized = normalized.split("|", 1)[1].strip() + normalized = re.sub(r"\s+\([^)]*\)$", "", normalized) + normalized = self._normalize_pattern(normalized) + return self._normalize_summary_text(normalized).lower() + + def _classify_history_trend( + self, + current_value: float, + average_prior_value: float, + tolerance: float, + ) -> str: + """ + Compare the current value against recent history. + + :param current_value: Current alert value. + :param average_prior_value: Average prior value. + :param tolerance: Minimum delta to call the trend changed. + :return: rising, falling, or stable. + """ + if current_value > average_prior_value + tolerance: + return "rising" + if current_value < average_prior_value - tolerance: + return "falling" + return "stable" + + def _build_grouped_evidence_items(self, evidences: list) -> list[str]: + """ + Group similar evidence descriptions into prompt-friendly digest lines. + + :param evidences: Evidence records for one alert. + :return: Ordered list of grouped evidence lines. + """ + grouped_evidences = defaultdict(list) + for evidence in evidences: + description = str( + getattr(evidence, "description", "") or "" + ).strip() + grouped_evidences[self._normalize_pattern(description)].append( + evidence + ) + + summaries = [] + for _, group in grouped_evidences.items(): + group.sort(key=self._get_evidence_sort_key) + first = group[0] + first_time = self._format_short_time(first.timestamp) + last_time = self._format_short_time(group[-1].timestamp) + time_range = ( + f"{first_time}-{last_time}" + if first_time and last_time and first_time != last_time + else first_time or last_time or "time-unknown" + ) + + description = str(getattr(first, "description", "") or "").strip() + sample_values = self._extract_sample_values( + [ + str(getattr(evidence, "description", "") or "") + for evidence in group[:3] + ] + ) + severity_counts = self._count_group_severities(group) + severity_text = self._format_severity_counts(severity_counts) + + if len(group) == 1: + line = f"{time_range} | {description}" + else: + line = ( + f"{time_range} | {description} " f"({len(group)}x similar" + ) + if severity_text: + line += f", severities: {severity_text}" + if sample_values: + line += ", samples: " + ", ".join( + sample_values[:MAX_SAMPLE_VALUES] + ) + line += ")" + + summaries.append( + { + "count": len(group), + "line": self._normalize_summary_text(line), + "sort_key": self._get_evidence_sort_key(first), + } + ) + + summaries.sort(key=lambda item: (-item["count"], item["sort_key"])) + return [item["line"] for item in summaries] + + def _normalize_pattern(self, description: str) -> str: + """ + Normalize variable values in descriptions before grouping. + + :param description: Raw evidence description. + :return: Normalized grouping key. + """ + pattern = description + pattern = re.sub(r"\b\d{1,3}(?:\.\d{1,3}){3}\b", "", pattern) + pattern = re.sub( + r"\b\d+/(TCP|UDP)\b", r"/\1", pattern, flags=re.IGNORECASE + ) + pattern = re.sub( + r"port[s]?:?\s*\d+(?:-\d+)?", + "port ", + pattern, + flags=re.IGNORECASE, + ) + pattern = re.sub(r"\b\d+\b", "", pattern) + return pattern + + def _extract_sample_values(self, descriptions: list[str]) -> list[str]: + """ + Extract useful IP and port examples from grouped descriptions. + + :param descriptions: Raw evidence descriptions from one group. + :return: Deduplicated example values. + """ + sample_values = [] + for description in descriptions: + sample_values.extend( + re.findall(r"\b\d{1,3}(?:\.\d{1,3}){3}\b", description) + ) + sample_values.extend( + [ + f"{port}/{proto.upper()}" + for port, proto in re.findall( + r"\b(\d+)/(TCP|UDP)\b", + description, + flags=re.IGNORECASE, + ) + ] + ) + + unique_samples = [] + seen = set() + for value in sample_values: + if value in seen: + continue + seen.add(value) + unique_samples.append(value) + return unique_samples + + def _count_group_severities(self, evidences: list) -> dict: + """ + Count threat levels inside one grouped evidence set. + + :param evidences: Evidence records in the group. + :return: Severity count mapping. + """ + severity_counts = {} + for evidence in evidences: + severity = str(getattr(evidence, "threat_level", "info")).lower() + severity_counts[severity] = severity_counts.get(severity, 0) + 1 + return severity_counts + + def _format_severity_counts(self, severity_counts: dict) -> str: + """ + Format grouped severity counts for prompt readability. + + :param severity_counts: Severity count mapping. + :return: Human-readable summary string. + """ + ordered = ["critical", "high", "medium", "low", "info"] + parts = [] + for severity in ordered: + count = severity_counts.get(severity, 0) + if count: + parts.append(f"{severity}={count}") + return ", ".join(parts) + + def _format_short_time(self, timestamp) -> str: + """ + Convert a timestamp into HH:MM when possible. + + :param timestamp: Timestamp value in any Slips-supported format. + :return: Short human-readable time. + """ + iso_timestamp = self._convert_timestamp_to_iso(timestamp) + if not iso_timestamp: + return str(timestamp or "") + + try: + parsed = datetime.fromisoformat(iso_timestamp) + except ValueError: + return str(timestamp or "") + return parsed.strftime("%H:%M") + + def _format_timestamp_for_prompt(self, timestamp) -> str: + """ + Convert a timestamp into the long prompt-friendly format. + + :param timestamp: Timestamp value in any Slips-supported format. + :return: Prompt-friendly timestamp string. + """ + iso_timestamp = self._convert_timestamp_to_iso(timestamp) + if not iso_timestamp: + return str(timestamp or "") + + try: + parsed = datetime.fromisoformat(iso_timestamp) + except ValueError: + return str(timestamp or "") + return parsed.strftime("%Y-%m-%d %H:%M:%S") + + def _convert_timestamp_to_iso(self, timestamp) -> str: + """ + Convert a timestamp into ISO format when possible. + + :param timestamp: Timestamp value in any Slips-supported format. + :return: ISO timestamp string or an empty string. + """ + if timestamp in ("", None): + return "" + try: + return str(utils.convert_ts_format(timestamp, "iso")) + except (TypeError, ValueError): + return "" + + def _chunk_items_for_reduction( + self, + alert, + items: list[str], + reduction_layer: int, + ) -> list[list[str]]: + """ + Split digest items into chunks that fit the reduction prompt budget. + + :param alert: Alert being summarized. + :param items: Current digest items to reduce. + :param reduction_layer: Zero-based current reduction layer. + :return: List of chunks, each chunk being a list of digest items. + """ + expanded_items = [] + for item in items: + if self._single_item_fits_reduction_prompt( + alert, item, reduction_layer + ): + expanded_items.append(item) + continue + + split_items = self._split_item_for_reduction( + alert, + item, + reduction_layer, + ) + self._log_operation( + f"Split oversized digest item for alert_id={alert.id} " + f"layer={reduction_layer + 1} " + f"parts={len(split_items)}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + expanded_items.extend(split_items) + + chunks = [] + current_chunk = [] + for item in expanded_items: + trial_chunk = current_chunk + [item] + if current_chunk and not self._chunk_fits_reduction_prompt( + alert, + trial_chunk, + reduction_layer, + len(items), + ): + chunks.append(current_chunk) + current_chunk = [item] + continue + current_chunk = trial_chunk + + if current_chunk: + chunks.append(current_chunk) + return chunks + + def _single_item_fits_reduction_prompt( + self, + alert, + item: str, + reduction_layer: int, + ) -> bool: + """ + Check whether one digest item fits in a reduction prompt. + + :param alert: Alert being summarized. + :param item: One digest item to test. + :param reduction_layer: Zero-based current reduction layer. + :return: True when the item fits without splitting. + """ + return self._chunk_fits_reduction_prompt( + alert, + [item], + reduction_layer, + 1, + ) + + def _chunk_fits_reduction_prompt( + self, + alert, + items: list[str], + reduction_layer: int, + source_item_count: int, + ) -> bool: + """ + Estimate whether a chunk fits the reduction prompt budget. + + :param alert: Alert being summarized. + :param items: Candidate chunk items. + :param reduction_layer: Zero-based current reduction layer. + :param source_item_count: Number of source digest items in this layer. + :return: True when the prompt estimate fits the configured budget. + """ + messages = self._build_reduction_messages( + alert, + items, + reduction_layer + 1, + 1, + 1, + source_item_count, + ) + return self._messages_fit( + messages, REDUCTION_PROMPT_INPUT_TOKEN_BUDGET + ) + + def _split_item_for_reduction( + self, + alert, + item: str, + reduction_layer: int, + ) -> list[str]: + """ + Split one oversized digest item into smaller parts without truncating. + + :param alert: Alert being summarized. + :param item: Oversized digest item text. + :param reduction_layer: Zero-based current reduction layer. + :return: List of smaller digest items. + """ + empty_messages = self._build_reduction_messages( + alert, + [], + reduction_layer + 1, + 1, + 1, + 1, + ) + overhead_tokens = self._estimate_messages_tokens(empty_messages) + available_tokens = max( + 120, + REDUCTION_PROMPT_INPUT_TOKEN_BUDGET - overhead_tokens - 64, + ) + parts = self._split_text_to_budget(item, available_tokens) + if len(parts) == 1: + return parts + return [ + f"{part} (continued segment {index}/{len(parts)})" + for index, part in enumerate(parts, start=1) + ] + + def _split_text_to_budget( + self, + text: str, + token_budget: int, + ) -> list[str]: + """ + Split a text block by sentence and word boundaries to fit a budget. + + :param text: Input text to split. + :param token_budget: Approximate token budget per part. + :return: Ordered list of text parts. + """ + normalized = self._normalize_summary_text(text) + if self._estimate_text_tokens(normalized) <= token_budget: + return [normalized] + + for separator in ("\n", "; ", ". ", ", "): + parts = self._split_text_by_separator( + normalized, + separator, + token_budget, + ) + if len(parts) > 1 and all( + self._estimate_text_tokens(part) <= token_budget + for part in parts + ): + return parts + + return self._split_text_by_words(normalized, token_budget) + + def _split_text_by_separator( + self, + text: str, + separator: str, + token_budget: int, + ) -> list[str]: + """ + Try to split a text block on one separator while honoring a budget. + + :param text: Input text to split. + :param separator: Separator to preserve between pieces. + :param token_budget: Approximate token budget per part. + :return: Split text parts. + """ + raw_parts = [ + part.strip() for part in text.split(separator) if part.strip() + ] + if len(raw_parts) <= 1: + return [text] + + merged_parts = [] + current = "" + for part in raw_parts: + candidate = part if not current else f"{current}{separator}{part}" + if self._estimate_text_tokens(candidate) <= token_budget: + current = candidate + continue + if current: + merged_parts.append(current) + current = part + + if current: + merged_parts.append(current) + return merged_parts + + def _split_text_by_words( + self, + text: str, + token_budget: int, + ) -> list[str]: + """ + Split a text block by words when coarser separators are not enough. + + :param text: Input text to split. + :param token_budget: Approximate token budget per part. + :return: Split text parts. + """ + words = text.split() + if not words: + return [text] + + parts = [] + current_words = [] + for word in words: + candidate_words = current_words + [word] + candidate = " ".join(candidate_words) + if ( + current_words + and self._estimate_text_tokens(candidate) > token_budget + ): + parts.append(" ".join(current_words)) + current_words = [word] + continue + current_words = candidate_words + + if current_words: + parts.append(" ".join(current_words)) + return parts + + def _estimate_messages_tokens(self, messages: list[dict]) -> int: + """ + Estimate the input token count for a message list. + + :param messages: Chat messages to estimate. + :return: Approximate token count. + """ + token_count = 0 + for message in messages: + token_count += 12 + token_count += self._estimate_text_tokens( + message.get("content", "") + ) + return token_count + + def _estimate_text_tokens(self, text: str) -> int: + """ + Estimate token count from text length using a conservative heuristic. + + :param text: Input text to estimate. + :return: Approximate token count. + """ + normalized = str(text or "") + return max( + 1, + (len(normalized) + APPROX_CHARS_PER_TOKEN - 1) + // APPROX_CHARS_PER_TOKEN, + ) + + def _truncate_text_to_budget(self, text: str, token_budget: int) -> str: + """ + Trim text to an approximate token budget without splitting mid-word. + + :param text: Text to trim. + :param token_budget: Approximate token budget. + :return: Trimmed text. + """ + normalized = self._normalize_summary_text(text) + if token_budget <= 0: + return "" + if self._estimate_text_tokens(normalized) <= token_budget: + return normalized + + words = normalized.split() + kept_words = [] + for word in words: + candidate = " ".join(kept_words + [word]) + if self._estimate_text_tokens(candidate) >= token_budget: + break + kept_words.append(word) + + trimmed = " ".join(kept_words).strip() + if not trimmed: + return normalized[: max(1, token_budget * APPROX_CHARS_PER_TOKEN)] + return f"{trimmed} ..." + + def _messages_fit(self, messages: list[dict], budget: int) -> bool: + """ + Return True when the prompt estimate fits the chosen budget. + + :param messages: Chat messages to estimate. + :param budget: Maximum estimated input tokens. + :return: True when the prompt should fit. + """ + return self._estimate_messages_tokens(messages) <= budget + + def _handle_pending_response(self): + """Consumes LLM responses published in the + self.db.channels.LLM_RESPONSE channel. + Each of these responses + """ + msg = self.get_msg(self.db.channels.LLM_RESPONSE) + if msg: + try: + response = json.loads(msg["data"]) + except (TypeError, json.JSONDecodeError): + self._log_operation( + "Received malformed llm_response payload. Ignoring.", + verbosity=LOG_VERBOSITY_DEBUG, + ) + return + + if ( + response.get("request_id") + != self.pending_request["request_id"] + ): + return + + self._finalize_request(response) + return + + if not self._is_response_timed_out(): + return + + if self.termination_event.is_set(): + if not self.pending_request.get("shutdown_wait_logged", False): + self._log_operation( + "Shutdown is in progress; keeping alert_summary alive " + f"for in-flight request_id={self.pending_request['request_id']} " + "until the shared LLM module replies.", + verbosity=LOG_VERBOSITY_SUMMARY, + ) + self.pending_request["shutdown_wait_logged"] = True + return + + self._finalize_request( + { + "request_id": self.pending_request["request_id"], + "backend": self.pending_request["backend"], + "success": False, + "error": "LLM summary request timed out.", + "text": "", + } + ) + + def _is_response_timed_out(self) -> bool: + """Return True when the active request exceeded the configured timeout.""" + if not self.pending_request or self.llm_response_timeout_seconds <= 0: + return False + + elapsed = time.time() - self.pending_request["sent_at"] + return elapsed >= self.llm_response_timeout_seconds + + def _finalize_request(self, response: dict): + """ + Continue the reduction pipeline or write the final alert summary. + + :param response: Shared LLM response payload. + :return: None + """ + request = self.pending_request + job = self.active_job + if not request or not job: + return + + phase = request["phase"] + usage = response.get("usage") or {} + usage_suffix = ( + " " f"usage={json.dumps(usage, sort_keys=True)}" if usage else "" + ) + + if response.get("success") and str(response.get("text", "")).strip(): + text = self._normalize_summary_text(response["text"]) + if phase == "final_summary": + self._write_summary_entry(job["alert"], f"LLM summary: {text}") + self._remember_alert_summary( + job["alert"], + text, + job.get("initial_grouped_items") + or job.get("current_items") + or [], + ) + self._log_operation( + f"Received successful llm_response request_id={request['request_id']} " + f"alert_id={job['alert'].id} phase={phase}{usage_suffix}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + self.pending_request = None + self.active_job = None + return + + job["completed_chunk_summaries"].append(text) + chunk_index = int(request["metadata"].get("chunk_index", 1)) + chunk_count = int(request["metadata"].get("chunk_count", 1)) + self._log_operation( + f"Received reduction digest request_id={request['request_id']} " + f"alert_id={job['alert'].id} " + f"layer={request['metadata'].get('reduction_layer')} " + f"chunk={chunk_index}/{chunk_count}{usage_suffix}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + self.pending_request = None + + if chunk_index < chunk_count: + self._dispatch_reduction_chunk(chunk_index) + return + + job["current_items"] = job["completed_chunk_summaries"] + job["completed_chunk_summaries"] = [] + job["current_chunks"] = [] + job["reduction_layer"] += 1 + self._log_operation( + f"Completed reduction layer={job['reduction_layer']} " + f"for alert_id={job['alert'].id} " + f"resulting_digest_items={len(job['current_items'])}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + self._advance_active_job() + return + + error = str(response.get("error", "Unknown LLM summary failure.")) + self._log_operation( + f"LLM request failed for alert_id={job['alert'].id} " + f"phase={phase}: {error}", + verbosity=LOG_VERBOSITY_SUMMARY, + ) + self._fail_active_job(error) + + def _normalize_summary_text(self, text: str) -> str: + """ + Collapse any multi-line reply into one plain-text paragraph. + + :param text: Raw model response. + :return: Single-paragraph normalized text. + """ + normalized = " ".join(str(text or "").split()) + return normalized.strip() + + def _fail_active_job(self, reason: str): + """ + Write a fallback summary for the active alert and clear its state. + + :param reason: Why the LLM pipeline failed. + :return: None + """ + if not self.active_job: + return + + alert = self.active_job["alert"] + summary_text = self._build_fallback_summary( + alert, + self.active_job["evidences"], + reason, + ) + self._write_summary_entry(alert, summary_text) + self._remember_alert_summary( + alert, + summary_text, + self.active_job.get("initial_grouped_items") + or self._build_grouped_evidence_items( + self.active_job["evidences"] + ), + ) + self.pending_request = None + self.active_job = None + + def _flush_queued_alerts_without_backend(self, reason: str): + """Write failure notes for queued alerts when shutdown happens first.""" + while self.pending_alerts: + queued_alert = self.pending_alerts.popleft() + summary_text = self._build_fallback_summary( + queued_alert["alert"], + queued_alert["evidences"], + reason, + ) + self._write_summary_entry(queued_alert["alert"], summary_text) + self._remember_alert_summary( + queued_alert["alert"], + summary_text, + self._build_grouped_evidence_items(queued_alert["evidences"]), + ) + self._log_operation( + f"Flushed alert_id={queued_alert['alert'].id}: {reason}", + verbosity=LOG_VERBOSITY_SUMMARY, + ) + + def _write_summary_entry(self, alert, summary_text: str): + """Append one human-readable summary line to alerts-summary.log.""" + if self.summary_log is None: + return + + profileid = str(alert.profile) + hostname = self.db.get_hostname_from_profile(profileid) or "" + profile = alert.profile.ip + if hostname: + profile = f"{profile} ({hostname})" + + try: + alert_time = utils.convert_ts_format( + alert.last_flow_datetime, utils.alerts_format + ) + except (TypeError, ValueError): + alert_time = alert.last_flow_datetime + + entry = ( + f"{alert_time}: " + f"Src IP {profile}. " + f"Alert {alert.id} on timewindow {alert.timewindow.number}. " + f"{summary_text}" + ) + self.summary_log.write(f"{entry}\n") + self.summary_log.flush() + os.fsync(self.summary_log.fileno()) + self._log_operation( + f"Wrote summary entry for alert_id={alert.id} " + f"timewindow={alert.timewindow.number}", + verbosity=LOG_VERBOSITY_REQUESTS, + ) + + def _log_operation( + self, message: str, verbosity: int = LOG_VERBOSITY_REQUESTS + ): + """Append one line to the module operation log.""" + if self.operation_log is None: + return + if verbosity > self.log_verbosity: + return + + timestamp = utils.get_human_readable_datetime() + self.operation_log.write(f"{timestamp} {message}\n") + self.operation_log.flush() + os.fsync(self.operation_log.fileno()) + + def _build_fallback_summary( + self, + alert, + evidences: list, + reason: str, + ) -> str: + """ + Generate a local one-paragraph summary when the LLM path fails. + + :param alert: Alert being summarized. + :param evidences: Evidence records correlated with the alert. + :param reason: Why the LLM summary was unavailable. + :return: Single-paragraph fallback summary. + """ + severity_counts = {"high": 0, "medium": 0, "low": 0, "info": 0} + for evidence in evidences: + severity = str(getattr(evidence, "threat_level", "info")).lower() + severity_counts[severity] = severity_counts.get(severity, 0) + 1 + + grouped_items = self._build_grouped_evidence_items(evidences) + strongest_indicators = "; ".join(grouped_items[:3]) or ( + "the correlated evidence set" + ) + history_analysis = self._analyze_recent_history( + alert, + self._get_recent_alert_history(alert), + grouped_items, + ) + + verdict = self._classify_alert_verdict( + alert, + evidences, + severity_counts, + history_analysis, + ) + risk = self._classify_alert_risk( + alert, severity_counts, history_analysis + ) + history_context = self._build_fallback_history_clause( + alert, history_analysis + ) + return ( + f"LLM summary unavailable ({reason}). " + f"Local heuristic summary: this alert correlates {len(evidences)} " + f"evidence records for source IP {alert.profile.ip}, with the " + f"strongest indicators being {strongest_indicators}. " + f"{history_context}" + f"The evidence mix includes {severity_counts.get('high', 0)} high, " + f"{severity_counts.get('medium', 0)} medium, " + f"{severity_counts.get('low', 0)} low, and " + f"{severity_counts.get('info', 0)} informational findings. " + f"Based on the accumulated threat level " + f"{alert.accumulated_threat_level:.2f} and confidence " + f"{alert.confidence:.2f}, this looks {verdict} and the " + f"operational risk appears {risk}." + ) + + def _classify_alert_verdict( + self, + alert, + evidences: list, + severity_counts: dict, + history_analysis: dict, + ) -> str: + """ + Estimate the analyst verdict for a fallback summary. + + :param alert: Alert being summarized. + :param evidences: Evidence records correlated with the alert. + :param severity_counts: Count of evidence severities. + :param history_analysis: Recent-history alignment metrics. + :return: Human-readable verdict label. + """ + if ( + history_analysis.get("matching_alert_count", 0) >= 2 + and history_analysis.get("repeated_pattern_count", 0) >= 1 + ): + return "increasingly like a likely true positive because the same pattern has repeated across prior alerts" + if severity_counts.get("high", 0) >= 3 or alert.confidence >= 0.8: + return "like a likely true positive" + if ( + evidences + and severity_counts.get("info", 0) >= len(evidences) + and alert.confidence < 0.4 + ): + return "uncertain and may be a false positive" + if alert.confidence >= 0.5 or severity_counts.get("medium", 0) >= 2: + return "concerning but still somewhat uncertain" + return "uncertain" + + def _build_fallback_history_clause( + self, alert, history_analysis: dict + ) -> str: + """ + Build one short history sentence for heuristic summaries. + + :param alert: Alert being summarized. + :param history_analysis: Recent-history alignment metrics. + :return: Short history clause or an empty string. + """ + recent_history = self._get_recent_alert_history(alert) + if not recent_history: + return "" + + top_history_patterns = [] + for entry in recent_history[:2]: + for pattern in entry.get("top_patterns") or []: + normalized = self._normalize_summary_text(pattern) + if normalized in top_history_patterns: + continue + top_history_patterns.append(normalized) + if len(top_history_patterns) >= 2: + break + if len(top_history_patterns) >= 2: + break + + if top_history_patterns: + pattern_text = "; ".join(top_history_patterns) + return ( + f"Recent related alert history for this source includes " + f"{len(recent_history)} prior summarized alerts, with " + f"{history_analysis.get('matching_alert_count', 0)} prior alerts " + f"showing overlapping dominant patterns and " + f"{history_analysis.get('repeated_pattern_count', 0)} repeated " + f"current-pattern matches. The repeated history most recently " + f"showed {pattern_text}, so recurrence should be treated as " + f"additional supporting context when the current evidence aligns. " + ) + + return ( + f"Recent related alert history for this source includes " + f"{len(recent_history)} prior summarized alerts, which should be " + f"treated as cumulative context for the current assessment. " + ) + + def _classify_alert_risk( + self, + alert, + severity_counts: dict, + history_analysis: dict, + ) -> str: + """ + Estimate operational risk for a fallback summary. + + :param alert: Alert being summarized. + :param severity_counts: Count of evidence severities. + :param history_analysis: Recent-history alignment metrics. + :return: Risk label for the summary paragraph. + """ + if ( + history_analysis.get("matching_alert_count", 0) >= 2 + and history_analysis.get("repeated_pattern_count", 0) >= 1 + ): + if ( + history_analysis.get("threat_trend") == "rising" + or severity_counts.get("high", 0) >= 1 + or alert.accumulated_threat_level >= 5 + ): + return "high" + return "medium-to-high" + if ( + severity_counts.get("high", 0) >= 3 + or alert.accumulated_threat_level >= 10 + ): + return "high" + if ( + severity_counts.get("medium", 0) >= 2 + or alert.accumulated_threat_level >= 5 + ): + return "medium" + return "low" + + def _remember_alert_summary( + self, + alert, + summary_text: str, + grouped_items: list[str], + ): + """ + Store one completed alert summary for later prompt context. + + :param alert: Alert that was summarized. + :param summary_text: Final summary text without file-log prefix handling. + :param grouped_items: Original grouped patterns for this alert. + :return: None + """ + if not self.history_enabled or self.history_max_alerts <= 0: + return + + profileid = str(alert.profile) + history = self.alert_history_by_profile[profileid] + top_patterns = [ + self._normalize_summary_text(item) + for item in (grouped_items or [])[ + : self.history_patterns_per_alert + ] + ] + pattern_signatures = self._build_pattern_signatures(top_patterns) + history.append( + { + "timewindow": getattr(alert.timewindow, "number", "?"), + "time_range": self._build_history_time_range(alert), + "accumulated_threat_level": float( + getattr(alert, "accumulated_threat_level", 0.0) or 0.0 + ), + "confidence": float(getattr(alert, "confidence", 0.0) or 0.0), + "top_patterns": top_patterns, + "pattern_signatures": pattern_signatures, + "summary_text": self._normalize_summary_text(summary_text), + } + ) + while len(history) > self.history_max_alerts: + history.popleft() + + def _build_history_time_range(self, alert) -> str: + """ + Format one compact time-range label for stored alert history. + + :param alert: Alert being summarized. + :return: Compact time range. + """ + start_time = self._format_short_time( + getattr(alert.timewindow, "start_time", "") + ) + end_time = self._format_short_time( + getattr(alert.timewindow, "end_time", "") + ) + if start_time and end_time: + return ( + f"{start_time}-{end_time}" + if start_time != end_time + else start_time + ) + return start_time or end_time or "time-unknown" diff --git a/modules/anomaly_detection_https/README.md b/modules/anomaly_detection_https/README.md index d97e7a15b5..18b05784aa 100644 --- a/modules/anomaly_detection_https/README.md +++ b/modules/anomaly_detection_https/README.md @@ -551,7 +551,7 @@ Every detection (`flow_detection` and `hourly_detection`) is emitted as Slips Ev Evidence design: -- `evidence_type`: `MALICIOUS_FLOW` +- `evidence_type`: `ANOMALOUS_FLOW` - `method`: `STATISTICAL` - `attacker`: source host (`profileid` IP, direction `SRC`) - `victim`: best available destination context (`SNI` domain first, otherwise destination IP/domain) diff --git a/modules/anomaly_detection_https/anomaly_detection_https.py b/modules/anomaly_detection_https/anomaly_detection_https.py index 726143f7f2..f1827e42b0 100644 --- a/modules/anomaly_detection_https/anomaly_detection_https.py +++ b/modules/anomaly_detection_https/anomaly_detection_https.py @@ -759,7 +759,7 @@ def emit_anomaly_evidence( src_port = None evidence = Evidence( - evidence_type=EvidenceType.MALICIOUS_FLOW, + evidence_type=EvidenceType.ANOMALOUS_FLOW, description=description, attacker=Attacker( direction=Direction.SRC, diff --git a/modules/exporting_alerts/stix_exporter.py b/modules/exporting_alerts/stix_exporter.py index 4a4605248f..84ac51689f 100644 --- a/modules/exporting_alerts/stix_exporter.py +++ b/modules/exporting_alerts/stix_exporter.py @@ -626,6 +626,8 @@ def _build_custom_properties( custom_properties: Dict[str, object] = { "x_slips_evidence_id": evidence.get("id"), "x_slips_threat_level": evidence.get("threat_level"), + "x_slips_evidence_signal": evidence.get("evidence_signal") + or "PAMP", "x_slips_profile_ip": profile.get("ip"), "x_slips_timewindow": timewindow.get("number"), "x_slips_attacker_direction": attacker.get("direction"), @@ -783,6 +785,7 @@ def _build_taxii1_package( threat_level = evidence.get("threat_level") if threat_level: meta["threat_level"] = threat_level + meta["evidence_signal"] = evidence.get("evidence_signal") or "PAMP" victim_value = (evidence.get("victim") or {}).get("value") if victim_value: meta["victim"] = victim_value diff --git a/modules/feeds_update_manager/whitelist_updater_mixin.py b/modules/feeds_update_manager/whitelist_updater_mixin.py index 0861b68ace..165cd22a97 100644 --- a/modules/feeds_update_manager/whitelist_updater_mixin.py +++ b/modules/feeds_update_manager/whitelist_updater_mixin.py @@ -19,8 +19,11 @@ def should_update_online_whitelist(self) -> bool: if not self.enable_online_whitelist: return False - if not self.db.is_tranco_whitelist_expired(): - # tranco whitelist not expired yet + if not self._did_update_period_pass( + self.online_whitelist_update_period, + "tranco_whitelist", + ): + self.loaded_ti_files += 1 return False # update period passed @@ -107,21 +110,23 @@ def update_mac_db(self): self._mark_feed_as_updated(self.mac_db_link) return True - def _update_online_whitelist(self): + def _update_online_whitelist(self) -> None: """ Updates online tranco whitelist defined in slips.yaml online_whitelist key """ - # delete the old ones - self.db.delete_tranco_whitelist() response = self.responses["tranco_whitelist"] domains = [] for line in response.text.splitlines(): - domain = line.split(",")[1].strip() + parts = line.split(",", 1) + if len(parts) != 2: + continue + domain = parts[1].strip().lower() + if not utils.is_valid_domain(domain): + continue domains.append(domain) - self.db.store_tranco_whitelisted_domains( - domains, ttl=self.online_whitelist_update_period - ) + + self.db.store_tranco_whitelisted_domains(domains) self._mark_feed_as_updated("tranco_whitelist") diff --git a/modules/flow_alerts/flow_alerts.py b/modules/flow_alerts/flow_alerts.py index 66295cc74f..108691aa7e 100644 --- a/modules/flow_alerts/flow_alerts.py +++ b/modules/flow_alerts/flow_alerts.py @@ -97,15 +97,9 @@ async def main(self): for analyzer in analyzers: # some analyzers are async functions if inspect.iscoroutinefunction(analyzer.analyze): - # analyzer will run normally, until it finishes. - # tasks inside this analyzer will run asynchrously, - # and finish whenever they finish, we'll not wait for them - loop = asyncio.get_event_loop() - task = loop.create_task(analyzer.analyze(msg)) - # because Async Tasks swallow exceptions. - task.add_done_callback(self.handle_task_exception) - # to wait for these functions before flow_alerts shuts down - self.tasks.append(task) + # Track async analyzer tasks through create_task so + # completed tasks are removed from self.tasks. + self.create_task(analyzer.analyze, msg) # Allow the event loop to run the scheduled task await asyncio.sleep(0) else: diff --git a/modules/llm_proxy/README.md b/modules/llm_proxy/README.md new file mode 100644 index 0000000000..73f26b4383 --- /dev/null +++ b/modules/llm_proxy/README.md @@ -0,0 +1,277 @@ +# LLM Module + +The `LLM` module is a shared service for other Slips modules. + +It reads configured backend connections from `config/slips.yaml`, listens for +requests on the Redis channel `llm_request`, sends the prompt to the selected +backend, and publishes the reply on `llm_response`. + +## Supported providers + +- `ollama` +- `openai` +- `anthropic` + +## Configuration + +Example: + +```yaml +llm: + enabled: true + default_backend: local_qwen + worker_threads: 2 + queue_size: 100 + backends: + local_qwen: + provider: ollama + model: qwen2.5:3b + base_url: http://127.0.0.1:11434 + timeout: 120 + openai_default: + provider: openai + model: gpt-4o-mini + base_url: https://api.openai.com/v1 + api_key_env: OPENAI_API_KEY + timeout: 60 + claude_default: + provider: anthropic + model: claude-sonnet-4-5 + base_url: https://api.anthropic.com + api_key_env: ANTHROPIC_API_KEY + timeout: 60 +``` + +Configuration reference: + +- `enabled`: enables or disables the shared LLM service module. +- `default_backend`: backend alias used when a request does not include + `backend`. +- `worker_threads`: number of requests the module can process in parallel. +- `queue_size`: maximum number of pending requests held in memory. +- `backends`: mapping of backend alias to backend connection settings. + +Per-backend options: + +- `provider`: one of `ollama`, `openai`, or `anthropic`. +- `model`: default model used by that backend alias. +- `base_url`: provider endpoint. If omitted, the module uses the provider + default. +- `timeout`: HTTP timeout in seconds. +- `api_key`: optional inline API key for `openai` or `anthropic`. +- `api_key_env`: optional environment variable name holding the API key. +- `api_key_file`: optional file path containing the API key. +- `anthropic_version`: optional Anthropic API version header. Default is + `2023-06-01`. + +Backend aliases are the names that caller modules use in the request field +`backend`. The alias is the stable selector. The `model` field inside a request +is only an optional override for that chosen backend. + +## Implementation Layout + +`llm.py` contains the `LLM` service class that implements `IModule`. Backend +support is split across these files: + +- `llm_backend_config.py`: backend configuration validation. +- `llm_backend.py`: shared HTTP and response helpers. +- `openai_backend_mixin.py`: `MixinOpenAIBackend`. +- `anthropic_backend_mixin.py`: `MixinAnthropicBackend`. +- `ollama_backend_mixin.py`: `MixinOllamaBackend`. +- `llm_errors.py`: shared LLM exceptions. + +Provider-specific behavior should live in a `_mixin.py` file and use a class +name prefixed with `Mixin`. + +## Request channel + +Channel: `llm_request` + +Minimal request: + +```json +{ + "request_id": "req-123", + "backend": "local_qwen", + "prompt": "Summarize this alert" +} +``` + +Request with explicit messages: + +```json +{ + "request_id": "req-456", + "requester": "Flow Alerts", + "backend": "openai_default", + "messages": [ + {"role": "system", "content": "You are a concise security analyst."}, + {"role": "user", "content": "Explain this incident."} + ], + "temperature": 0.2, + "max_tokens": 300, + "metadata": {"profileid": "profile_192.168.1.10"} +} +``` + +Fields: + +- `request_id`: technically optional, but caller modules should always set it. + This is the primary correlation key on the shared response channel. +- `requester`: optional module name for easier correlation. +- `backend`: optional if `default_backend` is configured. +- `prompt`: shortcut for a single user message. +- `messages`: list of text messages using `system`, `user`, or `assistant`. +- `model`: optional override of the configured model for that backend. +- `temperature`: optional float. +- `max_tokens`: optional integer. +- `metadata`: optional passthrough object returned unchanged in the response. + +## Discovery helper + +Caller modules can discover the runtime-ready backends using: + +```python +available = self.db.get_available_llm_backends() +``` + +The returned shape is: + +```json +{ + "default_backend": "local_qwen", + "backends": { + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b" + }, + "openai_default": { + "provider": "openai", + "model": "gpt-4o-mini" + } + } +} +``` + +If the LLM module is disabled, still starting, or no backend is runtime-ready +yet, the helper returns: + +```json +{ + "default_backend": "", + "backends": {} +} +``` + +Caller modules should retry later instead of treating an empty result as a +permanent failure. + +## How Caller Modules Should Use It + +This module uses one shared request channel and one shared response channel for +all of Slips. + +That means caller modules must follow this pattern: + +1. Subscribe to `llm_response` during module initialization. +2. Call `self.db.get_available_llm_backends()` before choosing a backend. +3. Pick a backend alias from `available["backends"]` or use + `available["default_backend"]`. +4. Generate a unique `request_id` before publishing. +5. Store local context keyed by `request_id` if the response must be matched + back to a flow, profile, or alert. +6. Publish the request to `llm_request`. +7. Read from `llm_response` and ignore responses whose `request_id` is not one + of yours. + +If two modules send requests at the same time, they separate replies by +matching on `request_id`. `requester` is only a human-readable label. It is not +the primary routing key. + +Recommended pattern: + +```python +import json +import uuid + +available = self.db.get_available_llm_backends() +backend = available["default_backend"] +if not backend: + return + +request_id = f"{self.name}-{uuid.uuid4()}" +pending_requests[request_id] = {"profileid": profileid} + +request = { + "request_id": request_id, + "requester": self.name, + "backend": backend, + "prompt": "Summarize this alert in 2 lines.", + "metadata": {"profileid": profileid}, +} +self.db.publish("llm_request", json.dumps(request)) +``` + +Response handling: + +```python +if msg := self.get_msg("llm_response"): + response = json.loads(msg["data"]) + request_id = response["request_id"] + if request_id not in pending_requests: + return + + context = pending_requests.pop(request_id) + text = response["text"] +``` + +Do not rely on the service to generate `request_id` for you. If the caller does +not generate it first, the caller cannot reliably match the reply later. + +## Response channel + +Channel: `llm_response` + +Success response: + +```json +{ + "request_id": "req-456", + "requester": "Flow Alerts", + "backend": "openai_default", + "provider": "openai", + "model": "gpt-4o-mini", + "success": true, + "text": "This alert shows repeated outbound connections...", + "usage": { + "input_tokens": 123, + "output_tokens": 57, + "total_tokens": 180 + }, + "metadata": {"profileid": "profile_192.168.1.10"}, + "ts": 1760000000.0 +} +``` + +Error response: + +```json +{ + "request_id": "req-789", + "backend": "missing_backend", + "success": false, + "error": "Unknown LLM backend requested: missing_backend", + "text": "", + "metadata": {}, + "ts": 1760000000.0 +} +``` + +## Notes + +- The module uses one shared response channel, so requesters must correlate + responses using `request_id`. +- Version 1 is text-only. It accepts plain string prompts and message content. +- Other modules can choose the backend per request by setting `backend`. +- The runtime discovery helper exposes only runtime-ready backends, not every + configured backend. diff --git a/modules/llm_proxy/__init__.py b/modules/llm_proxy/__init__.py new file mode 100644 index 0000000000..f436f14183 --- /dev/null +++ b/modules/llm_proxy/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only diff --git a/modules/llm_proxy/anthropic_backend_mixin.py b/modules/llm_proxy/anthropic_backend_mixin.py new file mode 100644 index 0000000000..9320a0fd2d --- /dev/null +++ b/modules/llm_proxy/anthropic_backend_mixin.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +"""Anthropic backend mixin for the shared LLM module.""" + +from typing import Any + +from modules.llm_proxy.llm_backend import LLMBackend + + +class MixinAnthropicBackend(LLMBackend): + """Generate responses through the Anthropic messages API.""" + + def generate(self, request: dict[str, Any]) -> dict[str, Any]: + """ + Send a normalized request to the Anthropic messages endpoint. + + Parameters: + request: Normalized LLM request payload. + + Returns: + Shared LLM result with text, usage, provider, and model. + """ + url = self._build_url("/v1/messages") + system_parts = [] + messages = [] + for message in request["messages"]: + role = message["role"] + content = message["content"] + if role == "system": + system_parts.append(content) + continue + messages.append({"role": role, "content": content}) + + payload = { + "model": request.get("model") or self.config.model, + "messages": messages, + "max_tokens": request.get("max_tokens") or 1024, + } + if system_parts: + payload["system"] = "\n\n".join(system_parts) + if request.get("temperature") is not None: + payload["temperature"] = request["temperature"] + + response = self._request_json( + "POST", + url, + payload, + headers={ + "x-api-key": self.config.api_key, + "anthropic-version": self.config.anthropic_version, + "Content-Type": "application/json", + }, + ) + + content = response.get("content") or [] + return { + "text": self._join_text_blocks(content), + "usage": self._normalize_usage(response.get("usage")), + "provider": self.config.provider, + "model": response.get("model") or payload["model"], + } diff --git a/modules/llm_proxy/llm_backend.py b/modules/llm_proxy/llm_backend.py new file mode 100644 index 0000000000..9758bc248d --- /dev/null +++ b/modules/llm_proxy/llm_backend.py @@ -0,0 +1,164 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +"""Base HTTP backend for shared LLM providers.""" + +import json +from typing import Any + +import certifi +import urllib3 + +from modules.llm_proxy.llm_backend_config import LLMBackendConfig +from modules.llm_proxy.llm_errors import LLMRequestError + + +class LLMBackend: + """Common HTTP and response helpers for LLM provider implementations.""" + + def __init__( + self, + config: LLMBackendConfig, + pool_maxsize: int = 2, + ) -> None: + """ + Initialize a backend with a validated config and HTTP pool. + + Parameters: + config: Validated backend configuration. + pool_maxsize: Maximum connections kept in the HTTP pool. + + Returns: + None. + """ + self.config = config + self.http = urllib3.PoolManager( + cert_reqs="CERT_REQUIRED", + ca_certs=certifi.where(), + maxsize=max(2, int(pool_maxsize)), + ) + + def generate(self, request: dict[str, Any]) -> dict[str, Any]: + """ + Generate a response for a normalized LLM request. + + Parameters: + request: Normalized request payload. + + Returns: + Provider response converted to the shared result shape. + """ + raise NotImplementedError + + def _request_json( + self, + method: str, + url: str, + payload: dict[str, Any], + headers: dict[str, str] | None = None, + ) -> dict[str, Any]: + """ + Send a JSON request and decode the JSON response. + + Parameters: + method: HTTP method to use. + url: Backend URL to call. + payload: JSON-serializable request body. + headers: Optional HTTP headers. + + Returns: + Decoded JSON response body. + """ + encoded_payload = json.dumps(payload).encode() + try: + response = self.http.request( + method, + url, + body=encoded_payload, + headers=headers or {"Content-Type": "application/json"}, + timeout=urllib3.Timeout( + connect=self.config.timeout, + read=self.config.timeout, + ), + ) + except urllib3.exceptions.HTTPError as exc: + raise LLMRequestError( + f"{self.config.alias} request failed: {exc}" + ) from exc + except OSError as exc: + raise LLMRequestError( + f"{self.config.alias} request failed: {exc}" + ) from exc + + try: + decoded = response.data.decode("utf-8") + except UnicodeDecodeError as exc: + raise LLMRequestError(f"Invalid backend response: {exc}") from exc + + if response.status >= 400: + raise LLMRequestError( + f"{self.config.alias} returned HTTP {response.status}: " + f"{decoded[:500]}" + ) + + try: + return json.loads(decoded) + except json.JSONDecodeError as exc: + raise LLMRequestError( + f"Backend {self.config.alias} returned invalid JSON." + ) from exc + + def _build_url(self, endpoint: str) -> str: + """ + Build a provider URL from the configured base URL and endpoint. + + Parameters: + endpoint: Provider endpoint path. + + Returns: + Full URL for the backend request. + """ + base_url = self.config.base_url.rstrip("/") + if endpoint.startswith("/v1/") and base_url.endswith("/v1"): + endpoint = endpoint[3:] + return f"{base_url}{endpoint}" + + @staticmethod + def _normalize_usage(usage: dict[str, Any] | None) -> dict[str, Any]: + """ + Convert provider usage counters to the shared token keys. + + Parameters: + usage: Provider usage payload. + + Returns: + Normalized usage mapping. + """ + usage = usage or {} + return { + "input_tokens": usage.get("prompt_tokens") + or usage.get("input_tokens"), + "output_tokens": usage.get("completion_tokens") + or usage.get("output_tokens"), + "total_tokens": usage.get("total_tokens"), + } + + @staticmethod + def _join_text_blocks(content: Any) -> str: + """ + Convert provider text blocks into one text string. + + Parameters: + content: Provider message content. + + Returns: + Joined response text. + """ + if isinstance(content, str): + return content + if isinstance(content, list): + text_parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + text_parts.append(str(item.get("text", ""))) + return "".join(text_parts) + return str(content or "") diff --git a/modules/llm_proxy/llm_backend_config.py b/modules/llm_proxy/llm_backend_config.py new file mode 100644 index 0000000000..b899aa3ff0 --- /dev/null +++ b/modules/llm_proxy/llm_backend_config.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +"""Configuration object for shared LLM backend connections.""" + +import os +from dataclasses import dataclass +from typing import Any + +from modules.llm_proxy.llm_errors import LLMConfigurationError + + +@dataclass +class LLMBackendConfig: + """Validated configuration for one LLM backend alias.""" + + alias: str + provider: str + model: str + base_url: str + timeout: int + api_key: str | None = None + anthropic_version: str = "2023-06-01" + + @classmethod + def from_dict(cls, alias: str, data: dict[str, Any]) -> "LLMBackendConfig": + """ + Build a backend configuration from raw config data. + + Parameters: + alias: Backend alias from the Slips configuration. + data: Raw backend configuration mapping. + + Returns: + Validated LLM backend configuration. + """ + if not isinstance(data, dict): + raise LLMConfigurationError(f"Backend {alias} must be a mapping.") + + provider = str(data.get("provider", "")).strip().lower() + if provider not in {"ollama", "openai", "anthropic"}: + raise LLMConfigurationError( + f"Backend {alias} has unsupported provider {provider!r}." + ) + + model = str(data.get("model", "")).strip() + if not model: + raise LLMConfigurationError(f"Backend {alias} is missing a model.") + + timeout = data.get("timeout", 60) + try: + timeout = int(timeout) + except (TypeError, ValueError): + timeout = 60 + timeout = max(1, timeout) + + base_url = str(data.get("base_url", "")).strip() + if not base_url: + base_url = { + "ollama": "http://127.0.0.1:11434", + "openai": "https://api.openai.com/v1", + "anthropic": "https://api.anthropic.com", + }[provider] + base_url = base_url.rstrip("/") + + api_key = cls._resolve_api_key(data) + if provider in {"openai", "anthropic"} and not api_key: + raise LLMConfigurationError( + f"Backend {alias} requires an API key." + ) + + anthropic_version = str( + data.get("anthropic_version", "2023-06-01") + ).strip() + + return cls( + alias=alias, + provider=provider, + model=model, + base_url=base_url, + timeout=timeout, + api_key=api_key, + anthropic_version=anthropic_version, + ) + + @staticmethod + def _resolve_api_key(data: dict[str, Any]) -> str | None: + """ + Resolve an API key from inline, environment, or file config. + + Parameters: + data: Raw backend configuration mapping. + + Returns: + API key string when one is configured and readable. + """ + api_key = data.get("api_key") + if isinstance(api_key, str) and api_key.strip(): + return api_key.strip() + + api_key_env = data.get("api_key_env") + if isinstance(api_key_env, str) and api_key_env.strip(): + env_value = os.environ.get(api_key_env.strip(), "").strip() + if env_value: + return env_value + + api_key_file = data.get("api_key_file") + if isinstance(api_key_file, str) and api_key_file.strip(): + try: + with open(api_key_file.strip(), "r", encoding="utf-8") as f: + return f.read().strip() or None + except OSError: + return None + + return None diff --git a/modules/llm_proxy/llm_errors.py b/modules/llm_proxy/llm_errors.py new file mode 100644 index 0000000000..97bcbd4e23 --- /dev/null +++ b/modules/llm_proxy/llm_errors.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +"""Exceptions raised by the shared LLM module.""" + + +class LLMConfigurationError(Exception): + """Raised when an LLM backend configuration is invalid.""" + + +class LLMRequestError(Exception): + """Raised when an LLM request cannot be prepared or completed.""" diff --git a/modules/llm_proxy/llm_proxy.py b/modules/llm_proxy/llm_proxy.py new file mode 100644 index 0000000000..09689bb70c --- /dev/null +++ b/modules/llm_proxy/llm_proxy.py @@ -0,0 +1,473 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +import os +import queue +import threading +import time +import uuid +from typing import Any, Dict, List + +from slips_files.common.abstracts.imodule import IModule +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import utils + +from modules.llm_proxy.anthropic_backend_mixin import MixinAnthropicBackend +from modules.llm_proxy.llm_backend import LLMBackend +from modules.llm_proxy.llm_backend_config import LLMBackendConfig +from modules.llm_proxy.llm_errors import LLMConfigurationError, LLMRequestError +from modules.llm_proxy.ollama_backend_mixin import MixinOllamaBackend +from modules.llm_proxy.openai_backend_mixin import MixinOpenAIBackend + + +AnthropicBackend = MixinAnthropicBackend +OllamaBackend = MixinOllamaBackend +OpenAIBackend = MixinOpenAIBackend + + +class LLMProxy(IModule): + name = "llm_proxy" + description = ( + "LLM proxy that forwards msgs from Slips modules to " + "local/remote configured LLMs." + ) + authors = ["Sebastian Garcia"] + + def init(self): + self.channels = {} + self.subscribe_to_channels() + self.request_queue: queue.Queue = queue.Queue() + self.worker_stop_event = threading.Event() + self.workers: List[threading.Thread] = [] + self.backends: Dict[str, LLMBackend] = {} + self.failed_backends: Dict[str, str] = {} + self.default_backend = "" + self.worker_threads = 2 + self.queue_size = 100 + self.last_request_activity = time.time() + self.operation_log = None + self.operation_log_path = self.get_module_specific_output_path( + "llm_proxy.log" + ) + self.waiting_for_upstream_modules_logged = False + self.read_configuration() + + def subscribe_to_channels(self): + """ + Subscribe to the Redis channels used by the shared LLM service. + """ + if self.channels: + return + + self.c1 = self.db.subscribe(self.db.channels.LLM_REQUEST) + self.channels = { + self.db.channels.LLM_REQUEST: self.c1, + } + + def read_configuration(self): + conf = ( + self.conf if hasattr(self.conf, "llm_enabled") else ConfigParser() + ) + self.enabled = conf.llm_enabled() + self.default_backend = conf.llm_default_backend().strip() + self.worker_threads = conf.llm_worker_threads() + self.queue_size = conf.llm_queue_size() + self.request_queue = queue.Queue(maxsize=self.queue_size) + + backend_data = conf.llm_backends() + for alias, data in backend_data.items(): + try: + config = LLMBackendConfig.from_dict(alias, data) + self.backends[alias] = self._create_backend(config) + except LLMConfigurationError as exc: + self.failed_backends[alias] = str(exc) + + def _create_backend(self, config: LLMBackendConfig) -> LLMBackend: + # Keep the reusable HTTP connection pool comfortably above the + # worker concurrency so busy runs do not spam pool-discard warnings. + pool_maxsize = max(2, self.worker_threads * 2) + if config.provider == "openai": + return OpenAIBackend(config, pool_maxsize=pool_maxsize) + if config.provider == "anthropic": + return AnthropicBackend(config, pool_maxsize=pool_maxsize) + return OllamaBackend(config, pool_maxsize=pool_maxsize) + + @staticmethod + def _empty_available_backends_registry() -> dict: + return {"default_backend": "", "backends": {}} + + def _get_available_backends_registry(self) -> dict: + available_backends = {} + for alias, backend in self.backends.items(): + available_backends[alias] = { + "provider": backend.config.provider, + "model": backend.config.model, + } + + default_backend = self.default_backend + if default_backend not in available_backends: + default_backend = "" + + return { + "default_backend": default_backend, + "backends": available_backends, + } + + def _store_available_backends_registry(self): + self.db.set_available_llm_backends( + self._get_available_backends_registry() + ) + + def _store_empty_available_backends_registry(self): + self.db.set_available_llm_backends( + self._empty_available_backends_registry() + ) + + def pre_main(self): + utils.drop_root_privs_permanently() + self._init_operation_log_file() + self.db.reset_pending_llm_request_counts() + + if not self.enabled: + self._store_empty_available_backends_registry() + self._log_operation("LLM module disabled in config.") + self.print("LLM module disabled in config.", 2, 0) + return True + + if self.failed_backends: + for alias, error in self.failed_backends.items(): + self._log_operation(f"Skipping backend alias={alias}: {error}") + self.print( + f"Skipping LLM backend {alias}: {error}", + 0, + 1, + ) + + if not self.backends: + self._store_empty_available_backends_registry() + self._log_operation("No valid LLM backends configured.") + self.print( + "No valid LLM backends configured. Stopping LLM module.", + 0, + 1, + ) + return True + + if self.default_backend and self.default_backend not in self.backends: + self._log_operation( + f"Configured default backend {self.default_backend} is unavailable." + ) + self.print( + f"Default LLM backend {self.default_backend} is not available.", + 0, + 1, + ) + self.default_backend = "" + + for idx in range(self.worker_threads): + worker = threading.Thread( + target=self._worker_loop, + name=f"llm_worker_{idx}", + daemon=True, + ) + worker.start() + self.workers.append(worker) + self._log_operation(f"Started worker thread name={worker.name}") + + self._store_available_backends_registry() + self._log_operation( + "LLM module ready. " + f"default_backend={self.default_backend or 'none'} " + f"available_backends={sorted(self.backends)} " + f"queue_size={self.queue_size} " + f"worker_threads={self.worker_threads}" + ) + self.print( + f"LLM module ready with backends: {list(self.backends)}", + 2, + 0, + ) + + def main(self): + if msg := self.get_msg(self.db.channels.LLM_REQUEST): + self._enqueue_request(msg) + + def shutdown_gracefully(self): + self._store_empty_available_backends_registry() + self.worker_stop_event.set() + for worker in self.workers: + try: + self.request_queue.put_nowait(None) + worker.join(timeout=1) + except queue.Full: + break + + self.db.reset_pending_llm_request_counts() + self._log_operation("LLM module stopped.") + if self.operation_log is not None: + self.operation_log.close() + return True + + def _enqueue_request(self, msg: dict): + try: + payload = json.loads(msg["data"]) + except json.JSONDecodeError: + self._publish_response( + { + "request_id": str(uuid.uuid4()), + "success": False, + "error": "Invalid JSON on llm_request channel.", + "text": "", + } + ) + return + + payload["request_id"] = str(payload.get("request_id") or uuid.uuid4()) + + try: + self.request_queue.put_nowait(payload) + self.db.increment_pending_llm_request_count( + payload.get("requester", "") + ) + self._record_request_activity() + self._log_operation( + "Queued llm_request " + f"request_id={payload['request_id']} " + f"requester={payload.get('requester', '')} " + f"backend={payload.get('backend') or self.default_backend} " + f"queue_size={self.request_queue.qsize()}" + ) + except queue.Full: + self._log_operation( + "Rejected llm_request because the queue is full " + f"request_id={payload['request_id']}" + ) + self._publish_response( + { + "request_id": payload["request_id"], + "requester": payload.get("requester"), + "backend": payload.get("backend"), + "success": False, + "error": "LLM request queue is full.", + "text": "", + "metadata": payload.get("metadata", {}), + } + ) + + def _worker_loop(self): + while not self.worker_stop_event.is_set(): + try: + payload = self.request_queue.get(timeout=0.2) + except queue.Empty: + continue + + if payload is None: + self.request_queue.task_done() + return + + self._record_request_activity() + try: + self._handle_request(payload) + finally: + self.request_queue.task_done() + self._record_request_activity() + + def _handle_request(self, payload: dict): + request_id = payload["request_id"] + requester = payload.get("requester") + metadata = payload.get("metadata", {}) + self._log_operation( + "Handling llm_request " + f"request_id={request_id} " + f"requester={requester or ''} " + f"backend={payload.get('backend') or self.default_backend}" + ) + + try: + request = self._prepare_request(payload) + backend = self.backends[request["backend"]] + result = backend.generate(request) + response = { + "request_id": request_id, + "requester": requester, + "backend": request["backend"], + "provider": result["provider"], + "model": result["model"], + "success": True, + "text": result["text"], + "usage": result["usage"], + "metadata": metadata, + "ts": time.time(), + } + self._log_operation( + "Completed llm_request " + f"request_id={request_id} " + f"backend={request['backend']} " + f"success=True " + f"output_chars={len(response['text'])}" + ) + except (LLMRequestError, KeyError, ValueError) as exc: + response = { + "request_id": request_id, + "requester": requester, + "backend": payload.get("backend"), + "success": False, + "error": str(exc), + "text": "", + "metadata": metadata, + "ts": time.time(), + } + self._log_operation( + "Completed llm_request " + f"request_id={request_id} " + f"backend={payload.get('backend')} " + f"success=False error={exc}" + ) + except Exception as exc: + response = { + "request_id": request_id, + "requester": requester, + "backend": payload.get("backend"), + "success": False, + "error": f"Unexpected LLM error: {exc}", + "text": "", + "metadata": metadata, + "ts": time.time(), + } + self._log_operation( + "Completed llm_request " + f"request_id={request_id} " + f"backend={payload.get('backend')} " + f"success=False error=Unexpected LLM error: {exc}" + ) + + self._publish_response(response) + + def _prepare_request(self, payload: dict) -> dict: + backend_name = str( + payload.get("backend") or self.default_backend + ).strip() + if not backend_name: + raise LLMRequestError("No backend specified for LLM request.") + if backend_name not in self.backends: + raise LLMRequestError( + f"Unknown LLM backend requested: {backend_name}" + ) + + messages = self._normalize_messages(payload) + request = { + "request_id": payload["request_id"], + "backend": backend_name, + "messages": messages, + "model": payload.get("model"), + "temperature": payload.get("temperature"), + "max_tokens": payload.get("max_tokens"), + } + return request + + def _normalize_messages(self, payload: dict) -> List[dict]: + messages = payload.get("messages") + if not messages: + prompt = payload.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise LLMRequestError( + "LLM request needs either messages or prompt." + ) + messages = [{"role": "user", "content": prompt}] + + if not isinstance(messages, list) or not messages: + raise LLMRequestError("LLM messages must be a non-empty list.") + + normalized_messages = [] + for message in messages: + if not isinstance(message, dict): + raise LLMRequestError("Each LLM message must be an object.") + role = str(message.get("role", "")).strip().lower() + if role not in {"system", "user", "assistant"}: + raise LLMRequestError(f"Invalid LLM role: {role!r}") + + content = self._normalize_message_content(message.get("content")) + if not content: + raise LLMRequestError("LLM message content cannot be empty.") + + normalized_messages.append({"role": role, "content": content}) + + return normalized_messages + + @staticmethod + def _normalize_message_content(content: Any) -> str: + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + parts.append(str(item.get("text", ""))) + return "".join(parts).strip() + if content is None: + return "" + return str(content).strip() + + def _publish_response(self, payload: dict): + requester = str(payload.get("requester") or "").strip() + try: + self._log_operation( + "Published llm_response " + f"request_id={payload.get('request_id')} " + f"requester={requester} " + f"backend={payload.get('backend')} " + f"success={payload.get('success')}" + ) + self.db.publish( + self.db.channels.LLM_RESPONSE, + json.dumps(payload), + ) + finally: + remaining = self.db.decrement_pending_llm_request_count(requester) + if requester: + self._log_operation( + "Updated requester inflight count " + f"requester={requester} remaining={remaining}" + ) + + def _record_request_activity(self): + """Update the timestamp used to keep the LLM service alive.""" + self.last_request_activity = time.time() + + def _init_operation_log_file(self): + """ + Create the per-run LLM operation log inside the module output dir. + + :return: None + """ + utils.initialize_logfile( + self.operation_log_path, + getattr(self.args, "is_slips_started_by_an_update", False), + ) + self.operation_log = open( + self.operation_log_path, + "a", + encoding="utf-8", + ) + + conf = ConfigParser() + utils.change_logfiles_ownership( + self.operation_log_path, + conf.get_UID(), + conf.get_GID(), + ) + + def _log_operation(self, message: str): + """ + Append one line to the LLM module operation log. + + :param message: Log message to append. + :return: None + """ + if self.operation_log is None: + return + + timestamp = utils.get_human_readable_datetime() + self.operation_log.write(f"{timestamp} {message}\n") + self.operation_log.flush() + os.fsync(self.operation_log.fileno()) diff --git a/modules/llm_proxy/ollama_backend_mixin.py b/modules/llm_proxy/ollama_backend_mixin.py new file mode 100644 index 0000000000..13e212a556 --- /dev/null +++ b/modules/llm_proxy/ollama_backend_mixin.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +"""Ollama backend mixin for the shared LLM module.""" + +from typing import Any + +from modules.llm_proxy.llm_backend import LLMBackend + + +class MixinOllamaBackend(LLMBackend): + """Generate responses through the Ollama chat API.""" + + def generate(self, request: dict[str, Any]) -> dict[str, Any]: + """ + Send a normalized request to the Ollama chat endpoint. + + Parameters: + request: Normalized LLM request payload. + + Returns: + Shared LLM result with text, usage, provider, and model. + """ + url = self._build_url("/api/chat") + payload = { + "model": request.get("model") or self.config.model, + "messages": request["messages"], + "stream": False, + } + options = {} + if request.get("temperature") is not None: + options["temperature"] = request["temperature"] + if request.get("max_tokens") is not None: + options["num_predict"] = request["max_tokens"] + if options: + payload["options"] = options + + response = self._request_json("POST", url, payload) + message = response.get("message", {}) + usage = { + "prompt_tokens": response.get("prompt_eval_count"), + "completion_tokens": response.get("eval_count"), + "total_tokens": None, + } + if ( + usage["prompt_tokens"] is not None + and usage["completion_tokens"] is not None + ): + usage["total_tokens"] = ( + usage["prompt_tokens"] + usage["completion_tokens"] + ) + return { + "text": self._join_text_blocks(message.get("content", "")), + "usage": self._normalize_usage(usage), + "provider": self.config.provider, + "model": response.get("model") or payload["model"], + } diff --git a/modules/llm_proxy/openai_backend_mixin.py b/modules/llm_proxy/openai_backend_mixin.py new file mode 100644 index 0000000000..6232e42a3f --- /dev/null +++ b/modules/llm_proxy/openai_backend_mixin.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +"""OpenAI-compatible backend mixin for the shared LLM module.""" + +from typing import Any + +from modules.llm_proxy.llm_backend import LLMBackend +from modules.llm_proxy.llm_errors import LLMRequestError + + +class MixinOpenAIBackend(LLMBackend): + """Generate responses through an OpenAI-compatible chat backend.""" + + def generate(self, request: dict[str, Any]) -> dict[str, Any]: + """ + Send a normalized request to an OpenAI-compatible chat endpoint. + + Parameters: + request: Normalized LLM request payload. + + Returns: + Shared LLM result with text, usage, provider, and model. + """ + url = self._build_url("/chat/completions") + payload = { + "model": request.get("model") or self.config.model, + "messages": request["messages"], + } + if request.get("temperature") is not None: + payload["temperature"] = request["temperature"] + if request.get("max_tokens") is not None: + payload["max_tokens"] = request["max_tokens"] + + response = self._request_json( + "POST", + url, + payload, + headers={ + "Authorization": f"Bearer {self.config.api_key}", + "Content-Type": "application/json", + }, + ) + + choices = response.get("choices") or [] + if not choices: + raise LLMRequestError( + f"Backend {self.config.alias} returned no choices." + ) + + message = choices[0].get("message", {}) + return { + "text": self._join_text_blocks(message.get("content", "")), + "usage": self._normalize_usage(response.get("usage")), + "provider": self.config.provider, + "model": response.get("model") or payload["model"], + } diff --git a/modules/regex_generator/README.md b/modules/regex_generator/README.md new file mode 100644 index 0000000000..421009f994 --- /dev/null +++ b/modules/regex_generator/README.md @@ -0,0 +1,416 @@ +# Regex Generator Module + +The `RegexGenerator` module continuously generates one pseudo-random regex at a +time for later Zeek-side matching. + +It uses the shared `LLM` module over the Redis channels `llm_request` and +`llm_response`, validates the generated regex against a benign corpus, and +stores accepted regexes in a local SQLite database that other modules can read +through `DBManager`. + +## Supported regex types + +- `dns_domain` +- `uri` +- `filename` +- `tls_sni` +- `certificate_cn` + +## Configuration + +Example: + +```yaml +regex_generator: + enabled: false + create_log_file: false + generation_interval_seconds: 5 + allowed_backends: [] + llm_temperature: 1.2 + llm_max_tokens: 80 + llm_response_timeout_seconds: 90 + recent_history_size: 0 + max_regex_length: 180 + regex_validation_timeout_seconds: 2 + benign_match_strength_threshold: 75 + type_weights: + dns_domain: 1 + uri: 1 + filename: 1 + tls_sni: 1 + certificate_cn: 1 + store_dir: output/regex_generator + persistent_store_dir: databases/regex_store + store_rejected_regexes: false + max_stored_rejected_regexes: 10000 + seed_benign_samples: true + +whitelists: + tranco_top_benign_limit: 1000 +``` + +Configuration reference: + +- `enabled`: enables or disables the module. +- `create_log_file`: creates `output/regex_generator.log` with detailed module + progress messages. This file rotates on the same global + `parameters.rotation` / `parameters.rotation_period` schedule used by the + current Slips run. +- `generation_interval_seconds`: delay between completed generation cycles. + Set `0` to start the next cycle immediately after the previous one finishes. +- `allowed_backends`: preferred LLM backend aliases for this module. +- `llm_temperature`: generation temperature. Kept high to encourage variation. +- `llm_max_tokens`: max tokens for the LLM reply. The module asks for one regex + line only, so this should stay small. +- `llm_response_timeout_seconds`: soft warning threshold while waiting for the + matching `llm_response`. The module keeps waiting after this. Set `0` to + disable the warning. +- `recent_history_size`: compatibility knob kept at `0`. Prompt history is not + sent to the LLM; repetition is checked locally. +- `max_regex_length`: hard reject longer regexes. +- `regex_validation_timeout_seconds`: hard wall-clock timeout for local regex + validation and benign-corpus matching. This prevents one pathological regex + from freezing the module. Set `0` to disable it. +- `benign_match_strength_threshold`: score from `0` to `100` used during the + benign scan. A regex is rejected only if its strongest benign match reaches + or exceeds this threshold. Higher values are more permissive. +- `type_weights`: weighted random choice among the five regex types. +- `store_dir`: directory containing `benign_corpus.sqlite` and + `generated_regexes.sqlite`. Absolute paths are used as-is. Relative paths are + resolved inside the current Slips run output directory. The default + `output/regex_generator` therefore becomes `/regex_generator`. +- `persistent_store_dir`: stable directory for the regex SQLite files. Relative + paths are resolved inside `parameters.permanent_dir`; the default + `databases/regex_store` therefore becomes + `/databases/regex_store`. If set, it takes precedence over + `store_dir` and lets the generator reuse the same DBs across many Slips + restarts. +- `store_rejected_regexes`: stores rejected regexes in SQLite for audit/debug + purposes. Default `false` so discarded candidates do not fill the disk. +- `max_stored_rejected_regexes`: retention cap for rejected rows when + `store_rejected_regexes` is enabled. Set `0` for unlimited retention. +- `seed_benign_samples`: seed the benign DB once with a small built-in sample. +- `whitelists.tranco_top_benign_limit`: number of ordered Tranco whitelist + domains reused as benign data by `RegexGenerator` and the offline coverage + report. + +## Runtime flow + +Each cycle does this: + +1. Discover runtime-ready LLM backends with + `self.db.get_available_llm_backends()`. +2. Choose one backend alias from `allowed_backends`, or fall back to the LLM + default backend. +3. Choose the next regex type using weighted random selection. +4. Build a minimal fixed prompt for that type. +5. Publish one request on `llm_request`. +6. Wait for the matching `llm_response` using `request_id`. + If the local LLM is slow, the module keeps waiting and only logs a warning + after `llm_response_timeout_seconds`. +7. Extract one regex line from the LLM reply. +8. Apply static safety validation. +9. Check local duplicate state with a bloom filter and exact DB lookup. +10. Stream the benign corpus for that type and compute a benign match-strength + score for each regex/string match. +11. Reject the regex only if some benign string reaches or exceeds + `benign_match_strength_threshold`. +12. Store accepted regexes in SQLite. Rejected regexes are only persisted if + `store_rejected_regexes` is enabled. + +V1 keeps only one LLM request in flight at a time. + +If `create_log_file` is enabled, the module writes detailed cycle logs to: + +```text +output/regex_generator.log +``` + +That file includes: + +- selected regex type +- selected backend +- published `llm_request` `request_id` +- slow-wait warnings while the LLM is still working +- accepted regexes +- rejected regexes and rejection reasons + +Accepted regexes are stored in the configured persistent store by default: + +```text +/databases/regex_store/generated_regexes.sqlite +``` + +If `persistent_store_dir` is empty, the fallback location is +`/regex_generator/generated_regexes.sqlite`. + +Rejected regexes are tracked in memory during the current run to prevent cheap +repeats, but they are not stored on disk unless `store_rejected_regexes` is +enabled. + +## LLM contract + +Request payload: + +```json +{ + "request_id": "RegexGenerator-...", + "requester": "RegexGenerator", + "backend": "local_qwen", + "messages": [...], + "temperature": 1.2, + "max_tokens": 80, + "metadata": { + "regex_type": "dns_domain", + "prompt_version": "regex-generator-v2", + "generation_nonce": "..." + } +} +``` + +The prompt requires the model to return exactly one regex line. No JSON, +explanation, or code fences. The parser still accepts JSON-shaped replies as a +fallback for compatibility, but the active prompt is raw-regex only. + +## Acceptance pipeline + +Static validation rejects: + +- non-ASCII regexes +- regexes longer than `max_regex_length` +- lookbehind +- backreferences +- unbounded prefix/suffix patterns like `.*...*` +- obviously broad patterns like `.*` or `.+` +- nested wildcard structures that risk catastrophic backtracking +- invalid Python/Zeek-compatible syntax + +After static validation, the module first checks for exact duplicate regexes +locally with a bloom filter and exact SQLite lookup, then scans the benign +corpus for the selected type and computes a benign match-strength score for +every regex/string match. The regex is rejected only if any benign string +reaches or exceeds `benign_match_strength_threshold`. + +The current benign match-strength score is an estimate from `0` to `100`. It +is computed per regex and per benign string using the strongest match span +found by Python `re.finditer()`. + +For one matched span, the score is: + +```text +score = + 40 * span_ratio + + 12 * start_bonus + + 12 * end_bonus + + 16 * full_bonus + + 30 * specificity_ratio + - 18 * wildcard_penalty +``` + +The result is clipped to the range `0..100`. The regex keeps the highest score +it obtains against that benign string. If any benign string reaches or exceeds +`benign_match_strength_threshold`, the regex is rejected. + +The terms mean: + +- `span_ratio = matched_span_length / benign_string_length` +- `start_bonus = 1` if the match starts at offset `0`, else `0` +- `end_bonus = 1` if the match ends at the final character, else `0` +- `full_bonus = 1` if the match covers the entire benign string, else `0` +- `specificity_ratio = literal_chars / (literal_chars + meta_tokens)` +- `wildcard_penalty = min(1.0, wildcard_points / ((literal_chars + meta_tokens) / 2))` + +Regex-specific features are measured from the regex text itself: + +- `literal_chars` counts explicit alphanumeric and common structural literal + characters such as `-`, `_`, `/`, `:`, `,`, `@`, and `=` +- escaped literals such as `\.` count as literal characters +- `meta_tokens` counts regex syntax such as `.`, `[]`, `*`, `+`, `?`, groups, + anchors, and generic escapes +- `wildcard_points` penalize broad constructs: + - `.*` or `.+` adds `2.5` + - bare `.` adds `1.5` + - `[` character classes add `1.2` + - `*`, `+`, and `?` add `1.0` + - generic escapes such as `\w` also add penalty + +Examples: + +- Regex `^google\.com$` against benign string `google.com` + - full span match, starts at `0`, ends at the end, full match bonus applies + - specificity is high because most of the pattern is literal + - wildcard penalty is low + - score is very high, so this benign match is rejected + +- Regex `google` against benign string `google.com` + - only part of the string is covered + - it starts at `0` but does not end at the final character + - no full-match bonus + - score is lower and may stay below the threshold + +- Regex `.*com` + - may match a long suffix, but it is penalized heavily by the wildcard term + - this keeps broad permissive patterns from automatically looking “strong” + +## Benign corpus and bloom filters + +The module creates a dedicated benign corpus DB once and can seed it with a +small built-in sample for all supported types. + +On each run, it also imports domain entries from the configured Slips local +whitelist file into the benign corpus for the matching domain-like regex +types: + +- `dns_domain` +- `tls_sni` +- `certificate_cn` + +If the daily Tranco whitelist has already been downloaded by Slips, the module +also imports the ordered configured Tranco top benign domains from Redis into +the same domain-like benign corpus. + +During runtime, the module also listens for `tw_closed`. When a finished time +window belongs to one of the host IPs of the machine running Slips, it checks +that host TW for alerts and evidence: + +- if the host TW has any alert or any evidence, nothing is imported +- if the host TW has zero alerts and zero evidence, the module learns extra + benign strings from that clean local TW + +The runtime benign import currently uses: + +- DNS query names -> `dns_domain` +- HTTP hostnames -> `dns_domain` +- TLS `server_name` -> `tls_sni` +- certificate `subject` CN -> `certificate_cn` +- filenames derived from HTTP URIs -> `filename` + +The module logs the total alert count, total evidence count, and a separate +best-effort anomaly-evidence count for that finished host TW. The anomaly +count is only for visibility; the import gate itself is strict: the TW must +have zero alerts and zero evidence. + +Redis storage note: + +- The original Tranco whitelist behavior is still present: Slips stores the + full downloaded Tranco whitelist in Redis under `tranco_whitelisted_domains` + while preserving download order. +- `RegexGenerator` reads the configured number of top-ranked entries from that + ordered whitelist cache so they can be treated as benign test data for + domain-like regexes. +- The number of domains read is configured with + `whitelists.tranco_top_benign_limit`. + +It builds one in-memory bloom filter per benign type and one additional bloom +filter for generated regex hashes. These filters speed up exact membership +checks, but they do not replace the benign corpus scan. Acceptance still +requires computing the benign match-strength score against the benign corpus +and rejecting the regex only if some benign string reaches or exceeds +`benign_match_strength_threshold`. + +## Stored regexes + +Accepted and rejected regexes are stored in `generated_regexes.sqlite`. + +Other modules should access accepted regexes through: + +```python +self.db.get_generated_regexes(regex_type="dns_domain", limit=100) +self.db.get_generated_regexes_count(regex_type="dns_domain") +``` + +These helpers read accepted regexes by default. + +## Offline coverage report + +To estimate how much the accepted regexes cover several reference +populations, run the offline report script by hand against a completed Slips +run output directory: + +```bash +./venv/bin/python scripts/regex_coverage_report.py \ + --run-output-dir output/eno1_2026-03-18_10:00:30 \ + --redis-port 6379 +``` + +By default, large populations are sampled so the script finishes in practical +time. It prints terminal progress while it runs, for example: + +```text +🧪 sampled estimate ███████░░░░░░░░░░░░ 31.62% | regex 247/781 | cmp 560,840/1,770,991 | type DNS Domain | ETA ⏳ 00:00:14 +``` + +In that progress line: + +- `regex 247/781` means 247 accepted regexes have been evaluated out of 781 total accepted regexes. +- `cmp 560,840/1,770,991` means regex-versus-string match operations, not raw TI entries. The number grows because many regexes are checked against many strings across the benign corpus, malicious TI, observed traffic, and reference-union populations. + +The report reuses the same `0..100` match-strength function as the live +module, but it applies it to every regex/string comparison in the selected +populations: + +- if the regex does not match the string, the score is `0` +- if it matches, the score is computed with the same span/anchor/specificity/ + wildcard formula used by the generator + +For each regex and each population, the report now computes: + +- `match_count`: how many strings matched at all +- `avg_all ± std_all`: average and standard deviation over all tested strings, + with non-matches counted as `0` +- `avg_match ± std_match`: average and standard deviation over only the strings + that matched + +The top-regex table ranks regexes by: + +```text +strength_gap = malicious_avg_all - benign_avg_all +``` + +This favors regexes that score strongly and/or broadly on malicious strings +while staying weak on benign strings. + +The HTML report also includes a `Strength Scatter` plot per regex type: + +- X axis: benign `avg_all` +- Y axis: malicious `avg_all` +- ideal area: upper-left + +That plot is useful when there are too many regexes for a table alone to be +read comfortably. + +If you want the exhaustive run for research, use: + +```bash +./venv/bin/python scripts/regex_coverage_report.py \ + --run-output-dir /path/to/regex_store \ + --redis-port 23456 \ + --ti-cache-port 6379 \ + --ti-cache-db 1 \ + --full-scan +``` + +Useful knobs: + +- `--sampling-ratio`: fraction of strings to evaluate from each regex-type population in estimate mode. This is applied separately to the benign corpus values, malicious TI values, observed traffic values, and reference-union values. Default: `0.1`. +- `--max-population-size`: hard cap on the number of strings evaluated for each regex type inside each population, after `--sampling-ratio` is applied. +- `--full-scan`: disable both `--sampling-ratio` and `--max-population-size`, and scan all strings in all populations for every regex type. +- `--match-timeout-seconds`: timeout for one regex tested against one regex-type population of strings. + +This generates: + +- `regex_generator_coverage_report.html` +- `regex_generator_coverage_report.json` + +inside the selected run output directory. + +The report estimates coverage against: + +- the local benign corpus DB, grouped by regex type +- the configured Tranco top benign domains from `whitelists.tranco_top_benign_limit` as extra benign data for domain-like types, when available in the Slips cache +- TI-derived malicious reference strings from Redis and TI files, grouped by regex type +- observed traffic strings from the same run, grouped by regex type and taken from Zeek logs or `flows.sqlite` +- the per-type reference union, which is `malicious TI ∪ observed traffic` + +The report is offline only. It is not part of the continuous RegexGenerator +loop. diff --git a/modules/regex_generator/__init__.py b/modules/regex_generator/__init__.py new file mode 100644 index 0000000000..f436f14183 --- /dev/null +++ b/modules/regex_generator/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only diff --git a/modules/regex_generator/blog_post.md b/modules/regex_generator/blog_post.md new file mode 100644 index 0000000000..e2c8ccba13 --- /dev/null +++ b/modules/regex_generator/blog_post.md @@ -0,0 +1,354 @@ +# Pseudo-Generated Regexes in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS): Adaptive Receptors with Negative Selection + +Network threats do not arrive as one fixed string that can be hard-coded once +and forgotten. Domains vary, URIs mutate, filenames drift, TLS SNI values +change, and certificate names are reused in slightly different forms. A system +that only depends on exact literals or hand-written pattern updates is always +reacting after the fact. + +That is one of the reasons +[Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) is moving +toward immunology concepts. The old detector structure is already a strong +innate immune system: it produces broad evidence quickly from many traffic +sources. What was missing was an adaptive way to keep building new candidate +recognizers continuously. + +`RegexGenerator` fills that gap. It gives +[Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) a mechanism to +continually propose, evaluate, reject, and retain regex-based receptors instead +of waiting for every new pattern to be manually engineered. + +Instead of hard-coding every future pattern in advance, [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) can now +pseudo-generate candidate regex detectors, validate them locally, reject the +dangerous or overly broad ones, and keep only the candidates that survive a +negative-selection step against benign data. + +That is the role of: + +- `modules/regex_generator/regex_generator.py` + +This module is one of the adaptive building blocks of the immune design in +[Slips](https://github.com/stratosphereips/StratosphereLinuxIPS). It is the +part that keeps building candidate receptors in the background so the adaptive +layer always has new hypotheses to test. + +## Why Generate Regexes Continually? + +Many relevant indicators in network evidence are not just IP addresses. They +are structured strings: + +- domains +- URIs +- filenames +- TLS SNI values +- certificate common names + +These strings often carry strong semantic information, but they also vary +heavily. A hard-coded exact match is often too narrow, while a naive wildcard +regex is often too broad. + +The idea behind `RegexGenerator` is to explore that middle ground: + +- generate candidate symbolic detectors +- keep them narrow enough to avoid matching benign traffic +- make them reusable by other modules + +The result is a growing local repertoire of candidate recognizers that can be +queried later by modules such as `T Cell`. + +Just as important, this is not a one-shot batch job. The module runs +continuously. As long as [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) is running, `RegexGenerator` keeps asking for a +new candidate, evaluating it, and either rejecting it or adding it to the +accepted receptor pool. + +The resulting repertoire is not opaque. The offline coverage report lets us +inspect how many accepted regexes exist per type, how they score against +reference populations, and how much benign spillover remains: + +![RegexGenerator coverage report overview](../../docs/images/regex_generator/coverage_report_overview.png) + +## Why "Pseudo-Generated" and Not Just "Generated"? + +The regexes are not accepted directly from the LLM. + +The LLM is used only as a hypothesis generator. It proposes one candidate regex +at a time. [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) then does the actual engineering work locally: + +1. parse the reply +2. validate syntax and safety +3. reject obvious bad constructions +4. compare the candidate against benign data +5. store only accepted regexes + +So the regex is pseudo-generated in the sense that the creative step comes from +the model, but acceptance is determined by [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) itself. + +This distinction matters. The module is not trusting model output as if it were +ground truth. It is treating it as a candidate detector that must pass +selection. + +## Using the Shared LLM Module + +`RegexGenerator` does not talk to one specific model API directly. It uses the +new shared `LLM` module in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) over the Redis channels: + +- `llm_request` +- `llm_response` + +That means [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) can take advantage of modern AI models without baking one +provider into the regex logic itself. + +The shared LLM layer decides how to talk to the configured backend. The regex +module just says: + +- I need one candidate regex +- for this exact regex type +- with this request ID + +This separation is important because it keeps the adaptive detector logic +independent from the transport and backend details of the LLM itself. + +## Why the Prompting Matters + +The prompt design is intentionally strict. + +The module does not ask the model for an explanation, JSON, or a list of +options. It asks for exactly one raw regex line for one exact regex type. + +It also sends a fresh generation nonce with every request. Combined with the +type-specific prompt, that nudges the model away from repeating the same answer +and toward producing a new candidate each cycle. + +So the system is not just "calling an LLM." It is using constrained prompting +to turn the model into a continual hypothesis generator: + +- one regex at a time +- for one target type at a time +- with a fresh nonce at every generation cycle + +That prompting strategy is then backed by local duplicate checks, local syntax +validation, and negative selection, so the model is encouraged to produce +novelty but [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) remains responsible for acceptance. + +## Five Supported Regex Types + +The module currently generates regexes for five structured data types: + +- `dns_domain` +- `uri` +- `filename` +- `tls_sni` +- `certificate_cn` + +These types were chosen because they are stable, meaningful, and already show +up naturally in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) evidence and linked altflows. + +That means the adaptive layer is not inventing arbitrary string targets. It is +working on fields that already matter operationally: + +- DNS query names and HTTP hosts +- HTTP request URIs +- filenames derived from HTTP paths +- TLS Server Name Indication values +- certificate Common Name values + +The same five types are also what later let the `T Cell` module do adaptive +antigen recognition. + +## How a Regex Candidate Is Created + +Each cycle of `RegexGenerator` is intentionally small and controlled. + +The module: + +1. chooses one regex type +2. selects an available LLM backend through the shared `LLM` service +3. sends a minimal typed prompt asking for exactly one regex +4. waits for the matching response by `request_id` +5. extracts a single regex line from the reply + +Only one request is in flight at a time. That keeps response correlation simple +and makes the continual generation process easy to audit. + +This is not bulk generation. It is a continuous stream of one candidate at a +time, because each candidate still has to pass a local selection pipeline. + +## Local Validation Before Selection + +Before the module even compares a regex against benign data, it runs a static +safety gate. + +Candidates are rejected if they are malformed or operationally unsafe, for +example: + +- non-ASCII patterns +- invalid syntax +- excessive length +- lookbehind +- backreferences +- obviously broad expressions such as `.*` or `.+` +- unbounded prefix/suffix wildcard structures +- nested wildcard constructions that risk catastrophic backtracking + +This step is important because it prevents the selection stage from wasting +time on candidates that are clearly unacceptable for runtime use. + +## Negative Selection Against Benign Traffic + +This is the core of the module. + +In immunology, negative selection removes detectors that react to what should be +tolerated. In [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS), that means candidate regexes are tested against a benign +corpus and rejected if they match benign strings too strongly. + +The module does not reject on any benign hit. That would be too strict. +Instead, it computes a benign match-strength score and rejects only when a +benign string crosses a configured threshold. + +That score considers: + +- how much of the benign string was covered +- whether the match starts at the beginning +- whether the match reaches the end +- whether it is a full-string match +- how literal or specific the regex looks +- how much wildcard power the regex uses + +So the selection rule is more nuanced than "matched something benign, reject +it." A candidate can survive weak or partial benign overlap while still being +rejected if it behaves like a broad general-purpose pattern. + +This is the operational negative selection algorithm in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS): + +- generate a candidate receptor +- expose it to benign data of the same type +- reject it if it reacts too strongly to benign strings +- keep it only if it stays below the benign threshold + +That is what turns free-form generation into a tolerized detector-building +process. + +## What Counts as the Benign Corpus? + +The benign corpus is not a single static file. + +The module can populate it from several sources: + +- a small built-in seed sample +- the local [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) whitelist +- the ordered top Tranco domains already cached by [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) +- clean local time windows from the host running [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) + +That last source is especially useful. When a time window from the local host +closes with zero alerts and zero evidence, the module can treat those observed +strings as clean operational context and import them into the benign corpus. + +This means the negative selection process is not only generic. It can adapt to +the real environment where [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) is running. + +In practice, that helps prevent the adaptive layer from learning detectors that +would constantly fire on the local normal baseline. + +## Duplicate Control and Storage + +The module also avoids wasting effort on repeated candidates. + +It uses: + +- bloom filters +- exact SQLite lookup + +to suppress cheap repeats and keep the accepted set cleaner over time. + +Accepted regexes are stored in: + +- `/regex_generator/generated_regexes.sqlite` + +Rejected regexes can be persisted too, but by default they are not written to +disk. That keeps the main store focused on the useful adaptive repertoire. + +The stored accepted regexes are then available through the DB helpers for other +modules. + +## Why This Matters for the Immune Design + +`RegexGenerator` is not just a regex toy. It is the adaptive receptor factory +for the larger immune model in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS). + +The existing [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) detectors produce the innate evidence layer: + +- fast +- broad +- always on + +But innate evidence alone does not create a reusable symbolic receptor +repertoire. + +That is what `RegexGenerator` provides: + +- a growing set of validated candidate recognizers +- scoped to structured string fields +- filtered by negative selection +- reusable by later modules + +Without this module, the adaptive layer would have no detector library to +consult. + +## The Connection to T Cell + +The clearest consumer of these accepted regexes is the `T Cell` module. + +When `T Cell` receives `PAMP` evidence, it extracts structured antigens such as +domains, URIs, filenames, SNI values, and certificate CNs. It then queries the +accepted regex repertoire built by `RegexGenerator`. + +That connection is the key architectural link: + +- `RegexGenerator` builds candidate receptors +- `T Cell` uses those receptors on live evidence + +So `RegexGenerator` is not making alerts by itself. It is preparing the +adaptive recognition layer that later allows T cells to say: + +- this antigen looks recognizable +- this recognizable thing is or is not dangerous in context + +## What Makes This Different from Plain IOC Matching? + +A normal IOC list gives exact values. + +`RegexGenerator` creates pattern detectors that can generalize within bounds. +That lets [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) work with suspicious neighborhoods of strings instead of only +exact literals. + +But that extra expressive power is also risky. A bad regex can easily become a +false-positive machine. + +That is why the negative-selection step is the real heart of the module. + +The important idea is not "LLM-generated regexes." + +The important idea is: + +- candidate regexes are cheap to propose +- acceptance is expensive and local +- only the selected ones become part of the adaptive repertoire + +That is a much safer and more defensible engineering design. + +## In Short + +The `RegexGenerator` module gives [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) an adaptive receptor-building process. + +It does that by: + +- generating one regex candidate at a time +- working across five meaningful network-data types +- validating candidates locally +- applying a negative-selection algorithm against benign corpora +- storing only accepted regexes for later reuse + +That makes it the first half of the adaptive immune system in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS). + +The second half is `T Cell`, which consumes those receptors during live +decision-making. diff --git a/modules/regex_generator/log_rotator.py b/modules/regex_generator/log_rotator.py new file mode 100644 index 0000000000..d2ddff376e --- /dev/null +++ b/modules/regex_generator/log_rotator.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import os +import re +import time + + +class LogRotator: + """ + Manage regex generator progress logging and time-based log rotation. + """ + + def __init__( + self, + output_dir: str, + log_file_path: str, + create_log_file: bool = False, + enable_log_rotation: bool = True, + log_rotation_period: int = 86400, + ) -> None: + """ + Initialize the log rotator. + + Parameters: + output_dir: Directory where the active log delete custom shutdown logic, + the moudle will never stop if it has pending redis msgs by defaultfile is stored. + log_file_path: Path to the active log file. + create_log_file: Whether progress logging is enabled. + enable_log_rotation: Whether time-based log rotation is enabled. + log_rotation_period: Rotation interval in seconds. + + Returns: + None + """ + self.output_dir = output_dir + self.log_file_path = log_file_path + self.create_log_file = create_log_file + self.enable_log_rotation = enable_log_rotation + self.log_rotation_period = log_rotation_period + self.last_log_rotation_time = time.time() + + def init_log_file(self) -> None: + """ + Create the active log file if progress logging is enabled. + + Returns: + None + """ + if not self.create_log_file: + return + + os.makedirs(self.output_dir, exist_ok=True) + if not os.path.exists(self.log_file_path): + with open(self.log_file_path, "w", encoding="utf-8") as log_file: + log_file.write("") + self.last_log_rotation_time = time.time() + + def rotate_log_file_if_needed(self) -> None: + """ + Rotate the active log file when the configured period has elapsed. + + Returns: + None + """ + if not self.enable_log_rotation or self.log_rotation_period <= 0: + return + + now = time.time() + if now - self.last_log_rotation_time < self.log_rotation_period: + return + + if ( + os.path.exists(self.log_file_path) + and os.path.getsize(self.log_file_path) > 0 + ): + timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime(now)) + rotated_path = f"{self.log_file_path}.{timestamp}" + os.replace(self.log_file_path, rotated_path) + + with open(self.log_file_path, "w", encoding="utf-8") as log_file: + log_file.write("") + self.last_log_rotation_time = now + + @staticmethod + def parse_rotation_period_seconds(rotation_period: object) -> int: + """ + Parse a rotation period value into seconds. + + Parameters: + rotation_period: Numeric seconds or a string with a supported time unit. + + Returns: + Rotation period in seconds, defaulting to one day for invalid values. + """ + if isinstance(rotation_period, (int, float)): + return max(1, int(rotation_period)) + + text = str(rotation_period or "").strip().lower().replace(" ", "") + match = re.fullmatch( + r"(?P\d+)(?Psec|secs|second|seconds|min|mins|minute|minutes|hr|hrs|hour|hours|day|days)", + text, + ) + if not match: + return 86400 + + value = int(match.group("value")) + unit = match.group("unit") + multipliers = { + "sec": 1, + "secs": 1, + "second": 1, + "seconds": 1, + "min": 60, + "mins": 60, + "minute": 60, + "minutes": 60, + "hr": 3600, + "hrs": 3600, + "hour": 3600, + "hours": 3600, + "day": 86400, + "days": 86400, + } + return max(1, value * multipliers[unit]) diff --git a/modules/regex_generator/match_strength.py b/modules/regex_generator/match_strength.py new file mode 100644 index 0000000000..1c7a8b36aa --- /dev/null +++ b/modules/regex_generator/match_strength.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import re + + +def measure_regex_specificity(regex_text: str) -> dict: + literal_chars = 0 + meta_tokens = 0 + wildcard_points = 0.0 + idx = 0 + while idx < len(regex_text): + char = regex_text[idx] + next_char = regex_text[idx + 1] if idx + 1 < len(regex_text) else "" + + if char == "\\": + token = regex_text[idx : idx + 2] + meta_tokens += 1 + if len(token) == 2 and token[1] in ".^$*+?{}[]()|\\": + literal_chars += 1 + else: + wildcard_points += 1.0 + idx += 2 if next_char else 1 + continue + + if char.isalnum() or char in "-_/:,@=": + literal_chars += 1 + idx += 1 + continue + + meta_tokens += 1 + if char == "." and next_char in {"*", "+"}: + wildcard_points += 2.5 + elif char == ".": + wildcard_points += 1.5 + elif char == "[": + wildcard_points += 1.2 + elif char in {"*", "+", "?"}: + wildcard_points += 1.0 + idx += 1 + + effective_length = max(1, literal_chars + meta_tokens) + specificity_ratio = min(1.0, literal_chars / effective_length) + wildcard_penalty = min(1.0, wildcard_points / max(1.0, effective_length / 2)) + return { + "specificity_ratio": specificity_ratio, + "wildcard_penalty": wildcard_penalty, + } + + +def compute_match_strength( + compiled_regex: re.Pattern, + value: str, + regex_features: dict | None = None, +) -> float: + value = str(value or "") + if not value: + return 0.0 + + if regex_features is None: + regex_features = measure_regex_specificity(compiled_regex.pattern) + + best_score = 0.0 + value_len = max(1, len(value)) + for match in compiled_regex.finditer(value): + start, end = match.span() + span_len = max(0, end - start) + if span_len <= 0: + continue + + span_ratio = min(1.0, span_len / value_len) + start_bonus = 1.0 if start == 0 else 0.0 + end_bonus = 1.0 if end == len(value) else 0.0 + full_bonus = 1.0 if start == 0 and end == len(value) else 0.0 + score = ( + 40.0 * span_ratio + + 12.0 * start_bonus + + 12.0 * end_bonus + + 16.0 * full_bonus + + 30.0 * regex_features["specificity_ratio"] + - 18.0 * regex_features["wildcard_penalty"] + ) + best_score = max(best_score, max(0.0, min(100.0, score))) + return best_score diff --git a/modules/regex_generator/regex_errors.py b/modules/regex_generator/regex_errors.py new file mode 100644 index 0000000000..044ebfe60f --- /dev/null +++ b/modules/regex_generator/regex_errors.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import signal +from types import FrameType, TracebackType +from typing import NoReturn + + +class _NullTimeout: + def __enter__(self) -> None: + """ + Enter a no-op timeout context. + + Returns: + None + """ + return None + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + """ + Exit a no-op timeout context. + + Args: + exc_type: Exception type raised in the context, if any. + exc: Exception raised in the context, if any. + exc_tb: Traceback attached to the exception, if any. + + Returns: + False to propagate any exception raised in the context. + """ + return False + + +class _SignalTimeout: + def __init__(self, timeout_seconds: float) -> None: + """ + Create a signal-backed timeout context. + + Args: + timeout_seconds: Number of seconds before timing out. + + Returns: + None + """ + self.timeout_seconds = timeout_seconds + self._previous_handler = None + + def __enter__(self) -> None: + """ + Start the timeout timer. + + Returns: + None + """ + self._previous_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, self._handle_timeout) + signal.setitimer(signal.ITIMER_REAL, self.timeout_seconds) + return None + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + """ + Stop the timeout timer and restore the previous signal handler. + + Args: + exc_type: Exception type raised in the context, if any. + exc: Exception raised in the context, if any. + exc_tb: Traceback attached to the exception, if any. + + Returns: + False to propagate any exception raised in the context. + """ + signal.setitimer(signal.ITIMER_REAL, 0) + if self._previous_handler is not None: + signal.signal(signal.SIGALRM, self._previous_handler) + return False + + @staticmethod + def _handle_timeout(signum: int, frame: FrameType | None) -> NoReturn: + """ + Raise an exception when the signal timer expires. + + Args: + signum: Signal number received from the operating system. + frame: Current stack frame when the signal was handled. + + Returns: + This method does not return. + """ + raise TimeoutError("regex validation timed out") diff --git a/modules/regex_generator/regex_generator.py b/modules/regex_generator/regex_generator.py new file mode 100644 index 0000000000..4c7dc25a6f --- /dev/null +++ b/modules/regex_generator/regex_generator.py @@ -0,0 +1,852 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +import random +import re +import time +import uuid +from hashlib import sha256 +from urllib.parse import urlparse + +from slips_files.common.abstracts.imodule import IModule +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import utils +from modules.regex_generator.match_strength import ( + compute_match_strength, + measure_regex_specificity, +) +from modules.regex_generator.log_rotator import LogRotator +from modules.regex_generator.regex_errors import _NullTimeout, _SignalTimeout +from slips_files.core.database.sqlite_db.regex_generator_db import ( + REGEX_TYPES, + RegexGeneratorStorage, +) + + +PROMPT_VERSION = "regex-generator-v2" +SYSTEM_PROMPT = """ +Return exactly one regex line. +No JSON. +No explanation. +No code fences. +Do not wrap the regex in slashes. +Use a conservative regex subset portable to Zeek and Python. +Do not use lookbehind, named groups, backreferences, or inline flags. +Avoid catastrophic backtracking and nested wildcards. +Keep it specific enough to avoid broad benign matching. +Keep it under 120 characters. +Model uncommon lexical structure, not explicit threat vocabulary. +Do not use literal threat words, brand names, or exact known IOCs. +""".strip() + +TYPE_PROMPTS = { + "dns_domain": """ +Generate one DNS domain regex for uncommon suspicious-looking lexical structure. +Target rare structure such as random-looking labels, encoded-looking subdomains, +digit-heavy tokens, awkward token boundaries, or unusual subdomain depth. +The input is only a domain name, not a URL. +Prefer anchors when useful. +Do not use words such as malware, trojan, virus, exploit, c2, bot, or ransom. +""".strip(), + "uri": """ +Generate one HTTP URI regex for uncommon suspicious-looking lexical structure. +Target rare path structure such as encoded segments, awkward separators, +unusual extension combinations, or long mixed-token segments. +Avoid ordinary website paths unless the lexical structure is clearly unusual. +Do not use words such as malware, trojan, virus, exploit, c2, bot, or ransom. +""".strip(), + "filename": """ +Generate one filename regex for uncommon suspicious-looking lexical structure. +Target rare structure such as double extensions, deceptive token boundaries, +random-looking names, or unusual risky extension combinations. +Prefer anchors when useful. +Do not use words such as malware, trojan, virus, exploit, c2, bot, or ransom. +""".strip(), + "tls_sni": """ +Generate one TLS SNI hostname regex for uncommon suspicious-looking lexical structure. +Target rare structure such as disposable subdomains, random-looking host labels, +awkward token composition, or deceptive naming without using explicit threat words. +The input is only the SNI hostname. +Prefer anchors when useful. +Do not use words such as malware, trojan, virus, exploit, c2, bot, or ransom. +""".strip(), + "certificate_cn": """ +Generate one X.509 certificate Common Name regex for uncommon suspicious-looking lexical structure. +Target rare structure such as deceptive hostnames, awkward token combinations, +random or encoded-looking names, or unusual service-like naming patterns. +The input is only the CN text. +Prefer anchors when useful. +Do not use words such as malware, trojan, virus, exploit, c2, bot, or ransom. +""".strip(), +} + + +class RegexGenerator(IModule): + name = "regex_generator" + description = "Continuously generates and validates pseudo-random regexes" + authors = ["Sebastian Garcia"] + + def init(self): + self.channels = {} + self.subscribe_to_channels() + self.storage = None + self.enabled = False + self.log_rotator = LogRotator( + self.output_dir, + self.get_module_specific_output_path("regex_generator.log"), + ) + self.generation_interval_seconds = 5.0 + self.allowed_backends = [] + self.llm_temperature = 1.2 + self.llm_max_tokens = 80 + self.llm_response_timeout_seconds = 90 + self.recent_history_size = 0 + self.max_regex_length = 180 + self.regex_validation_timeout_seconds = 2.0 + self.benign_match_strength_threshold = 75.0 + self.type_weights = {regex_type: 1.0 for regex_type in REGEX_TYPES} + self.pending_request = None + self.next_generation_at = 0.0 + self._rng = random.Random() + self.read_configuration() + + def subscribe_to_channels(self): + """ + Subscribe to the Redis channels used by the regex generator. + + Returns: + None + """ + if self.channels: + return + + self.c_llm = self.db.subscribe(self.db.channels.LLM_RESPONSE) + self.c_tw_closed = self.db.subscribe("tw_closed") + self.channels = { + self.db.channels.LLM_RESPONSE: self.c_llm, + "tw_closed": self.c_tw_closed, + } + + def read_configuration(self): + conf = ( + self.conf + if hasattr(self.conf, "regex_generator_enabled") + else ConfigParser() + ) + rotation_period_getter = getattr( + conf, "default_rotation_interval", None + ) or getattr(conf, "rotation_period") + self.enabled = conf.regex_generator_enabled() + self.log_rotator.create_log_file = ( + conf.regex_generator_create_log_file() + ) + self.log_rotator.enable_log_rotation = conf.rotation() + self.log_rotator.log_rotation_period = ( + LogRotator.parse_rotation_period_seconds(rotation_period_getter()) + ) + self.generation_interval_seconds = ( + conf.regex_generator_generation_interval_seconds() + ) + self.allowed_backends = conf.regex_generator_allowed_backends() + self.llm_temperature = conf.regex_generator_llm_temperature() + self.llm_max_tokens = conf.regex_generator_llm_max_tokens() + self.llm_response_timeout_seconds = ( + conf.regex_generator_llm_response_timeout_seconds() + ) + self.recent_history_size = conf.regex_generator_recent_history_size() + self.max_regex_length = conf.regex_generator_max_regex_length() + self.regex_validation_timeout_seconds = ( + conf.regex_generator_regex_validation_timeout_seconds() + ) + self.benign_match_strength_threshold = ( + conf.regex_generator_benign_match_strength_threshold() + ) + self.type_weights = conf.regex_generator_type_weights() + + def pre_main(self): + utils.drop_root_privs_permanently() + + if not self.enabled: + self.print("RegexGenerator module disabled in config.", 2, 0) + return True + + self.log_rotator.output_dir = self.output_dir + self.log_rotator.init_log_file() + self.storage = RegexGeneratorStorage( + self.logger, + self.conf, + self.output_dir, + self.ppid, + self.db, + ) + self.next_generation_at = time.time() + self.log_detail("RegexGenerator module ready.") + self.log_detail( + f"Using storage at {self.storage.store_dir}. " + f"Benign corpus DB: {self.storage.benign_db.db_path}. " + f"Generated regex DB: {self.storage.generated_db.db_path}." + ) + self.log_detail( + "Rejected regex persistence is " + f"{'enabled' if self.storage.store_rejected_regexes else 'disabled'}." + ) + self.print("RegexGenerator module ready.", 2, 0) + + def shutdown_gracefully(self): + if self.storage: + self.storage.close() + return True + + def main(self): + self._handle_one_tw_closed_message() + + now = time.time() + if self.pending_request: + self._handle_pending_response(now) + return + + if now < self.next_generation_at: + time.sleep(min(0.5, self.next_generation_at - now)) + return + + available_backends = self.db.get_available_llm_backends() + backend = self._select_backend(available_backends) + if not backend: + self.log_detail( + "No runtime-ready LLM backend available yet. Waiting for discovery." + ) + self.print( + "RegexGenerator is waiting for a runtime-ready LLM backend.", + 2, + 0, + ) + self.next_generation_at = now + self.generation_interval_seconds + time.sleep(min(0.5, self.generation_interval_seconds)) + return + + regex_type = self._choose_regex_type() + self.log_detail( + f"Starting generation cycle. regex_type={regex_type} backend={backend}" + ) + self._send_generation_request(regex_type, backend) + + def _handle_one_tw_closed_message(self): + if self.storage is None: + return + + msg = self.get_msg("tw_closed") + if not msg: + return + + profileid, twid = self._split_profileid_twid(msg.get("data", "")) + if not profileid or not twid: + return + + if not self._is_host_profile(profileid): + return + + alerts = self.db.get_profileid_twid_alerts(profileid, twid) or {} + evidence = self._normalize_evidence_records( + self.db.get_twid_evidence(profileid, twid) or {} + ) + anomaly_evidence_count = self._count_anomaly_evidence(evidence) + self.log_detail( + f"Finished host TW profileid={profileid} twid={twid} " + f"alerts={len(alerts)} evidence={len(evidence)} " + f"anomaly_evidence={anomaly_evidence_count}" + ) + + if alerts or evidence: + return + + learned = self._extract_benign_candidates_from_twid(profileid, twid) + learned_counts = {} + source = f"clean_client_tw:{profileid}:{twid}" + for regex_type, values in learned.items(): + inserted = self.storage.add_benign_strings( + regex_type, values, source + ) + if inserted: + learned_counts[regex_type] = inserted + + if learned_counts: + summary = ", ".join( + f"{regex_type}={count}" + for regex_type, count in sorted(learned_counts.items()) + ) + self.log_detail( + f"Imported runtime benign strings from clean host TW " + f"profileid={profileid} twid={twid}: {summary}" + ) + + @staticmethod + def _split_profileid_twid(profileid_twid: str) -> tuple[str, str]: + text = str(profileid_twid or "").strip() + if not text or "_" not in text: + return "", "" + profileid, twid = text.rsplit("_", 1) + return profileid, twid + + def _is_host_profile(self, profileid: str) -> bool: + profile_ip = str(profileid or "").split("_", 1)[-1] + host_ips = {str(ip).strip() for ip in self.db.get_all_host_ips() or []} + return profile_ip in host_ips + + @staticmethod + def _normalize_evidence_records(raw_evidence: dict) -> dict[str, dict]: + normalized = {} + for evidence_id, payload in (raw_evidence or {}).items(): + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + normalized[evidence_id] = payload + return normalized + + @staticmethod + def _count_anomaly_evidence(evidence_records: dict[str, dict]) -> int: + anomaly_evidence_types = {"ANOMALOUS_FLOW", "MALICIOUS_FLOW"} + count = 0 + for evidence in evidence_records.values(): + evidence_type = str(evidence.get("evidence_type", "")) + description = str(evidence.get("description", "")).lower() + if ( + evidence_type in anomaly_evidence_types + or "anomaly" in evidence_type.lower() + or "anomaly" in description + ): + count += 1 + return count + + def _extract_benign_candidates_from_twid( + self, profileid: str, twid: str + ) -> dict[str, set[str]]: + learned = {regex_type: set() for regex_type in REGEX_TYPES} + altflows = ( + self.db.get_all_altflows_in_profileid_twid(profileid, twid) or [] + ) + for row in altflows: + flow = row.get("flow", {}) + flow_type = row.get("flow_type") or flow.get("type_") + if flow_type == "dns": + domain = self._normalize_domain(flow.get("query", "")) + if domain: + learned["dns_domain"].add(domain) + elif flow_type == "http": + host = self._normalize_domain(flow.get("host", "")) + if host: + learned["dns_domain"].add(host) + filename = self._extract_filename_from_uri(flow.get("uri", "")) + if filename: + learned["filename"].add(filename) + elif flow_type == "ssl": + server_name = self._normalize_domain( + flow.get("server_name", "") + ) + if server_name: + learned["tls_sni"].add(server_name) + cn = self._extract_cn(flow.get("subject", "")) + if cn: + learned["certificate_cn"].add(cn) + return learned + + @staticmethod + def _normalize_domain(value: str) -> str: + domain = str(value or "").strip().rstrip(".").lower() + if not domain or not utils.is_valid_domain(domain): + return "" + return domain + + @staticmethod + def _extract_cn(subject: str) -> str: + match = re.search(r"(?:^|,)CN=([^,]+)", str(subject or "")) + if not match: + return "" + return match.group(1).strip() + + @staticmethod + def _extract_filename_from_uri(uri: str) -> str: + value = str(uri or "").strip() + if not value: + return "" + parsed = urlparse(value) + path = parsed.path or value + filename = path.rsplit("/", 1)[-1].strip() + if not filename or "." not in filename: + return "" + return filename + + def log_detail(self, text: str) -> None: + """ + Append one timestamped progress line to the regex generator log. + + Parameters: + text: Log message text to append. + """ + if not self.log_rotator.create_log_file: + return + + self.log_rotator.rotate_log_file_if_needed() + human_readable_datetime = utils.convert_ts_format( + time.time(), utils.alerts_format + ) + with open( + self.log_rotator.log_file_path, "a", encoding="utf-8" + ) as log_file: + log_file.write(f"{human_readable_datetime} - {text}\n") + + def _select_backend(self, available_backends: dict) -> str: + available = available_backends.get("backends", {}) + if not available: + return "" + + for backend in self.allowed_backends: + if backend in available: + return backend + + default_backend = available_backends.get("default_backend", "") + if default_backend in available: + return default_backend + + if self.allowed_backends: + return "" + + return sorted(available)[0] + + def _choose_regex_type(self) -> str: + regex_types = list(self.type_weights) + weights = [self.type_weights[regex_type] for regex_type in regex_types] + return self._rng.choices(regex_types, weights=weights, k=1)[0] + + def _send_generation_request(self, regex_type: str, backend: str): + request_id = f"{self.name}-{uuid.uuid4()}" + generation_nonce = str(uuid.uuid4()) + request = { + "request_id": request_id, + "requester": self.name, + "backend": backend, + "messages": self._build_prompt_messages( + regex_type, generation_nonce + ), + "temperature": self.llm_temperature, + "max_tokens": self.llm_max_tokens, + "metadata": { + "regex_type": regex_type, + "prompt_version": PROMPT_VERSION, + "generation_nonce": generation_nonce, + }, + } + self.db.publish( + self.db.channels.LLM_REQUEST, + json.dumps(request), + ) + self.log_detail( + f"Published llm_request request_id={request_id} " + f"regex_type={regex_type} backend={backend}" + ) + self.pending_request = { + "request_id": request_id, + "regex_type": regex_type, + "backend": backend, + "sent_at": time.time(), + "generation_nonce": generation_nonce, + "last_warning_at": 0.0, + } + + def _build_prompt_messages( + self, + regex_type: str, + generation_nonce: str, + ) -> list: + user_prompt = ( + f"Type: {regex_type}\n" + f"Prompt version: {PROMPT_VERSION}\n" + f"Nonce: {generation_nonce}\n" + "Goal: generate a regex for uncommon suspicious-looking lexical structure.\n" + "Prefer structural contrast over explicit malicious words.\n" + "Do not repeat previous generations.\n" + f"{TYPE_PROMPTS[regex_type]}\n" + "Return one regex only." + ) + return [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ] + + def _handle_pending_response(self, now: float): + self._warn_if_llm_is_slow(now) + + if not (msg := self.get_msg(self.db.channels.LLM_RESPONSE)): + time.sleep(0.1) + return + + try: + response = json.loads(msg["data"]) + except (TypeError, json.JSONDecodeError): + return + + if response.get("request_id") != self.pending_request["request_id"]: + return + + self.log_detail( + f"Received matching llm_response request_id={response.get('request_id')}" + ) + self._finalize_request(response) + self.pending_request = None + self.next_generation_at = ( + time.time() + self.generation_interval_seconds + ) + + def _warn_if_llm_is_slow(self, now: float): + if self.llm_response_timeout_seconds <= 0: + return + + elapsed = now - self.pending_request["sent_at"] + if elapsed <= self.llm_response_timeout_seconds: + return + + last_warning_at = self.pending_request.get("last_warning_at", 0.0) + warning_interval = max(30.0, float(self.llm_response_timeout_seconds)) + if last_warning_at and now - last_warning_at < warning_interval: + return + + self.print( + f"RegexGenerator is still waiting for llm_response after {elapsed:.1f}s.", + 2, + 0, + ) + self.log_detail( + f"Still waiting for llm_response request_id=" + f"{self.pending_request['request_id']} elapsed={elapsed:.1f}s" + ) + self.pending_request["last_warning_at"] = now + + def _finalize_request(self, response: dict): + if not response.get("success"): + self.log_detail( + f"LLM response failed request_id={response.get('request_id')} " + f"error={response.get('error', 'unknown')}" + ) + self.print( + f"RegexGenerator LLM error: {response.get('error', 'unknown')}", + 0, + 1, + ) + return + + llm_text = response.get("text", "") + regex, rejection_reason = self._extract_regex_from_llm_text(llm_text) + if rejection_reason: + return + + record = { + "regex_type": self.pending_request["regex_type"], + "regex": regex, + "regex_hash": self._hash_regex(regex), + "backend_alias": self.pending_request["backend"], + "provider": response.get("provider"), + "model": response.get("model"), + "temperature": self.llm_temperature, + "prompt_version": PROMPT_VERSION, + "request_id": self.pending_request["request_id"], + "created_at": time.time(), + } + self._validate_and_store_regex(record) + + def _extract_regex_from_llm_text( + self, llm_text: str + ) -> tuple[str, str | None]: + raw_regex = self._extract_raw_regex_candidate(llm_text) + if raw_regex: + return raw_regex, None + + payload = self._extract_json_payload(llm_text) + if payload is None: + return "", "invalid_response" + + if not isinstance(payload, dict): + return "", "response_not_object" + + regex = payload.get("regex") + if not isinstance(regex, str) or not regex.strip(): + return "", "missing_regex" + + return regex.strip(), None + + @staticmethod + def _extract_raw_regex_candidate(llm_text: str) -> str: + if not isinstance(llm_text, str): + return "" + + text = RegexGenerator._strip_code_fences(llm_text).strip() + if not text: + return "" + + for line in text.splitlines(): + candidate = line.strip().strip("`").strip() + if not candidate: + continue + if candidate.lower().startswith("regex:"): + candidate = candidate.split(":", 1)[1].strip() + candidate = candidate.strip().strip('"').strip("'") + if ( + candidate.startswith("/") + and candidate.endswith("/") + and len(candidate) > 1 + ): + candidate = candidate[1:-1].strip() + if not candidate or " " in candidate or candidate.startswith("{"): + continue + if not re.search(r"[\^\$\[\]\(\)\{\}\\\.\|\*\+\?]", candidate): + continue + return candidate + + return "" + + @staticmethod + def _strip_code_fences(text: str) -> str: + stripped = text.strip() + if not stripped.startswith("```"): + return stripped + + lines = stripped.splitlines() + if lines and lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + return "\n".join(lines).strip() + + @staticmethod + def _extract_json_payload(llm_text: str) -> dict | None: + if not isinstance(llm_text, str): + return None + + candidates = [llm_text.strip()] + fenced_match = re.search( + r"```(?:json)?\s*(\{.*?\})\s*```", + llm_text, + flags=re.DOTALL, + ) + if fenced_match: + candidates.append(fenced_match.group(1).strip()) + + object_text = RegexGenerator._extract_first_json_object(llm_text) + if object_text: + candidates.append(object_text) + + for candidate in candidates: + if not candidate: + continue + try: + return json.loads(candidate) + except (TypeError, json.JSONDecodeError): + continue + + return None + + @staticmethod + def _extract_first_json_object(text: str) -> str | None: + start = text.find("{") + while start != -1: + depth = 0 + in_string = False + escaped = False + for idx in range(start, len(text)): + char = text[idx] + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + + if char == '"': + in_string = True + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return text[start : idx + 1].strip() + + start = text.find("{", start + 1) + + return None + + @staticmethod + def _short_preview(text: str, limit: int = 200) -> str: + text = " ".join(str(text).split()) + if len(text) <= limit: + return text + return f"{text[:limit]}..." + + @staticmethod + def _hash_regex(regex: str) -> str: + return sha256(regex.encode("utf-8")).hexdigest() + + def _validate_and_store_regex(self, record: dict): + try: + with self._regex_validation_timeout(): + validation_error = self._validate_regex(record["regex"]) + except TimeoutError: + self._store_rejected_regex(record, "regex_validation_timeout") + return + + if validation_error: + self._store_rejected_regex(record, validation_error) + return + + if self.storage.might_have_generated_regex(record["regex_hash"]): + if self.storage.get_existing_generated_regex( + record["regex_hash"] + ) or self.storage.was_rejected_in_current_run( + record["regex_hash"] + ): + self.log_detail( + f"Rejected duplicate regex request_id={record['request_id']} " + f"regex_type={record['regex_type']} regex={record['regex']}" + ) + self.print( + f"RegexGenerator rejected duplicate regex: {record['regex']}", + 2, + 0, + ) + return + + try: + with self._regex_validation_timeout(): + compiled_regex = re.compile(record["regex"]) + matched_benign, benign_match_score = ( + self._find_strong_benign_match( + record["regex_type"], + record["regex"], + compiled_regex, + ) + ) + except TimeoutError: + self._store_rejected_regex(record, "regex_validation_timeout") + return + + if matched_benign: + self._store_rejected_regex( + record, + "matched_benign_data_too_strong", + matched_benign_value=matched_benign, + benign_match_score=benign_match_score, + ) + return + + record["status"] = "accepted" + record["rejection_reason"] = None + record["matched_benign_value"] = None + self.storage.store_generated_regex(record) + self.log_detail( + f"Accepted regex request_id={record['request_id']} " + f"regex_type={record['regex_type']} regex={record['regex']}" + ) + + def _store_rejected_regex( + self, + record: dict, + rejection_reason: str, + matched_benign_value: str | None = None, + benign_match_score: float | None = None, + ): + record["status"] = "rejected" + record["rejection_reason"] = rejection_reason + record["matched_benign_value"] = matched_benign_value + self.storage.store_generated_regex(record) + extra = ( + f" matched_benign_value={matched_benign_value}" + if matched_benign_value + else "" + ) + if benign_match_score is not None: + extra += f" benign_match_score={benign_match_score:.2f}" + self.log_detail( + f"Rejected regex request_id={record['request_id']} " + f"regex_type={record['regex_type']} reason={rejection_reason}" + f"{extra} regex={record['regex']}" + ) + + def _regex_validation_timeout(self): + timeout = float(self.regex_validation_timeout_seconds) + if timeout <= 0: + return _NullTimeout() + return _SignalTimeout(timeout) + + def _validate_regex(self, regex: str) -> str | None: + try: + regex.encode("ascii") + except UnicodeEncodeError: + return "non_ascii_regex" + + if len(regex) > self.max_regex_length: + return "regex_too_long" + + if regex in {".*", ".+", "^.*$", "^.+$"}: + return "regex_too_broad" + + if "(?<=" in regex or "(? bool: + stripped_regex = regex.strip("^$()") + if "|" not in stripped_regex: + return False + + parts = [ + part.strip("()[]{}?+*.^$") for part in stripped_regex.split("|") + ] + parts = [part for part in parts if part] + if len(parts) < 4: + return False + return all(len(part) <= 2 for part in parts) + + def _find_strong_benign_match( + self, regex_type: str, regex_text: str, compiled_regex + ) -> tuple[str | None, float | None]: + regex_features = self._measure_regex_specificity(regex_text) + for value in self.storage.iter_benign_strings(regex_type): + score = self._compute_match_strength( + compiled_regex, value, regex_features + ) + if score >= self.benign_match_strength_threshold: + return value, score + return None, None + + def _compute_match_strength( + self, compiled_regex, value: str, regex_features: dict + ) -> float: + return compute_match_strength(compiled_regex, value, regex_features) + + @staticmethod + def _measure_regex_specificity(regex_text: str) -> dict: + return measure_regex_specificity(regex_text) diff --git a/modules/t_cell/README.md b/modules/t_cell/README.md new file mode 100644 index 0000000000..3a17280763 --- /dev/null +++ b/modules/t_cell/README.md @@ -0,0 +1,107 @@ +# T Cell Module + +`modules/t_cell/t_cell.py` implements an immune-inspired responder for Slips. + +It does not modify detector modules. Instead, it subscribes to the shared +`evidence_added` channel, reads the centrally assigned `evidence_signal`, and +creates one T Cell per: + +- responsible IP +- regex type +- normalized antigen value + +Main behavior: + +- only `PAMP` evidence starts antigen recognition and cell creation +- antigens are extracted from evidence fields plus linked DNS/HTTP/SSL altflows +- accepted regexes come from the existing RegexGenerator SQLite store +- `evidence.profile.ip` is the related host context, while containment and + T-cell ownership use the evidence's responsible IP +- stored `DAMP` observations raise the danger pressure used by + co-stimulation and context for the same responsible IP, and each new DAMP + reevaluates waiting cells on that responsible IP +- optional decision tracing writes a separate JSONL audit file showing which + evidence IDs contributed to threshold calculations +- co-stimulation and context scores decide whether the cell becomes tolerant, + activates, requests containment, or stores memory +- state `1 - antigen-recognized` and state `3 - activated` can each wait for + at most one configured Slips time window before timing out to `2 - anergic` + or `0 - mature`; waiting cells are explicitly marked as + `waiting for co-stimulation` or `waiting for context` +- once a cell reaches `5 - memory`, later matching evidence keeps it in memory + without emitting repeated `memory_stored` actions +- containment reuses the existing `new_blocking` payload shape +- all T Cell state is stored in its own SQLite DB and log file + +## State Machine + +```mermaid +stateDiagram-v2 + [*] --> S0 + + state "0 - mature" as S0 + state "1 - antigen-recognized" as S1 + state "2 - anergic" as S2 + state "3 - activated" as S3 + state "4 - effector" as S4 + state "5 - memory" as S5 + + S0 --> S1 : PAMP + antigen + regex match + S0 --> S2 : PAMP + antigen + no regex match + S0 --> S0 : DAMP only or no antigen + S2 --> S0 : anergy TTL expired + S1 --> S3 : co-stimulation threshold met + S1 --> S1 : later PAMP or DAMP re-check + S1 --> S2 : co-stimulation timeout + S3 --> S4 : context -> contain + S3 --> S5 : context -> remember + S3 --> S3 : later PAMP or DAMP re-check + S3 --> S0 : context timeout + S5 --> S5 : later matching evidence retained +``` + +Artifacts: + +- module log: `/t_cell/t_cell.log` +- optional trace file: `/t_cell/t_cell_trace.jsonl` + The configured trace path is always forced under the t_cell module output + directory. +- module DB: `/t_cell/t_cell.sqlite` +- offline HTML report: `/t_cell_report.html` + +## Local HTML Report + +Use the included offline report generator to build a static HTML page from a +completed or running Slips output directory: + +```bash +./venv/bin/python modules/t_cell/analyze_t_cell.py \ + --run-output-dir output/ +``` + +By default it writes: + +```text +output//t_cell_report.html +``` + +Open that HTML file locally in a browser. If you want a different filename, +pass `--out `. + +The report reads the T Cell SQLite DB first, then enriches the page with the +module log and decision trace when those files exist. That means it still gives +useful summaries when `log_verbosity` is `1` or `2`, and becomes more detailed +when verbosity `3` or decision tracing is enabled. + +What the report tells you: + +- whether the run was dominated by `PAMP`, `DAMP`, or both +- which evidence types, responsible IPs, targets, and antigens drove the run +- which T-cell state transitions happened and how many times +- which cells are currently waiting, activated, anergic, effector, or memory +- why thresholds were crossed when decision tracing was enabled +- which raw observations reached the T Cell module, even when log verbosity was + low + +See [docs/t_cell_module.md](../../docs/t_cell_module.md) for the full design, +configuration, formulas, and DB schema. diff --git a/modules/t_cell/__init__.py b/modules/t_cell/__init__.py new file mode 100644 index 0000000000..f436f14183 --- /dev/null +++ b/modules/t_cell/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only diff --git a/modules/t_cell/analyze_t_cell.py b/modules/t_cell/analyze_t_cell.py new file mode 100644 index 0000000000..3e3dc3097b --- /dev/null +++ b/modules/t_cell/analyze_t_cell.py @@ -0,0 +1,4508 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +""" +Generate a standalone local HTML report for a T Cell module run. + +Usage: + python3 modules/t_cell/analyze_t_cell.py \ + --run-output-dir output/ + +By default the script writes: + output//t_cell_report.html +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import sqlite3 +from collections import Counter, defaultdict, deque +from datetime import datetime, timezone +from html import escape +from pathlib import Path +from typing import Any, Iterable + +try: + import yaml +except ImportError: # pragma: no cover - optional runtime dependency + yaml = None + + +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +STATE_LABELS = { + 0: "0 - mature", + 1: "1 - antigen-recognized", + 2: "2 - anergic", + 3: "3 - activated", + 4: "4 - effector", + 5: "5 - memory", +} +STATE_CLASS = { + 0: "state-mature", + 1: "state-recognized", + 2: "state-anergic", + 3: "state-activated", + 4: "state-effector", + 5: "state-memory", +} +STATE_COLORS = { + "state-mature": "#0f766e", + "state-recognized": "#d97706", + "state-anergic": "#2563eb", + "state-activated": "#a21caf", + "state-effector": "#b91c1c", + "state-memory": "#15803d", +} +SIGNAL_COLORS = {"PAMP": "#c2410c", "DAMP": "#0369a1"} +TRACE_STAGE_COLORS = {"co_stimulation": "#b45309", "context": "#7c3aed"} +WAITING_LABELS = { + "co_stimulation": "waiting for co-stimulation", + "context": "waiting for context", +} +DEFAULT_COSTIM_WEIGHTS = { + "confidence": 0.35, + "related_pamps": 0.25, + "danger": 0.40, +} +DEFAULT_PRIMING_PROFILES = { + "PAMP": { + "strength": 1.0, + "co_stimulation_threshold_offset": 0.0, + "effector_threshold_offset": 0.0, + "memory_threshold_offset": 0.0, + "state_wait_timeout_factor": 1.0, + "effector_min_related_count_offset": 0, + "memory_min_related_count_offset": 0, + }, + "DAMP": { + "strength": 0.6, + "co_stimulation_threshold_offset": 0.15, + "effector_threshold_offset": 0.10, + "memory_threshold_offset": 0.05, + "state_wait_timeout_factor": 0.5, + "effector_min_related_count_offset": 1, + "memory_min_related_count_offset": 1, + }, +} +DEFAULT_DOC_CONFIG = { + "anergy_ttl_seconds": 21600.0, + "related_lookback_seconds": 3600.0, + "related_pamps_saturation": 5.0, + "danger_saturation": 2.5, + "damp_danger_weight": 1.5, + "co_stimulation_threshold": 0.65, + "co_stimulation_weights": DEFAULT_COSTIM_WEIGHTS, + "novelty_window_seconds": 86400.0, + "context_recent_window_seconds": 1800.0, + "effector_threshold": 0.70, + "effector_min_related_count": 4, + "effector_cooldown_seconds": 1800.0, + "memory_threshold": 0.60, + "memory_trend_ratio_max": 0.60, + "memory_min_related_count": 3, + "state_wait_timeout_seconds": 3600.0, + "priming_profiles": DEFAULT_PRIMING_PROFILES, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate an offline T Cell HTML report." + ) + parser.add_argument( + "--run-output-dir", + required=True, + help=( + "Slips run output directory. The T Cell DB is resolved from " + "metadata when it is stored under permanent_dir." + ), + ) + parser.add_argument( + "--out", + default="", + help="Output HTML path. Default: /t_cell_report.html", + ) + parser.add_argument( + "--max-observations", + type=int, + default=200, + help="Maximum recent observations to render in the report.", + ) + parser.add_argument( + "--max-log-lines", + type=int, + default=400, + help="Maximum recent module log lines to embed in the report.", + ) + parser.add_argument( + "--max-trace-rows", + type=int, + default=200, + help="Maximum recent trace rows to render in the report.", + ) + return parser.parse_args() + + +def load_json(raw_value: str, fallback): + try: + return json.loads(raw_value) + except (TypeError, ValueError): + return fallback + + +def parse_alerts_timestamp(raw_value: str | None) -> float | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + + fmts = ( + "%Y/%m/%d %H:%M:%S.%f%z", + "%Y/%m/%d %H:%M:%S.%f", + "%Y/%m/%d %H:%M:%S%z", + "%Y/%m/%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + ) + for fmt in fmts: + try: + value = datetime.strptime(text, fmt) + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.timestamp() + except ValueError: + continue + try: + return float(text) + except (TypeError, ValueError): + return None + + +def ts_to_iso(ts: float | None) -> str: + if ts is None: + return "n/a" + return ( + datetime.fromtimestamp(float(ts), tz=timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def state_label(state: int | None) -> str: + return STATE_LABELS.get(state, f"unknown:{state}") + + +def state_class(state: int | None) -> str: + return STATE_CLASS.get(state, "state-unknown") + + +def cell_waiting_label(cell: dict | None) -> str: + context = (cell or {}).get("context") or {} + return WAITING_LABELS.get(context.get("waiting_for"), "") + + +def display_cell_state(cell: dict) -> str: + label = state_label(cell.get("state")) + waiting_label = cell_waiting_label(cell) + if waiting_label: + return f"{label} ({waiting_label})" + return label + + +def shorten(value: Any, limit: int = 96) -> str: + text = str(value or "") + if len(text) <= limit: + return text + return text[: limit - 1] + "…" + + +def format_float(value: Any, digits: int = 3) -> str: + if value is None or value == "": + return "n/a" + try: + numeric = float(value) + except (TypeError, ValueError): + return str(value) + if math.isfinite(numeric) and abs(numeric - round(numeric)) < 1e-9: + return str(int(round(numeric))) + return f"{numeric:.{digits}f}" + + +def load_yaml_config(metadata_path: Path) -> dict: + if not metadata_path.exists() or yaml is None: + return {} + try: + return yaml.safe_load(metadata_path.read_text(encoding="utf-8")) or {} + except Exception: + return {} + + +def find_metadata_path(run_output_dir: Path) -> Path: + """ + Return the Slips metadata YAML path for a run output directory. + + Parameters: + run_output_dir: Slips run output directory. + + Returns: + Path to the preferred or first available metadata YAML file. + """ + metadata_dir = run_output_dir / "metadata" + preferred_path = metadata_dir / "slips.yaml" + if preferred_path.exists(): + return preferred_path + + yaml_paths = sorted(metadata_dir.glob("*.yaml")) + yaml_paths.extend(sorted(metadata_dir.glob("*.yml"))) + if yaml_paths: + return yaml_paths[0] + return preferred_path + + +def resolve_persistent_store_dir(raw_store_dir: str, metadata: dict) -> Path: + """ + Resolve a persistent store directory using the run metadata. + + Parameters: + raw_store_dir: Configured persistent store directory. + metadata: Parsed Slips metadata YAML. + + Returns: + Absolute path to the persistent store directory. + """ + store_dir = Path(str(raw_store_dir or "t_cell").strip()).expanduser() + if store_dir.is_absolute(): + return store_dir + + parameters = metadata.get("parameters", {}) + permanent_dir = Path(str(parameters.get("permanent_dir") or "permanent")) + permanent_dir = permanent_dir.expanduser() + if not permanent_dir.is_absolute(): + permanent_dir = Path.cwd() / permanent_dir + return permanent_dir.joinpath(*store_dir.parts) + + +def resolve_t_cell_db_path(run_output_dir: Path, metadata: dict) -> Path: + """ + Resolve the T Cell SQLite DB path for current and legacy runs. + + Parameters: + run_output_dir: Slips run output directory. + metadata: Parsed Slips metadata YAML. + + Returns: + Existing DB path when found, otherwise the run-local DB path. + """ + config = metadata.get("t_cell", {}) + persistent_store_dir = config.get("persistent_store_dir", "") + candidates = [] + if isinstance(persistent_store_dir, str) and persistent_store_dir.strip(): + candidates.append( + resolve_persistent_store_dir(persistent_store_dir, metadata) + / "t_cell.sqlite" + ) + candidates.append(run_output_dir / "t_cell" / "t_cell.sqlite") + + for db_path in candidates: + if db_path.exists(): + return db_path + return candidates[-1] + + +def sanitize_module_relative_path(raw_path: str, default_filename: str) -> Path: + """ + Normalize a configured module-local file path. + + Parameters: + raw_path: Configured path value. + default_filename: Filename to use when the configured value is empty. + + Returns: + Safe relative path inside the module output directory. + """ + normalized = str(raw_path or "").strip() or default_filename + normalized = normalized.replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + if Path(normalized).is_absolute(): + normalized = Path(normalized).name + if normalized.startswith("output/"): + normalized = normalized[len("output/") :] + + safe_parts = [] + for part in normalized.split("/"): + if not part or part in (".", ".."): + continue + if part.endswith(":"): + continue + safe_parts.append(part) + if not safe_parts: + safe_parts = [default_filename] + return Path(*safe_parts) + + +def first_existing_path(paths: Iterable[Path]) -> Path: + """ + Return the first existing path from a list of candidates. + + Parameters: + paths: Candidate paths ordered by preference. + + Returns: + First existing path, or the first candidate when none exist. + """ + paths = list(paths) + for path in paths: + if path.exists(): + return path + return paths[0] + + +def _row_to_observation(row: sqlite3.Row) -> dict: + return { + "id": row["id"], + "evidence_id": row["evidence_id"], + "evidence_type": row["evidence_type"], + "evidence_signal": row["evidence_signal"], + "responsible_ip": row["profile_ip"], + "timewindow_number": row["timewindow_number"], + "timestamp": row["timestamp"], + "observed_at": row["observed_at"], + "confidence": row["confidence"], + "threat_level": row["threat_level"], + "threat_level_value": row["threat_level_value"], + "interface": row["interface"], + "uids": load_json(row["uid_json"], []), + "antigen_count": row["antigen_count"], + "antigens": load_json(row["antigens_json"], []), + "matched_regexes": load_json(row["matched_regexes_json"], []), + "raw_evidence": load_json(row["raw_evidence_json"], {}), + } + + +def _row_to_cell(row: sqlite3.Row) -> dict: + return { + "cell_key": row["cell_key"], + "responsible_ip": row["profile_ip"], + "regex_type": row["regex_type"], + "antigen_value": row["antigen_value"], + "state": row["state"], + "state_name": row["state_name"], + "matched_regex_hash": row["matched_regex_hash"], + "matched_regex": row["matched_regex"], + "matched_value": row["matched_value"], + "anergic_until": row["anergic_until"], + "effector_cooldown_until": row["effector_cooldown_until"], + "last_observation_id": row["last_observation_id"], + "last_evidence_id": row["last_evidence_id"], + "last_transition_at": row["last_transition_at"], + "last_co_stimulation": row["last_co_stimulation"], + "last_effector_score": row["last_effector_score"], + "last_memory_score": row["last_memory_score"], + "context": load_json(row["context_json"], {}), + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def _row_to_transition(row: sqlite3.Row) -> dict: + return { + "id": row["id"], + "cell_key": row["cell_key"], + "responsible_ip": row["profile_ip"], + "regex_type": row["regex_type"], + "antigen_value": row["antigen_value"], + "evidence_id": row["evidence_id"], + "observation_id": row["observation_id"], + "from_state": row["from_state"], + "to_state": row["to_state"], + "reason": row["reason"], + "matched_regex_hash": row["matched_regex_hash"], + "matched_regex": row["matched_regex"], + "matched_value": row["matched_value"], + "scores": load_json(row["scores_json"], {}), + "created_at": row["created_at"], + } + + +def _row_to_memory(row: sqlite3.Row) -> dict: + return { + "cell_key": row["cell_key"], + "responsible_ip": row["profile_ip"], + "regex_type": row["regex_type"], + "antigen_value": row["antigen_value"], + "regex_hash": row["regex_hash"], + "regex": row["regex"], + "matched_value": row["matched_value"], + "context": load_json(row["context_json"], {}), + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def load_db_records(db_path: Path) -> dict: + if not db_path.exists(): + raise FileNotFoundError(f"T Cell DB not found: {db_path}") + + with sqlite3.connect(db_path) as conn: + conn.row_factory = sqlite3.Row + observations = [ + _row_to_observation(row) + for row in conn.execute( + "SELECT * FROM observations ORDER BY observed_at ASC, id ASC" + ) + ] + cells = [ + _row_to_cell(row) + for row in conn.execute( + "SELECT * FROM cells ORDER BY updated_at DESC, created_at DESC" + ) + ] + transitions = [ + _row_to_transition(row) + for row in conn.execute( + "SELECT * FROM transitions ORDER BY created_at ASC, id ASC" + ) + ] + memories = [ + _row_to_memory(row) + for row in conn.execute( + "SELECT * FROM memories ORDER BY updated_at DESC, created_at DESC" + ) + ] + return { + "observations": observations, + "cells": cells, + "transitions": transitions, + "memories": memories, + } + + +def parse_log_line(raw_line: str) -> dict | None: + line = ANSI_RE.sub("", raw_line.strip()) + if not line: + return None + + parts = [part.strip() for part in line.split(" | ")] + record = {"raw": line, "wall": parts[0], "ts": parse_alerts_timestamp(parts[0])} + extras = [] + for part in parts[1:]: + if "=" in part: + key, value = part.split("=", 1) + record[key] = value + else: + extras.append(part) + if extras: + record["details"] = " | ".join(extras) + return record + + +def load_log_entries(log_path: Path, max_lines: int) -> dict: + if not log_path.exists(): + return {"entries": [], "tail": []} + + entries = [] + tail = deque(maxlen=max(1, max_lines)) + with log_path.open("r", encoding="utf-8", errors="replace") as handle: + for raw_line in handle: + line = raw_line.rstrip("\n") + tail.append(ANSI_RE.sub("", line)) + parsed = parse_log_line(line) + if parsed: + entries.append(parsed) + return {"entries": entries, "tail": list(tail)} + + +def load_trace_entries(trace_path: Path) -> list[dict]: + if not trace_path.exists(): + return [] + entries = [] + with trace_path.open("r", encoding="utf-8", errors="replace") as handle: + for line in handle: + text = line.strip() + if not text: + continue + try: + entry = json.loads(text) + except json.JSONDecodeError: + continue + entry["_ts"] = parse_alerts_timestamp(entry.get("ts")) + entries.append(entry) + return entries + + +def entity_ip(entity: dict | None) -> str: + if not isinstance(entity, dict): + return "" + raw_type = str(entity.get("ioc_type") or "").upper() + if raw_type.endswith("IP") or raw_type == "IP": + return str(entity.get("value") or "") + return "" + + +def observation_related_profile(observation: dict) -> str: + raw = observation.get("raw_evidence") or {} + profile = raw.get("profile") or {} + if isinstance(profile, dict) and profile.get("ip"): + return str(profile.get("ip")) + return observation.get("responsible_ip") or "" + + +def observation_target_ip(observation: dict) -> str: + raw = observation.get("raw_evidence") or {} + return entity_ip(raw.get("victim")) + + +def observation_description(observation: dict) -> str: + raw = observation.get("raw_evidence") or {} + return str(raw.get("description") or "") + + +def summarize_antigens(antigens: list[dict], limit: int = 4) -> str: + if not antigens: + return "none" + return ", ".join( + f"{item.get('regex_type')}:{item.get('value')}" + for item in antigens[:limit] + ) + + +def summarize_matched_regexes(matches: list[dict], limit: int = 2) -> str: + if not matches: + return "none" + return ", ".join( + f"{item.get('regex_type')}:{item.get('value')}" + for item in matches[:limit] + ) + + +def categorize_observation(observation: dict, transition_map: dict[int, list[dict]]) -> str: + signal = observation.get("evidence_signal") + if signal != "PAMP": + matches = observation.get("matched_regexes") or [] + if matches: + return "DAMP with regex match" + transitions = transition_map.get(observation["id"], []) + if any(item.get("reason") == "no_regex_match" for item in transitions): + return "DAMP with no regex match" + if observation.get("antigen_count", 0) > 0: + return "DAMP with extracted antigens" + return "DAMP with no antigen" + + if observation.get("antigen_count", 0) <= 0: + return "PAMP with no antigen" + + matches = observation.get("matched_regexes") or [] + if matches: + return "PAMP with regex match" + + transitions = transition_map.get(observation["id"], []) + if any(item.get("reason") == "no_regex_match" for item in transitions): + return "PAMP with no regex match" + return "PAMP with antigens but no stored match" + + +def top_counts(counter: Counter, limit: int = 12) -> list[dict]: + return [ + {"label": label, "count": count} + for label, count in counter.most_common(limit) + ] + + +def safe_div(num: float, den: float) -> float: + if not den: + return 0.0 + return num / den + + +def normalize_costim_weights(weights: Any) -> dict[str, float]: + if not isinstance(weights, dict): + weights = {} + sanitized = {} + for key, default_value in DEFAULT_COSTIM_WEIGHTS.items(): + raw_value = weights.get(key, default_value) + try: + raw_value = float(raw_value) + except (TypeError, ValueError): + raw_value = default_value + sanitized[key] = max(0.0, raw_value) + + total = sum(sanitized.values()) + if total <= 0: + total = sum(DEFAULT_COSTIM_WEIGHTS.values()) + sanitized = DEFAULT_COSTIM_WEIGHTS.copy() + return {key: value / total for key, value in sanitized.items()} + + +def normalize_priming_profiles(profiles: Any) -> dict[str, dict[str, float | int]]: + raw_profiles = profiles if isinstance(profiles, dict) else {} + normalized: dict[str, dict[str, float | int]] = {} + for signal_name, defaults in DEFAULT_PRIMING_PROFILES.items(): + raw_profile = raw_profiles.get(signal_name, {}) + if not isinstance(raw_profile, dict): + raw_profile = {} + + profile: dict[str, float | int] = {} + for key, default_value in defaults.items(): + raw_value = raw_profile.get(key, default_value) + if isinstance(default_value, int) and not isinstance( + default_value, bool + ): + try: + raw_value = int(raw_value) + except (TypeError, ValueError): + raw_value = int(default_value) + else: + try: + raw_value = float(raw_value) + except (TypeError, ValueError): + raw_value = float(default_value) + profile[key] = raw_value + + profile["strength"] = max(0.0, min(1.0, float(profile["strength"]))) + profile["state_wait_timeout_factor"] = max( + 0.01, float(profile["state_wait_timeout_factor"]) + ) + profile["co_stimulation_threshold_offset"] = float( + profile["co_stimulation_threshold_offset"] + ) + profile["effector_threshold_offset"] = float( + profile["effector_threshold_offset"] + ) + profile["memory_threshold_offset"] = float( + profile["memory_threshold_offset"] + ) + profile["effector_min_related_count_offset"] = int( + profile["effector_min_related_count_offset"] + ) + profile["memory_min_related_count_offset"] = int( + profile["memory_min_related_count_offset"] + ) + normalized[signal_name] = profile + return normalized + + +def effective_priming_profile(config: dict, signal_name: str) -> dict: + priming_profiles = normalize_priming_profiles(config.get("priming_profiles")) + normalized_signal = str(signal_name or "PAMP").strip().upper() + if normalized_signal not in priming_profiles: + normalized_signal = "PAMP" + profile = priming_profiles[normalized_signal] + return { + "signal": normalized_signal, + "label": f"{normalized_signal.lower()}-primed", + "strength": float(profile["strength"]), + "base_co_stimulation_threshold": float(config["co_stimulation_threshold"]), + "base_effector_threshold": float(config["effector_threshold"]), + "base_memory_threshold": float(config["memory_threshold"]), + "base_state_wait_timeout_seconds": float( + config["state_wait_timeout_seconds"] + ), + "base_effector_min_related_count": int( + config["effector_min_related_count"] + ), + "base_memory_min_related_count": int(config["memory_min_related_count"]), + "co_stimulation_threshold_offset": float( + profile["co_stimulation_threshold_offset"] + ), + "effector_threshold_offset": float(profile["effector_threshold_offset"]), + "memory_threshold_offset": float(profile["memory_threshold_offset"]), + "state_wait_timeout_factor": float(profile["state_wait_timeout_factor"]), + "effector_min_related_count_offset": int( + profile["effector_min_related_count_offset"] + ), + "memory_min_related_count_offset": int( + profile["memory_min_related_count_offset"] + ), + "co_stimulation_threshold": max( + 0.0, + min( + 1.0, + float(config["co_stimulation_threshold"]) + + float(profile["co_stimulation_threshold_offset"]), + ), + ), + "effector_threshold": max( + 0.0, + min( + 1.0, + float(config["effector_threshold"]) + + float(profile["effector_threshold_offset"]), + ), + ), + "memory_threshold": max( + 0.0, + min( + 1.0, + float(config["memory_threshold"]) + + float(profile["memory_threshold_offset"]), + ), + ), + "state_wait_timeout_seconds": max( + 1.0, + float(config["state_wait_timeout_seconds"]) + * float(profile["state_wait_timeout_factor"]), + ), + "effector_min_related_count": max( + 1, + int(config["effector_min_related_count"]) + + int(profile["effector_min_related_count_offset"]), + ), + "memory_min_related_count": max( + 1, + int(config["memory_min_related_count"]) + + int(profile["memory_min_related_count_offset"]), + ), + } + + +def effective_priming_profiles(config: dict) -> dict[str, dict]: + return { + signal_name: effective_priming_profile(config, signal_name) + for signal_name in DEFAULT_PRIMING_PROFILES + } + + +def coerce_time_window_width(raw_value: Any) -> float: + if raw_value in (None, ""): + return float(DEFAULT_DOC_CONFIG["state_wait_timeout_seconds"]) + try: + return float(raw_value) + except (TypeError, ValueError): + text = str(raw_value) + if "only_one_tw" in text: + return 9999999999.0 + return float(DEFAULT_DOC_CONFIG["state_wait_timeout_seconds"]) + + +def report_config_with_defaults(report: dict) -> dict: + config = dict(report.get("config") or {}) + merged = {} + for key, default_value in DEFAULT_DOC_CONFIG.items(): + raw_value = config.get(key, default_value) + if key == "co_stimulation_weights": + merged[key] = normalize_costim_weights(raw_value) + continue + if key == "priming_profiles": + merged[key] = normalize_priming_profiles(raw_value) + continue + if key == "state_wait_timeout_seconds": + merged[key] = coerce_time_window_width(raw_value) + continue + if raw_value in (None, "", {}): + raw_value = default_value + merged[key] = raw_value + merged["effective_priming_profiles"] = effective_priming_profiles(merged) + return merged + + +def build_findings(report: dict) -> list[str]: + totals = report["totals"] + categories = report["observation_categories"] + current_states = report["cell_states"] + findings = [] + + if totals["observations"] == 0: + findings.append("No T-cell observations were stored for this run.") + return findings + + damp_count = totals["signals"].get("DAMP", 0) + pamp_count = totals["signals"].get("PAMP", 0) + if damp_count: + ratio = safe_div(damp_count, totals["observations"]) * 100.0 + findings.append( + f"Most evidence was DAMP: {damp_count}/{totals['observations']} " + f"observations ({ratio:.1f}%)." + ) + if pamp_count and categories.get("PAMP with no antigen", 0): + findings.append( + f"{categories['PAMP with no antigen']} PAMP observations stopped before " + "regex matching because no supported antigen could be extracted." + ) + if categories.get("PAMP with no regex match", 0): + findings.append( + f"{categories['PAMP with no regex match']} PAMP observations reached " + "antigen extraction but did not match any accepted regex." + ) + if categories.get("DAMP with no regex match", 0): + findings.append( + f"{categories['DAMP with no regex match']} DAMP observations reached " + "antigen extraction but did not match any accepted regex, so the " + "cell became anergic." + ) + if totals["cells"] == 0 and totals["transitions"] == 0: + findings.append( + "No T-cell was ever created, so no state transition, effector action, " + "or memory write could happen." + ) + if totals["transitions_to_state"].get("3 - activated", 0): + findings.append( + f"{totals['transitions_to_state']['3 - activated']} activation " + "transition(s) reached state 3." + ) + if totals["transitions_to_state"].get("4 - effector", 0): + findings.append( + f"{totals['transitions_to_state']['4 - effector']} effector " + "transition(s) requested containment." + ) + if totals["memories"]: + findings.append( + f"{totals['memories']} memory cell(s) were stored for later reuse." + ) + if current_states.get("1 - antigen-recognized", 0): + findings.append( + f"{current_states['1 - antigen-recognized']} cell(s) are currently waiting " + "for co-stimulation." + ) + if current_states.get("3 - activated", 0): + findings.append( + f"{current_states['3 - activated']} cell(s) are currently waiting " + "for context." + ) + if report["sources"]["trace_enabled"] and not report["trace"]["rows"]: + findings.append( + "Decision tracing was enabled, but no trace rows were written." + ) + if not report["sources"]["trace_enabled"]: + findings.append( + "Decision trace was off for this run, so threshold-by-threshold " + "explanations are not available." + ) + return findings[:8] + + +def build_timelines( + observations: list[dict], + transitions: list[dict], + trace_rows: list[dict], +) -> dict: + observation_items = [ + {"ts": item["observed_at"], "signal": item["evidence_signal"]} + for item in observations + if item.get("observed_at") is not None + ] + transition_items = [ + {"ts": item["created_at"], "to_state": state_label(item.get("to_state"))} + for item in transitions + if item.get("created_at") is not None + ] + trace_items = [ + { + "ts": item.get("_ts"), + "stage": item.get("stage"), + "action": item.get("action"), + } + for item in trace_rows + if item.get("_ts") is not None + ] + + return { + "observations": bucket_items( + observation_items, + { + "PAMP observations": lambda item: item["signal"] == "PAMP", + "DAMP observations": lambda item: item["signal"] == "DAMP", + }, + ), + "transitions": bucket_items( + transition_items, + { + "recognized": lambda item: item["to_state"] == "1 - antigen-recognized", + "anergic": lambda item: item["to_state"] == "2 - anergic", + "activated": lambda item: item["to_state"] == "3 - activated", + "effector": lambda item: item["to_state"] == "4 - effector", + "memory": lambda item: item["to_state"] == "5 - memory", + }, + ), + "trace": bucket_items( + trace_items, + { + "co-stimulation": lambda item: item["stage"] == "co_stimulation", + "context": lambda item: item["stage"] == "context", + }, + ), + } + + +def bucket_items( + items: list[dict], series_predicates: dict[str, Any], bin_count: int = 36 +) -> dict: + timed = [item for item in items if item.get("ts") is not None] + if not timed: + return {} + + min_ts = min(float(item["ts"]) for item in timed) + max_ts = max(float(item["ts"]) for item in timed) + if max_ts <= min_ts: + max_ts = min_ts + 1.0 + bin_count = max(8, min(bin_count, 72)) + width = (max_ts - min_ts) / bin_count + if width <= 0: + width = 1.0 + + labels = [] + series = {name: [0] * bin_count for name in series_predicates} + for index in range(bin_count): + center = min_ts + ((index + 0.5) * width) + labels.append(ts_to_iso(center)) + + for item in timed: + idx = int((float(item["ts"]) - min_ts) / width) + idx = max(0, min(idx, bin_count - 1)) + for name, predicate in series_predicates.items(): + if predicate(item): + series[name][idx] += 1 + + return { + "labels": labels, + "series": series, + "min_ts": min_ts, + "max_ts": max_ts, + "width": width, + "bin_count": bin_count, + } + + +def trace_row_cell_key(entry: dict) -> str: + if entry.get("cell_key"): + return str(entry.get("cell_key")) + candidate = entry.get("candidate") or {} + responsible_ip = str(entry.get("responsible_ip") or "") + regex_type = str(candidate.get("regex_type") or "") + antigen_value = str(candidate.get("value") or "") + if responsible_ip and regex_type and antigen_value: + return f"{responsible_ip}|{regex_type}|{antigen_value}" + return "" + + +def describe_current_evidence(current_evidence: dict | None) -> str: + current_evidence = current_evidence or {} + evidence_id = current_evidence.get("evidence_id") or "n/a" + evidence_type = current_evidence.get("evidence_type") or "unknown" + signal = current_evidence.get("signal") or "unknown" + confidence = format_float(current_evidence.get("confidence")) + threat_level = current_evidence.get("threat_level") or "unknown" + threat_level_value = format_float(current_evidence.get("threat_level_value")) + danger = format_float(current_evidence.get("danger_contribution")) + observation_id = current_evidence.get("observation_id") + observation_part = ( + f"obs={observation_id} | " if observation_id not in (None, "") else "" + ) + return ( + f"{observation_part}eid={evidence_id} | {evidence_type} | {signal} | " + f"conf={confidence} | threat={threat_level} ({threat_level_value}) | " + f"danger={danger}" + ) + + +def describe_observation_row(observation: dict | None) -> str: + observation = observation or {} + if not observation: + return "no linked observation row" + return ( + f"obs={observation.get('id')} | eid={observation.get('evidence_id')} | " + f"{observation.get('evidence_type')} | {observation.get('evidence_signal')} | " + f"conf={format_float(observation.get('confidence'))} | " + f"threat={observation.get('threat_level')} " + f"({format_float(observation.get('threat_level_value'))}) | " + f"antigens={summarize_antigens(observation.get('antigens') or [])} | " + f"matches={summarize_matched_regexes(observation.get('matched_regexes') or [])}" + ) + + +def describe_trace_contributor(prefix: str, contributor: dict) -> str: + relations = contributor.get("relations") or [] + relations_text = f" | relations={','.join(relations)}" if relations else "" + return ( + f"{prefix}: obs={contributor.get('observation_id')} | " + f"eid={contributor.get('evidence_id')} | {contributor.get('evidence_type')} | " + f"{contributor.get('signal')} | conf={format_float(contributor.get('confidence'))} | " + f"threat={contributor.get('threat_level')} " + f"({format_float(contributor.get('threat_level_value'))}) | " + f"danger={format_float(contributor.get('danger_contribution'))}" + f"{relations_text}" + ) + + +def summarize_lines(lines: list[str], fallback: str = "n/a", limit: int = 2) -> str: + cleaned = [str(line).strip() for line in lines if str(line).strip()] + if not cleaned: + return fallback + summary = " | ".join(cleaned[:limit]) + if len(cleaned) > limit: + summary += f" | +{len(cleaned) - limit} more" + return summary + + +def generic_threshold_result(scores: dict) -> tuple[str, list[str]]: + if not isinstance(scores, dict): + return ("n/a", ["No threshold snapshot was stored for this transition."]) + + if "value" in scores and "threshold" in scores: + value = scores.get("value") + threshold = scores.get("threshold") + passed = float(value) >= float(threshold) + comparator = ">=" if passed else "<" + status = "passed" if passed else "failed" + return ( + f"{status}: {format_float(value)} {comparator} {format_float(threshold)}", + [ + f"value={format_float(value)}", + f"threshold={format_float(threshold)}", + ], + ) + + result_lines = [] + summary_bits = [] + if "effector_score" in scores and "effector_threshold" in scores: + passed = float(scores["effector_score"]) >= float(scores["effector_threshold"]) + summary_bits.append( + "effector " + + ("passed" if passed else "failed") + + f": {format_float(scores['effector_score'])} " + + (">=" if passed else "<") + + f" {format_float(scores['effector_threshold'])}" + ) + result_lines.append(summary_bits[-1]) + if "memory_score" in scores and "memory_threshold" in scores: + passed = float(scores["memory_score"]) >= float(scores["memory_threshold"]) + summary_bits.append( + "memory " + + ("passed" if passed else "failed") + + f": {format_float(scores['memory_score'])} " + + (">=" if passed else "<") + + f" {format_float(scores['memory_threshold'])}" + ) + result_lines.append(summary_bits[-1]) + if not summary_bits: + return ("n/a", ["No threshold keys were stored in this score snapshot."]) + return (" | ".join(summary_bits), result_lines) + + +def build_trace_threshold_result(entry: dict) -> tuple[str, list[str]]: + formula = entry.get("formula") or {} + stage = entry.get("stage") + action = entry.get("action") or "" + if stage == "co_stimulation": + value = formula.get("value") + threshold = formula.get("threshold") + if value is None or threshold is None: + return ("n/a", ["Missing co-stimulation value or threshold."]) + passed = float(value) >= float(threshold) + comparator = ">=" if passed else "<" + return ( + f"{'passed' if passed else 'failed'}: " + f"{format_float(value)} {comparator} {format_float(threshold)}", + [ + f"action={action}", + f"value={format_float(value)}", + f"threshold={format_float(threshold)}", + ], + ) + + if stage == "context": + decision = formula.get("decision") or {} + effector = bool(decision.get("effector")) + memory = bool(decision.get("memory")) + effector_score = formula.get("effector_score") + effector_threshold = formula.get("effector_threshold") + memory_score = formula.get("memory_score") + memory_threshold = formula.get("memory_threshold") + summary = ( + f"effector={'yes' if effector else 'no'} " + f"({format_float(effector_score)} / {format_float(effector_threshold)}) | " + f"memory={'yes' if memory else 'no'} " + f"({format_float(memory_score)} / {format_float(memory_threshold)})" + ) + return ( + summary, + [ + f"action={action}", + f"effector decision={'passed' if effector else 'failed'}", + f"memory decision={'passed' if memory else 'failed'}", + f"effector_score={format_float(effector_score)} threshold={format_float(effector_threshold)}", + f"memory_score={format_float(memory_score)} threshold={format_float(memory_threshold)}", + ], + ) + return ("n/a", ["No threshold formatter for this trace stage."]) + + +def build_trace_considered_evidence(entry: dict) -> tuple[str, list[str]]: + formula = entry.get("formula") or {} + stage = entry.get("stage") + lines = [] + current_evidence = entry.get("current_evidence") or {} + if current_evidence: + lines.append("current: " + describe_current_evidence(current_evidence)) + + components = formula.get("components") or {} + if stage == "co_stimulation": + related = (components.get("related_pamps") or {}).get("contributors") or [] + danger = components.get("danger") or {} + pamp_contributors = danger.get("pamp_contributors") or [] + damp_contributors = danger.get("damp_contributors") or [] + for contributor in related: + lines.append(describe_trace_contributor("related_pamp", contributor)) + for contributor in pamp_contributors: + lines.append(describe_trace_contributor("danger_pamp", contributor)) + for contributor in damp_contributors: + lines.append(describe_trace_contributor("danger_damp", contributor)) + elif stage == "context": + recent_related = (components.get("recent_related") or {}).get( + "contributors" + ) or [] + recent_pressure = components.get("recent_pressure") or {} + previous_pressure = components.get("previous_pressure") or {} + for contributor in recent_related: + lines.append(describe_trace_contributor("recent_related", contributor)) + for contributor in recent_pressure.get("pamp_contributors") or []: + lines.append(describe_trace_contributor("recent_pressure_pamp", contributor)) + for contributor in recent_pressure.get("damp_contributors") or []: + lines.append(describe_trace_contributor("recent_pressure_damp", contributor)) + for contributor in previous_pressure.get("pamp_contributors") or []: + lines.append(describe_trace_contributor("previous_pressure_pamp", contributor)) + for contributor in previous_pressure.get("damp_contributors") or []: + lines.append(describe_trace_contributor("previous_pressure_damp", contributor)) + + if not lines: + lines.append("No contributor evidence snapshot was stored for this event.") + return (summarize_lines(lines, fallback="no stored evidence inputs"), lines) + + +def build_trace_computation_lines(entry: dict) -> tuple[str, list[str]]: + formula = entry.get("formula") or {} + stage = entry.get("stage") + priming = formula.get("priming") or {} + priming_lines = [] + if priming: + priming_lines = [ + ( + "priming: label=" + f"{priming.get('label', 'n/a')} signal=" + f"{priming.get('signal', 'n/a')} strength=" + f"{format_float(priming.get('strength'))}" + ), + ( + "effective thresholds: co=" + f"{format_float(priming.get('co_stimulation_threshold'))} " + "effector=" + f"{format_float(priming.get('effector_threshold'))} memory=" + f"{format_float(priming.get('memory_threshold'))}" + ), + ( + "effective wait/counts: wait=" + f"{format_float(priming.get('state_wait_timeout_seconds'))} " + "effector_related>=" + f"{format_float(priming.get('effector_min_related_count'))} " + "memory_related>=" + f"{format_float(priming.get('memory_min_related_count'))}" + ), + ] + if stage == "co_stimulation": + components = formula.get("components") or {} + confidence = components.get("confidence") or {} + related = components.get("related_pamps") or {} + danger = components.get("danger") or {} + lines = [ + f"value={format_float(formula.get('value'))}", + f"threshold={format_float(formula.get('threshold'))}", + ( + "confidence: value=" + f"{format_float(confidence.get('value'))} weighted=" + f"{format_float(confidence.get('weighted'))}" + ), + ( + "related_pamps: count=" + f"{related.get('count', 'n/a')} saturation=" + f"{format_float(related.get('saturation'))} score=" + f"{format_float(related.get('score'))} weighted=" + f"{format_float(related.get('weighted'))}" + ), + ( + "danger: score=" + f"{format_float(danger.get('score'))} weighted=" + f"{format_float(danger.get('weighted'))} pamp_score=" + f"{format_float(danger.get('pamp_score'))} damp_score=" + f"{format_float(danger.get('damp_score'))} damp_weight=" + f"{format_float(danger.get('damp_weight'))} saturation=" + f"{format_float(danger.get('danger_saturation'))}" + ), + ] + lines.extend(priming_lines) + return (summarize_trace_formula(formula, stage), lines) + + if stage == "context": + components = formula.get("components") or {} + novelty = components.get("novelty") or {} + recent_related = components.get("recent_related") or {} + recent_pressure = components.get("recent_pressure") or {} + previous_pressure = components.get("previous_pressure") or {} + lines = [ + ( + "effector_score=" + f"{format_float(formula.get('effector_score'))} threshold=" + f"{format_float(formula.get('effector_threshold'))}" + ), + ( + "memory_score=" + f"{format_float(formula.get('memory_score'))} threshold=" + f"{format_float(formula.get('memory_threshold'))}" + ), + ( + "decision flags: effector=" + f"{'yes' if (formula.get('decision') or {}).get('effector') else 'no'} " + "memory=" + f"{'yes' if (formula.get('decision') or {}).get('memory') else 'no'}" + ), + ( + "novelty: score=" + f"{format_float(novelty.get('score'))} has_memory=" + f"{'yes' if novelty.get('has_memory_for_regex') else 'no'} " + "recent_activity=" + f"{'yes' if novelty.get('has_recent_regex_activity') else 'no'}" + ), + ( + "recent_related: count=" + f"{recent_related.get('count', 'n/a')} saturation=" + f"{format_float(recent_related.get('saturation'))} score=" + f"{format_float(recent_related.get('score'))}" + ), + ( + "recent_pressure: combined=" + f"{format_float(recent_pressure.get('combined_score'))} pamp=" + f"{format_float(recent_pressure.get('pamp_score'))} damp=" + f"{format_float(recent_pressure.get('damp_score'))} " + "raw_pamp=" + f"{format_float(recent_pressure.get('pamp_total_raw'))} raw_damp=" + f"{format_float(recent_pressure.get('damp_total_raw'))}" + ), + ( + "previous_pressure: combined=" + f"{format_float(previous_pressure.get('combined_score'))} pamp=" + f"{format_float(previous_pressure.get('pamp_score'))} damp=" + f"{format_float(previous_pressure.get('damp_score'))} " + "raw_pamp=" + f"{format_float(previous_pressure.get('pamp_total_raw'))} raw_damp=" + f"{format_float(previous_pressure.get('damp_total_raw'))}" + ), + f"trend_ratio={format_float(components.get('trend_ratio'))}", + f"decrease_score={format_float(components.get('decrease_score'))}", + f"familiarity_score={format_float(components.get('familiarity_score'))}", + f"stability_score={format_float(components.get('stability_score'))}", + ] + lines.extend(priming_lines) + return (summarize_trace_formula(formula, stage), lines) + + return ("n/a", ["No computation formatter for this trace stage."]) + + +def build_transition_computation_lines(transition: dict) -> tuple[str, list[str]]: + scores = transition.get("scores") or {} + if not scores: + return ("no score snapshot", ["This transition stored no score payload."]) + lines = [f"{key}={format_float(value)}" for key, value in sorted(scores.items())] + return (summarize_lines(lines, fallback="score snapshot"), lines) + + +def build_transition_event( + transition: dict, observations_by_id: dict[int, dict] +) -> dict: + observation = observations_by_id.get(int(transition.get("observation_id") or 0), {}) + threshold_summary, threshold_lines = generic_threshold_result( + transition.get("scores") or {} + ) + computation_summary, computation_lines = build_transition_computation_lines( + transition + ) + evidence_lines = [describe_observation_row(observation)] + return { + "ts": transition.get("created_at"), + "wall": ts_to_iso(transition.get("created_at")), + "source": "State transition", + "step": transition.get("reason") or "transition", + "stage": "transition", + "state_path": ( + f"{state_label(transition.get('from_state'))} → " + f"{state_label(transition.get('to_state'))}" + ), + "evidence_id": transition.get("evidence_id") or "", + "threshold_summary": threshold_summary, + "threshold_lines": threshold_lines, + "considered_summary": summarize_lines(evidence_lines), + "considered_lines": evidence_lines, + "computation_summary": computation_summary, + "computation_lines": computation_lines, + "priority": 2, + } + + +def build_trace_event(entry: dict) -> dict: + threshold_summary, threshold_lines = build_trace_threshold_result(entry) + considered_summary, considered_lines = build_trace_considered_evidence(entry) + computation_summary, computation_lines = build_trace_computation_lines(entry) + current_evidence = entry.get("current_evidence") or {} + evidence_id = current_evidence.get("evidence_id") or "" + evidence_type = current_evidence.get("evidence_type") or "" + signal = current_evidence.get("signal") or "" + if evidence_type or signal: + evidence_label = f"{evidence_id} | {evidence_type} | {signal}".strip(" |") + else: + evidence_label = evidence_id or "n/a" + return { + "ts": entry.get("_ts"), + "wall": entry.get("ts") or ts_to_iso(entry.get("_ts")), + "source": "Decision trace", + "step": f"{entry.get('stage') or 'trace'}: {entry.get('action') or 'event'}", + "stage": entry.get("stage") or "trace", + "state_path": ( + f"{entry.get('from_state') or 'n/a'} → {entry.get('to_state') or 'n/a'}" + ), + "evidence_id": evidence_label, + "threshold_summary": threshold_summary, + "threshold_lines": threshold_lines, + "considered_summary": considered_summary, + "considered_lines": considered_lines, + "computation_summary": computation_summary, + "computation_lines": computation_lines, + "priority": 1, + } + + +def build_life_path( + transitions_for_cell: list[dict], current_state_label: str | None +) -> str: + ordered = sorted( + transitions_for_cell, + key=lambda item: (float(item.get("created_at") or 0.0), int(item.get("id") or 0)), + ) + states = [] + for transition in ordered: + from_label = state_label(transition.get("from_state")) + to_label = state_label(transition.get("to_state")) + if not states: + states.append(from_label) + if states[-1] != from_label: + states.append(from_label) + if states[-1] != to_label: + states.append(to_label) + if not states and current_state_label: + states = [current_state_label] + elif current_state_label and states[-1] != current_state_label: + states.append(current_state_label) + return " → ".join(states) if states else "no recorded state changes" + + +def extract_history_priming( + cell: dict, + transitions_for_cell: list[dict], + report_config: dict, +) -> dict: + context = (cell or {}).get("context") or {} + priming_profile = context.get("priming_profile") or {} + if isinstance(priming_profile, dict) and priming_profile.get("signal"): + return { + "label": priming_profile.get("label") or "", + "signal": priming_profile.get("signal") or "", + "strength": priming_profile.get("strength"), + "profile": priming_profile, + } + + for transition in transitions_for_cell: + scores = transition.get("scores") or {} + signal_name = str(scores.get("priming_signal") or "").strip().upper() + if not signal_name: + continue + return { + "label": scores.get("priming_label") + or f"{signal_name.lower()}-primed", + "signal": signal_name, + "strength": scores.get("priming_strength"), + "profile": effective_priming_profile(report_config, signal_name), + } + + default_profile = effective_priming_profile(report_config, "PAMP") + return { + "label": default_profile["label"], + "signal": default_profile["signal"], + "strength": default_profile["strength"], + "profile": default_profile, + } + + +def build_cell_histories( + observations: list[dict], + cells: list[dict], + transitions: list[dict], + trace_rows: list[dict], + report_config: dict, +) -> list[dict]: + observations_by_id = { + int(observation["id"]): observation + for observation in observations + if observation.get("id") is not None + } + cells_by_key = { + str(cell.get("cell_key")): cell + for cell in cells + if cell.get("cell_key") + } + transitions_by_cell: dict[str, list[dict]] = defaultdict(list) + for transition in transitions: + cell_key = str(transition.get("cell_key") or "") + if cell_key: + transitions_by_cell[cell_key].append(transition) + + traces_by_cell: dict[str, list[dict]] = defaultdict(list) + for entry in trace_rows: + cell_key = trace_row_cell_key(entry) + if cell_key: + traces_by_cell[cell_key].append(entry) + + cell_keys = set(cells_by_key) | set(transitions_by_cell) | set(traces_by_cell) + histories = [] + for cell_key in sorted(cell_keys): + cell = cells_by_key.get(cell_key, {}) + cell_transitions = transitions_by_cell.get(cell_key, []) + cell_traces = traces_by_cell.get(cell_key, []) + + events = [build_trace_event(entry) for entry in cell_traces] + events.extend( + build_transition_event(transition, observations_by_id) + for transition in cell_transitions + ) + events.sort( + key=lambda item: ( + item.get("ts") is None, + float(item.get("ts") or 0.0), + int(item.get("priority") or 9), + item.get("step") or "", + ) + ) + + current_state_label = state_label(cell.get("state")) if cell else None + waiting_label = cell_waiting_label(cell) if cell else "" + first_ts_candidates = [ + float(item.get("ts")) + for item in events + if item.get("ts") is not None + ] + if cell.get("created_at") is not None: + first_ts_candidates.append(float(cell.get("created_at"))) + last_ts_candidates = [ + float(item.get("ts")) + for item in events + if item.get("ts") is not None + ] + if cell.get("updated_at") is not None: + last_ts_candidates.append(float(cell.get("updated_at"))) + first_seen = min(first_ts_candidates) if first_ts_candidates else None + last_seen = max(last_ts_candidates) if last_ts_candidates else None + priming = extract_history_priming(cell, cell_transitions, report_config) + + histories.append( + { + "cell_key": cell_key, + "responsible_ip": cell.get("responsible_ip") + or ( + cell_transitions[0].get("profile_ip") + if cell_transitions + else (cell_traces[0].get("responsible_ip") if cell_traces else "") + ), + "regex_type": cell.get("regex_type") + or ( + cell_transitions[0].get("regex_type") + if cell_transitions + else ((cell_traces[0].get("candidate") or {}).get("regex_type", "")) + ), + "antigen_value": cell.get("antigen_value") + or ( + cell_transitions[0].get("antigen_value") + if cell_transitions + else ((cell_traces[0].get("candidate") or {}).get("value", "")) + ), + "matched_value": cell.get("matched_value") + or ( + cell_transitions[-1].get("matched_value") + if cell_transitions + else ((cell_traces[-1].get("match") or {}).get("value", "")) + ), + "current_state": current_state_label or "unknown", + "current_state_class": state_class(cell.get("state")) + if cell + else "state-unknown", + "waiting_label": waiting_label, + "priming_label": priming.get("label") or "", + "priming_signal": priming.get("signal") or "", + "priming_strength": priming.get("strength"), + "priming_profile": priming.get("profile") or {}, + "life_path": build_life_path(cell_transitions, current_state_label), + "first_seen": ts_to_iso(first_seen), + "last_seen": ts_to_iso(last_seen), + "event_count": len(events), + "transition_count": len(cell_transitions), + "trace_count": len(cell_traces), + "events": events, + } + ) + + return histories + + +def build_report_payload( + run_output_dir: Path, + max_observations: int = 200, + max_log_lines: int = 400, + max_trace_rows: int = 200, +) -> dict: + run_output_dir = run_output_dir.expanduser().resolve() + metadata_path = find_metadata_path(run_output_dir) + metadata = load_yaml_config(metadata_path) + config = metadata.get("t_cell", {}) + db_path = resolve_t_cell_db_path(run_output_dir, metadata) + log_path = first_existing_path( + [ + run_output_dir / "t_cell" / "t_cell.log", + run_output_dir / "T Cell" / "t_cell.log", + ] + ) + trace_relative_path = sanitize_module_relative_path( + config.get("decision_trace_file"), "t_cell_trace.jsonl" + ) + trace_path = first_existing_path( + [ + run_output_dir / "t_cell" / trace_relative_path, + run_output_dir / "T Cell" / trace_relative_path, + run_output_dir / "t_cell" / "t_cell_trace.jsonl", + run_output_dir / "T Cell" / "t_cell_trace.jsonl", + ] + ) + db_records = load_db_records(db_path) + observations = db_records["observations"] + cells = db_records["cells"] + transitions = db_records["transitions"] + memories = db_records["memories"] + log_data = load_log_entries(log_path, max_log_lines) + trace_rows = load_trace_entries(trace_path) + parameters = metadata.get("parameters", {}) + report_config = report_config_with_defaults( + { + "config": { + "related_lookback_seconds": config.get( + "related_lookback_seconds" + ), + "related_pamps_saturation": config.get( + "related_pamps_saturation" + ), + "danger_saturation": config.get("danger_saturation"), + "damp_danger_weight": config.get("damp_danger_weight"), + "co_stimulation_threshold": config.get( + "co_stimulation_threshold" + ), + "co_stimulation_weights": config.get( + "co_stimulation_weights" + ), + "novelty_window_seconds": config.get( + "novelty_window_seconds" + ), + "context_recent_window_seconds": config.get( + "context_recent_window_seconds" + ), + "effector_threshold": config.get("effector_threshold"), + "effector_min_related_count": config.get( + "effector_min_related_count" + ), + "effector_cooldown_seconds": config.get( + "effector_cooldown_seconds" + ), + "memory_threshold": config.get("memory_threshold"), + "memory_trend_ratio_max": config.get( + "memory_trend_ratio_max" + ), + "memory_min_related_count": config.get( + "memory_min_related_count" + ), + "anergy_ttl_seconds": config.get("anergy_ttl_seconds"), + "state_wait_timeout_seconds": coerce_time_window_width( + parameters.get("time_window_width") + ), + "priming_profiles": config.get("priming_profiles"), + } + } + ) + + transitions_by_observation: dict[int, list[dict]] = defaultdict(list) + for transition in transitions: + observation_id = transition.get("observation_id") + if observation_id is not None: + transitions_by_observation[int(observation_id)].append(transition) + + signal_counts = Counter() + evidence_type_counts = Counter() + observation_categories = Counter() + responsible_ip_counts = Counter() + related_profile_counts = Counter() + target_ip_counts = Counter() + antigen_counts = Counter() + unmatched_pamp_antigens = Counter() + matched_regex_counts = Counter() + + recent_observations = [] + for observation in observations: + signal_counts[observation["evidence_signal"]] += 1 + evidence_type_counts[ + (observation["evidence_type"], observation["evidence_signal"]) + ] += 1 + responsible_ip_counts[observation["responsible_ip"]] += 1 + + related_profile = observation_related_profile(observation) + target_ip = observation_target_ip(observation) + if related_profile: + related_profile_counts[related_profile] += 1 + if target_ip: + target_ip_counts[target_ip] += 1 + + category = categorize_observation(observation, transitions_by_observation) + observation_categories[category] += 1 + + for antigen in observation["antigens"]: + key = f"{antigen.get('regex_type')}:{antigen.get('value')}" + antigen_counts[key] += 1 + if ( + observation["evidence_signal"] == "PAMP" + and not observation["matched_regexes"] + ): + unmatched_pamp_antigens[key] += 1 + for match in observation["matched_regexes"]: + matched_regex_counts[ + f"{match.get('regex_type')}:{match.get('value')}" + ] += 1 + + recent_observations.append( + { + "ts": observation["observed_at"], + "wall": ts_to_iso(observation["observed_at"]), + "evidence_id": observation["evidence_id"], + "evidence_type": observation["evidence_type"], + "signal": observation["evidence_signal"], + "responsible_ip": observation["responsible_ip"], + "related_profile": related_profile, + "target_ip": target_ip, + "category": category, + "antigens": summarize_antigens(observation["antigens"]), + "matched_regexes": summarize_matched_regexes( + observation["matched_regexes"] + ), + "description": observation_description(observation), + "timewindow": observation["timewindow_number"], + "confidence": observation["confidence"], + } + ) + + recent_observations.sort( + key=lambda item: (float(item["ts"]), item["evidence_id"]), reverse=True + ) + + transition_reason_counts = Counter() + transition_path_counts = Counter() + transitions_to_state = Counter() + recent_transitions = [] + for transition in transitions: + transition_reason_counts[transition["reason"]] += 1 + from_label = state_label(transition.get("from_state")) + to_label = state_label(transition.get("to_state")) + transition_path_counts[f"{from_label} -> {to_label}"] += 1 + transitions_to_state[to_label] += 1 + recent_transitions.append( + { + "ts": transition["created_at"], + "wall": ts_to_iso(transition["created_at"]), + "cell_key": transition["cell_key"], + "responsible_ip": transition["responsible_ip"], + "regex_type": transition["regex_type"], + "antigen_value": transition["antigen_value"], + "evidence_id": transition["evidence_id"], + "from_state": from_label, + "to_state": to_label, + "from_state_order": transition.get("from_state", -1), + "to_state_order": transition.get("to_state", -1), + "reason": transition["reason"], + "matched_value": transition.get("matched_value") or "", + "scores": transition.get("scores") or {}, + } + ) + recent_transitions.sort( + key=lambda item: ( + item["cell_key"].lower(), + float(item["ts"]), + int(item["from_state_order"]), + int(item["to_state_order"]), + item["evidence_id"], + ) + ) + + current_state_counts = Counter() + recent_cells = [] + for cell in cells: + label = state_label(cell["state"]) + current_state_counts[label] += 1 + recent_cells.append( + { + "ts": cell["updated_at"], + "wall": ts_to_iso(cell["updated_at"]), + "cell_key": cell["cell_key"], + "responsible_ip": cell["responsible_ip"], + "state": label, + "state_display": display_cell_state(cell), + "state_class": state_class(cell["state"]), + "regex_type": cell["regex_type"], + "antigen_value": cell["antigen_value"], + "matched_value": cell.get("matched_value") or "", + "last_co_stimulation": cell.get("last_co_stimulation"), + "last_effector_score": cell.get("last_effector_score"), + "last_memory_score": cell.get("last_memory_score"), + "last_evidence_id": cell.get("last_evidence_id") or "", + "waiting_label": cell_waiting_label(cell), + } + ) + recent_cells.sort(key=lambda item: item["ts"], reverse=True) + + recent_memories = [] + for memory in memories: + recent_memories.append( + { + "ts": memory["updated_at"], + "wall": ts_to_iso(memory["updated_at"]), + "cell_key": memory["cell_key"], + "responsible_ip": memory["responsible_ip"], + "regex_type": memory["regex_type"], + "antigen_value": memory["antigen_value"], + "matched_value": memory["matched_value"], + "regex_hash": memory["regex_hash"], + "context": memory.get("context") or {}, + } + ) + recent_memories.sort(key=lambda item: item["ts"], reverse=True) + + trace_action_counts = Counter() + recent_trace_rows = [] + for entry in trace_rows: + trace_action_counts[entry.get("action") or "unknown"] += 1 + formula = entry.get("formula") or {} + recent_trace_rows.append( + { + "ts": entry.get("_ts"), + "wall": entry.get("ts") or ts_to_iso(entry.get("_ts")), + "stage": entry.get("stage") or "", + "action": entry.get("action") or "", + "from_state": entry.get("from_state") or "", + "to_state": entry.get("to_state") or "", + "responsible_ip": entry.get("responsible_ip") or "", + "candidate": entry.get("candidate") or {}, + "match": entry.get("match") or {}, + "formula": formula, + "score_summary": summarize_trace_formula(formula, entry.get("stage")), + } + ) + recent_trace_rows.sort( + key=lambda item: (item["ts"] is None, item["ts"] or 0.0), reverse=True + ) + + log_action_counts = Counter( + entry.get("action", "unknown") for entry in log_data["entries"] if entry + ) + recent_log_rows = [ + { + "ts": entry.get("ts"), + "wall": entry.get("wall") or "", + "action": entry.get("action", ""), + "signal": entry.get("signal", ""), + "evidence": entry.get("evidence", ""), + "responsible": entry.get("responsible", ""), + "raw": entry.get("raw", ""), + } + for entry in log_data["entries"][-max(1, max_log_lines) :] + ] + cell_histories = build_cell_histories( + observations=observations, + cells=cells, + transitions=transitions, + trace_rows=trace_rows, + report_config=report_config, + ) + + report = { + "generated_at": now_iso(), + "run_output_dir": str(run_output_dir), + "sources": { + "db_path": str(db_path), + "log_path": str(log_path), + "trace_path": str(trace_path), + "metadata_path": str(metadata_path), + "trace_enabled": bool(trace_path.exists()), + "log_present": log_path.exists(), + "metadata_present": metadata_path.exists(), + }, + "config": { + "enabled": config.get("enabled"), + "log_verbosity": config.get("log_verbosity"), + "decision_trace_mode": config.get("decision_trace_mode"), + "related_lookback_seconds": config.get("related_lookback_seconds"), + "related_pamps_saturation": config.get("related_pamps_saturation"), + "danger_saturation": config.get("danger_saturation"), + "damp_danger_weight": config.get("damp_danger_weight"), + "co_stimulation_threshold": config.get("co_stimulation_threshold"), + "co_stimulation_weights": normalize_costim_weights( + config.get("co_stimulation_weights") + ), + "novelty_window_seconds": config.get("novelty_window_seconds"), + "context_recent_window_seconds": config.get( + "context_recent_window_seconds" + ), + "effector_threshold": config.get("effector_threshold"), + "effector_min_related_count": config.get( + "effector_min_related_count" + ), + "memory_threshold": config.get("memory_threshold"), + "memory_trend_ratio_max": config.get("memory_trend_ratio_max"), + "memory_min_related_count": config.get("memory_min_related_count"), + "anergy_ttl_seconds": config.get("anergy_ttl_seconds"), + "effector_cooldown_seconds": config.get("effector_cooldown_seconds"), + "state_wait_timeout_seconds": coerce_time_window_width( + parameters.get("time_window_width") + ), + "priming_profiles": config.get("priming_profiles"), + }, + "totals": { + "observations": len(observations), + "cells": len(cells), + "transitions": len(transitions), + "memories": len(memories), + "trace_rows": len(trace_rows), + "log_rows": len(log_data["entries"]), + "observations_with_antigens": sum( + 1 for item in observations if item["antigen_count"] > 0 + ), + "observations_with_matches": sum( + 1 for item in observations if item["matched_regexes"] + ), + "signals": dict(signal_counts), + "transitions_to_state": dict(transitions_to_state), + }, + "observation_categories": dict(observation_categories), + "cell_states": dict(current_state_counts), + "top_signals_by_type": [ + { + "evidence_type": evidence_type, + "signal": signal, + "count": count, + } + for (evidence_type, signal), count in evidence_type_counts.most_common(20) + ], + "top_responsible_ips": top_counts(responsible_ip_counts), + "top_related_profiles": top_counts(related_profile_counts), + "top_targets": top_counts(target_ip_counts), + "top_antigens": top_counts(antigen_counts, limit=20), + "top_unmatched_pamp_antigens": top_counts(unmatched_pamp_antigens, limit=20), + "top_matched_regexes": top_counts(matched_regex_counts, limit=20), + "transition_reasons": top_counts(transition_reason_counts, limit=20), + "transition_paths": top_counts(transition_path_counts, limit=20), + "trace_action_counts": top_counts(trace_action_counts, limit=20), + "log_action_counts": top_counts(log_action_counts, limit=20), + "recent_observations": recent_observations[: max(1, max_observations)], + "recent_transitions": recent_transitions[: max(1, max_observations)], + "recent_cells": recent_cells[: max(1, max_observations)], + "recent_memories": recent_memories[: max(1, max_observations)], + "trace": { + "rows": recent_trace_rows[: max(1, max_trace_rows)], + "total_rows": len(trace_rows), + }, + "cell_histories": cell_histories, + "log": { + "rows": recent_log_rows, + "tail_text": "\n".join(log_data["tail"]), + }, + } + report["timelines"] = build_timelines(observations, transitions, trace_rows) + report["findings"] = build_findings(report) + return report + + +def summarize_trace_formula(formula: dict, stage: str | None) -> str: + if not isinstance(formula, dict): + return "n/a" + priming = formula.get("priming") or {} + priming_suffix = "" + if priming.get("label"): + priming_suffix = f" | {priming.get('label')}" + if stage == "co_stimulation": + return ( + f"value={format_float(formula.get('value'))} / " + f"threshold={format_float(formula.get('threshold'))}" + f"{priming_suffix}" + ) + if stage == "context": + return ( + f"effector={format_float(formula.get('effector_score'))}/" + f"{format_float(formula.get('effector_threshold'))}, " + f"memory={format_float(formula.get('memory_score'))}/" + f"{format_float(formula.get('memory_threshold'))}" + f"{priming_suffix}" + ) + return "n/a" + + +def render_badge(text: str, css_class: str) -> str: + return f'{escape(text)}' + + +def render_counter_cards(report: dict) -> str: + totals = report["totals"] + signals = totals["signals"] + cards = [ + ("Observations", totals["observations"], "warm"), + ("PAMP", signals.get("PAMP", 0), "pamp"), + ("DAMP", signals.get("DAMP", 0), "damp"), + ("With Antigens", totals["observations_with_antigens"], "neutral"), + ("Regex Matches", totals["observations_with_matches"], "neutral"), + ("Cells", totals["cells"], "neutral"), + ("Transitions", totals["transitions"], "neutral"), + ("Memories", totals["memories"], "memory"), + ] + return "".join( + f""" +
+

{escape(label)}

+

{escape(str(value))}

+
+ """ + for label, value, css_class in cards + ) + + +def render_simple_table(columns: list[str], rows: list[dict], empty_text: str) -> str: + if not rows: + return f'

{escape(empty_text)}

' + head = "".join(f"{escape(column)}" for column in columns) + body_rows = [] + for row in rows: + body_cells = "".join( + f"{row.get(column, '')}" for column in columns + ) + body_rows.append(f"{body_cells}") + body = "".join(body_rows) + return ( + '
' + f"{head}{body}
" + ) + + +def render_sortable_cell_table(rows: list[dict]) -> str: + if not rows: + return '

No cells are stored.

' + + columns = [ + "Updated", + "State", + "Responsible", + "T Cell", + "Antigen", + "Matched value", + "Scores", + ] + head = "".join( + ( + "" + f"" + "" + ) + for index, column in enumerate(columns) + ) + + body_rows = [] + for index, row in enumerate(rows): + score_parts = [ + f"co={format_float(row['last_co_stimulation'])}" + if row["last_co_stimulation"] is not None + else "", + f"eff={format_float(row['last_effector_score'])}" + if row["last_effector_score"] is not None + else "", + f"mem={format_float(row['last_memory_score'])}" + if row["last_memory_score"] is not None + else "", + ] + score_summary = ", ".join(part for part in score_parts if part) or "n/a" + waiting_html = "" + if row["waiting_label"]: + waiting_html = ( + f"
{escape(row['waiting_label'])}
" + ) + cells = [ + (escape(row["wall"]), row["ts"]), + ( + "
" + f"{render_badge(row['state'], row['state_class'])}" + f"{waiting_html}" + "
", + row["state"], + ), + (escape(row["responsible_ip"]), row["responsible_ip"]), + ( + f"
{escape(shorten(row['cell_key'], 72))}
", + row["cell_key"], + ), + ( + f"
{escape(row['regex_type'])}:" + f"{escape(shorten(row['antigen_value'], 52))}
", + f"{row['regex_type']}:{row['antigen_value']}", + ), + ( + f"
{escape(shorten(row['matched_value'], 52))}
", + row["matched_value"], + ), + (escape(score_summary), score_summary), + ] + body_cells = "".join( + f"{html_value}" + for html_value, sort_value in cells + ) + body_rows.append(f"{body_cells}") + + body = "".join(body_rows) + return ( + "
" + "" + f"{head}{body}
" + ) + + +def render_sortable_observation_table(rows: list[dict]) -> str: + if not rows: + return '

No observations available.

' + + columns = [ + "Observed at", + "Category", + "Signal", + "Evidence", + "Responsible", + "Related profile", + "Target", + "Antigens", + "Matches", + ] + head = "".join( + ( + "" + f"" + "" + ) + for index, column in enumerate(columns) + ) + + body_rows = [] + for index, row in enumerate(rows): + cells = [ + (escape(row["wall"]), row["ts"]), + (escape(row["category"]), row["category"]), + (render_badge(row["signal"], row["signal"].lower()), row["signal"]), + ( + escape(f"{row['evidence_type']} · {shorten(row['evidence_id'], 16)}"), + f"{row['evidence_type']} {row['evidence_id']}", + ), + (escape(row["responsible_ip"]), row["responsible_ip"]), + (escape(row["related_profile"]), row["related_profile"]), + (escape(row["target_ip"]), row["target_ip"]), + (escape(shorten(row["antigens"], 120)), row["antigens"]), + (escape(shorten(row["matched_regexes"], 120)), row["matched_regexes"]), + ] + body_cells = "".join( + f"{html_value}" + for html_value, sort_value in cells + ) + body_rows.append(f"{body_cells}") + + body = "".join(body_rows) + return ( + "
" + "" + f"{head}{body}
" + ) + + +def render_sortable_transition_table(rows: list[dict]) -> str: + if not rows: + return '

No state transitions were recorded.

' + + columns = [ + "When", + "Path", + "Reason", + "Responsible", + "T Cell", + "Evidence", + "Scores", + ] + head = "".join( + ( + "" + f"" + "" + ) + for index, column in enumerate(columns) + ) + + body_rows = [] + for index, row in enumerate(rows): + score_summary = ", ".join( + f"{key}={value}" for key, value in sorted((row["scores"] or {}).items()) + ) or "n/a" + cells = [ + (escape(row["wall"]), row["ts"]), + ( + f"{render_badge(row['from_state'], state_class_name(row['from_state']))} " + f"→ {render_badge(row['to_state'], state_class_name(row['to_state']))}", + f"{row['from_state_order']:02d}->{row['to_state_order']:02d}", + ), + (escape(row["reason"]), row["reason"]), + (escape(row["responsible_ip"]), row["responsible_ip"]), + (escape(shorten(row["cell_key"], 54)), row["cell_key"]), + (escape(shorten(row["evidence_id"], 20)), row["evidence_id"]), + ( + f"
show
{render_pretty_json(row['scores'])}
", + score_summary, + ), + ] + body_cells = "".join( + f"{html_value}" + for html_value, sort_value in cells + ) + body_rows.append(f"{body_cells}") + + body = "".join(body_rows) + return ( + "
" + "" + f"{head}{body}
" + ) + + +def render_svg_timeline(title: str, timeline: dict, series_order: list[str], color_map: dict[str, str]) -> str: + if not timeline: + return ( + f"

{escape(title)}

" + "

No timed data available.

" + ) + + labels = timeline["labels"] + series = timeline["series"] + width = 960 + height = 220 + padding_top = 18 + padding_bottom = 28 + padding_side = 20 + plot_width = width - (padding_side * 2) + plot_height = height - padding_top - padding_bottom + bars = [] + + max_total = 0 + for idx in range(len(labels)): + total = sum(series.get(name, [0] * len(labels))[idx] for name in series_order) + max_total = max(max_total, total) + max_total = max(max_total, 1) + bar_width = plot_width / max(1, len(labels)) + + for idx, label in enumerate(labels): + x = padding_side + (idx * bar_width) + stack_height = 0.0 + tooltip_lines = [label] + for name in series_order: + value = series.get(name, [0] * len(labels))[idx] + tooltip_lines.append(f"{name}: {value}") + if value <= 0: + continue + rect_height = (float(value) / float(max_total)) * plot_height + y = padding_top + (plot_height - stack_height - rect_height) + bars.append( + "" + f"{escape(' | '.join(tooltip_lines))}" + "" + ) + stack_height += rect_height + + legend = "".join( + f"
  • {escape(name)}
    • " + for name in series_order + ) + return f""" +
    +
    +

    {escape(title)}

    +

    {escape(ts_to_iso(timeline['min_ts']))} to {escape(ts_to_iso(timeline['max_ts']))} + · {int(round(timeline['width']))}s per bucket

    +
    + + + + {''.join(bars)} + +
      {legend}
    +
    + """ + + +def hex_to_rgba(hex_color: str, alpha: float) -> str: + color = hex_color.lstrip("#") + if len(color) != 6: + return f"rgba(31, 41, 55, {alpha})" + red = int(color[0:2], 16) + green = int(color[2:4], 16) + blue = int(color[4:6], 16) + return f"rgba({red}, {green}, {blue}, {alpha})" + + +def render_state_machine_graph(report: dict) -> str: + node_layout = { + 0: {"x": 40, "y": 122}, + 1: {"x": 320, "y": 44}, + 2: {"x": 320, "y": 244}, + 3: {"x": 600, "y": 122}, + 4: {"x": 880, "y": 30}, + 5: {"x": 880, "y": 214}, + } + node_width = 210 + node_height = 68 + transition_counts = { + row["label"]: row["count"] for row in report.get("transition_paths", []) + } + current_state_counts = report.get("cell_states", {}) + + edges = [ + { + "from": 0, + "to": 1, + "trigger": "accepted regex match", + "path": "M 250 156 C 275 156, 286 120, 320 104", + "label_x": 258, + "label_y": 116, + }, + { + "from": 0, + "to": 2, + "trigger": "no accepted regex match", + "path": "M 250 156 C 275 156, 286 286, 320 278", + "label_x": 236, + "label_y": 252, + }, + { + "from": 2, + "to": 0, + "trigger": "anergy TTL", + "path": "M 320 306 C 248 338, 178 322, 146 190", + "label_x": 182, + "label_y": 330, + }, + { + "from": 1, + "to": 1, + "trigger": "co-stimulation below threshold", + "path": "M 392 44 C 350 4, 502 4, 460 44", + "label_x": 350, + "label_y": 12, + }, + { + "from": 1, + "to": 3, + "trigger": "co-stimulation", + "path": "M 530 78 L 600 156", + "label_x": 542, + "label_y": 94, + }, + { + "from": 1, + "to": 2, + "trigger": "no co-stimulation timeout", + "path": "M 425 112 L 425 244", + "label_x": 438, + "label_y": 184, + }, + { + "from": 3, + "to": 3, + "trigger": "wait", + "path": "M 672 122 C 630 82, 782 82, 740 122", + "label_x": 706, + "label_y": 90, + }, + { + "from": 3, + "to": 4, + "trigger": "contain", + "path": "M 810 144 L 880 86", + "label_x": 828, + "label_y": 112, + }, + { + "from": 3, + "to": 5, + "trigger": "remember", + "path": "M 810 168 L 880 248", + "label_x": 824, + "label_y": 214, + }, + { + "from": 3, + "to": 0, + "trigger": "context timeout", + "path": "M 600 156 C 536 236, 286 236, 250 156", + "label_x": 430, + "label_y": 260, + }, + { + "from": 4, + "to": 4, + "trigger": "cooldown", + "path": "M 952 30 C 914 -8, 1088 -8, 1050 30", + "label_x": 1000, + "label_y": 2, + }, + { + "from": 5, + "to": 5, + "trigger": "retained", + "path": "M 952 282 C 914 320, 1088 320, 1050 282", + "label_x": 998, + "label_y": 334, + }, + ] + + node_svg = [] + for state_id, label in STATE_LABELS.items(): + node = node_layout[state_id] + color = STATE_COLORS[state_class(state_id)] + count = current_state_counts.get(label, 0) + node_svg.append( + f""" + + + {escape(label)} + current cells: {count} + + """ + ) + + edge_svg = [] + for edge in edges: + from_label = STATE_LABELS[edge["from"]] + to_label = STATE_LABELS[edge["to"]] + path_key = f"{from_label} -> {to_label}" + count = int(transition_counts.get(path_key, 0)) + active = count > 0 + stroke = STATE_COLORS[state_class(edge["to"])] + edge_svg.append( + f""" + + + + {escape(edge['trigger'])} · {count} + + + """ + ) + + return f""" +
    +
    +

    T Cell State Machine

    +

    Node badges show current cells in each state. Arrow labels show how many times each transition happened in this run.

    +
    + + + + + + + + {''.join(edge_svg)} + {''.join(node_svg)} + +
      +
    • 0 - mature -> 1 - antigen-recognized: `PAMP` or `DAMP` with an extracted antigen and an accepted regex match. `DAMP` creates a weaker priming profile.
      • +
    • 0 - mature -> 2 - anergic: `PAMP` or `DAMP` with extracted antigen but no accepted regex match.
      • +
    • 0 - mature -> 0 - mature: any evidence with no extracted antigen leaves the mature cell unchanged, so no recognition transition happens.
      • +
    • 1 - antigen-recognized -> 1 - antigen-recognized: co-stimulation is still below threshold, so the cell stays recognized and waits for re-evaluation.
      • +
    • 1 - antigen-recognized -> 2 - anergic: co-stimulation never reached threshold before the waiting window expired.
      • +
    +
    + """ + + +def render_pretty_json(value: Any) -> str: + return escape(json.dumps(value, indent=2, sort_keys=True)) + + +def render_formula_box(lines: list[str]) -> str: + return ( + "
    "
+        + escape("\n".join(lines))
+        + "
    " + ) + + +def render_term_cards(terms: list[dict]) -> str: + return "".join( + f""" +
    +

    {escape(term['label'])}

    +

    {escape(term['formula'])}

    +

    {escape(term['description'])}

    +
    + """ + for term in terms + ) + + +def render_formula_tree_node(node: dict) -> str: + children = node.get("children") or [] + child_class = "formula-children" + if len(children) > 1: + child_class += " has-multiple" + tooltip = node.get("tooltip") or "" + formula = node.get("formula") or "" + summary = node.get("summary") or "" + children_html = "" + if children: + children_html = ( + f"
    " + + "".join( + "
    " + + render_formula_tree_node(child) + + "
    " + for child in children + ) + + "
    " + ) + return f""" +
    +
    + {escape(node.get('label', 'value'))} + {f"{escape(formula)}" if formula else ""} + {f"{escape(summary)}" if summary else ""} + {f"{escape(tooltip)}" if tooltip else ""} +
    + {children_html} +
    + """ + + +def render_formula_tree(node: dict) -> str: + return f"
    {render_formula_tree_node(node)}
    " + + +def render_decision_doc_card(card: dict) -> str: + equation_html = render_formula_box(card["equation_lines"]) + gate_html = render_formula_box(card["gate_lines"]) + term_cards_html = render_term_cards(card["terms"]) + tree_html = render_formula_tree(card["tree"]) + notes_html = "".join( + f"

    {escape(note)}

    " + for note in card.get("notes", []) + ) + return f""" +
    +
    +

    {escape(card['title'])}

    +

    {escape(card['summary'])}

    +
    +
    +
    +

    Exact Equation

    + {equation_html} +
    +
    +

    Decision Gate

    + {gate_html} +
    +
    + {notes_html} +
    + {term_cards_html} +
    +
    +
    +

    Input Tree

    +

    Hover or focus a node to see where that term comes from.

    +
    + {tree_html} +
    +
    + """ + + +def render_rule_cards(cards: list[dict]) -> str: + return "".join( + f""" +
    +

    {escape(card['title'])}

    +

    {escape(card['rule'])}

    +

    {escape(card['description'])}

    +
    + """ + for card in cards + ) + + +def render_decision_reference(report: dict) -> str: + config = report_config_with_defaults(report) + weights = config["co_stimulation_weights"] + priming_profiles = config["effective_priming_profiles"] + pamp_profile = priming_profiles["PAMP"] + damp_profile = priming_profiles["DAMP"] + related_lookback = format_float(config["related_lookback_seconds"]) + related_saturation = format_float(config["related_pamps_saturation"]) + danger_saturation = format_float(config["danger_saturation"]) + damp_weight = format_float(config["damp_danger_weight"]) + novelty_window = format_float(config["novelty_window_seconds"]) + context_window = format_float(config["context_recent_window_seconds"]) + wait_limit = format_float(config["state_wait_timeout_seconds"]) + effector_cooldown = format_float(config["effector_cooldown_seconds"]) + memory_ratio_max = format_float(config["memory_trend_ratio_max"]) + anergy_ttl = format_float(config["anergy_ttl_seconds"]) + + decision_cards = [ + { + "title": "Recognition & Priming: 0 -> 1 setup", + "summary": "Both PAMP and DAMP can recognize an antigen and create state 1, but the signal that primed the cell decides how hard the later gates will be.", + "equation_lines": [ + "recognized when extracted_antigen matches an accepted regex", + "priming_profile = signal_specific_profile(PAMP or DAMP)", + ( + "effective gates = base thresholds/counts/wait " + "+ offsets or factors from priming_profile" + ), + ], + "gate_lines": [ + "PAMP or DAMP + extracted antigen + accepted regex => 0 -> 1", + "PAMP or DAMP + extracted antigen + no accepted regex => 0 -> 2", + "no extracted antigen => stay in 0 - mature", + ], + "notes": [ + ( + "PAMP keeps the base profile: co-stimulation " + f"{format_float(pamp_profile['co_stimulation_threshold'])}, effector " + f"{format_float(pamp_profile['effector_threshold'])}, memory " + f"{format_float(pamp_profile['memory_threshold'])}, wait " + f"{format_float(pamp_profile['state_wait_timeout_seconds'])}s." + ), + ( + "DAMP creates a weaker cell: co-stimulation " + f"{format_float(damp_profile['co_stimulation_threshold'])}, effector " + f"{format_float(damp_profile['effector_threshold'])}, memory " + f"{format_float(damp_profile['memory_threshold'])}, wait " + f"{format_float(damp_profile['state_wait_timeout_seconds'])}s." + ), + "The recognition observation is consumed for that cell after the state change, so it cannot also count toward the next transition.", + ], + "terms": [ + { + "label": "PAMP profile", + "formula": ( + f"strength={format_float(pamp_profile['strength'])}, " + f"co={format_float(pamp_profile['co_stimulation_threshold'])}, " + f"effector={format_float(pamp_profile['effector_threshold'])}, " + f"memory={format_float(pamp_profile['memory_threshold'])}" + ), + "description": "PAMP-primed cells use the base thresholds, base counts, and the full wait window.", + }, + { + "label": "DAMP profile", + "formula": ( + f"strength={format_float(damp_profile['strength'])}, " + f"co={format_float(damp_profile['co_stimulation_threshold'])}, " + f"effector={format_float(damp_profile['effector_threshold'])}, " + f"memory={format_float(damp_profile['memory_threshold'])}" + ), + "description": "DAMP-primed cells raise later thresholds, require more supporting related PAMPs, and shorten the wait window.", + }, + { + "label": "state_wait_timeout_seconds", + "formula": ( + "base_wait_timeout * priming_profile.state_wait_timeout_factor" + ), + "description": "The same priming profile controls how long the cell can remain waiting in state 1 or state 3.", + }, + { + "label": "consumed transition evidence", + "formula": "exclude consumed observation IDs from later counts and danger", + "description": "Once evidence causes a state change for a cell, that observation is excluded from later related-count, danger, and novelty calculations for the same cell.", + }, + ], + "tree": { + "label": "signal_specific_priming", + "formula": "PAMP or DAMP decides the effective thresholds, counts, and wait limit", + "summary": "Recognition creates a primed cell profile that shapes every later decision.", + "tooltip": "Both signals can create state 1, but they do not create equally strong cells.", + "children": [ + { + "label": "PAMP", + "formula": "base thresholds and counts", + "summary": ( + f"co={format_float(pamp_profile['co_stimulation_threshold'])}, " + f"effector={format_float(pamp_profile['effector_threshold'])}, " + f"memory={format_float(pamp_profile['memory_threshold'])}" + ), + "tooltip": "PAMP priming keeps the baseline gates and full wait window.", + "children": [ + { + "label": "wait_limit", + "formula": format_float( + pamp_profile["state_wait_timeout_seconds"] + ), + "summary": "Full wait window", + "tooltip": "PAMP-primed cells can wait for the full configured time window.", + }, + { + "label": "related counts", + "formula": ( + f"effector>={format_float(pamp_profile['effector_min_related_count'])}, " + f"memory>={format_float(pamp_profile['memory_min_related_count'])}" + ), + "summary": "Base supporting PAMP requirements", + "tooltip": "No extra related-count offset is added for PAMP priming.", + }, + ], + }, + { + "label": "DAMP", + "formula": "base values + offsets/factor", + "summary": ( + f"co={format_float(damp_profile['co_stimulation_threshold'])}, " + f"effector={format_float(damp_profile['effector_threshold'])}, " + f"memory={format_float(damp_profile['memory_threshold'])}" + ), + "tooltip": "DAMP priming deliberately makes later activation harder and the wait window shorter.", + "children": [ + { + "label": "wait_limit", + "formula": ( + f"{wait_limit} * " + f"{format_float(damp_profile['state_wait_timeout_factor'])}" + ), + "summary": format_float( + damp_profile["state_wait_timeout_seconds"] + ) + + "s effective wait", + "tooltip": "DAMP priming shortens how long the cell can wait before timing out.", + }, + { + "label": "related counts", + "formula": ( + f"effector>={format_float(damp_profile['effector_min_related_count'])}, " + f"memory>={format_float(damp_profile['memory_min_related_count'])}" + ), + "summary": "Extra corroboration required", + "tooltip": "DAMP priming adds one more related PAMP requirement for both effector and memory.", + }, + ], + }, + ], + }, + }, + { + "title": "Co-Stimulation: 1 -> 3 activation", + "summary": "This score is evaluated after antigen recognition to decide whether the cell activates. The effective threshold depends on the cell’s priming profile.", + "equation_lines": [ + ( + "co_stimulation = " + f"{format_float(weights['confidence'])} * confidence" + ), + ( + f" + {format_float(weights['related_pamps'])} " + "* related_pamp_score" + ), + ( + f" + {format_float(weights['danger'])} " + "* profile_danger_score" + ), + ], + "gate_lines": [ + "activate when co_stimulation >= priming_profile.co_stimulation_threshold", + ( + "PAMP threshold=" + f"{format_float(pamp_profile['co_stimulation_threshold'])} | " + "DAMP threshold=" + f"{format_float(damp_profile['co_stimulation_threshold'])}" + ), + "otherwise stay in 1 - antigen-recognized", + ( + "timeout to 2 - anergic after the priming-specific wait limit " + f"(PAMP {format_float(pamp_profile['state_wait_timeout_seconds'])}s, " + f"DAMP {format_float(damp_profile['state_wait_timeout_seconds'])}s)" + ), + ], + "notes": [ + f"Related PAMPs are counted over the last {related_lookback}s for the same responsible IP.", + "A related PAMP shares either the same antigen value or the same matched regex hash. The current observation is excluded from that count.", + "DAMP observations also contribute to the mixed danger term, and DAMP-primed cells use the stricter threshold from their stored priming profile.", + "Previously consumed transition evidence for this cell is excluded from the related-count and danger inputs so one observation cannot chain multiple state changes.", + ], + "terms": [ + { + "label": "confidence", + "formula": "current evidence.confidence", + "description": "The confidence carried by the observation that is being evaluated right now.", + }, + { + "label": "related_pamp_score", + "formula": f"clamp01(related_pamp_count / {related_saturation})", + "description": "How much recent, related PAMP evidence reinforces the same antigen or regex identity.", + }, + { + "label": "profile_danger_score", + "formula": ( + "clamp01((pamp_raw + " + f"{damp_weight} * damp_raw) / {danger_saturation})" + ), + "description": "The mixed danger pressure for the same responsible IP, with DAMP raw danger amplified before normalization.", + }, + { + "label": "pamp_raw / damp_raw", + "formula": "sum(threat_level_value * confidence)", + "description": "Raw danger is the sum of threat level value multiplied by confidence across recent PAMP or DAMP observations.", + }, + { + "label": "priming_profile.co_stimulation_threshold", + "formula": "base_co_stimulation_threshold + co_stimulation_threshold_offset", + "description": "The effective activation threshold comes from the profile stored when the cell entered state 1.", + }, + ], + "tree": { + "label": "co_stimulation", + "formula": ( + f"{format_float(weights['confidence'])} * confidence + " + f"{format_float(weights['related_pamps'])} * related_pamp_score + " + f"{format_float(weights['danger'])} * profile_danger_score" + ), + "summary": ( + "Activation score. Threshold depends on priming: " + f"PAMP {format_float(pamp_profile['co_stimulation_threshold'])}, " + f"DAMP {format_float(damp_profile['co_stimulation_threshold'])}" + ), + "tooltip": "Final co-stimulation score used for the 1 -> 3 decision.", + "children": [ + { + "label": "confidence", + "formula": "current evidence.confidence", + "summary": "Current PAMP confidence", + "tooltip": "Read directly from the observation currently being processed.", + }, + { + "label": "related_pamp_score", + "formula": f"clamp01(related_pamp_count / {related_saturation})", + "summary": "Recent related PAMP reinforcement", + "tooltip": "Normalized count of related PAMP observations in the related lookback window.", + "children": [ + { + "label": "related_pamp_count", + "formula": "count of related recent PAMPs", + "summary": "Same antigen value or same matched regex hash", + "tooltip": ( + f"Counted over the last {related_lookback}s for the same responsible IP. " + "The current observation is excluded." + ), + }, + { + "label": "related_pamps_saturation", + "formula": related_saturation, + "summary": "Count where the score saturates at 1", + "tooltip": "If the count reaches this value, related_pamp_score stops increasing.", + }, + ], + }, + { + "label": "profile_danger_score", + "formula": ( + "clamp01((pamp_raw + " + f"{damp_weight} * damp_raw) / {danger_saturation})" + ), + "summary": "Normalized mixed danger for the responsible IP", + "tooltip": "Recent PAMP and DAMP danger are combined, then clamped into the 0..1 range.", + "children": [ + { + "label": "pamp_raw", + "formula": "sum(threat_level_value * confidence)", + "summary": "Recent PAMP raw danger", + "tooltip": ( + f"Summed over PAMP observations for the same responsible IP within the last {related_lookback}s." + ), + }, + { + "label": "damp_raw", + "formula": "sum(threat_level_value * confidence)", + "summary": "Recent DAMP raw danger", + "tooltip": ( + f"Summed over DAMP observations for the same responsible IP within the last {related_lookback}s." + ), + }, + { + "label": "damp_danger_weight", + "formula": damp_weight, + "summary": "Amplifies DAMP raw danger before normalization", + "tooltip": "DAMP pressure is scaled before it is added into the mixed danger term.", + }, + { + "label": "danger_saturation", + "formula": danger_saturation, + "summary": "Raw danger amount that maps to score 1", + "tooltip": "The combined raw danger is divided by this value before clamp01 is applied.", + }, + ], + }, + ], + }, + }, + { + "title": "Context Effector: 3 -> 4 containment", + "summary": "This score evaluates whether an activated cell should escalate into an effector response. The threshold and minimum related count depend on priming.", + "equation_lines": [ + "effector_score = 0.45 * recent_pressure", + " + 0.25 * recent_related_score", + " + 0.30 * novelty_score", + ], + "gate_lines": [ + "effector = (novelty_score > 0)", + " and (recent_related_count >= priming_profile.effector_min_related_count)", + " and (effector_score >= priming_profile.effector_threshold)", + ( + "PAMP gate: related>=" + f"{format_float(pamp_profile['effector_min_related_count'])}, " + "threshold=" + f"{format_float(pamp_profile['effector_threshold'])}" + ), + ( + "DAMP gate: related>=" + f"{format_float(damp_profile['effector_min_related_count'])}, " + "threshold=" + f"{format_float(damp_profile['effector_threshold'])}" + ), + ], + "notes": [ + f"recent_pressure is computed over the last {context_window}s and uses the same mixed PAMP + weighted DAMP danger model as co-stimulation.", + f"novelty_score is binary: it becomes 1 only if the matched regex has no stored memory row and no recent transition activity in the last {novelty_window}s.", + f"If the cell reaches state 4, repeated containment is still gated by an effector cooldown of {effector_cooldown}s.", + "Consumed transition observations for that cell are excluded from novelty, pressure, and recent-related calculations.", + ], + "terms": [ + { + "label": "recent_pressure", + "formula": ( + "clamp01((recent_pamp_raw + " + f"{damp_weight} * recent_damp_raw) / {danger_saturation})" + ), + "description": "The normalized mixed danger in the recent context window for the same responsible IP.", + }, + { + "label": "recent_related_score", + "formula": f"clamp01(recent_related_count / {related_saturation})", + "description": "How much recent PAMP evidence in the context window still points to the same antigen or regex identity.", + }, + { + "label": "novelty_score", + "formula": "1 if no memory and no recent regex activity else 0", + "description": "A binary novelty gate. If the regex is already familiar, the effector path is blocked immediately.", + }, + { + "label": "priming_profile.effector_threshold", + "formula": "base_effector_threshold + effector_threshold_offset", + "description": "The effective effector threshold is higher for weaker priming profiles such as DAMP.", + }, + ], + "tree": { + "label": "effector_score", + "formula": "0.45 * recent_pressure + 0.25 * recent_related_score + 0.30 * novelty_score", + "summary": ( + "Containment score. Priming-specific gate: " + f"PAMP>={format_float(pamp_profile['effector_threshold'])}, " + f"DAMP>={format_float(damp_profile['effector_threshold'])}" + ), + "tooltip": "Final context score used to decide whether state 3 escalates to state 4.", + "children": [ + { + "label": "recent_pressure", + "formula": ( + "clamp01((recent_pamp_raw + " + f"{damp_weight} * recent_damp_raw) / {danger_saturation})" + ), + "summary": f"Mixed danger during the most recent {context_window}s window", + "tooltip": "Computed from the recent context window immediately before the current decision.", + "children": [ + { + "label": "recent_pamp_raw", + "formula": "sum(threat_level_value * confidence)", + "summary": "Recent PAMP raw danger", + "tooltip": "Summed over recent PAMP observations in the context window.", + }, + { + "label": "recent_damp_raw", + "formula": "sum(threat_level_value * confidence)", + "summary": "Recent DAMP raw danger", + "tooltip": "Summed over recent DAMP observations in the context window.", + }, + ], + }, + { + "label": "recent_related_score", + "formula": f"clamp01(recent_related_count / {related_saturation})", + "summary": "Recent supporting PAMP count normalized to 0..1", + "tooltip": "Counts related PAMP observations in the recent context window.", + "children": [ + { + "label": "recent_related_count", + "formula": "count of related recent PAMPs", + "summary": "Same antigen value or same matched regex hash", + "tooltip": ( + f"Counted only inside the recent context window of {context_window}s." + ), + }, + { + "label": "related_pamps_saturation", + "formula": related_saturation, + "summary": "Cap for the normalized related score", + "tooltip": "The count is divided by this saturation value before clamp01 is applied.", + }, + ], + }, + { + "label": "novelty_score", + "formula": "1 if no memory and no recent activity else 0", + "summary": "Binary novelty gate", + "tooltip": "Effector requires the regex to still look new for this responsible IP.", + "children": [ + { + "label": "has_memory_for_regex", + "formula": "memory row exists for regex_hash", + "summary": "If true, novelty_score becomes 0", + "tooltip": "A stored memory for the regex marks it as familiar immediately.", + }, + { + "label": "has_recent_regex_activity", + "formula": f"transition activity within {novelty_window}s", + "summary": "If true, novelty_score becomes 0", + "tooltip": "Any recent transition for the same responsible IP and regex hash removes novelty.", + }, + ], + }, + ], + }, + }, + { + "title": "Context Memory: 3 -> 5 storage", + "summary": "This score evaluates whether an activated cell should store memory instead of escalating to containment. The threshold and minimum related count depend on priming.", + "equation_lines": [ + "memory_score = 0.60 * decrease_score", + " + 0.25 * familiarity_score", + " + 0.15 * stability_score", + ], + "gate_lines": [ + "memory = (familiarity_score > 0)", + " and (recent_related_count >= priming_profile.memory_min_related_count)", + f" and (trend_ratio <= {memory_ratio_max})", + " and (memory_score >= priming_profile.memory_threshold)", + ( + "PAMP gate: related>=" + f"{format_float(pamp_profile['memory_min_related_count'])}, " + "threshold=" + f"{format_float(pamp_profile['memory_threshold'])}" + ), + ( + "DAMP gate: related>=" + f"{format_float(damp_profile['memory_min_related_count'])}, " + "threshold=" + f"{format_float(damp_profile['memory_threshold'])}" + ), + ], + "notes": [ + "Memory is the cooling-down path: the same pattern is no longer novel, pressure is lower than before, and enough related evidence still supports the match.", + "trend_ratio compares the recent mixed pressure window against the previous adjacent window. Lower is better for memory.", + ( + "If neither effector nor memory passes, the cell stays in 3 - activated until the priming-specific context timeout expires: " + f"PAMP {format_float(pamp_profile['state_wait_timeout_seconds'])}s, " + f"DAMP {format_float(damp_profile['state_wait_timeout_seconds'])}s." + ), + ], + "terms": [ + { + "label": "decrease_score", + "formula": "clamp01(1 - trend_ratio)", + "description": "Rewards situations where recent pressure is clearly lower than previous pressure.", + }, + { + "label": "trend_ratio", + "formula": "recent_pressure / max(previous_pressure, 0.01)", + "description": "Measures whether the mixed danger is falling, flat, or rising between adjacent context windows.", + }, + { + "label": "familiarity_score", + "formula": "1 - novelty_score", + "description": "Memory requires the regex to already be familiar rather than novel.", + }, + { + "label": "stability_score", + "formula": "clamp01(recent_related_count / priming_profile.memory_min_related_count)", + "description": "Ensures there is still enough related recent evidence to justify storing memory.", + }, + { + "label": "priming_profile.memory_threshold", + "formula": "base_memory_threshold + memory_threshold_offset", + "description": "The effective memory threshold is profile-specific, so DAMP-primed cells require stronger memory evidence.", + }, + ], + "tree": { + "label": "memory_score", + "formula": "0.60 * decrease_score + 0.25 * familiarity_score + 0.15 * stability_score", + "summary": ( + "Memory score. Priming-specific gate: " + f"PAMP>={format_float(pamp_profile['memory_threshold'])}, " + f"DAMP>={format_float(damp_profile['memory_threshold'])}" + ), + "tooltip": "Final context score used to decide whether state 3 transitions into state 5.", + "children": [ + { + "label": "decrease_score", + "formula": "clamp01(1 - trend_ratio)", + "summary": "Higher when recent pressure is falling", + "tooltip": "A falling trend pushes the memory score up.", + "children": [ + { + "label": "trend_ratio", + "formula": "recent_pressure / max(previous_pressure, 0.01)", + "summary": f"Must stay <= {memory_ratio_max} for memory", + "tooltip": "Compares the most recent context window against the immediately preceding one.", + "children": [ + { + "label": "recent_pressure", + "formula": ( + "clamp01((recent_pamp_raw + " + f"{damp_weight} * recent_damp_raw) / {danger_saturation})" + ), + "summary": f"Mixed danger over the last {context_window}s", + "tooltip": "Same recent pressure value also used by effector_score.", + }, + { + "label": "previous_pressure", + "formula": ( + "clamp01((previous_pamp_raw + " + f"{damp_weight} * previous_damp_raw) / {danger_saturation})" + ), + "summary": f"Mixed danger over the previous {context_window}s window", + "tooltip": "Computed over the context window immediately before the recent one.", + }, + ], + } + ], + }, + { + "label": "familiarity_score", + "formula": "1 - novelty_score", + "summary": "Higher when the regex is already familiar", + "tooltip": "Memory is only allowed once novelty has disappeared.", + "children": [ + { + "label": "novelty_score", + "formula": "1 if no memory and no recent activity else 0", + "summary": "Same novelty gate used by the effector path", + "tooltip": "If novelty_score stays 1, familiarity_score stays 0 and memory fails.", + } + ], + }, + { + "label": "stability_score", + "formula": "clamp01(recent_related_count / priming_profile.memory_min_related_count)", + "summary": "Recent evidence stability", + "tooltip": "Memory still requires enough related recent PAMPs to support the pattern.", + "children": [ + { + "label": "recent_related_count", + "formula": "count of related recent PAMPs", + "summary": ( + "Must satisfy the priming-specific minimum related count" + ), + "tooltip": "Related means same antigen value or same matched regex hash in the recent context window.", + } + ], + }, + ], + }, + }, + ] + + rule_cards = [ + { + "title": "Recognition", + "rule": "0 -> 1 when PAMP or DAMP has an extracted antigen and an accepted regex match", + "description": "Both signals can create a recognized cell, but the stored priming profile marks whether that cell is PAMP-primed or DAMP-primed.", + }, + { + "title": "Signal-Specific No-Match", + "rule": "PAMP or DAMP no-match => 0 -> 2 when an extracted antigen has no accepted regex", + "description": "If a mature cell receives an extracted antigen but no accepted regex matches it, the cell becomes anergic regardless of whether the evidence was PAMP or DAMP.", + }, + { + "title": "Consumed Transition Evidence", + "rule": "exclude transition-causing observation IDs from later counts, danger, and novelty", + "description": "The same observation should not drive a whole chain of activations for the same cell, so transition-causing evidence is remembered and excluded from later steps.", + }, + { + "title": "Anergy Expiry", + "rule": f"2 -> 0 when anergy_ttl_seconds ({anergy_ttl}s) has elapsed", + "description": "Once the anergy TTL expires, the cell returns to mature and can be evaluated again.", + }, + { + "title": "Co-Stimulation Timeout", + "rule": "1 -> 2 when the co-stimulation wait reaches the priming-specific wait limit", + "description": ( + "The cell can keep waiting in 1 - antigen-recognized while later PAMP or DAMP evidence reevaluates the score, " + f"but only until its stored wait limit expires. PAMP uses {format_float(pamp_profile['state_wait_timeout_seconds'])}s and DAMP uses {format_float(damp_profile['state_wait_timeout_seconds'])}s." + ), + }, + { + "title": "Context Timeout", + "rule": "3 -> 0 when the context wait reaches the priming-specific wait limit", + "description": "If neither effector nor memory passes before the waiting window ends, the cell falls back to 0 - mature and keeps no active waiting context.", + }, + { + "title": "Effector Cooldown", + "rule": f"4 -> 4 suppress repeated containment until {effector_cooldown}s passes", + "description": "The state can stay effector while repeated blocking publications are suppressed by cooldown.", + }, + { + "title": "Memory Retention", + "rule": "5 -> 5 keep the memory state on later matching evidence", + "description": "Once memory is stored for that cell, later hits retain state 5 without writing repeated memory_stored events.", + }, + ] + + decision_cards_html = "".join( + render_decision_doc_card(card) for card in decision_cards + ) + rule_cards_html = render_rule_cards(rule_cards) + return f""" +
    +
    +

    Decision Reference

    +

    Bottom-of-report explanation of how the T Cell equations and branch conditions are computed.

    +
    +

    + This section documents the exact values, thresholds, and helper rules used by the report and by the T Cell module decision logic. + Normalization uses clamp01(x) = max(0, min(1, x)). Thresholds and wait windows can change per cell because the priming profile stored at 0 -> 1 is carried into later decisions. Hover or focus a node in each tree to inspect where that term comes from. +

    +
    + {decision_cards_html} +
    +
    +
    +

    Rule-Based Decisions

    +

    These branches are not weighted equations, but they still change state or suppress actions.

    +
    +
    + {rule_cards_html} +
    +
    +
    + """ + + +def render_history_details(summary: str, lines: list[str]) -> str: + details_body = escape("\n".join(lines or ["n/a"])) + return ( + f"
    {escape(summary)}" + f"
    {details_body}
    " + ) + + +def render_history_event_table(events: list[dict]) -> str: + if not events: + return '

    No history events were recorded for this T cell.

    ' + + head = "".join( + f"{escape(column)}" + for column in [ + "When", + "Source", + "Step", + "State path", + "Evidence", + "Threshold result", + "Evidence considered", + "Computation", + ] + ) + rows = [] + for event in events: + row_cells = [ + escape(event.get("wall") or "n/a"), + escape(event.get("source") or "unknown"), + escape(event.get("step") or "event"), + escape(event.get("state_path") or "n/a"), + escape(event.get("evidence_id") or "n/a"), + render_history_details( + event.get("threshold_summary") or "n/a", + event.get("threshold_lines") or [], + ), + render_history_details( + event.get("considered_summary") or "n/a", + event.get("considered_lines") or [], + ), + render_history_details( + event.get("computation_summary") or "n/a", + event.get("computation_lines") or [], + ), + ] + rows.append( + "" + + "".join(f"{cell}" for cell in row_cells) + + "" + ) + return ( + "
    " + "" + f"{head}" + f"{''.join(rows)}
    " + ) + + +def render_cell_histories(report: dict) -> str: + histories = report.get("cell_histories") or [] + if not histories: + return """ +
    +

    T Cell Histories

    +

    No T-cell histories were available for this run.

    +
    + """ + + index_rows = [ + { + "Responsible": escape(item.get("responsible_ip") or ""), + "Cell": escape(shorten(item.get("cell_key") or "", 64)), + "Current state": render_badge( + item.get("current_state") or "unknown", + item.get("current_state_class") or "state-unknown", + ), + "Life path": escape(shorten(item.get("life_path") or "", 88)), + "Events": escape(str(item.get("event_count") or 0)), + } + for item in histories + ] + index_table = render_simple_table( + ["Responsible", "Cell", "Current state", "Life path", "Events"], + index_rows, + "No T-cell history index available.", + ) + + trace_mode = (report.get("config") or {}).get("decision_trace_mode") + if trace_mode in (None, "", {}): + trace_note = ( + "Decision trace configuration was not found in metadata, so histories rely on whatever trace rows and transitions were stored." + ) + elif str(trace_mode).lower() in {"0", "off"}: + trace_note = ( + "Decision trace was off for this run, so histories can only show state transitions and any score snapshots saved with those transitions." + ) + elif str(trace_mode).lower() in {"1", "transitions"}: + trace_note = ( + "Decision trace was limited to transition events, so waiting reevaluations may be missing from the lifecycle view." + ) + else: + trace_note = ( + "Decision trace was fully enabled, so histories include both state changes and intermediate decision evaluations when available." + ) + + history_cards = [] + for index, item in enumerate(histories): + priming_profile = item.get("priming_profile") or {} + title = ( + f"{item.get('responsible_ip') or 'unknown'} | " + f"{item.get('regex_type') or 'unknown'}:{item.get('antigen_value') or ''}" + ) + meta_bits = [ + f"current={item.get('current_state') or 'unknown'}", + f"priming={item.get('priming_label') or 'n/a'}", + f"life path={item.get('life_path') or 'n/a'}", + f"first seen={item.get('first_seen') or 'n/a'}", + f"last seen={item.get('last_seen') or 'n/a'}", + f"events={item.get('event_count') or 0}", + f"transitions={item.get('transition_count') or 0}", + f"trace rows={item.get('trace_count') or 0}", + ] + if item.get("waiting_label"): + meta_bits.insert(1, f"waiting={item['waiting_label']}") + table_html = render_history_event_table(item.get("events") or []) + waiting_html = "" + if item.get("waiting_label"): + waiting_html = ( + f"
    {escape(item['waiting_label'])}
    " + ) + history_cards.append( + f""" +
    + +
    +
    +

    T Cell

    +

    {escape(title)}

    +

    {escape(' | '.join(meta_bits))}

    +
    +
    +
    + {render_badge(item.get("current_state") or "unknown", item.get("current_state_class") or "state-unknown")} + {waiting_html} +
    +
    +
    +     +
    +

    Cell key: {escape(item.get('cell_key') or '')}

    +

    Matched value: {escape(item.get('matched_value') or 'n/a')}

    +

    Priming: {escape(item.get('priming_label') or 'n/a')} | signal={escape(item.get('priming_signal') or 'n/a')} | strength={escape(format_float(item.get('priming_strength')))}

    +

    Effective profile: co_threshold={escape(format_float(priming_profile.get('co_stimulation_threshold')))} | effector_threshold={escape(format_float(priming_profile.get('effector_threshold')))} | memory_threshold={escape(format_float(priming_profile.get('memory_threshold')))} | wait_limit={escape(format_float(priming_profile.get('state_wait_timeout_seconds')))} | effector_related>={escape(format_float(priming_profile.get('effector_min_related_count')))} | memory_related>={escape(format_float(priming_profile.get('memory_min_related_count')))}

    + {table_html} +
    +
    + """ + ) + + return f""" +
    +
    +

    T Cell Histories

    +

    Chronological lifecycle view for each cell, combining stored state transitions with decision-trace computations.

    +
    +

    {escape(trace_note)}

    +
    +

    History Index

    + {index_table} +
    +
    + {''.join(history_cards)} +
    +
    + """ + + +def render_html(report: dict) -> str: + findings_html = "".join( + f"
  • {escape(item)}
    • " for item in report.get("findings", []) + ) or "
  • No notable findings.
    • " + + config_rows = [] + for key, value in report["config"].items(): + if value in (None, "", {}): + continue + config_rows.append( + { + "Key": escape(str(key)), + "Value": escape(str(value)), + } + ) + + signals_table = render_simple_table( + ["Signal", "Count", "Share"], + [ + { + "Signal": render_badge(signal, signal.lower()), + "Count": escape(str(count)), + "Share": escape( + f"{safe_div(count, report['totals']['observations']) * 100.0:.1f}%" + ), + } + for signal, count in sorted( + report["totals"]["signals"].items(), key=lambda item: item[0] + ) + ], + "No observations were stored.", + ) + + evidence_type_table = render_simple_table( + ["Evidence type", "Signal", "Count"], + [ + { + "Evidence type": escape(row["evidence_type"]), + "Signal": render_badge(row["signal"], row["signal"].lower()), + "Count": escape(str(row["count"])), + } + for row in report["top_signals_by_type"] + ], + "No evidence rows available.", + ) + + observation_table = render_sortable_observation_table( + report["recent_observations"] + ) + + transition_table = render_sortable_transition_table( + report["recent_transitions"] + ) + + cell_table = render_sortable_cell_table(report["recent_cells"]) + + memory_table = render_simple_table( + ["Updated", "Responsible", "Cell", "Regex", "Matched value", "Context"], + [ + { + "Updated": escape(row["wall"]), + "Responsible": escape(row["responsible_ip"]), + "Cell": escape(shorten(row["cell_key"], 56)), + "Regex": escape(shorten(row["regex_hash"], 24)), + "Matched value": escape(shorten(row["matched_value"], 40)), + "Context": ( + f"
    show
    {render_pretty_json(row['context'])}
    " + ), + } + for row in report["recent_memories"] + ], + "No memories are stored.", + ) + + trace_section = render_simple_table( + ["When", "Stage", "Action", "Path", "Responsible", "Candidate", "Scores"], + [ + { + "When": escape(row["wall"]), + "Stage": render_badge( + row["stage"] or "unknown", + f"trace-{(row['stage'] or 'unknown').replace('_', '-')}", + ), + "Action": escape(row["action"]), + "Path": escape(f"{row['from_state']} → {row['to_state']}"), + "Responsible": escape(row["responsible_ip"]), + "Candidate": escape( + f"{row['candidate'].get('regex_type', '')}:" + f"{shorten(row['candidate'].get('value', ''), 48)}" + ), + "Scores": ( + f"
    {escape(row['score_summary'])}" + f"
    {render_pretty_json(row['formula'])}
    " + ), + } + for row in report["trace"]["rows"] + ], + "No decision trace rows were stored.", + ) + + action_tables = { + "Top responsible IPs": render_simple_table( + ["Label", "Count"], + [ + {"Label": escape(row["label"]), "Count": escape(str(row["count"]))} + for row in report["top_responsible_ips"] + ], + "No responsible IP data.", + ), + "Top related profiles": render_simple_table( + ["Label", "Count"], + [ + {"Label": escape(row["label"]), "Count": escape(str(row["count"]))} + for row in report["top_related_profiles"] + ], + "No related profile data.", + ), + "Top targets": render_simple_table( + ["Label", "Count"], + [ + {"Label": escape(row["label"]), "Count": escape(str(row["count"]))} + for row in report["top_targets"] + ], + "No target data.", + ), + "Top antigens": render_simple_table( + ["Label", "Count"], + [ + {"Label": escape(row["label"]), "Count": escape(str(row["count"]))} + for row in report["top_antigens"] + ], + "No extracted antigens.", + ), + "Top unmatched PAMP antigens": render_simple_table( + ["Label", "Count"], + [ + {"Label": escape(row["label"]), "Count": escape(str(row["count"]))} + for row in report["top_unmatched_pamp_antigens"] + ], + "No unmatched PAMP antigens.", + ), + "Transition reasons": render_simple_table( + ["Label", "Count"], + [ + {"Label": escape(row["label"]), "Count": escape(str(row["count"]))} + for row in report["transition_reasons"] + ], + "No transition reasons available.", + ), + "Log action counts": render_simple_table( + ["Label", "Count"], + [ + {"Label": escape(row["label"]), "Count": escape(str(row["count"]))} + for row in report["log_action_counts"] + ], + "No module log actions available.", + ), + } + + action_sections = "".join( + f""" +
    +

    {escape(title)}

    + {html} +
    + """ + for title, html in action_tables.items() + ) + + config_section = render_simple_table( + ["Key", "Value"], + config_rows, + "No metadata configuration found.", + ) + + observation_timeline = render_svg_timeline( + "Observation Timeline", + report["timelines"]["observations"], + ["PAMP observations", "DAMP observations"], + { + "PAMP observations": SIGNAL_COLORS["PAMP"], + "DAMP observations": SIGNAL_COLORS["DAMP"], + }, + ) + transition_timeline = render_svg_timeline( + "Transition Timeline", + report["timelines"]["transitions"], + ["recognized", "anergic", "activated", "effector", "memory"], + { + "recognized": STATE_COLORS["state-recognized"], + "anergic": STATE_COLORS["state-anergic"], + "activated": STATE_COLORS["state-activated"], + "effector": STATE_COLORS["state-effector"], + "memory": STATE_COLORS["state-memory"], + }, + ) + trace_timeline = render_svg_timeline( + "Decision Trace Timeline", + report["timelines"]["trace"], + ["co-stimulation", "context"], + TRACE_STAGE_COLORS, + ) + state_machine_graph = render_state_machine_graph(report) + decision_reference = render_decision_reference(report) + histories_section = render_cell_histories(report) + + return f""" + + + + + T Cell Report + + + +
    +
    +

    T Cell HTML Report

    +
    +

    T Cell Run Report

    +

    Static analysis of observations, signals, transitions, memories, and optional decision traces. Generated at {escape(report['generated_at'])}

    +
    +
    +

    Run Output

    {escape(report['run_output_dir'])}
    +

    Database

    {escape(report['sources']['db_path'])}
    +

    Module Log

    {escape(report['sources']['log_path'])}
    +

    Decision Trace

    {escape(report['sources']['trace_path'])}
    +
    +
    + +
    + + +
    + +
    + +
    +
    +

    Quick Summary

    +
    + {render_counter_cards(report)} +
    +
    +
    +

    Run Findings

    +
      {findings_html}
    +
    +
    + +
    + {observation_timeline} + {transition_timeline} + {trace_timeline} +
    + + {state_machine_graph} + +
    +
    +

    Signals

    + {signals_table} +
    +
    +

    Evidence Types

    + {evidence_type_table} +
    +
    + +
    + {action_sections} +
    + +
    +

    Transitions

    +

    Click a column header to sort. Default order groups rows by T cell so each cell's path stays together.

    + {transition_table} +
    + +
    +
    +

    Current Cells

    +

    Click a column header to sort. Waiting cells keep the main state badge and show the wait condition underneath.

    + {cell_table} +
    +
    +

    Stored Memories

    + {memory_table} +
    +
    + +
    +

    Decision Trace

    +

    If decision tracing was off for the run, this section will stay empty even when the rest of the report is populated.

    + {trace_section} +
    + +
    +

    Recent Observations

    +

    These rows come from the T Cell SQLite DB, so they remain available even when module log verbosity was low. Click a column header to sort.

    + {observation_table} +
    + + {decision_reference} + +
    +
    + Run configuration snapshot +

    Compact copy of the T Cell-related metadata used for this report.

    + {config_section} +
    +
    +
    + + +
    + + + +""" + + +def state_class_name(label: str) -> str: + mapping = {value: STATE_CLASS[key] for key, value in STATE_LABELS.items()} + return mapping.get(label, "state-unknown") + + +def write_report(run_output_dir: Path, output_html: Path, args: argparse.Namespace) -> Path: + report = build_report_payload( + run_output_dir, + max_observations=args.max_observations, + max_log_lines=args.max_log_lines, + max_trace_rows=args.max_trace_rows, + ) + output_html.parent.mkdir(parents=True, exist_ok=True) + output_html.write_text(render_html(report), encoding="utf-8") + return output_html + + +def main() -> int: + args = parse_args() + run_output_dir = Path(args.run_output_dir).expanduser().resolve() + output_html = ( + Path(args.out).expanduser().resolve() + if args.out + else run_output_dir / "t_cell_report.html" + ) + report_path = write_report(run_output_dir, output_html, args) + print(f"Report written to: {report_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/modules/t_cell/blog_post.md b/modules/t_cell/blog_post.md new file mode 100644 index 0000000000..f81f00c7f5 --- /dev/null +++ b/modules/t_cell/blog_post.md @@ -0,0 +1,427 @@ +# T Cells in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS): Adaptive Response on Top of Innate Evidence + +The original +[Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) pipeline is +good at turning traffic into evidence. It can detect scans, suspicious ports, +DNS anomalies, HTTP oddities, TLS issues, and many other behaviors, and it can +do that across large volumes of traffic. But producing evidence is not the same +thing as having an immune response. + +If every suspicious event is treated in isolation, the system misses some of +the most important immune ideas: + +- recognition should be separated from activation +- danger should matter, not just pattern match +- tolerance should be explicit +- memory should be explicit +- containment should happen only after enough evidence and context accumulate + +That is the problem the T Cell module is trying to solve. + +[Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) is now moving +from a pure detection pipeline toward a system that implements immunology +concepts directly. The existing detector structure remains the innate layer, +and the adaptive layer is added on top. + +[Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) now has an immune-style responder that sits on top of the existing +evidence pipeline and decides when recognition should stay tolerant, when it +should activate, when it should contain, and when it should store memory. + +That responder is: + +- `modules/t_cell/t_cell.py` + +It is not a replacement for the old detectors. It is an adaptive layer that +uses the old detectors as its innate immune system and uses the accepted regex +repertoire from `RegexGenerator` as its recognition library. + +That connection matters because the adaptive side in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) is now split into +two cooperating pieces: + +- `RegexGenerator` continually generates new candidate receptors through the + shared `LLM` module and keeps only the regexes that survive local selection +- `T Cell` consumes those accepted receptors against live `PAMP` evidence and + combines them with `DAMP` danger context to decide what to do + +![T Cell HTML report overview](../../docs/images/t_cell/t_cell_report_overview.png) + +## Innate and Adaptive Immunity in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) + +The immune split in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) is now clear: + +- the innate immune system is the existing [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) detection structure plus the + central `PAMP` / `DAMP` signal tagging +- the adaptive immune system is the combination of `RegexGenerator` and + `T Cell` + +This is the important architectural idea. + +The old [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) modules still do what they always did: + +- inspect network behavior +- detect suspicious conditions +- emit evidence + +What changed is that [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) now centrally tags evidence with an +`evidence_signal`: + +- `PAMP` +- `DAMP` + +That gives the T Cell module a biologically meaningful input vocabulary. + +The adaptive part then adds two capabilities: + +1. `RegexGenerator` creates a validated receptor library. +2. `T Cell` uses that library plus live danger context to decide what to do. + +## What the T Cell Module Actually Does + +The T Cell module subscribes to the shared `evidence_added` channel. + +For each relevant evidence, it stores its own observation and then decides +whether a T cell should: + +- stay mature +- recognize an antigen +- become tolerant +- activate +- contain +- remember + +It tracks one cell per: + +- responsible IP +- regex type +- normalized antigen value + +That means the unit of response is not "all evidence for a profile." It is a +more precise adaptive unit tied to a responsible source and one structured +antigen candidate. + +## Why the Responsible IP Matters + +One of the key implementation details is that T Cell does not simply act on +`evidence.profile.ip`. + +[Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) evidence has several roles at once: + +- `profile.ip`: the related profile bucket +- `attacker`: the responsible entity +- `victim`: the target entity +- `direction`: whether a network entity appeared on the `SRC` or `DST` side + +Those are not the same thing. + +T Cell derives a responsible IP from the evidence and uses that value for: + +- cell ownership +- danger aggregation +- reevaluation +- containment + +So when the evidence says one machine was the victim but another was the +attacker, containment targets the responsible source, not just the profile +bucket that happened to hold the evidence. + +## The Connection to RegexGenerator + +The T Cell module depends directly on the regex repertoire built by +`RegexGenerator`. + +When a `PAMP` arrives, T Cell extracts structured antigens such as: + +- domains +- URIs +- filenames +- TLS SNI values +- certificate common names + +It then checks whether any accepted regex of the same type matches that +antigen. + +This is the adaptive recognition step. + +Without `RegexGenerator`, T Cell would still see `PAMP` and `DAMP`, but it +would not have a receptor library to consult. With `RegexGenerator`, the T Cell +can treat live evidence as candidate antigens and test them against an accepted +adaptive repertoire. + +That is why the two modules belong together conceptually: + +- `RegexGenerator` builds the receptors +- `T Cell` uses them in live response + +## State 0: Mature + +Every cell starts in: + +- `0 - mature` + +At this point there is no recognition yet. + +The module stores the observation and checks whether the evidence can even +produce a usable antigen candidate. If not, the evidence is logged and kept as +observation data, but it does not create a useful recognition event. + +## From Mature to Antigen Recognition + +Only `PAMP` evidence can start recognition from `0 - mature`. + +That is deliberate. `PAMP` is the structured trigger that tells the adaptive +layer there may be something pathogen-like worth recognizing. + +T Cell extracts antigen candidates from: + +- evidence entities +- linked DNS altflows +- linked HTTP altflows +- linked SSL/TLS altflows + +If an accepted regex matches one of those antigens, the cell moves to: + +- `1 - antigen-recognized` + +If an antigen is present but no accepted regex matches, the cell moves to: + +- `2 - anergic` + +That is the tolerance path. The system saw a candidate antigen, checked its +adaptive repertoire, and found no reason to escalate. + +## Co-Stimulation: Recognition Is Not Enough + +A recognized antigen still does not automatically become an active response. + +The next question is whether there is enough danger to justify activation. + +The module computes co-stimulation from: + +- the confidence of the current `PAMP` +- the number of related `PAMP` observations +- the weighted cumulative danger seen for the same responsible IP + +That danger term includes both: + +- `PAMP` +- `DAMP` + +This is exactly where the innate and adaptive layers meet. + +The regex match says: + +- this pattern looks recognizable + +The co-stimulation score says: + +- is the surrounding danger level high enough to matter? + +If the threshold is crossed, the cell becomes: + +- `3 - activated` + +If not, the cell can wait for one [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) time window in an explicit waiting +substatus. + +## Why DAMP Matters So Much + +`DAMP` evidence does not create a new T cell by itself. + +But it is still crucial. + +`DAMP` does three important things: + +- it is stored as danger context +- it contributes to the cumulative danger term in co-stimulation +- it re-triggers reevaluation of cells that are already waiting + +That means the innate danger layer is not passive background data. It actively +shapes how the adaptive layer behaves. + +Without `DAMP`, T Cell would only know that something matched. + +With `DAMP`, T Cell can tell whether the surrounding situation is intensifying, +stable, or cooling down. + +## Context: Activated Is Still Not the End + +Once a cell reaches: + +- `3 - activated` + +it still needs to decide what kind of response makes sense. + +The context stage looks at: + +- novelty +- related evidence volume +- recent pressure versus previous pressure +- weighted `PAMP` + `DAMP` danger + +The purpose of this stage is to distinguish between two very different +situations: + +- a new, intense threat that should be stopped quickly +- a familiar threat pattern that is still visible but already cooling down + +## Effector vs Memory + +If the context signals say the threat is new and intense enough, the cell moves +to: + +- `4 - effector` + +At that point T Cell reuses the existing [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) containment path and can trigger +blocking behavior such as ARP-poisoning-based isolation through the normal +blocking flow. + +If the context signals say the threat is familiar and decreasing, the cell +moves to: + +- `5 - memory` + +That path stores: + +- the matched regex +- the matched value +- the context snapshot + +in the T Cell SQLite store for later reuse. + +This is what makes the module more than a simple thresholded blocker. It is +explicitly modeling both response and retention. + +## Waiting Is a First-Class Runtime Condition + +Two states can wait: + +- `1 - antigen-recognized` +- `3 - activated` + +Those are not extra numbered states. They are explicit waiting substates +recorded on the cell context: + +- waiting for co-stimulation +- waiting for context + +That makes the runtime easier to interpret because a cell can be: + +- recognized but still not dangerous enough +- activated but still not ready for effector or memory + +And both of those waiting conditions are reevaluated on later `PAMP` or later +`DAMP` arrivals for the same responsible IP. + +## Why This Is More Than a Fancy Alert Filter + +The T Cell module is not just filtering alerts after the fact. + +It adds an actual stateful decision layer: + +- recognition +- tolerance +- co-stimulation +- activation +- context +- effector response +- memory + +That matters because it lets [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) reason about events across time instead of +treating each evidence row as a fully independent trigger. + +The module is effectively asking: + +- does this look recognizable? +- is it dangerous enough? +- is it urgent enough? +- or is it something we should remember instead? + +That is a very different model from direct one-shot escalation. + +## Why the T Cell Module Needs the Regex Module + +The T Cell module can only be adaptive because it has something adaptive to +consult. + +That comes from `RegexGenerator`. + +If the regex module did not exist, T Cell would still have: + +- `PAMP` +- `DAMP` +- danger aggregation +- thresholds + +But it would not have a selected symbolic recognition library. + +It would know danger, but not adaptive antigen recognition. + +So the full adaptive design requires both modules: + +- `RegexGenerator` creates the detector repertoire through pseudo-generation and + negative selection +- `T Cell` uses that repertoire inside a state machine that decides whether to + tolerate, activate, contain, or remember + +## Why This Is an Immune System and Not Just a Metaphor + +The mapping is close enough to be useful in engineering terms: + +- old [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) detectors -> innate sensing +- central `PAMP` / `DAMP` tagging -> innate danger language +- `RegexGenerator` -> adaptive receptor repertoire generation +- benign-corpus rejection -> negative selection +- `T Cell` state machine -> activation, tolerance, context, effector, memory + +The value of the metaphor is not decoration. It gives [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) a concrete design +for how to: + +- add adaptive recognition without replacing the old system +- separate recognition from activation +- separate activation from action +- keep explicit tolerance and explicit memory + +## What You Can Inspect + +The module stores its own artifacts per run: + +- `t_cell.log` +- `t_cell.sqlite` +- optional `t_cell_trace.jsonl` +- `t_cell_report.html` + +The report gives a static explanation of what happened: + +- how much of the run was `PAMP` versus `DAMP` +- which antigens were extracted +- which regexes matched +- which cells moved through the state machine +- which cells are waiting now +- which cells became memory or effector +- why thresholds were crossed when decision tracing was enabled + +So the module is not only stateful. It is also inspectable. + +## In Short + +The T Cell module is the adaptive response engine in [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS). + +It takes: + +- innate evidence from the old detector structure +- `PAMP` / `DAMP` tags from the central evidence pipeline +- accepted regex receptors from `RegexGenerator` + +and turns them into a stateful decision process that can: + +- recognize +- tolerate +- activate +- contain +- and remember + +That is what makes the current [Slips](https://github.com/stratosphereips/StratosphereLinuxIPS) immune architecture coherent: + +- innate sensing from the legacy evidence layer +- adaptive receptor generation from `RegexGenerator` +- adaptive live response from `T Cell` diff --git a/modules/t_cell/t_cell.py b/modules/t_cell/t_cell.py new file mode 100644 index 0000000000..74722fc33c --- /dev/null +++ b/modules/t_cell/t_cell.py @@ -0,0 +1,2957 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +import os +import re +import time +from dataclasses import dataclass +from urllib.parse import urlparse + +from modules.regex_generator.match_strength import ( + compute_match_strength, + measure_regex_specificity, +) +from slips_files.common.abstracts.imodule import IModule +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import utils +from slips_files.core.structures.evidence import ( + EvidenceSignal, + dict_to_evidence, +) + +STATE_MATURE = 0 +STATE_ANTIGEN_RECOGNIZED = 1 +STATE_ANERGIC = 2 +STATE_ACTIVATED = 3 +STATE_EFFECTOR = 4 +STATE_MEMORY = 5 + +STATE_INFO = { + STATE_MATURE: {"label": "0 - mature", "color": "\033[36m"}, + STATE_ANTIGEN_RECOGNIZED: { + "label": "1 - antigen-recognized", + "color": "\033[33m", + }, + STATE_ANERGIC: {"label": "2 - anergic", "color": "\033[34m"}, + STATE_ACTIVATED: {"label": "3 - activated", "color": "\033[35m"}, + STATE_EFFECTOR: {"label": "4 - effector", "color": "\033[31m"}, + STATE_MEMORY: {"label": "5 - memory", "color": "\033[32m"}, +} +COLOR_RESET = "\033[0m" +SUPPORTED_REGEX_TYPES = ( + "dns_domain", + "uri", + "filename", + "tls_sni", + "certificate_cn", +) +DEFAULT_COSTIM_WEIGHTS = { + "confidence": 0.35, + "related_pamps": 0.25, + "danger": 0.40, +} +DEFAULT_PRIMING_PROFILES = { + "PAMP": { + "strength": 1.0, + "co_stimulation_threshold_offset": 0.0, + "effector_threshold_offset": 0.0, + "memory_threshold_offset": 0.0, + "state_wait_timeout_factor": 1.0, + "effector_min_related_count_offset": 0, + "memory_min_related_count_offset": 0, + }, + "DAMP": { + "strength": 0.6, + "co_stimulation_threshold_offset": 0.15, + "effector_threshold_offset": 0.10, + "memory_threshold_offset": 0.05, + "state_wait_timeout_factor": 0.5, + "effector_min_related_count_offset": 1, + "memory_min_related_count_offset": 1, + }, +} +LOG_VERBOSITY_SUMMARY = 1 +LOG_VERBOSITY_DECISIONS = 2 +LOG_VERBOSITY_DEBUG = 3 +TRACE_MODE_OFF = 0 +TRACE_MODE_TRANSITIONS = 1 +TRACE_MODE_ALL = 2 +CONTEXT_REMOVE = object() +WAITING_CO_STIMULATION = "co_stimulation" +WAITING_CONTEXT = "context" +WAITING_LABELS = { + WAITING_CO_STIMULATION: "waiting for co-stimulation", + WAITING_CONTEXT: "waiting for context", +} + + +@dataclass(frozen=True) +class AntigenCandidate: + regex_type: str + value: str + + def as_dict(self) -> dict: + return {"regex_type": self.regex_type, "value": self.value} + + +@dataclass(frozen=True) +class RegexMatch: + regex_type: str + value: str + regex_hash: str + regex: str + created_at: float + specificity: float + + def as_dict(self) -> dict: + return { + "regex_type": self.regex_type, + "value": self.value, + "regex_hash": self.regex_hash, + "regex": self.regex, + "created_at": self.created_at, + "specificity": self.specificity, + } + + +class TCell(IModule): + name = "t_cell" + description = ( + "Immune-style responder that matches PAMP antigens to regexes and " + "uses both PAMP and DAMP danger pressure to escalate to blocking " + "or memory." + ) + authors = ["Sebastian Garcia"] + + def init(self): + self.channels = {} + self.subscribe_to_channels() + self.enabled = False + self.output_dir = os.path.join(self.parent_output_dir, "t_cell") + self.create_log_file = True + self.log_colors = True + self.log_verbosity = LOG_VERBOSITY_SUMMARY + self.log_file_path = self.get_module_specific_output_path("t_cell.log") + self.decision_trace_mode = TRACE_MODE_OFF + self.decision_trace_max_evidence = 10 + self.trace_file_path = os.path.join( + self.output_dir, "t_cell_trace.jsonl" + ) + self.storage = None + self.state_wait_timeout_seconds = 3600.0 + self.observation_retention_seconds = 604800 + self.anergy_ttl_seconds = 21600 + self.related_lookback_seconds = 3600 + self.related_pamps_saturation = 5.0 + self.danger_saturation = 2.5 + self.damp_danger_weight = 1.5 + self.co_stimulation_threshold = 0.65 + self.co_stimulation_weights = DEFAULT_COSTIM_WEIGHTS.copy() + self.priming_profiles = self._normalize_priming_profiles( + DEFAULT_PRIMING_PROFILES + ) + self.novelty_window_seconds = 86400 + self.context_recent_window_seconds = 1800 + self.effector_threshold = 0.70 + self.effector_min_related_count = 4 + self.effector_cooldown_seconds = 1800 + self.memory_threshold = 0.60 + self.memory_trend_ratio_max = 0.60 + self.memory_min_related_count = 3 + self.simulate_effector_without_blocking = True + self.read_configuration() + + def subscribe_to_channels(self): + """ + Subscribe to the evidence stream consumed by the T Cell module. + + Returns: + None + """ + if self.channels: + return + + self.c_evidence = self.db.subscribe("evidence_added") + self.channels = {"evidence_added": self.c_evidence} + + def read_configuration(self): + conf = ( + self.conf + if hasattr(self.conf, "t_cell_enabled") + else ConfigParser() + ) + self.enabled = conf.t_cell_enabled() + self.create_log_file = conf.t_cell_create_log_file() + self.log_colors = conf.t_cell_log_colors() + self.log_verbosity = conf.t_cell_log_verbosity() + self.decision_trace_mode = conf.t_cell_decision_trace_mode() + self.decision_trace_max_evidence = ( + conf.t_cell_decision_trace_max_evidence() + ) + self.trace_file_path = self._resolve_trace_file_path( + conf.t_cell_decision_trace_file() + ) + try: + self.state_wait_timeout_seconds = float( + conf.get_tw_width_in_seconds() + ) + except Exception: + self.state_wait_timeout_seconds = 3600.0 + self.observation_retention_seconds = ( + conf.t_cell_observation_retention_seconds() + ) + self.anergy_ttl_seconds = conf.t_cell_anergy_ttl_seconds() + self.related_lookback_seconds = conf.t_cell_related_lookback_seconds() + self.related_pamps_saturation = conf.t_cell_related_pamps_saturation() + self.danger_saturation = conf.t_cell_danger_saturation() + self.damp_danger_weight = conf.t_cell_damp_danger_weight() + self.co_stimulation_threshold = conf.t_cell_co_stimulation_threshold() + self.co_stimulation_weights = self._normalize_weights( + conf.t_cell_co_stimulation_weights() + ) + self.priming_profiles = self._normalize_priming_profiles( + conf.t_cell_priming_profiles() + ) + self.novelty_window_seconds = conf.t_cell_novelty_window_seconds() + self.context_recent_window_seconds = ( + conf.t_cell_context_recent_window_seconds() + ) + self.effector_threshold = conf.t_cell_effector_threshold() + self.effector_min_related_count = ( + conf.t_cell_effector_min_related_count() + ) + self.effector_cooldown_seconds = ( + conf.t_cell_effector_cooldown_seconds() + ) + self.memory_threshold = conf.t_cell_memory_threshold() + self.memory_trend_ratio_max = conf.t_cell_memory_trend_ratio_max() + self.memory_min_related_count = conf.t_cell_memory_min_related_count() + self.simulate_effector_without_blocking = ( + conf.t_cell_simulate_effector_without_blocking() + ) + + def pre_main(self): + utils.drop_root_privs_permanently() + if not self.enabled: + self.print("T Cell module disabled in config.", 2, 0) + return True + + self.storage = self.db.get_t_cell_storage() + self._init_log_file() + self._init_trace_file() + self._log_detail("T Cell module ready.") + return False + + def main(self): + if msg := self.get_msg("evidence_added"): + self._process_evidence_message(msg) + + @staticmethod + def _normalize_weights(weights: dict) -> dict: + sanitized = {} + for key, default_value in DEFAULT_COSTIM_WEIGHTS.items(): + raw_value = weights.get(key, default_value) + try: + raw_value = float(raw_value) + except (TypeError, ValueError): + raw_value = default_value + sanitized[key] = max(0.0, raw_value) + + total = sum(sanitized.values()) + if total <= 0: + total = sum(DEFAULT_COSTIM_WEIGHTS.values()) + sanitized = DEFAULT_COSTIM_WEIGHTS.copy() + return {key: value / total for key, value in sanitized.items()} + + @staticmethod + def _normalize_priming_profiles(profiles: dict | None) -> dict: + raw_profiles = profiles if isinstance(profiles, dict) else {} + normalized = {} + for signal_name, defaults in DEFAULT_PRIMING_PROFILES.items(): + raw_profile = raw_profiles.get(signal_name, {}) + if not isinstance(raw_profile, dict): + raw_profile = {} + + profile = {} + for key, default_value in defaults.items(): + raw_value = raw_profile.get(key, default_value) + if isinstance(default_value, int) and not isinstance( + default_value, bool + ): + try: + raw_value = int(raw_value) + except (TypeError, ValueError): + raw_value = int(default_value) + else: + try: + raw_value = float(raw_value) + except (TypeError, ValueError): + raw_value = float(default_value) + profile[key] = raw_value + + profile["strength"] = max( + 0.0, min(1.0, float(profile["strength"])) + ) + profile["state_wait_timeout_factor"] = max( + 0.01, float(profile["state_wait_timeout_factor"]) + ) + profile["co_stimulation_threshold_offset"] = float( + profile["co_stimulation_threshold_offset"] + ) + profile["effector_threshold_offset"] = float( + profile["effector_threshold_offset"] + ) + profile["memory_threshold_offset"] = float( + profile["memory_threshold_offset"] + ) + profile["effector_min_related_count_offset"] = int( + profile["effector_min_related_count_offset"] + ) + profile["memory_min_related_count_offset"] = int( + profile["memory_min_related_count_offset"] + ) + normalized[signal_name] = profile + return normalized + + def _process_evidence_message(self, message: dict): + try: + raw_evidence = json.loads(message["data"]) + evidence = dict_to_evidence(raw_evidence) + except Exception: + self.print_traceback() + return + + now = time.time() + responsible_ip = self._get_responsible_ip(evidence) + antigens = self._extract_antigen_candidates(evidence) + if antigens: + self._log_event( + action="antigens_extracted", + state=None, + evidence=evidence, + details=( + "antigens=" + + ", ".join( + f"{candidate.regex_type}:{candidate.value}" + for candidate in antigens + ) + ), + verbosity=LOG_VERBOSITY_DEBUG, + ) + observation_id = self.storage.insert_observation( + { + "evidence_id": evidence.id, + "evidence_type": str(evidence.evidence_type), + "evidence_signal": str(evidence.evidence_signal), + "profile_ip": responsible_ip, + "timewindow_number": evidence.timewindow.number, + "timestamp": evidence.timestamp, + "observed_at": now, + "confidence": evidence.confidence, + "threat_level": str(evidence.threat_level), + "threat_level_value": float(evidence.threat_level.value), + "interface": evidence.interface, + "uids": evidence.uid, + "antigen_count": len(antigens), + "antigens": [candidate.as_dict() for candidate in antigens], + "matched_regexes": [], + "raw_evidence": raw_evidence, + } + ) + matched_regexes = [] + + reevaluated_count = 0 + if evidence.evidence_signal == EvidenceSignal.DAMP: + reevaluated_count = self._reevaluate_waiting_cells( + evidence=evidence, + observation_id=observation_id, + responsible_ip=responsible_ip, + now=now, + ) + self._log_event( + action="damp_reverification", + state=None, + evidence=evidence, + metrics={"reevaluated_cells": reevaluated_count}, + details=( + "stored DAMP danger and rechecked waiting cells for this " + "responsible IP" + ), + verbosity=LOG_VERBOSITY_DECISIONS, + ) + elif evidence.evidence_signal != EvidenceSignal.PAMP: + self._log_event( + action="ignored_non_pamp", + state=None, + evidence=evidence, + verbosity=LOG_VERBOSITY_DECISIONS, + ) + self._prune_observations(now) + return + + if not antigens: + self._log_event( + action="no_antigen_extracted", + state=None, + evidence=evidence, + details=( + "no supported dns_domain/uri/filename/tls_sni/" + "certificate_cn values found" + ), + verbosity=LOG_VERBOSITY_DECISIONS, + ) + self._prune_observations(now) + return + + for candidate in antigens: + match = self._process_candidate( + evidence, + observation_id, + candidate, + now, + responsible_ip, + ) + if match: + matched_regexes.append(match.as_dict()) + + self.storage.update_observation_matches( + observation_id, matched_regexes + ) + self._prune_observations(now) + + def _process_candidate( + self, + evidence, + observation_id: int, + candidate: AntigenCandidate, + now: float, + responsible_ip: str, + ) -> RegexMatch | None: + signal_name = self._signal_name(evidence.evidence_signal) + cell = self._get_or_create_cell( + responsible_ip, candidate.regex_type, candidate.value, now + ) + + if ( + cell["state"] == STATE_ANERGIC + and cell.get("anergic_until") + and now < cell["anergic_until"] + ): + self._log_event( + action="anergy_suppressed", + state=cell["state"], + evidence=evidence, + cell=cell, + details=f"until={cell['anergic_until']:.3f}", + verbosity=LOG_VERBOSITY_DECISIONS, + ) + return None + + if signal_name == "DAMP" and cell["state"] != STATE_MATURE: + self._log_event( + action="damp_match_skipped_existing_cell", + state=cell["state"], + evidence=evidence, + cell=cell, + details=( + "existing non-mature cells are reevaluated through the " + "waiting-cell path; skipping direct DAMP candidate " + "advancement for this cell" + ), + verbosity=LOG_VERBOSITY_DEBUG, + ) + return None + + if ( + cell["state"] == STATE_ANERGIC + and cell.get("anergic_until") + and now >= cell["anergic_until"] + ): + cell = self._transition_cell( + cell=cell, + to_state=STATE_MATURE, + reason="anergy_expired", + evidence=evidence, + observation_id=observation_id, + now=now, + scores={"anergic_until": None}, + extra_updates={"anergic_until": None}, + ) + + match = self._find_best_regex_match(candidate) + if not match: + if cell["state"] == STATE_MATURE: + cell = self._transition_cell( + cell=cell, + to_state=STATE_ANERGIC, + reason="no_regex_match", + evidence=evidence, + observation_id=observation_id, + now=now, + scores={"anergic_until": now + self.anergy_ttl_seconds}, + extra_updates={ + "anergic_until": now + self.anergy_ttl_seconds + }, + ) + else: + self._update_cell( + cell, + now, + last_observation_id=observation_id, + last_evidence_id=evidence.id, + ) + self._update_cell_context( + cell, + now, + reason="no_regex_match_after_activation", + observation_id=observation_id, + ) + self._log_event( + action="no_regex_match", + state=cell["state"], + evidence=evidence, + cell=cell, + details=( + "cell already active; keeping current state without " + "a new regex match" + ), + verbosity=LOG_VERBOSITY_DECISIONS, + ) + return None + + match_updates = { + "matched_regex_hash": match.regex_hash, + "matched_regex": match.regex, + "matched_value": match.value, + "last_observation_id": observation_id, + "last_evidence_id": evidence.id, + "anergic_until": None, + } + newly_recognized = False + if cell["state"] == STATE_MATURE: + priming_profile = self._build_effective_priming_profile( + signal_name + ) + cell = self._transition_cell( + cell=cell, + to_state=STATE_ANTIGEN_RECOGNIZED, + reason="antigen_recognized", + evidence=evidence, + observation_id=observation_id, + now=now, + match=match, + scores={ + "priming_label": priming_profile["label"], + "regex_specificity": match.specificity, + "priming_signal": signal_name, + "priming_strength": priming_profile["strength"], + "base_co_stimulation_threshold": priming_profile[ + "base_co_stimulation_threshold" + ], + "co_stimulation_threshold": priming_profile[ + "co_stimulation_threshold" + ], + "co_stimulation_threshold_offset": priming_profile[ + "co_stimulation_threshold_offset" + ], + "base_effector_threshold": priming_profile[ + "base_effector_threshold" + ], + "effector_threshold": priming_profile[ + "effector_threshold" + ], + "effector_threshold_offset": priming_profile[ + "effector_threshold_offset" + ], + "base_memory_threshold": priming_profile[ + "base_memory_threshold" + ], + "memory_threshold": priming_profile["memory_threshold"], + "memory_threshold_offset": priming_profile[ + "memory_threshold_offset" + ], + "base_state_wait_timeout_seconds": priming_profile[ + "base_state_wait_timeout_seconds" + ], + "state_wait_timeout_seconds": priming_profile[ + "state_wait_timeout_seconds" + ], + "state_wait_timeout_factor": priming_profile[ + "state_wait_timeout_factor" + ], + "base_effector_min_related_count": priming_profile[ + "base_effector_min_related_count" + ], + "effector_min_related_count": priming_profile[ + "effector_min_related_count" + ], + "effector_min_related_count_offset": priming_profile[ + "effector_min_related_count_offset" + ], + "base_memory_min_related_count": priming_profile[ + "base_memory_min_related_count" + ], + "memory_min_related_count": priming_profile[ + "memory_min_related_count" + ], + "memory_min_related_count_offset": priming_profile[ + "memory_min_related_count_offset" + ], + }, + extra_updates=match_updates, + ) + cell = self._remember_consumed_transition_observation( + cell, + now, + observation_id, + ) + newly_recognized = True + cell = self._remember_priming_context( + cell, + now, + observation_id, + evidence.id, + signal_name, + ) + else: + cell = self._update_cell( + cell, + now, + **match_updates, + ) + cell = self._remember_match_context( + cell, + now, + observation_id, + evidence.id, + match, + ) + if newly_recognized: + cell = self._set_waiting_context( + cell, + now, + WAITING_CO_STIMULATION, + evidence, + observation_id, + ) + self._log_event( + action="waiting_for_co_stimulation", + state=cell["state"], + evidence=evidence, + cell=cell, + match=match, + details=( + "recognized the antigen and stored the match context; " + "co-stimulation is deferred until future evidence so this " + "same observation cannot drive the next state change" + ), + metrics={ + "consumed_observation_id": observation_id, + "consumed_evidence_id": evidence.id, + }, + verbosity=LOG_VERBOSITY_DECISIONS, + ) + return match + + if cell["state"] == STATE_MEMORY: + self._update_cell_context( + cell, + now, + reason="memory_retained", + observation_id=observation_id, + matched_regex_hash=match.regex_hash, + ) + self._log_event( + action="memory_retained", + state=STATE_MEMORY, + evidence=evidence, + cell=cell, + match=match, + details=( + "memory already exists for this cell; keeping the memory " + "state without storing a new memory event" + ), + verbosity=LOG_VERBOSITY_DEBUG, + ) + return match + + return self._advance_cell_with_match( + cell=cell, + evidence=evidence, + observation_id=observation_id, + candidate=candidate, + match=match, + now=now, + responsible_ip=responsible_ip, + ) + + def _get_or_create_cell( + self, profile_ip: str, regex_type: str, antigen_value: str, now: float + ) -> dict: + cell_key = self._make_cell_key(profile_ip, regex_type, antigen_value) + cell = self.storage.get_cell(cell_key) + if cell: + return cell + + return { + "cell_key": cell_key, + "profile_ip": profile_ip, + "regex_type": regex_type, + "antigen_value": antigen_value, + "state": STATE_MATURE, + "state_name": STATE_INFO[STATE_MATURE]["label"], + "matched_regex_hash": None, + "matched_regex": None, + "matched_value": None, + "anergic_until": None, + "effector_cooldown_until": None, + "last_observation_id": None, + "last_evidence_id": None, + "last_transition_at": None, + "last_co_stimulation": None, + "last_effector_score": None, + "last_memory_score": None, + "context": {}, + "created_at": now, + "updated_at": now, + } + + def _transition_cell( + self, + cell: dict, + to_state: int, + reason: str, + evidence, + observation_id: int, + now: float, + match: RegexMatch | None = None, + scores: dict | None = None, + extra_updates: dict | None = None, + ) -> dict: + from_state = cell["state"] + updates = { + "state": to_state, + "state_name": STATE_INFO[to_state]["label"], + "last_observation_id": observation_id, + "last_evidence_id": evidence.id, + "last_transition_at": now, + } + if match: + updates.update( + { + "matched_regex_hash": match.regex_hash, + "matched_regex": match.regex, + "matched_value": match.value, + } + ) + if extra_updates: + updates.update(extra_updates) + + cell = self._update_cell(cell, now, **updates) + self.storage.insert_transition( + { + "cell_key": cell["cell_key"], + "profile_ip": cell["profile_ip"], + "regex_type": cell["regex_type"], + "antigen_value": cell["antigen_value"], + "evidence_id": evidence.id, + "observation_id": observation_id, + "from_state": from_state, + "to_state": to_state, + "reason": reason, + "matched_regex_hash": cell.get("matched_regex_hash"), + "matched_regex": cell.get("matched_regex"), + "matched_value": cell.get("matched_value"), + "scores": scores or {}, + "created_at": now, + } + ) + self._log_event( + action=reason, + state=to_state, + evidence=evidence, + cell=cell, + match=match, + metrics=scores, + verbosity=LOG_VERBOSITY_SUMMARY, + ) + return cell + + def _update_cell(self, cell: dict, now: float, **updates) -> dict: + cell.update(updates) + cell["updated_at"] = now + self.storage.upsert_cell(cell) + return cell + + @staticmethod + def _merge_cell_context_values(cell: dict, **updates) -> dict: + merged = dict(cell.get("context") or {}) + for key, value in updates.items(): + if value is CONTEXT_REMOVE: + merged.pop(key, None) + continue + merged[key] = value + return merged + + def _update_cell_context(self, cell: dict, now: float, **updates) -> dict: + return self._update_cell( + cell, + now, + context=self._merge_cell_context_values(cell, **updates), + ) + + def _remember_match_context( + self, + cell: dict, + now: float, + observation_id: int, + evidence_id: str, + match: RegexMatch, + ) -> dict: + return self._update_cell_context( + cell, + now, + recognition_observation_id=observation_id, + recognition_evidence_id=evidence_id, + matched_regex_created_at=match.created_at, + matched_regex_specificity=match.specificity, + ) + + @staticmethod + def _signal_name(signal_value) -> str: + text = str(signal_value or "").strip().upper() + if "." in text: + text = text.rsplit(".", 1)[-1] + if text in DEFAULT_PRIMING_PROFILES: + return text + return "PAMP" + + def _build_effective_priming_profile(self, signal_name: str) -> dict: + profile = self.priming_profiles.get( + signal_name, self.priming_profiles["PAMP"] + ) + return { + "signal": signal_name, + "label": f"{signal_name.lower()}-primed", + "strength": float(profile["strength"]), + "base_co_stimulation_threshold": self.co_stimulation_threshold, + "base_effector_threshold": self.effector_threshold, + "base_memory_threshold": self.memory_threshold, + "base_state_wait_timeout_seconds": self.state_wait_timeout_seconds, + "base_effector_min_related_count": self.effector_min_related_count, + "base_memory_min_related_count": self.memory_min_related_count, + "co_stimulation_threshold_offset": float( + profile["co_stimulation_threshold_offset"] + ), + "effector_threshold_offset": float( + profile["effector_threshold_offset"] + ), + "memory_threshold_offset": float( + profile["memory_threshold_offset"] + ), + "state_wait_timeout_factor": float( + profile["state_wait_timeout_factor"] + ), + "effector_min_related_count_offset": int( + profile["effector_min_related_count_offset"] + ), + "memory_min_related_count_offset": int( + profile["memory_min_related_count_offset"] + ), + "co_stimulation_threshold": self._clamp01( + self.co_stimulation_threshold + + float(profile["co_stimulation_threshold_offset"]) + ), + "effector_threshold": self._clamp01( + self.effector_threshold + + float(profile["effector_threshold_offset"]) + ), + "memory_threshold": self._clamp01( + self.memory_threshold + + float(profile["memory_threshold_offset"]) + ), + "state_wait_timeout_seconds": max( + 1.0, + self.state_wait_timeout_seconds + * float(profile["state_wait_timeout_factor"]), + ), + "effector_min_related_count": max( + 1, + self.effector_min_related_count + + int(profile["effector_min_related_count_offset"]), + ), + "memory_min_related_count": max( + 1, + self.memory_min_related_count + + int(profile["memory_min_related_count_offset"]), + ), + } + + def _remember_priming_context( + self, + cell: dict, + now: float, + observation_id: int, + evidence_id: str, + signal_name: str, + ) -> dict: + priming_profile = self._build_effective_priming_profile(signal_name) + return self._update_cell_context( + cell, + now, + priming_signal=signal_name, + priming_label=priming_profile["label"], + priming_strength=priming_profile["strength"], + priming_observation_id=observation_id, + priming_evidence_id=evidence_id, + priming_profile=priming_profile, + ) + + def _get_cell_priming_profile(self, cell: dict) -> dict: + context = cell.get("context") or {} + priming_profile = context.get("priming_profile") + if isinstance(priming_profile, dict) and priming_profile.get("signal"): + return priming_profile + signal_name = self._signal_name(context.get("priming_signal")) + return self._build_effective_priming_profile(signal_name) + + @staticmethod + def _normalize_observation_ids(values) -> set[int]: + if values is None: + return set() + if isinstance(values, (list, tuple, set)): + raw_values = values + else: + raw_values = [values] + + normalized = set() + for value in raw_values: + try: + normalized.add(int(value)) + except (TypeError, ValueError): + continue + return normalized + + def _get_consumed_transition_observation_ids(self, cell: dict) -> set[int]: + context = cell.get("context") or {} + return self._normalize_observation_ids( + context.get("consumed_transition_observation_ids") + ) + + def _remember_consumed_transition_observation( + self, + cell: dict, + now: float, + observation_id: int, + ) -> dict: + consumed_ids = self._get_consumed_transition_observation_ids(cell) + consumed_ids.add(int(observation_id)) + return self._update_cell_context( + cell, + now, + consumed_transition_observation_ids=sorted(consumed_ids), + ) + + def _clear_waiting_context(self, cell: dict, now: float) -> dict: + return self._update_cell_context( + cell, + now, + waiting_for=CONTEXT_REMOVE, + waiting_label=CONTEXT_REMOVE, + waiting_since=CONTEXT_REMOVE, + wait_deadline=CONTEXT_REMOVE, + wait_trigger_signal=CONTEXT_REMOVE, + wait_trigger_evidence_id=CONTEXT_REMOVE, + wait_trigger_observation_id=CONTEXT_REMOVE, + ) + + def _effective_wait_limit(self, cell: dict) -> float: + return float( + self._get_cell_priming_profile(cell).get( + "state_wait_timeout_seconds", self.state_wait_timeout_seconds + ) + ) + + def _set_waiting_context( + self, + cell: dict, + now: float, + waiting_for: str, + evidence, + observation_id: int, + ) -> dict: + context = cell.get("context") or {} + waiting_since = context.get("waiting_since") + if context.get("waiting_for") != waiting_for or waiting_since is None: + waiting_since = ( + cell.get("last_transition_at") or cell.get("created_at") or now + ) + try: + waiting_since = float(waiting_since) + except (TypeError, ValueError): + waiting_since = float(now) + return self._update_cell_context( + cell, + now, + waiting_for=waiting_for, + waiting_label=WAITING_LABELS.get(waiting_for, waiting_for), + waiting_since=waiting_since, + wait_deadline=waiting_since + self._effective_wait_limit(cell), + wait_trigger_signal=str(evidence.evidence_signal), + wait_trigger_evidence_id=evidence.id, + wait_trigger_observation_id=observation_id, + ) + + def _build_match_from_cell(self, cell: dict) -> RegexMatch | None: + regex_hash = str(cell.get("matched_regex_hash") or "").strip() + regex = str(cell.get("matched_regex") or "").strip() + regex_type = str(cell.get("regex_type") or "").strip() + value = str( + cell.get("matched_value") or cell.get("antigen_value") or "" + ).strip() + if not (regex_hash and regex and regex_type and value): + return None + + context = cell.get("context") or {} + created_at = context.get("matched_regex_created_at") or 0.0 + try: + created_at = float(created_at) + except (TypeError, ValueError): + created_at = 0.0 + + specificity = context.get("matched_regex_specificity") + try: + specificity = float(specificity) + except (TypeError, ValueError): + specificity = measure_regex_specificity(regex) + + return RegexMatch( + regex_type=regex_type, + value=value, + regex_hash=regex_hash, + regex=regex, + created_at=created_at, + specificity=specificity, + ) + + def _reevaluate_waiting_cells( + self, + evidence, + observation_id: int, + responsible_ip: str, + now: float, + ) -> int: + waiting_cells = self.storage.get_cells_for_profile_states( + responsible_ip, + [STATE_ANTIGEN_RECOGNIZED, STATE_ACTIVATED], + ) + reevaluated = 0 + for cell in waiting_cells: + match = self._build_match_from_cell(cell) + if not match: + self._log_event( + action="waiting_cell_missing_match", + state=cell["state"], + evidence=evidence, + cell=cell, + details=( + "cannot reevaluate waiting cell because the stored " + "regex match metadata is incomplete" + ), + verbosity=LOG_VERBOSITY_DEBUG, + ) + continue + + candidate = AntigenCandidate( + regex_type=cell["regex_type"], + value=cell["antigen_value"], + ) + self._advance_cell_with_match( + cell=cell, + evidence=evidence, + observation_id=observation_id, + candidate=candidate, + match=match, + now=now, + responsible_ip=responsible_ip, + ) + reevaluated += 1 + return reevaluated + + def _advance_cell_with_match( + self, + cell: dict, + evidence, + observation_id: int, + candidate: AntigenCandidate, + match: RegexMatch, + now: float, + responsible_ip: str, + ) -> RegexMatch: + if ( + cell.get("last_observation_id") != observation_id + or cell.get("last_evidence_id") != evidence.id + ): + cell = self._update_cell( + cell, + now, + last_observation_id=observation_id, + last_evidence_id=evidence.id, + ) + + consumed_observation_ids = ( + self._get_consumed_transition_observation_ids(cell) + ) + priming_profile = self._get_cell_priming_profile(cell) + wait_limit = float( + priming_profile.get( + "state_wait_timeout_seconds", self.state_wait_timeout_seconds + ) + ) + if cell["state"] == STATE_MEMORY: + cell = self._update_cell_context( + cell, + now, + reason="memory_retained", + observation_id=observation_id, + matched_regex_hash=match.regex_hash, + ) + self._log_event( + action="memory_retained", + state=STATE_MEMORY, + evidence=evidence, + cell=cell, + match=match, + details=( + "memory already exists for this cell; keeping the memory " + "state without storing a new memory event" + ), + verbosity=LOG_VERBOSITY_DEBUG, + ) + return match + + co_stimulation = self._compute_co_stimulation( + responsible_ip, + observation_id, + candidate, + match, + now, + priming_profile=priming_profile, + exclude_observation_ids=consumed_observation_ids, + ) + cell = self._update_cell( + cell, + now, + last_co_stimulation=co_stimulation["value"], + ) + cell = self._update_cell_context( + cell, + now, + co_stimulation=co_stimulation, + ) + + if cell["state"] < STATE_ACTIVATED: + wait_elapsed = self._get_state_wait_elapsed(cell, now) + if co_stimulation["value"] >= co_stimulation["threshold"]: + self._maybe_trace_co_stimulation( + action="co_stimulation_threshold_met", + evidence=evidence, + cell=cell, + candidate=candidate, + match=match, + co_stimulation=co_stimulation, + responsible_ip=responsible_ip, + observation_id=observation_id, + now=now, + from_state=cell["state"], + to_state=STATE_ACTIVATED, + ) + cell = self._transition_cell( + cell=cell, + to_state=STATE_ACTIVATED, + reason="co_stimulation_threshold_met", + evidence=evidence, + observation_id=observation_id, + now=now, + match=match, + scores=co_stimulation, + ) + cell = self._remember_consumed_transition_observation( + cell, + now, + observation_id, + ) + cell = self._set_waiting_context( + cell, + now, + WAITING_CONTEXT, + evidence, + observation_id, + ) + self._log_event( + action="waiting_for_context", + state=cell["state"], + evidence=evidence, + cell=cell, + match=match, + details=( + "co-stimulation activated the cell; context is " + "deferred until future evidence so this same " + "observation cannot drive the next state change" + ), + metrics={ + "consumed_observation_id": observation_id, + "consumed_evidence_id": evidence.id, + }, + verbosity=LOG_VERBOSITY_DECISIONS, + ) + return match + elif cell[ + "state" + ] == STATE_ANTIGEN_RECOGNIZED and self._state_wait_expired( + cell, now + ): + self._maybe_trace_co_stimulation( + action="co_stimulation_timeout", + evidence=evidence, + cell=cell, + candidate=candidate, + match=match, + co_stimulation={ + **co_stimulation, + "elapsed": wait_elapsed, + "wait_limit": wait_limit, + "anergic_until": now + self.anergy_ttl_seconds, + }, + responsible_ip=responsible_ip, + observation_id=observation_id, + now=now, + from_state=cell["state"], + to_state=STATE_ANERGIC, + ) + cell = self._transition_cell( + cell=cell, + to_state=STATE_ANERGIC, + reason="co_stimulation_timeout", + evidence=evidence, + observation_id=observation_id, + now=now, + match=match, + scores={ + **co_stimulation, + "elapsed": wait_elapsed, + "wait_limit": wait_limit, + "anergic_until": now + self.anergy_ttl_seconds, + }, + extra_updates={ + "anergic_until": now + self.anergy_ttl_seconds, + }, + ) + cell = self._clear_waiting_context(cell, now) + return match + else: + cell = self._set_waiting_context( + cell, + now, + WAITING_CO_STIMULATION, + evidence, + observation_id, + ) + self._maybe_trace_co_stimulation( + action="waiting_for_co_stimulation", + evidence=evidence, + cell=cell, + candidate=candidate, + match=match, + co_stimulation={ + **co_stimulation, + "elapsed": wait_elapsed, + "wait_limit": wait_limit, + }, + responsible_ip=responsible_ip, + observation_id=observation_id, + now=now, + from_state=cell["state"], + to_state=cell["state"], + ) + self._log_event( + action="waiting_for_co_stimulation", + state=cell["state"], + evidence=evidence, + cell=cell, + match=match, + details=( + "score below threshold; keeping the cell in " + "antigen-recognized state and reevaluating on future " + "PAMP or DAMP evidence" + ), + metrics={ + "score": co_stimulation["value"], + "threshold": co_stimulation["threshold"], + "gap": max( + 0.0, + co_stimulation["threshold"] + - co_stimulation["value"], + ), + "confidence": co_stimulation["confidence"], + "related_pamps": co_stimulation["related_pamp_count"], + "related_score": co_stimulation["related_pamp_score"], + "danger_score": co_stimulation["profile_danger_score"], + "pamp_danger": co_stimulation["pamp_danger_score"], + "damp_danger": co_stimulation["damp_danger_score"], + "damp_weight": co_stimulation["damp_danger_weight"], + "elapsed": wait_elapsed, + "wait_limit": wait_limit, + }, + verbosity=LOG_VERBOSITY_DECISIONS, + ) + return match + + context = self._compute_context_signals( + responsible_ip, + observation_id, + candidate, + match, + now, + priming_profile=priming_profile, + exclude_observation_ids=consumed_observation_ids, + ) + cell = self._update_cell( + cell, + now, + last_effector_score=context["effector_score"], + last_memory_score=context["memory_score"], + ) + cell = self._update_cell_context( + cell, + now, + co_stimulation=co_stimulation, + context=context, + ) + + if context["effector"]: + self._maybe_trace_context( + action="context_effector", + evidence=evidence, + cell=cell, + candidate=candidate, + match=match, + context=context, + responsible_ip=responsible_ip, + observation_id=observation_id, + now=now, + from_state=cell["state"], + to_state=STATE_EFFECTOR, + ) + if cell["state"] != STATE_EFFECTOR: + cell = self._transition_cell( + cell=cell, + to_state=STATE_EFFECTOR, + reason="context_effector", + evidence=evidence, + observation_id=observation_id, + now=now, + match=match, + scores=context, + ) + cell = self._remember_consumed_transition_observation( + cell, + now, + observation_id, + ) + cell = self._clear_waiting_context(cell, now) + self._apply_effector( + cell, + evidence, + match, + context, + now, + responsible_ip, + ) + return match + + if context["memory"]: + self._maybe_trace_context( + action="context_memory", + evidence=evidence, + cell=cell, + candidate=candidate, + match=match, + context=context, + responsible_ip=responsible_ip, + observation_id=observation_id, + now=now, + from_state=cell["state"], + to_state=STATE_MEMORY, + ) + if cell["state"] != STATE_MEMORY: + cell = self._transition_cell( + cell=cell, + to_state=STATE_MEMORY, + reason="context_memory", + evidence=evidence, + observation_id=observation_id, + now=now, + match=match, + scores=context, + ) + cell = self._remember_consumed_transition_observation( + cell, + now, + observation_id, + ) + cell = self._clear_waiting_context(cell, now) + self._store_memory(cell, match, context, now) + self._log_event( + action="memory_stored", + state=STATE_MEMORY, + evidence=evidence, + cell=cell, + match=match, + metrics={"memory_score": context["memory_score"]}, + verbosity=LOG_VERBOSITY_SUMMARY, + ) + return match + + wait_elapsed = self._get_state_wait_elapsed(cell, now) + if cell["state"] == STATE_ACTIVATED and self._state_wait_expired( + cell, now + ): + self._maybe_trace_context( + action="context_timeout", + evidence=evidence, + cell=cell, + candidate=candidate, + match=match, + context={ + **context, + "elapsed": wait_elapsed, + "wait_limit": wait_limit, + }, + responsible_ip=responsible_ip, + observation_id=observation_id, + now=now, + from_state=cell["state"], + to_state=STATE_MATURE, + ) + cell = self._transition_cell( + cell=cell, + to_state=STATE_MATURE, + reason="context_timeout", + evidence=evidence, + observation_id=observation_id, + now=now, + match=match, + scores={ + **context, + "elapsed": wait_elapsed, + "wait_limit": wait_limit, + }, + ) + cell = self._clear_waiting_context(cell, now) + return match + + cell = self._set_waiting_context( + cell, + now, + WAITING_CONTEXT, + evidence, + observation_id, + ) + self._maybe_trace_context( + action="waiting_for_context", + evidence=evidence, + cell=cell, + candidate=candidate, + match=match, + context={ + **context, + "elapsed": wait_elapsed, + "wait_limit": wait_limit, + }, + responsible_ip=responsible_ip, + observation_id=observation_id, + now=now, + from_state=cell["state"], + to_state=cell["state"], + ) + self._log_event( + action="waiting_for_context", + state=cell["state"], + evidence=evidence, + cell=cell, + match=match, + details=( + "context is not strong enough yet for effector or memory; " + "keeping the current state and reevaluating on future PAMP " + "or DAMP evidence" + ), + metrics={ + "effector_score": context["effector_score"], + "effector_threshold": context["effector_threshold"], + "memory_score": context["memory_score"], + "memory_threshold": context["memory_threshold"], + "novelty_score": context["novelty_score"], + "related_pamps": context["recent_related_count"], + "recent_pamp_pressure": context["recent_pamp_pressure"], + "recent_damp_pressure": context["recent_damp_pressure"], + "previous_pamp_pressure": context["previous_pamp_pressure"], + "previous_damp_pressure": context["previous_damp_pressure"], + "damp_weight": context["damp_danger_weight"], + "trend_ratio": context["trend_ratio"], + "elapsed": wait_elapsed, + "wait_limit": wait_limit, + }, + verbosity=LOG_VERBOSITY_DECISIONS, + ) + return match + + def _compute_co_stimulation( + self, + profile_ip: str, + observation_id: int, + candidate: AntigenCandidate, + match: RegexMatch, + now: float, + priming_profile: dict, + exclude_observation_ids: set[int] | None = None, + ) -> dict: + exclude_observation_ids = self._normalize_observation_ids( + exclude_observation_ids + ) + pamp_observations = self.storage.get_recent_observations( + profile_ip, + now - self.related_lookback_seconds, + evidence_signal="PAMP", + ) + damp_observations = self.storage.get_recent_observations( + profile_ip, + now - self.related_lookback_seconds, + evidence_signal="DAMP", + ) + current_observation = ( + self.storage.get_observation(observation_id) or {} + ) + confidence = float(current_observation.get("confidence", 0.0)) + related_exclusions = set(exclude_observation_ids) + related_exclusions.add(int(observation_id)) + related_pamp_count = self._count_related_observations( + pamp_observations, + candidate, + match.regex_hash, + exclude_observation_ids=related_exclusions, + ) + related_pamp_score = self._clamp01( + related_pamp_count / self.related_pamps_saturation + ) + danger_scores = self._compute_danger_scores( + pamp_observations, + damp_observations, + exclude_observation_ids=exclude_observation_ids, + ) + profile_danger_score = danger_scores["combined_score"] + value = ( + self.co_stimulation_weights["confidence"] * confidence + + self.co_stimulation_weights["related_pamps"] * related_pamp_score + + self.co_stimulation_weights["danger"] * profile_danger_score + ) + return { + "value": value, + "confidence": confidence, + "confidence_observation_id": current_observation.get("id"), + "confidence_evidence_id": current_observation.get("evidence_id"), + "related_pamp_count": related_pamp_count, + "related_pamp_score": related_pamp_score, + "profile_danger_score": profile_danger_score, + "pamp_danger_score": danger_scores["pamp_score"], + "damp_danger_score": danger_scores["damp_score"], + "damp_danger_weight": self.damp_danger_weight, + "threshold": priming_profile["co_stimulation_threshold"], + "priming_signal": priming_profile["signal"], + "priming_label": priming_profile["label"], + "priming_strength": priming_profile["strength"], + "base_co_stimulation_threshold": priming_profile[ + "base_co_stimulation_threshold" + ], + "co_stimulation_threshold_offset": priming_profile[ + "co_stimulation_threshold_offset" + ], + "base_effector_threshold": priming_profile[ + "base_effector_threshold" + ], + "effector_threshold": priming_profile["effector_threshold"], + "effector_threshold_offset": priming_profile[ + "effector_threshold_offset" + ], + "base_memory_threshold": priming_profile["base_memory_threshold"], + "memory_threshold": priming_profile["memory_threshold"], + "memory_threshold_offset": priming_profile[ + "memory_threshold_offset" + ], + "base_state_wait_timeout_seconds": priming_profile[ + "base_state_wait_timeout_seconds" + ], + "state_wait_timeout_seconds": priming_profile[ + "state_wait_timeout_seconds" + ], + "state_wait_timeout_factor": priming_profile[ + "state_wait_timeout_factor" + ], + "base_effector_min_related_count": priming_profile[ + "base_effector_min_related_count" + ], + "effector_min_related_count": priming_profile[ + "effector_min_related_count" + ], + "effector_min_related_count_offset": priming_profile[ + "effector_min_related_count_offset" + ], + "base_memory_min_related_count": priming_profile[ + "base_memory_min_related_count" + ], + "memory_min_related_count": priming_profile[ + "memory_min_related_count" + ], + "memory_min_related_count_offset": priming_profile[ + "memory_min_related_count_offset" + ], + } + + def _compute_context_signals( + self, + profile_ip: str, + observation_id: int, + candidate: AntigenCandidate, + match: RegexMatch, + now: float, + priming_profile: dict, + exclude_observation_ids: set[int] | None = None, + ) -> dict: + exclude_observation_ids = self._normalize_observation_ids( + exclude_observation_ids + ) + recent_start = now - self.context_recent_window_seconds + previous_start = now - (2 * self.context_recent_window_seconds) + + recent_pamp_observations = self.storage.get_recent_observations( + profile_ip, + recent_start, + evidence_signal="PAMP", + ) + recent_damp_observations = self.storage.get_recent_observations( + profile_ip, + recent_start, + evidence_signal="DAMP", + ) + previous_pamp_observations = self.storage.get_recent_observations( + profile_ip, + previous_start, + until_ts=recent_start, + evidence_signal="PAMP", + ) + previous_damp_observations = self.storage.get_recent_observations( + profile_ip, + previous_start, + until_ts=recent_start, + evidence_signal="DAMP", + ) + related_exclusions = set(exclude_observation_ids) + related_exclusions.add(int(observation_id)) + recent_related_count = self._count_related_observations( + recent_pamp_observations, + candidate, + match.regex_hash, + exclude_observation_ids=related_exclusions, + ) + recent_related_score = self._clamp01( + recent_related_count / self.related_pamps_saturation + ) + recent_danger = self._compute_danger_scores( + recent_pamp_observations, + recent_damp_observations, + exclude_observation_ids=exclude_observation_ids, + ) + previous_danger = self._compute_danger_scores( + previous_pamp_observations, + previous_damp_observations, + exclude_observation_ids=exclude_observation_ids, + ) + recent_pressure = recent_danger["combined_score"] + previous_pressure = previous_danger["combined_score"] + trend_ratio = recent_pressure / max(previous_pressure, 0.01) + novelty_score = ( + 1.0 + if self._is_novel_regex( + profile_ip, + match, + observation_id, + now, + exclude_observation_ids=exclude_observation_ids, + ) + else 0.0 + ) + effector_score = ( + (0.45 * recent_pressure) + + (0.25 * recent_related_score) + + (0.30 * novelty_score) + ) + decrease_score = self._clamp01(1.0 - trend_ratio) + familiarity_score = 1.0 - novelty_score + stability_score = self._clamp01( + recent_related_count / priming_profile["memory_min_related_count"] + ) + memory_score = ( + (0.60 * decrease_score) + + (0.25 * familiarity_score) + + (0.15 * stability_score) + ) + effector = ( + novelty_score > 0 + and recent_related_count + >= priming_profile["effector_min_related_count"] + and effector_score >= priming_profile["effector_threshold"] + ) + memory = ( + familiarity_score > 0 + and recent_related_count + >= priming_profile["memory_min_related_count"] + and trend_ratio <= self.memory_trend_ratio_max + and memory_score >= priming_profile["memory_threshold"] + ) + return { + "novelty_score": novelty_score, + "recent_pressure": recent_pressure, + "previous_pressure": previous_pressure, + "recent_pamp_pressure": recent_danger["pamp_score"], + "recent_damp_pressure": recent_danger["damp_score"], + "previous_pamp_pressure": previous_danger["pamp_score"], + "previous_damp_pressure": previous_danger["damp_score"], + "damp_danger_weight": self.damp_danger_weight, + "trend_ratio": trend_ratio, + "recent_related_count": recent_related_count, + "recent_related_score": recent_related_score, + "effector_score": effector_score, + "memory_score": memory_score, + "decrease_score": decrease_score, + "familiarity_score": familiarity_score, + "stability_score": stability_score, + "effector_threshold": priming_profile["effector_threshold"], + "memory_threshold": priming_profile["memory_threshold"], + "effector_min_related_count": priming_profile[ + "effector_min_related_count" + ], + "memory_min_related_count": priming_profile[ + "memory_min_related_count" + ], + "priming_signal": priming_profile["signal"], + "priming_label": priming_profile["label"], + "priming_strength": priming_profile["strength"], + "base_co_stimulation_threshold": priming_profile[ + "base_co_stimulation_threshold" + ], + "co_stimulation_threshold": priming_profile[ + "co_stimulation_threshold" + ], + "co_stimulation_threshold_offset": priming_profile[ + "co_stimulation_threshold_offset" + ], + "base_effector_threshold": priming_profile[ + "base_effector_threshold" + ], + "effector_threshold_offset": priming_profile[ + "effector_threshold_offset" + ], + "base_memory_threshold": priming_profile["base_memory_threshold"], + "memory_threshold_offset": priming_profile[ + "memory_threshold_offset" + ], + "base_state_wait_timeout_seconds": priming_profile[ + "base_state_wait_timeout_seconds" + ], + "state_wait_timeout_seconds": priming_profile[ + "state_wait_timeout_seconds" + ], + "state_wait_timeout_factor": priming_profile[ + "state_wait_timeout_factor" + ], + "base_effector_min_related_count": priming_profile[ + "base_effector_min_related_count" + ], + "effector_min_related_count_offset": priming_profile[ + "effector_min_related_count_offset" + ], + "base_memory_min_related_count": priming_profile[ + "base_memory_min_related_count" + ], + "memory_min_related_count_offset": priming_profile[ + "memory_min_related_count_offset" + ], + "effector": effector, + "memory": memory, + } + + def _maybe_trace_co_stimulation( + self, + action: str, + evidence, + cell: dict, + candidate: AntigenCandidate, + match: RegexMatch, + co_stimulation: dict, + responsible_ip: str, + observation_id: int, + now: float, + from_state: int, + to_state: int, + ): + if not self._should_write_decision_trace(action): + return + + since_ts = now - self.related_lookback_seconds + consumed_observation_ids = ( + self._get_consumed_transition_observation_ids(cell) + ) + related_exclusions = set(consumed_observation_ids) + related_exclusions.add(int(observation_id)) + pamp_observations = self.storage.get_recent_observations( + responsible_ip, + since_ts, + evidence_signal="PAMP", + ) + damp_observations = self.storage.get_recent_observations( + responsible_ip, + since_ts, + evidence_signal="DAMP", + ) + current_observation = ( + self.storage.get_observation(observation_id) or {} + ) + related_trace = self._build_related_trace( + pamp_observations, + candidate, + match.regex_hash, + exclude_observation_ids=related_exclusions, + ) + pamp_danger_trace = self._build_danger_trace( + pamp_observations, + exclude_observation_ids=consumed_observation_ids, + ) + damp_danger_trace = self._build_danger_trace( + damp_observations, + exclude_observation_ids=consumed_observation_ids, + ) + priming_profile = self._get_cell_priming_profile(cell) + + entry = { + "ts": utils.convert_ts_format(now, utils.alerts_format), + "stage": "co_stimulation", + "action": action, + "from_state": STATE_INFO[from_state]["label"], + "to_state": STATE_INFO[to_state]["label"], + "cell_key": cell["cell_key"], + "profile_ip": evidence.profile.ip, + "responsible_ip": responsible_ip, + "target_ip": self._get_target_ip(evidence), + "candidate": candidate.as_dict(), + "match": match.as_dict(), + "current_evidence": self._summarize_current_observation( + evidence, current_observation + ), + "formula": { + "value": co_stimulation["value"], + "threshold": co_stimulation["threshold"], + "weights": self.co_stimulation_weights, + "priming": { + "signal": priming_profile["signal"], + "label": priming_profile["label"], + "strength": priming_profile["strength"], + "base_co_stimulation_threshold": priming_profile[ + "base_co_stimulation_threshold" + ], + "co_stimulation_threshold": priming_profile[ + "co_stimulation_threshold" + ], + "co_stimulation_threshold_offset": priming_profile[ + "co_stimulation_threshold_offset" + ], + "base_effector_threshold": priming_profile[ + "base_effector_threshold" + ], + "effector_threshold": priming_profile[ + "effector_threshold" + ], + "effector_threshold_offset": priming_profile[ + "effector_threshold_offset" + ], + "base_memory_threshold": priming_profile[ + "base_memory_threshold" + ], + "memory_threshold": priming_profile["memory_threshold"], + "memory_threshold_offset": priming_profile[ + "memory_threshold_offset" + ], + "base_state_wait_timeout_seconds": priming_profile[ + "base_state_wait_timeout_seconds" + ], + "state_wait_timeout_seconds": priming_profile[ + "state_wait_timeout_seconds" + ], + "state_wait_timeout_factor": priming_profile[ + "state_wait_timeout_factor" + ], + "base_effector_min_related_count": priming_profile[ + "base_effector_min_related_count" + ], + "effector_min_related_count": priming_profile[ + "effector_min_related_count" + ], + "effector_min_related_count_offset": priming_profile[ + "effector_min_related_count_offset" + ], + "base_memory_min_related_count": priming_profile[ + "base_memory_min_related_count" + ], + "memory_min_related_count": priming_profile[ + "memory_min_related_count" + ], + "memory_min_related_count_offset": priming_profile[ + "memory_min_related_count_offset" + ], + }, + "components": { + "confidence": { + "value": co_stimulation["confidence"], + "weighted": ( + self.co_stimulation_weights["confidence"] + * co_stimulation["confidence"] + ), + "evidence_id": co_stimulation.get( + "confidence_evidence_id" + ) + or evidence.id, + "observation_id": co_stimulation.get( + "confidence_observation_id" + ), + }, + "related_pamps": { + "count": co_stimulation["related_pamp_count"], + "saturation": self.related_pamps_saturation, + "score": co_stimulation["related_pamp_score"], + "weighted": ( + self.co_stimulation_weights["related_pamps"] + * co_stimulation["related_pamp_score"] + ), + "contributors": related_trace["contributors"], + "omitted_count": related_trace["omitted_count"], + }, + "danger": { + "score": co_stimulation["profile_danger_score"], + "weighted": ( + self.co_stimulation_weights["danger"] + * co_stimulation["profile_danger_score"] + ), + "danger_saturation": self.danger_saturation, + "damp_weight": self.damp_danger_weight, + "pamp_score": co_stimulation["pamp_danger_score"], + "damp_score": co_stimulation["damp_danger_score"], + "pamp_total_raw": pamp_danger_trace["total_raw"], + "damp_total_raw": damp_danger_trace["total_raw"], + "pamp_contributors": pamp_danger_trace["contributors"], + "pamp_omitted_count": pamp_danger_trace[ + "omitted_count" + ], + "damp_contributors": damp_danger_trace["contributors"], + "damp_omitted_count": damp_danger_trace[ + "omitted_count" + ], + }, + }, + }, + } + self._write_decision_trace(entry) + + def _maybe_trace_context( + self, + action: str, + evidence, + cell: dict, + candidate: AntigenCandidate, + match: RegexMatch, + context: dict, + responsible_ip: str, + observation_id: int, + now: float, + from_state: int, + to_state: int, + ): + if not self._should_write_decision_trace(action): + return + + recent_start = now - self.context_recent_window_seconds + previous_start = now - (2 * self.context_recent_window_seconds) + recent_pamp_observations = self.storage.get_recent_observations( + responsible_ip, + recent_start, + evidence_signal="PAMP", + ) + recent_damp_observations = self.storage.get_recent_observations( + responsible_ip, + recent_start, + evidence_signal="DAMP", + ) + previous_pamp_observations = self.storage.get_recent_observations( + responsible_ip, + previous_start, + until_ts=recent_start, + evidence_signal="PAMP", + ) + previous_damp_observations = self.storage.get_recent_observations( + responsible_ip, + previous_start, + until_ts=recent_start, + evidence_signal="DAMP", + ) + consumed_observation_ids = ( + self._get_consumed_transition_observation_ids(cell) + ) + related_exclusions = set(consumed_observation_ids) + related_exclusions.add(int(observation_id)) + current_observation = ( + self.storage.get_observation(observation_id) or {} + ) + related_trace = self._build_related_trace( + recent_pamp_observations, + candidate, + match.regex_hash, + exclude_observation_ids=related_exclusions, + ) + priming_profile = self._get_cell_priming_profile(cell) + has_memory = self.storage.has_memory_for_regex(match.regex_hash) + has_recent_activity = self.storage.has_recent_regex_activity( + responsible_ip, + match.regex_hash, + now - self.novelty_window_seconds, + exclude_observation_ids=related_exclusions, + ) + + entry = { + "ts": utils.convert_ts_format(now, utils.alerts_format), + "stage": "context", + "action": action, + "from_state": STATE_INFO[from_state]["label"], + "to_state": STATE_INFO[to_state]["label"], + "cell_key": cell["cell_key"], + "profile_ip": evidence.profile.ip, + "responsible_ip": responsible_ip, + "target_ip": self._get_target_ip(evidence), + "candidate": candidate.as_dict(), + "match": match.as_dict(), + "current_evidence": self._summarize_current_observation( + evidence, current_observation + ), + "formula": { + "effector_score": context["effector_score"], + "effector_threshold": context["effector_threshold"], + "memory_score": context["memory_score"], + "memory_threshold": context["memory_threshold"], + "priming": { + "signal": priming_profile["signal"], + "label": priming_profile["label"], + "strength": priming_profile["strength"], + "base_co_stimulation_threshold": priming_profile[ + "base_co_stimulation_threshold" + ], + "co_stimulation_threshold": priming_profile[ + "co_stimulation_threshold" + ], + "co_stimulation_threshold_offset": priming_profile[ + "co_stimulation_threshold_offset" + ], + "base_effector_threshold": priming_profile[ + "base_effector_threshold" + ], + "effector_threshold": priming_profile[ + "effector_threshold" + ], + "effector_threshold_offset": priming_profile[ + "effector_threshold_offset" + ], + "base_memory_threshold": priming_profile[ + "base_memory_threshold" + ], + "memory_threshold": priming_profile["memory_threshold"], + "memory_threshold_offset": priming_profile[ + "memory_threshold_offset" + ], + "base_state_wait_timeout_seconds": priming_profile[ + "base_state_wait_timeout_seconds" + ], + "state_wait_timeout_seconds": priming_profile[ + "state_wait_timeout_seconds" + ], + "state_wait_timeout_factor": priming_profile[ + "state_wait_timeout_factor" + ], + "base_effector_min_related_count": priming_profile[ + "base_effector_min_related_count" + ], + "effector_min_related_count": priming_profile[ + "effector_min_related_count" + ], + "effector_min_related_count_offset": priming_profile[ + "effector_min_related_count_offset" + ], + "base_memory_min_related_count": priming_profile[ + "base_memory_min_related_count" + ], + "memory_min_related_count": priming_profile[ + "memory_min_related_count" + ], + "memory_min_related_count_offset": priming_profile[ + "memory_min_related_count_offset" + ], + }, + "decision": { + "effector": context["effector"], + "memory": context["memory"], + }, + "components": { + "novelty": { + "score": context["novelty_score"], + "has_memory_for_regex": has_memory, + "has_recent_regex_activity": has_recent_activity, + "novelty_window_seconds": self.novelty_window_seconds, + }, + "recent_related": { + "count": context["recent_related_count"], + "saturation": self.related_pamps_saturation, + "score": context["recent_related_score"], + "contributors": related_trace["contributors"], + "omitted_count": related_trace["omitted_count"], + }, + "recent_pressure": self._build_pressure_trace( + recent_pamp_observations, + recent_damp_observations, + context["recent_pressure"], + context["recent_pamp_pressure"], + context["recent_damp_pressure"], + exclude_observation_ids=consumed_observation_ids, + ), + "previous_pressure": self._build_pressure_trace( + previous_pamp_observations, + previous_damp_observations, + context["previous_pressure"], + context["previous_pamp_pressure"], + context["previous_damp_pressure"], + exclude_observation_ids=consumed_observation_ids, + ), + "trend_ratio": context["trend_ratio"], + "decrease_score": context["decrease_score"], + "familiarity_score": context["familiarity_score"], + "stability_score": context["stability_score"], + }, + }, + } + self._write_decision_trace(entry) + + def _build_pressure_trace( + self, + pamp_observations: list[dict], + damp_observations: list[dict], + combined_score: float, + pamp_score: float, + damp_score: float, + exclude_observation_ids: set[int] | None = None, + ) -> dict: + pamp_trace = self._build_danger_trace( + pamp_observations, + exclude_observation_ids=exclude_observation_ids, + ) + damp_trace = self._build_danger_trace( + damp_observations, + exclude_observation_ids=exclude_observation_ids, + ) + return { + "combined_score": combined_score, + "pamp_score": pamp_score, + "damp_score": damp_score, + "danger_saturation": self.danger_saturation, + "damp_weight": self.damp_danger_weight, + "pamp_total_raw": pamp_trace["total_raw"], + "damp_total_raw": damp_trace["total_raw"], + "pamp_contributors": pamp_trace["contributors"], + "pamp_omitted_count": pamp_trace["omitted_count"], + "damp_contributors": damp_trace["contributors"], + "damp_omitted_count": damp_trace["omitted_count"], + } + + def _summarize_current_observation( + self, evidence, observation: dict | None + ) -> dict: + observation = observation or {} + return { + "observation_id": observation.get("id"), + "evidence_id": evidence.id, + "evidence_type": evidence.evidence_type.name, + "signal": str(evidence.evidence_signal), + "confidence": observation.get("confidence", evidence.confidence), + "threat_level": observation.get( + "threat_level", str(evidence.threat_level) + ), + "threat_level_value": observation.get( + "threat_level_value", float(evidence.threat_level.value) + ), + "danger_contribution": ( + self._observation_danger_contribution(observation) + if observation + else float(evidence.confidence) + * float(evidence.threat_level.value) + ), + } + + def _build_related_trace( + self, + observations: list[dict], + candidate: AntigenCandidate, + regex_hash: str, + exclude_observation_ids: set[int] | None = None, + ) -> dict: + exclude_observation_ids = self._normalize_observation_ids( + exclude_observation_ids + ) + contributors = [] + for observation in observations: + if self._observation_is_excluded( + observation, + exclude_observation_ids, + ): + continue + relations = self._get_observation_relations( + observation, candidate, regex_hash + ) + if not relations: + continue + contributors.append( + self._summarize_observation( + observation, + relations=relations, + ) + ) + + return self._limit_trace_items(contributors) + + def _build_danger_trace( + self, + observations: list[dict], + exclude_observation_ids: set[int] | None = None, + ) -> dict: + exclude_observation_ids = self._normalize_observation_ids( + exclude_observation_ids + ) + contributors = [ + self._summarize_observation(observation) + for observation in observations + if not self._observation_is_excluded( + observation, + exclude_observation_ids, + ) + ] + limited = self._limit_trace_items(contributors) + limited["total_raw"] = sum( + item["danger_contribution"] for item in contributors + ) + return limited + + def _summarize_observation( + self, + observation: dict, + relations: list[str] | None = None, + ) -> dict: + summary = { + "observation_id": observation.get("id"), + "evidence_id": observation.get("evidence_id"), + "evidence_type": observation.get("evidence_type"), + "signal": observation.get("evidence_signal"), + "observed_at": observation.get("observed_at"), + "confidence": float(observation.get("confidence", 0.0)), + "threat_level": observation.get("threat_level"), + "threat_level_value": float( + observation.get("threat_level_value", 0.0) + ), + "danger_contribution": self._observation_danger_contribution( + observation + ), + } + if relations: + summary["relations"] = relations + return summary + + @staticmethod + def _observation_danger_contribution(observation: dict) -> float: + return float(observation.get("threat_level_value", 0.0)) * float( + observation.get("confidence", 0.0) + ) + + def _limit_trace_items(self, items: list[dict]) -> dict: + ordered_items = sorted( + items, + key=lambda item: ( + float(item.get("danger_contribution", 0.0)), + float(item.get("observed_at", 0.0) or 0.0), + ), + reverse=True, + ) + return { + "contributors": ordered_items[: self.decision_trace_max_evidence], + "omitted_count": max( + 0, + len(ordered_items) - self.decision_trace_max_evidence, + ), + } + + def _should_write_decision_trace(self, action: str) -> bool: + if self.decision_trace_mode == TRACE_MODE_OFF: + return False + if self.decision_trace_mode >= TRACE_MODE_ALL: + return True + return action not in { + "waiting_for_co_stimulation", + "waiting_for_context", + } + + def _write_decision_trace(self, entry: dict): + if self.decision_trace_mode == TRACE_MODE_OFF: + return + trace_dir = os.path.dirname(self.trace_file_path) + if trace_dir: + os.makedirs(trace_dir, exist_ok=True) + with open(self.trace_file_path, "a", encoding="utf-8") as trace_file: + trace_file.write(json.dumps(entry, sort_keys=True)) + trace_file.write("\n") + + def _is_novel_regex( + self, + profile_ip: str, + match: RegexMatch, + observation_id: int, + now: float, + exclude_observation_ids: set[int] | None = None, + ) -> bool: + excluded_ids = self._normalize_observation_ids(exclude_observation_ids) + excluded_ids.add(int(observation_id)) + if self.storage.has_memory_for_regex(match.regex_hash): + return False + return not self.storage.has_recent_regex_activity( + profile_ip, + match.regex_hash, + now - self.novelty_window_seconds, + exclude_observation_ids=excluded_ids, + ) + + def _apply_effector( + self, + cell: dict, + evidence, + match: RegexMatch, + context: dict, + now: float, + responsible_ip: str, + ): + cooldown_until = cell.get("effector_cooldown_until") or 0 + if now < cooldown_until: + self._log_event( + action="effector_cooldown", + state=STATE_EFFECTOR, + evidence=evidence, + cell=cell, + match=match, + metrics={"cooldown_until": cooldown_until}, + details=( + "effector already fired recently for this cell; " + "suppressing repeated blocking" + ), + verbosity=LOG_VERBOSITY_DECISIONS, + ) + return + + blocking_data = { + "ip": responsible_ip, + "block": True, + "tw": evidence.timewindow.number, + "interface": utils.get_interface_of_ip( + responsible_ip, self.db, self.args + ), + } + next_cooldown = now + self.effector_cooldown_seconds + self._update_cell( + cell, + now, + effector_cooldown_until=next_cooldown, + ) + self._update_cell_context( + cell, + now, + context=context, + effector_payload=blocking_data, + ) + + if self._blocking_modules_available(): + self.db.publish("new_blocking", json.dumps(blocking_data)) + self._log_event( + action="effector_published", + state=STATE_EFFECTOR, + evidence=evidence, + cell=cell, + match=match, + metrics={"effector_score": context["effector_score"]}, + verbosity=LOG_VERBOSITY_SUMMARY, + ) + return + + if self.simulate_effector_without_blocking: + self._log_event( + action="effector_simulated", + state=STATE_EFFECTOR, + evidence=evidence, + cell=cell, + match=match, + details=json.dumps(blocking_data, sort_keys=True), + metrics={"effector_score": context["effector_score"]}, + verbosity=LOG_VERBOSITY_SUMMARY, + ) + return + + self._log_event( + action="effector_unavailable", + state=STATE_EFFECTOR, + evidence=evidence, + cell=cell, + match=match, + metrics={"effector_score": context["effector_score"]}, + details="blocking modules are not running and simulation is disabled", + verbosity=LOG_VERBOSITY_SUMMARY, + ) + + def _store_memory( + self, cell: dict, match: RegexMatch, context: dict, now: float + ): + self.storage.upsert_memory( + { + "cell_key": cell["cell_key"], + "profile_ip": cell["profile_ip"], + "regex_type": cell["regex_type"], + "antigen_value": cell["antigen_value"], + "regex_hash": match.regex_hash, + "regex": match.regex, + "matched_value": match.value, + "context": context, + "created_at": now, + "updated_at": now, + } + ) + + def _blocking_modules_available(self) -> bool: + blocking_pid = self.db.get_pid_of("Blocking") + arp_pid = self.db.get_pid_of("ARP Poisoner") + return self._pid_is_running(blocking_pid) or self._pid_is_running( + arp_pid + ) + + @staticmethod + def _pid_is_running(pid) -> bool: + if isinstance(pid, int): + return pid > 0 + if isinstance(pid, str): + return pid.isdigit() and int(pid) > 0 + return False + + def _find_best_regex_match( + self, candidate: AntigenCandidate + ) -> RegexMatch | None: + regex_records = self.db.get_generated_regexes( + regex_type=candidate.regex_type, status="accepted" + ) + best_match = None + best_key = None + for record in regex_records or []: + regex_text = str(record.get("regex", "")) + if not regex_text: + continue + try: + compiled_regex = re.compile(regex_text) + if not compiled_regex.search(candidate.value): + continue + except re.error: + continue + + specificity_features = measure_regex_specificity(regex_text) + specificity = float( + specificity_features.get("specificity_ratio", 0.0) + ) + wildcard_penalty = float( + specificity_features.get("wildcard_penalty", 1.0) + ) + match_strength = compute_match_strength( + compiled_regex, + candidate.value, + regex_features=specificity_features, + ) + created_at = float(record.get("created_at") or 0.0) + sort_key = ( + match_strength, + specificity, + -wildcard_penalty, + created_at, + ) + if best_key is not None and sort_key <= best_key: + continue + + best_key = sort_key + best_match = RegexMatch( + regex_type=candidate.regex_type, + value=candidate.value, + regex_hash=str(record.get("regex_hash", "")), + regex=regex_text, + created_at=created_at, + specificity=specificity, + ) + return best_match + + def _extract_antigen_candidates(self, evidence) -> list[AntigenCandidate]: + candidates = {} + + for entity in (evidence.attacker, evidence.victim): + self._extract_from_entity(entity, candidates) + + for uid in evidence.uid: + flow = self._unwrap_flow_record(self.db.get_altflow_from_uid(uid)) + if not flow: + continue + + flow_type = str( + flow.get("flow_type") or flow.get("type_") or "" + ).lower() + if flow_type == "dns" or "query" in flow: + self._add_candidate( + candidates, + "dns_domain", + self._normalize_domain(flow.get("query")), + ) + if flow_type == "http" or "uri" in flow or "host" in flow: + self._add_candidate( + candidates, + "dns_domain", + self._normalize_domain(flow.get("host")), + ) + uri = self._normalize_uri(flow.get("uri")) + self._add_candidate(candidates, "uri", uri) + self._add_candidate( + candidates, + "filename", + self._extract_filename_from_uri(uri), + ) + if ( + flow_type == "ssl" + or "server_name" in flow + or "subject" in flow + ): + self._add_candidate( + candidates, + "tls_sni", + self._normalize_domain(flow.get("server_name")), + ) + self._add_candidate( + candidates, + "certificate_cn", + self._extract_cn(flow.get("subject")), + ) + + return [ + AntigenCandidate(regex_type=regex_type, value=value) + for regex_type, value in sorted(candidates.keys()) + ] + + def _extract_from_entity(self, entity, candidates: dict): + if not entity: + return + + ioc_type = self._enum_name(getattr(entity, "ioc_type", None)) + if ioc_type == "DOMAIN": + self._add_candidate( + candidates, "dns_domain", self._normalize_domain(entity.value) + ) + elif ioc_type == "URL": + parsed = urlparse(str(entity.value or "").strip()) + self._add_candidate( + candidates, + "dns_domain", + self._normalize_domain(parsed.hostname), + ) + uri = self._normalize_uri(entity.value) + self._add_candidate(candidates, "uri", uri) + self._add_candidate( + candidates, "filename", self._extract_filename_from_uri(uri) + ) + + self._add_candidate( + candidates, + "tls_sni", + self._normalize_domain(getattr(entity, "SNI", "")), + ) + + @staticmethod + def _enum_name(value) -> str: + if hasattr(value, "name"): + return str(value.name).upper() + raw_value = str(value or "").strip() + if "." in raw_value: + raw_value = raw_value.rsplit(".", 1)[-1] + return raw_value.upper() + + def _get_entity_ip(self, entity) -> str: + if not entity: + return "" + if self._enum_name(getattr(entity, "ioc_type", None)) != "IP": + return "" + value = str(getattr(entity, "value", "") or "").strip() + if not utils.is_valid_ip(value): + return "" + return value + + def _get_responsible_ip(self, evidence) -> str: + attacker_ip = self._get_entity_ip(getattr(evidence, "attacker", None)) + if attacker_ip: + return attacker_ip + + for entity in ( + getattr(evidence, "attacker", None), + getattr(evidence, "victim", None), + ): + if self._enum_name(getattr(entity, "direction", None)) != "SRC": + continue + entity_ip = self._get_entity_ip(entity) + if entity_ip: + return entity_ip + + return str(getattr(getattr(evidence, "profile", None), "ip", "") or "") + + def _get_target_ip(self, evidence) -> str: + victim_ip = self._get_entity_ip(getattr(evidence, "victim", None)) + if victim_ip: + return victim_ip + return "" + + def _count_related_observations( + self, + observations: list[dict], + candidate: AntigenCandidate, + regex_hash: str, + exclude_observation_ids: set[int] | None = None, + ) -> int: + exclude_observation_ids = self._normalize_observation_ids( + exclude_observation_ids + ) + count = 0 + for observation in observations: + if self._observation_is_excluded( + observation, + exclude_observation_ids, + ): + continue + if self._is_related_observation( + observation, candidate, regex_hash + ): + count += 1 + return count + + def _is_related_observation( + self, + observation: dict, + candidate: AntigenCandidate, + regex_hash: str, + ) -> bool: + return bool( + self._get_observation_relations(observation, candidate, regex_hash) + ) + + @staticmethod + def _get_observation_relations( + observation: dict, + candidate: AntigenCandidate, + regex_hash: str, + ) -> list[str]: + relations = [] + for antigen in observation.get("antigens", []): + if ( + antigen.get("regex_type") == candidate.regex_type + and antigen.get("value") == candidate.value + ): + relations.append("same_antigen") + break + for match in observation.get("matched_regexes", []): + if regex_hash and match.get("regex_hash") == regex_hash: + relations.append("same_regex_hash") + break + return relations + + @staticmethod + def _observation_is_excluded( + observation: dict, + exclude_observation_ids: set[int] | None, + ) -> bool: + if not exclude_observation_ids: + return False + try: + return int(observation.get("id")) in exclude_observation_ids + except (TypeError, ValueError): + return False + + def _sum_danger( + self, + observations: list[dict], + exclude_observation_ids: set[int] | None = None, + ) -> float: + exclude_observation_ids = self._normalize_observation_ids( + exclude_observation_ids + ) + return sum( + float(obs.get("threat_level_value", 0.0)) + * float(obs.get("confidence", 0.0)) + for obs in observations + if not self._observation_is_excluded( + obs, + exclude_observation_ids, + ) + ) + + def _compute_danger_scores( + self, + pamp_observations: list[dict], + damp_observations: list[dict], + exclude_observation_ids: set[int] | None = None, + ) -> dict: + pamp_raw = self._sum_danger( + pamp_observations, + exclude_observation_ids=exclude_observation_ids, + ) + damp_raw = self._sum_danger( + damp_observations, + exclude_observation_ids=exclude_observation_ids, + ) + combined_raw = pamp_raw + (self.damp_danger_weight * damp_raw) + return { + "pamp_score": self._normalize_danger(pamp_raw), + "damp_score": self._normalize_danger(damp_raw), + "combined_score": self._normalize_danger(combined_raw), + } + + def _normalize_danger(self, raw_value: float) -> float: + return self._clamp01(raw_value / self.danger_saturation) + + @staticmethod + def _clamp01(value: float) -> float: + return round(max(0.0, min(1.0, float(value))), 10) + + @staticmethod + def _get_state_wait_elapsed(cell: dict, now: float) -> float: + start_ts = ( + cell.get("last_transition_at") or cell.get("created_at") or now + ) + try: + start_ts = float(start_ts) + except (TypeError, ValueError): + start_ts = now + return max(0.0, float(now) - start_ts) + + def _state_wait_expired(self, cell: dict, now: float) -> bool: + return self._get_state_wait_elapsed( + cell, now + ) >= self._effective_wait_limit(cell) + + @staticmethod + def _make_cell_key( + profile_ip: str, regex_type: str, antigen_value: str + ) -> str: + return f"{profile_ip}|{regex_type}|{antigen_value}" + + @staticmethod + def _unwrap_flow_record(flow_record) -> dict: + if not isinstance(flow_record, dict): + return {} + if isinstance(flow_record.get("flow"), dict): + flow = dict(flow_record["flow"]) + flow["flow_type"] = flow_record.get("flow_type") or flow.get( + "flow_type" + ) + return flow + return dict(flow_record) + + @staticmethod + def _add_candidate(candidates: dict, regex_type: str, value: str): + normalized = str(value or "").strip() + if regex_type not in SUPPORTED_REGEX_TYPES or not normalized: + return + candidates[(regex_type, normalized)] = True + + @staticmethod + def _normalize_domain(value: str) -> str: + domain = str(value or "").strip().rstrip(".").lower() + if not domain or not utils.is_valid_domain(domain): + return "" + return domain + + @staticmethod + def _normalize_uri(value: str) -> str: + raw_value = str(value or "").strip() + if not raw_value: + return "" + parsed = urlparse(raw_value) + if parsed.scheme or parsed.netloc: + uri = parsed.path or "/" + if parsed.query: + uri = f"{uri}?{parsed.query}" + return uri + return raw_value + + @staticmethod + def _extract_cn(subject: str) -> str: + match = re.search(r"(?:^|,)CN=([^,]+)", str(subject or "")) + if not match: + return "" + return match.group(1).strip() + + @staticmethod + def _extract_filename_from_uri(uri: str) -> str: + value = str(uri or "").strip() + if not value: + return "" + parsed = urlparse(value) + path = parsed.path or value + filename = path.rsplit("/", 1)[-1].strip() + if not filename or "." not in filename: + return "" + return filename + + def _prune_observations(self, now: float): + cutoff = now - self.observation_retention_seconds + self.storage.prune_observations(cutoff) + + def _init_log_file(self): + if not self.create_log_file: + return + with open(self.log_file_path, "w", encoding="utf-8") as log_file: + log_file.write("") + + def _init_trace_file(self): + if self.decision_trace_mode == TRACE_MODE_OFF: + return + trace_dir = os.path.dirname(self.trace_file_path) + if trace_dir: + os.makedirs(trace_dir, exist_ok=True) + with open(self.trace_file_path, "w", encoding="utf-8") as trace_file: + trace_file.write("") + + def _resolve_trace_file_path(self, raw_path: str) -> str: + normalized = str(raw_path or "").strip() + if not normalized: + normalized = "t_cell_trace.jsonl" + + normalized = normalized.replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + if os.path.isabs(normalized): + normalized = os.path.basename(normalized) + if normalized.startswith("output/"): + normalized = normalized[len("output/") :] + + safe_parts = [] + for part in normalized.split("/"): + if not part or part in (".", ".."): + continue + if part.endswith(":"): + continue + safe_parts.append(part) + if not safe_parts: + safe_parts = ["t_cell_trace.jsonl"] + return os.path.join(self.output_dir, *safe_parts) + + @staticmethod + def _get_waiting_label(cell: dict | None) -> str: + context = (cell or {}).get("context") or {} + waiting_for = context.get("waiting_for") + return WAITING_LABELS.get(waiting_for, "") + + def _format_state_label(self, state: int, cell: dict | None = None) -> str: + label = STATE_INFO[state]["label"] + waiting_label = self._get_waiting_label(cell) + if waiting_label: + return f"{label} ({waiting_label})" + return label + + def _colorize_state(self, state: int, cell: dict | None = None) -> str: + label = self._format_state_label(state, cell) + if not self.log_colors: + return label + return f"{STATE_INFO[state]['color']}{label}{COLOR_RESET}" + + def _log_event( + self, + action: str, + evidence, + state: int | None, + cell: dict | None = None, + match: RegexMatch | None = None, + details: str | None = None, + metrics: dict | None = None, + verbosity: int = LOG_VERBOSITY_DECISIONS, + ): + if verbosity > self.log_verbosity: + return + parts = [ + utils.convert_ts_format(time.time(), utils.alerts_format), + f"action={action}", + ] + if state is not None: + parts.append(f"state={self._format_state_label(state, cell=cell)}") + if self.log_colors: + parts.append( + f"state_display={self._colorize_state(state, cell=cell)}" + ) + if evidence: + parts.append(f"evidence={evidence.evidence_type.name}") + parts.append(f"eid={evidence.id}") + parts.append(f"signal={evidence.evidence_signal}") + parts.append(f"profile={evidence.profile.ip}") + responsible_ip = self._get_responsible_ip(evidence) + if responsible_ip: + parts.append(f"responsible={responsible_ip}") + target_ip = self._get_target_ip(evidence) + if target_ip: + parts.append(f"target={target_ip}") + if cell: + parts.append(f"cell={cell['cell_key']}") + waiting_label = self._get_waiting_label(cell) + if waiting_label: + parts.append(f"waiting={waiting_label}") + if match: + parts.append(f"regex={match.regex_hash}") + parts.append(f"value={match.value}") + if metrics: + metric_text = ",".join( + ( + f"{key}={value:.3f}" + if isinstance(value, float) + else f"{key}={value}" + ) + for key, value in metrics.items() + ) + parts.append(metric_text) + if details: + parts.append(details) + self._log_detail(" | ".join(parts)) + + def _log_detail(self, text: str): + if not self.create_log_file: + return + with open(self.log_file_path, "a", encoding="utf-8") as log_file: + log_file.write(f"{text}\n") diff --git a/output/test-llm-summary/llm-summary/blog_post.md b/output/test-llm-summary/llm-summary/blog_post.md new file mode 100644 index 0000000000..0e48652c62 --- /dev/null +++ b/output/test-llm-summary/llm-summary/blog_post.md @@ -0,0 +1,553 @@ +# How the Slips Alert Summary Method Works + +Slips already detects suspicious behavior and correlates multiple findings into +one alert. The alert summary layer is the part that turns that correlated +evidence into a short explanation that a human can read quickly. + +This is not a general-purpose chatbot feature. It is a tightly scoped method +for translating one structured IDS alert into one evidence-bound analyst +paragraph. + +## The Core Idea + +The method starts from a simple observation: an IDS alert is often technically +correct, but still hard for a person to consume. + +One alert may contain many repeated detections: + +- connections to many similar destinations +- repeated attempts on the same port +- several evidence records that describe the same behavior in slightly + different wording + +If all of that is passed to a language model as raw text, the result is waste, +noise, and unstable output. The method therefore has to do real engineering +before the model sees anything. + +The alert summary process has five stages: + +1. reconstruct the full correlated evidence set behind the alert +2. collapse repetitive evidence into a compact incident digest +3. retrieve recent alert history for the same source as extra context +4. build a constrained analyst-oriented prompt +5. reduce oversized alerts recursively when needed +6. generate one plain-text summary with a local fine-tuned model + +That is the method. + +## 1. Reconstruct the Whole Alert, Not Just the Final Label + +The summary process does not start from a final verdict string. It starts from +the correlated evidence that Slips already attached to the alert. + +Yes: the method uses the alert and all of its related evidence records. + +That matters because a good explanation depends on: + +- what behaviors were observed +- how often they happened +- how they were distributed over time +- which pieces of evidence are strong +- which pieces of evidence weaken confidence + +So the method first rebuilds the alert context from the correlated evidence set +and orders it by time. If the full set is incomplete, it still falls back to +the most recent evidence so the alert is never left without context. + +This keeps the summarizer tied to the actual IDS evidence instead of turning it +into a free-form text generator. + +In practical terms, the alert carries the identifiers of the evidence records +that belong to it. The summarization layer uses those identifiers to fetch the +full correlated evidence set for that source IP and time window, then selects +the evidence records referenced by that alert. + +So the context is not: + +- one title +- one severity +- one last event + +The context is: + +- the alert metadata +- the full set of evidence records linked to that alert +- their timestamps +- their descriptions +- their severities +- the threat level and confidence already computed by Slips + +Example: + +```text +Alert: +- Source IP: 192.168.1.113 +- Time window: 8 +- Threat level: 15.1 +- Confidence: 0.84 +- Related evidence IDs: 31 records + +Related evidence: +- 07:00 Horizontal port scan to 443/TCP from 192.168.1.113 to 5 unique IPs +- 07:02 Connection to unknown destination port 449/TCP to 76.16.105.16 +- 07:03 Connection to unknown destination port 449/TCP to 177.251.27.6 +- 07:10 Connection to unknown destination port 449/TCP to 209.205.188.238 +- ... +``` + +That full correlated set is what the summary method works from. + +## 2. Collapse Repetition into Incident Patterns + +This is the most important part of the method. + +Security alerts are usually repetitive. The same basic event can appear many +times with only a changed IP address, port number, or counter. A small local +model should not waste context reading fifty versions of the same pattern. + +The method therefore groups similar evidence descriptions together after +normalizing variable fields. + +In practice, this means: + +- IP addresses are abstracted into placeholders +- port expressions are abstracted into placeholders +- raw counts are abstracted into placeholders + +After normalization, descriptions that represent the same underlying behavior +collapse into one group. + +Example of normalization: + +```text +Before: +- Connection to unknown destination port 449/TCP to 76.16.105.16 +- Connection to unknown destination port 449/TCP to 177.251.27.6 +- Connection to unknown destination port 449/TCP to 209.205.188.238 + +Normalized pattern: +- Connection to unknown destination port /TCP to +``` + +For each group, the method keeps the details that matter operationally: + +- the time range of the group +- one representative behavior description +- how many similar records belong to that group +- the severity mix inside the group +- a few sample IPs or ports when examples are useful + +The result is no longer a flat list of events. It becomes an incident digest: +a short list of dominant behavior patterns, ordered by importance. + +Example of a grouped digest: + +```text +- 07:02-07:22 | Connection to unknown destination port 449/TCP + (21x similar, severities: medium=21, samples: 76.16.105.16, 177.251.27.6, 209.205.188.238, 449/TCP) +- 07:00-07:03 | Horizontal port scan to port 443/TCP + (2x similar, severities: high=2, samples: 443/TCP) +``` + +This is the main reason the summary layer works well with local models. It +removes repetition without removing the shape of the incident. + +## 3. Build a Constrained Analyst Prompt + +The current alert is no longer summarized in isolation. + +Before the final prompt is built, the method can retrieve a bounded memory of +recent prior alerts for the same source or profile. That history is not the new +evidence itself. It is supporting context that helps answer questions such as: + +- is this a continuation of the same activity? +- is the behavior escalating? +- is the source expanding into new patterns? +- does the current alert look isolated or part of a sequence? + +The history kept for each prior alert is compact. It includes: + +- the time window and time range +- the accumulated threat level +- the alert confidence +- a few dominant grouped patterns +- the final summary text of that earlier alert + +Example of recent history context: + +```text +Recent alert history: +- TW 6 | 08:00-09:00 | threat=9.20 | conf=0.76 | + top patterns: Horizontal port scan to port 443/TCP; + repeated unknown-port traffic | + prior summary: Earlier scanning activity suggests reconnaissance. + +- TW 7 | 09:00-10:00 | threat=12.40 | conf=0.81 | + top patterns: Repeated connections to unknown destination port 449/TCP | + prior summary: The source continued unusual outbound probing across multiple + external IPs. +``` + +That history is then added to the final prompt as context only. The current +alert evidence remains the primary source of truth. + +## 4. Build a Constrained Analyst Prompt + +Once the incident digest exists, the method builds a prompt that is narrow, +structured, and conservative. + +The prompt contains two kinds of information. + +First, it provides metadata about the alert: + +- source IP +- time window +- time range +- accumulated threat level +- alert confidence +- number of correlated evidence records +- number of grouped evidence patterns +- number of reduction layers already applied + +Second, when available, it provides the recent alert history for the same +source/profile. + +Third, it provides the evidence digest itself: the grouped incident patterns +described above. + +Then it asks for a very specific output: + +- explain the main suspicious behavior +- identify the strongest supporting or weakening evidence +- say whether the alert looks likely true positive, likely false positive, or + uncertain +- state the likely operational risk or urgency +- explain whether the current alert looks like a continuation, escalation, + diversification, or a different pattern relative to recent activity + +The output is constrained to one paragraph of plain text. + +This is important. The system is not asking for a report, bullets, JSON, or +creative prose. It is asking for one concise analyst paragraph that can sit +next to the alert. + +So if the question is "what exactly is the context and what exactly is the +question?", the answer is: + +The context is the alert metadata plus the grouped digest of all evidence +linked to that alert, plus recent prior summarized alerts for the same source +when history is available. + +The question is essentially: + +```text +Here is one Slips alert with its correlated evidence. +Here is recent alert history for the same source, if available. +Explain the main suspicious behavior. +Identify what evidence most strongly supports or weakens the alert. +Say whether it looks likely true positive, likely false positive, or uncertain. +State the likely operational risk. +Explain whether this looks like a continuation, escalation, diversification, or +a different pattern relative to recent activity. +Write exactly one plain-text paragraph. +Use only the provided data. +Do not invent missing facts. +``` + +Example of prompt context: + +```text +Incident metadata: +- Source IP: 192.168.1.113 +- Time window: 8 +- Time range: 07:00 to 07:22 +- Threat level: 15.1 +- Confidence: 0.84 +- Correlated evidence records: 31 +- Grouped evidence patterns: 2 + +Evidence digest: +- 07:02-07:22 | Connection to unknown destination port 449/TCP (21x similar) +- 07:00-07:03 | Horizontal port scan to port 443/TCP (2x similar) + +Recent alert history: +- TW 7 | 06:00-07:00 | threat=8.70 | conf=0.71 | top patterns: repeated + unknown-port traffic | prior summary: Earlier unusual outbound probing was + already observed from the same source. +``` + +Example of the kind of answer the method is trying to produce: + +```text +This alert shows repeated connections from 192.168.1.113 to an unusual +destination port 449/TCP across multiple external IPs, together with a +horizontal scan to port 443/TCP, which makes the activity look more consistent +with reconnaissance than with a single benign connection. The repeated pattern +and high-severity scan evidence strengthen the alert, and the recent alert +history suggests this is a continuation and escalation of earlier probing from +the same source. Although the absence of host-side context leaves some +uncertainty, this looks like a likely true positive and should be treated as +medium-to-high operational risk. +``` + +## 5. Guard the Model Against Hallucinations + +Using an LLM inside an IDS only makes sense if the output is grounded. + +The summary method adds several guardrails before and during generation: + +- the prompt explicitly says to use only the provided alert and evidence data +- the prompt explicitly says not to invent missing facts +- the output is restricted to one paragraph +- repetitive evidence is grouped before inference so the model sees a cleaner + representation of the incident +- uncertainty is required when the evidence is weak, incomplete, or + contradictory +- the generation temperature is kept low to reduce drift + +This does not mean hallucinations are impossible. It means the method is built +to make them less likely, easier to detect, and less operationally dangerous. + +The training pipeline also helps here. The local models were not trained on +random free-form responses. They were trained on selected, higher-quality +security summaries derived from real Slips incidents. That gives the model a +much tighter target behavior. + +## 6. Control the Context Budget Explicitly + +Small local models are useful only if the input stays under control. + +The method therefore estimates prompt size explicitly and uses separate budgets +for: + +- recent alert history +- the final analyst summary prompt +- intermediate reduction prompts +- the final answer length + +This is not cosmetic. It is required for reliable local deployment. + +If the grouped digest already fits inside the final prompt budget, the model is +called directly. + +If it does not fit, the method does not simply cut the alert in half and hope +for the best. Instead, it performs recursive evidence reduction. + +The history itself is also bounded. Only a small number of recent alerts are +kept per source/profile, and only a bounded token budget is reserved for that +history in the final prompt. This prevents memory from growing into prompt +pollution. + +## 7. Use Recursive Reduction Instead of Blind Truncation + +When an alert is too large, the evidence digest is split into smaller chunks. +Each chunk is summarized into a shorter intermediate digest, and those +intermediate digests are then combined into the next layer. + +This can repeat several times until the final prompt fits. + +The reduction step is carefully scoped: + +- it preserves behaviors, time ranges, counts, and suspicious indicators +- it preserves false-positive clues +- it is not allowed to make the final verdict for the whole alert + +If one digest item is still too large by itself, it is split at natural +boundaries: + +- line boundaries +- semicolons +- sentence breaks +- commas +- finally words if necessary + +This matters because it keeps the reduction process information-preserving. +Instead of throwing evidence away, the method compresses it layer by layer. + +Example: + +```text +Original digest: +- 18 grouped items covering scans, repeated unknown-port traffic, DNS failures, + TLS anomalies, and HTTP evidence + +Reduction layer 1: +- chunk 1 summary: dominant scanning and unknown-port behavior +- chunk 2 summary: DNS and TLS anomalies +- chunk 3 summary: HTTP and supporting evidence + +Final prompt: +- 3 reduced digest items instead of the original 18 +``` + +That is a much stronger design for security alerts than naive truncation. + +## 8. Use a Shared Local Model Service + +The alert summary layer does not hard-code one model API into the summarization +logic. + +Instead, it sends requests through a shared model service that all AI-enabled +parts of Slips can use. That service is responsible for: + +- backend selection +- queueing +- worker concurrency +- request tracking +- response delivery + +This separation is important for engineering quality. + +It means the alert summary method can stay focused on: + +- evidence reconstruction +- grouping +- prompt design +- reduction +- fallback behavior + +while the model service handles transport and backend management. + +It also means Slips can expose several local model variants with different +speed and quality tradeoffs without changing the summary method itself. + +## 9. Prefer Local Fine-Tuned Models + +For this use case, local models are the right default. + +The reasons are practical: + +- alert data stays inside the monitored environment +- the feature works offline +- the cost is predictable +- deployment is possible on the same systems where Slips already runs + +This is especially relevant for edge deployments. Slips was designed to support +local inference even on constrained hardware, which is why model size, +quantization, context budget, and prompt efficiency matter so much in this +method. + +The summary layer also benefits directly from fine-tuning. General-purpose +models are not optimized for compact, evidence-bound security explanations. +Fine-tuned local models are better aligned with the actual task: + +- summarize correlated alerts +- reason about supporting and weakening evidence +- express uncertainty correctly +- stay concise under tight context limits + +The public fine-tuning pipeline behind these models used real Slips incidents, +best-of-N supervision, and judge-based selection to create higher-quality +training targets. The result is a family of local models specialized for alert +summary and risk-oriented security analysis. + +## 10. Match Replies Strictly + +In a shared model architecture, many requests may be in flight at once. The +summary method therefore tracks each request with its own unique identifier and +accepts only the matching reply. + +This is a simple but critical detail. + +Without strict reply matching, one module could accidentally consume another +module's response. In a security system, that would be unacceptable. + +By keeping one active summary request at a time and matching replies +explicitly, the method stays deterministic and auditable. + +## 11. Fail Safely When the Model Is Unavailable + +An IDS cannot simply return nothing because a model times out. + +So the summary layer includes a local fallback path. + +If the model request fails, the method still produces one paragraph based on: + +- the strongest grouped indicators +- the severity distribution +- the alert confidence +- the accumulated threat level +- recent alert history when available + +This fallback is intentionally simpler than the model output, but it preserves +an essential property: every alert still gets an explanation. + +Example fallback style: + +```text +LLM summary unavailable. Local heuristic summary: this alert correlates 31 +evidence records for one source IP, with the strongest indicators being +repeated connections to an unknown destination port and a horizontal scan to +443/TCP. The evidence mix includes high and medium severity findings. Based on +the accumulated threat level and confidence, this looks like a likely true +positive and the operational risk appears medium to high. +``` + +That is the correct design choice for an operational security tool. + +## 12. Shut Down Without Dropping Explanations + +Another subtle part of the method is shutdown behavior. + +If the system is stopping while a model request is still in flight, the summary +layer does not immediately abandon that request. It waits for the shared model +service to finish delivering the reply. + +Only if it truly cannot complete the request does it flush the alert with the +fallback paragraph. + +This protects against a common failure mode in asynchronous systems: the model +finishes, but the caller has already exited, so the answer is lost. + +For an IDS, losing the explanation even though the alert exists is a real +operational bug. The method is designed to avoid that. + +## 13. Why This Improves Explainability + +The real value of the summary layer is not that it adds AI to an IDS. + +The real value is that it improves the last mile of human consumption. + +Slips already knows a lot about an incident. What this method adds is a bridge +from machine evidence to human understanding. + +That improves: + +- analyst triage speed +- clarity of why an alert matters +- explainability for non-specialists +- usability of a behavioral IDS in practice + +Instead of forcing the human to read a large correlated evidence set, the +system provides one evidence-bound explanation that can be read in seconds. + +## 14. Why This Matters Beyond Slips + +To our knowledge, Slips is the first IDS deployed with a local model +fine-tuned specifically for security analysis of alerts. + +That matters because it shows a different path for AI in security. + +The interesting contribution is not "an IDS that can call a model." The +interesting contribution is this combination: + +- local inference +- fine-tuned security-specific models +- structured evidence compression +- recursive reduction for large incidents +- explicit anti-hallucination guardrails +- analyst-facing explanations + +That is a real method, not a demo. + +## Final Thought + +The alert summary layer is a narrow, technical, and carefully engineered piece +of Slips. + +It reconstructs the correlated evidence behind an alert, compresses repetition +into incident patterns, adds bounded recent-history context, controls the +prompt budget, uses recursive reduction when needed, and generates one +grounded explanation with a local fine-tuned model. + +That is why it works. And that is why this feature is useful. diff --git a/scripts/analyze_alert_creation_delay.py b/scripts/analyze_alert_creation_delay.py new file mode 100755 index 0000000000..aeba7cb18d --- /dev/null +++ b/scripts/analyze_alert_creation_delay.py @@ -0,0 +1,843 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +""" +Analyze alert creation delay from Slips alerts exports. + +This script measures the delay between each alert's CreateTime and StartTime, +then summarizes the distribution and how it evolves over time. It supports the +newline-delimited JSON format used by alerts.json as well as plain JSON arrays. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import sys +from collections import defaultdict +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path + + +DEFAULT_RESOLUTIONS = ("day", "hour", "minute") +VALID_RESOLUTIONS = set(DEFAULT_RESOLUTIONS) +DELAY_BANDS = ( + ("negative", None, 0.0), + ("0s-1s", 0.0, 1.0), + ("1s-10s", 1.0, 10.0), + ("10s-60s", 10.0, 60.0), + ("1m-5m", 60.0, 300.0), + ("5m-1h", 300.0, 3600.0), + ("1h-1d", 3600.0, 86400.0), + (">=1d", 86400.0, None), +) + + +@dataclass(frozen=True) +class AlertDelayRecord: + record_number: int + alert_id: str + severity: str + create_time: str + start_time: str + delay_seconds: float + description: str + + +@dataclass(frozen=True) +class SummaryStats: + count: int + min_seconds: float + mean_seconds: float + p50_seconds: float + p90_seconds: float + p95_seconds: float + p99_seconds: float + max_seconds: float + + +@dataclass(frozen=True) +class BucketSummary: + bucket_start: str + count: int + min_seconds: float + mean_seconds: float + p50_seconds: float + p95_seconds: float + p99_seconds: float + max_seconds: float + + +def parse_args() -> argparse.Namespace: + class HelpFormatter( + argparse.ArgumentDefaultsHelpFormatter, + argparse.RawDescriptionHelpFormatter, + ): + pass + + parser = argparse.ArgumentParser( + description=( + "Analyze alert creation delay in Slips alerts exports.\n\n" + "The script reads alerts.json, computes the per-alert delay as\n" + "CreateTime - StartTime, then summarizes the overall distribution\n" + "and how that delay evolves over time by day, hour, and minute." + ), + epilog=( + "Input format:\n" + " alerts.json can be newline-delimited JSON (one alert per line)\n" + " or a regular JSON array of alert objects.\n\n" + "Outputs:\n" + " The terminal output shows overall statistics, delay bands,\n" + " the alerts with the largest delays, and trend tables.\n" + " If --output-dir is given, the script also writes CSV files for\n" + " each selected time resolution, a summary.json file, and a\n" + " Markdown analysis report.\n\n" + "Example:\n" + " python3 scripts/analyze_alert_creation_delay.py \\\n" + " output/test-tcell-8/alerts.json \\\n" + " --output-dir output/test-tcell-8/alert_creation_delay_report" + ), + formatter_class=HelpFormatter, + ) + parser.add_argument( + "alerts_path", + help="Path to alerts.json (JSONL or JSON array).", + ) + parser.add_argument( + "--bucket-time", + choices=("create", "start"), + default="create", + help=( + "Which timestamp to use for trend buckets. Default: create " + "(group by CreateTime)." + ), + ) + parser.add_argument( + "--resolution", + action="append", + choices=sorted(VALID_RESOLUTIONS), + help=( + "Trend resolution to emit. Repeat to select a subset. " + "Default: day, hour, minute." + ), + ) + parser.add_argument( + "--output-dir", + default="", + help=( + "Optional directory where CSV trend files, top-delays CSV, and " + "summary.json and a Markdown report will be written." + ), + ) + parser.add_argument( + "--print-limit", + type=int, + default=120, + help=( + "Print all buckets when a resolution has at most this many buckets. " + "Default: 120." + ), + ) + parser.add_argument( + "--top-buckets", + type=int, + default=10, + help=( + "When a resolution has many buckets, print this many worst buckets " + "and this many most recent buckets. Default: 10." + ), + ) + parser.add_argument( + "--top-alerts", + type=int, + default=10, + help="Show this many alerts with the largest delays. Default: 10.", + ) + parser.add_argument( + "--description-width", + type=int, + default=110, + help="Maximum description width in the top-alerts section. Default: 110.", + ) + return parser.parse_args() + + +def detect_input_format(path: Path) -> str: + with path.open(encoding="utf-8") as handle: + while True: + char = handle.read(1) + if not char: + raise ValueError(f"{path} is empty") + if char.isspace(): + continue + return "json-array" if char == "[" else "jsonl" + + +def iter_alert_records(path: Path): + input_format = detect_input_format(path) + if input_format == "json-array": + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, list): + raise ValueError(f"{path} is a JSON array file but did not contain a list") + for index, alert in enumerate(payload, start=1): + if not isinstance(alert, dict): + raise ValueError(f"Record {index} is not a JSON object") + yield input_format, index, alert + return + + with path.open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + stripped = line.strip() + if not stripped: + continue + try: + alert = json.loads(stripped) + except json.JSONDecodeError as exc: + raise ValueError( + f"Invalid JSON on line {line_number}: {exc.msg}" + ) from exc + if not isinstance(alert, dict): + raise ValueError(f"Line {line_number} is not a JSON object") + yield input_format, line_number, alert + + +def parse_timestamp(value: str) -> datetime: + normalized = value.replace("Z", "+00:00") + return datetime.fromisoformat(normalized) + + +def truncate_datetime(value: datetime, resolution: str) -> datetime: + if resolution == "day": + return value.replace(hour=0, minute=0, second=0, microsecond=0) + if resolution == "hour": + return value.replace(minute=0, second=0, microsecond=0) + if resolution == "minute": + return value.replace(second=0, microsecond=0) + raise ValueError(f"Unsupported resolution: {resolution}") + + +def percentile(sorted_values: list[float], fraction: float) -> float: + if not sorted_values: + raise ValueError("percentile() requires at least one value") + if len(sorted_values) == 1: + return sorted_values[0] + position = (len(sorted_values) - 1) * fraction + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return sorted_values[lower] + lower_value = sorted_values[lower] + upper_value = sorted_values[upper] + return lower_value + (upper_value - lower_value) * (position - lower) + + +def build_summary(values: list[float]) -> SummaryStats: + if not values: + raise ValueError("No values available to summarize") + ordered = sorted(values) + return SummaryStats( + count=len(ordered), + min_seconds=ordered[0], + mean_seconds=sum(ordered) / len(ordered), + p50_seconds=percentile(ordered, 0.50), + p90_seconds=percentile(ordered, 0.90), + p95_seconds=percentile(ordered, 0.95), + p99_seconds=percentile(ordered, 0.99), + max_seconds=ordered[-1], + ) + + +def build_bucket_summaries( + bucket_values: dict[datetime, list[float]] +) -> list[BucketSummary]: + summaries: list[BucketSummary] = [] + for bucket_start, values in sorted(bucket_values.items()): + ordered = sorted(values) + summaries.append( + BucketSummary( + bucket_start=bucket_start.isoformat(), + count=len(ordered), + min_seconds=ordered[0], + mean_seconds=sum(ordered) / len(ordered), + p50_seconds=percentile(ordered, 0.50), + p95_seconds=percentile(ordered, 0.95), + p99_seconds=percentile(ordered, 0.99), + max_seconds=ordered[-1], + ) + ) + return summaries + + +def delay_band_label(delay_seconds: float) -> str: + for label, lower, upper in DELAY_BANDS: + if lower is None and delay_seconds < upper: + return label + if upper is None and delay_seconds >= lower: + return label + if lower is not None and upper is not None and lower <= delay_seconds < upper: + return label + return "unclassified" + + +def ellipsize(text: str, width: int) -> str: + if width <= 3 or len(text) <= width: + return text + return text[: width - 3] + "..." + + +def print_summary_stats(summary: SummaryStats): + print("Overall delay statistics (CreateTime - StartTime, in seconds)") + print(f" alerts: {summary.count:,}") + print(f" min_s: {summary.min_seconds:.6f}") + print(f" mean_s: {summary.mean_seconds:.6f}") + print(f" p50_s: {summary.p50_seconds:.6f}") + print(f" p90_s: {summary.p90_seconds:.6f}") + print(f" p95_s: {summary.p95_seconds:.6f}") + print(f" p99_s: {summary.p99_seconds:.6f}") + print(f" max_s: {summary.max_seconds:.6f}") + + +def print_delay_bands(band_counts: dict[str, int], total: int): + print("\nDelay bands") + for label, _, _ in DELAY_BANDS: + count = band_counts.get(label, 0) + percentage = (count / total * 100) if total else 0.0 + print(f" {label:>8}: {count:>9,} ({percentage:6.2f}%)") + + +def print_top_alerts(top_alerts: list[AlertDelayRecord], description_width: int): + if not top_alerts: + return + print("\nLargest per-alert delays") + for rank, item in enumerate(top_alerts, start=1): + description = ellipsize(item.description.replace("\n", " "), description_width) + print( + f" {rank:>2}. delay_s={item.delay_seconds:>12.6f} " + f"record={item.record_number:<8} severity={item.severity or '-':<6} " + f"id={item.alert_id or '-'}" + ) + print( + f" start={item.start_time} create={item.create_time} " + f"description={description}" + ) + + +def print_bucket_table(rows: list[BucketSummary]): + if not rows: + print(" no buckets") + return + header = ( + f"{'bucket_start':<25} {'count':>8} {'min_s':>12} {'mean_s':>12} " + f"{'p50_s':>12} {'p95_s':>12} {'p99_s':>12} {'max_s':>12}" + ) + print(header) + print("-" * len(header)) + for row in rows: + print( + f"{row.bucket_start:<25} {row.count:>8,} " + f"{row.min_seconds:>12.3f} {row.mean_seconds:>12.3f} " + f"{row.p50_seconds:>12.3f} {row.p95_seconds:>12.3f} " + f"{row.p99_seconds:>12.3f} {row.max_seconds:>12.3f}" + ) + + +def print_resolution_summary( + resolution: str, + rows: list[BucketSummary], + print_limit: int, + top_buckets: int, + csv_path: Path | None, +): + print(f"\nBy {resolution}") + if not rows: + print(" no data") + return + + first_row = rows[0] + last_row = rows[-1] + print( + f" buckets: {len(rows):,}; first={first_row.bucket_start}; " + f"last={last_row.bucket_start}" + ) + print( + f" first mean/p50/p95: {first_row.mean_seconds:.3f} / " + f"{first_row.p50_seconds:.3f} / {first_row.p95_seconds:.3f} seconds" + ) + print( + f" last mean/p50/p95: {last_row.mean_seconds:.3f} / " + f"{last_row.p50_seconds:.3f} / {last_row.p95_seconds:.3f} seconds" + ) + if csv_path is not None: + print(f" csv: {csv_path}") + + if len(rows) <= print_limit: + print_bucket_table(rows) + return + + worst_rows = sorted( + rows, + key=lambda row: (row.p95_seconds, row.max_seconds, row.mean_seconds), + reverse=True, + )[:top_buckets] + recent_rows = rows[-top_buckets:] + + print(f" {len(rows):,} buckets exceed --print-limit={print_limit}.") + print(f" Worst {len(worst_rows)} buckets by p95_s") + print_bucket_table(sorted(worst_rows, key=lambda row: row.bucket_start)) + print(f"\n Most recent {len(recent_rows)} buckets") + print_bucket_table(recent_rows) + + +def write_bucket_csv(path: Path, rows: list[BucketSummary]): + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow( + [ + "bucket_start", + "count", + "min_s", + "mean_s", + "p50_s", + "p95_s", + "p99_s", + "max_s", + ] + ) + for row in rows: + writer.writerow( + [ + row.bucket_start, + row.count, + f"{row.min_seconds:.6f}", + f"{row.mean_seconds:.6f}", + f"{row.p50_seconds:.6f}", + f"{row.p95_seconds:.6f}", + f"{row.p99_seconds:.6f}", + f"{row.max_seconds:.6f}", + ] + ) + + +def write_top_alerts_csv(path: Path, rows: list[AlertDelayRecord]): + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow( + [ + "record_number", + "alert_id", + "severity", + "create_time", + "start_time", + "delay_s", + "description", + ] + ) + for row in rows: + writer.writerow( + [ + row.record_number, + row.alert_id, + row.severity, + row.create_time, + row.start_time, + f"{row.delay_seconds:.6f}", + row.description, + ] + ) + + +def ensure_output_dir(output_dir: str) -> Path | None: + if not output_dir: + return None + path = Path(output_dir).expanduser().resolve() + path.mkdir(parents=True, exist_ok=True) + return path + + +def markdown_escape(text: str) -> str: + return text.replace("\\", "\\\\").replace("|", "\\|").replace("\n", " ") + + +def append_markdown_bucket_table(lines: list[str], rows: list[BucketSummary]): + lines.append( + "| bucket_start | count | min_s | mean_s | p50_s | p95_s | p99_s | max_s |" + ) + lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") + for row in rows: + lines.append( + f"| `{row.bucket_start}` | {row.count:,} | {row.min_seconds:.3f} | " + f"{row.mean_seconds:.3f} | {row.p50_seconds:.3f} | " + f"{row.p95_seconds:.3f} | {row.p99_seconds:.3f} | " + f"{row.max_seconds:.3f} |" + ) + lines.append("") + + +def write_markdown_report( + path: Path, + alerts_path: Path, + input_format: str | None, + bucket_time: str, + resolutions: tuple[str, ...], + overall_summary: SummaryStats, + band_counts: dict[str, int], + top_delay_records: list[AlertDelayRecord], + bucket_summaries: dict[str, list[BucketSummary]], + artifact_paths: dict[str, Path], + skipped_missing_timestamps: int, + skipped_invalid_timestamps: int, + negative_count: int, + zero_count: int, + trend_min: datetime | None, + trend_max: datetime | None, + print_limit: int, + top_buckets: int, + description_width: int, +): + lines = [ + "# Alert Creation Delay Analysis", + "", + "## Run Summary", + f"- Input: `{alerts_path}`", + f"- Input format: `{input_format}`", + f"- Trend bucket timestamp: `{bucket_time}` time", + f"- Valid alerts: {overall_summary.count:,}", + f"- Skipped missing timestamps: {skipped_missing_timestamps:,}", + f"- Skipped invalid timestamps: {skipped_invalid_timestamps:,}", + f"- Negative delays: {negative_count:,}", + f"- Zero delays: {zero_count:,}", + ] + if trend_min is not None and trend_max is not None: + lines.append( + f"- Trend range: `{trend_min.isoformat()}` to `{trend_max.isoformat()}`" + ) + + lines.extend( + [ + "", + "## Overall Delay Statistics", + "", + "| metric | seconds |", + "| --- | ---: |", + f"| min_s | {overall_summary.min_seconds:.6f} |", + f"| mean_s | {overall_summary.mean_seconds:.6f} |", + f"| p50_s | {overall_summary.p50_seconds:.6f} |", + f"| p90_s | {overall_summary.p90_seconds:.6f} |", + f"| p95_s | {overall_summary.p95_seconds:.6f} |", + f"| p99_s | {overall_summary.p99_seconds:.6f} |", + f"| max_s | {overall_summary.max_seconds:.6f} |", + "", + "## Delay Bands", + "", + "| band | count | percentage |", + "| --- | ---: | ---: |", + ] + ) + for label, _, _ in DELAY_BANDS: + count = band_counts.get(label, 0) + percentage = count / overall_summary.count * 100 if overall_summary.count else 0.0 + lines.append(f"| {label} | {count:,} | {percentage:.2f}% |") + + lines.extend( + [ + "", + "## Largest Per-Alert Delays", + "", + "| rank | delay_s | record | severity | id | start | create | description |", + "| ---: | ---: | ---: | --- | --- | --- | --- | --- |", + ] + ) + for rank, item in enumerate(top_delay_records, start=1): + description = ellipsize( + markdown_escape(item.description), description_width + ) + lines.append( + f"| {rank} | {item.delay_seconds:.6f} | {item.record_number} | " + f"{markdown_escape(item.severity or '-')} | " + f"{markdown_escape(item.alert_id or '-')} | " + f"`{item.start_time}` | `{item.create_time}` | {description} |" + ) + + lines.extend( + [ + "", + "## Artifacts", + "", + f"- JSON summary: [{artifact_paths['summary_json'].name}]({artifact_paths['summary_json'].name})", + f"- Top delays CSV: [{artifact_paths['top_alerts'].name}]({artifact_paths['top_alerts'].name})", + ] + ) + for resolution in resolutions: + lines.append( + f"- {resolution.capitalize()} CSV: " + f"[{artifact_paths[resolution].name}]({artifact_paths[resolution].name})" + ) + + lines.append("") + lines.append("## Trend Summary") + lines.append("") + + for resolution in resolutions: + rows = bucket_summaries[resolution] + lines.append(f"### By {resolution}") + if not rows: + lines.extend(["No data.", ""]) + continue + + first_row = rows[0] + last_row = rows[-1] + lines.append(f"- Buckets: {len(rows):,}") + lines.append( + f"- First bucket: `{first_row.bucket_start}` with mean/p50/p95 " + f"`{first_row.mean_seconds:.3f} / {first_row.p50_seconds:.3f} / " + f"{first_row.p95_seconds:.3f}` seconds" + ) + lines.append( + f"- Last bucket: `{last_row.bucket_start}` with mean/p50/p95 " + f"`{last_row.mean_seconds:.3f} / {last_row.p50_seconds:.3f} / " + f"{last_row.p95_seconds:.3f}` seconds" + ) + lines.append( + f"- Full CSV: [{artifact_paths[resolution].name}]({artifact_paths[resolution].name})" + ) + lines.append("") + + if len(rows) <= print_limit: + append_markdown_bucket_table(lines, rows) + continue + + worst_rows = sorted( + rows, + key=lambda row: (row.p95_seconds, row.max_seconds, row.mean_seconds), + reverse=True, + )[:top_buckets] + recent_rows = rows[-top_buckets:] + + lines.append( + f"Full series omitted here because it has {len(rows):,} buckets and " + f"`--print-limit` is {print_limit}. The CSV contains every bucket." + ) + lines.append("") + lines.append(f"#### Worst {len(worst_rows)} Buckets by p95_s") + lines.append("") + append_markdown_bucket_table( + lines, sorted(worst_rows, key=lambda row: row.bucket_start) + ) + lines.append(f"#### Most Recent {len(recent_rows)} Buckets") + lines.append("") + append_markdown_bucket_table(lines, recent_rows) + + with path.open("w", encoding="utf-8") as handle: + handle.write("\n".join(lines)) + handle.write("\n") + + +def main() -> int: + args = parse_args() + alerts_path = Path(args.alerts_path).expanduser().resolve() + if not alerts_path.exists(): + print(f"alerts file not found: {alerts_path}", file=sys.stderr) + return 1 + + resolutions = tuple(args.resolution or DEFAULT_RESOLUTIONS) + output_dir = ensure_output_dir(args.output_dir) + + overall_delays: list[float] = [] + bucket_values = { + resolution: defaultdict(list) for resolution in resolutions + } + band_counts: dict[str, int] = defaultdict(int) + top_delay_records: list[AlertDelayRecord] = [] + skipped_missing_timestamps = 0 + skipped_invalid_timestamps = 0 + negative_count = 0 + zero_count = 0 + trend_min: datetime | None = None + trend_max: datetime | None = None + input_format: str | None = None + + for current_format, record_number, alert in iter_alert_records(alerts_path): + input_format = current_format + create_time_raw = alert.get("CreateTime") + start_time_raw = alert.get("StartTime") + if not create_time_raw or not start_time_raw: + skipped_missing_timestamps += 1 + continue + + try: + create_time = parse_timestamp(create_time_raw) + start_time = parse_timestamp(start_time_raw) + except ValueError: + skipped_invalid_timestamps += 1 + continue + + delay_seconds = (create_time - start_time).total_seconds() + overall_delays.append(delay_seconds) + band_counts[delay_band_label(delay_seconds)] += 1 + if delay_seconds < 0: + negative_count += 1 + elif delay_seconds == 0: + zero_count += 1 + + top_delay_records.append( + AlertDelayRecord( + record_number=record_number, + alert_id=str(alert.get("ID") or ""), + severity=str(alert.get("Severity") or ""), + create_time=create_time_raw, + start_time=start_time_raw, + delay_seconds=delay_seconds, + description=str(alert.get("Description") or ""), + ) + ) + + trend_time = create_time if args.bucket_time == "create" else start_time + if trend_min is None or trend_time < trend_min: + trend_min = trend_time + if trend_max is None or trend_time > trend_max: + trend_max = trend_time + for resolution in resolutions: + bucket_values[resolution][ + truncate_datetime(trend_time, resolution) + ].append(delay_seconds) + + if not overall_delays: + print( + ( + "No alerts with valid CreateTime and StartTime were found in " + f"{alerts_path}" + ), + file=sys.stderr, + ) + return 1 + + overall_summary = build_summary(overall_delays) + top_delay_records = sorted( + top_delay_records, + key=lambda item: item.delay_seconds, + reverse=True, + )[: args.top_alerts] + bucket_summaries = { + resolution: build_bucket_summaries(bucket_values[resolution]) + for resolution in resolutions + } + + artifact_paths: dict[str, Path] = {} + if output_dir is not None: + for resolution in resolutions: + artifact_paths[resolution] = ( + output_dir / f"alert_creation_delay_by_{resolution}.csv" + ) + + artifact_paths["top_alerts"] = ( + output_dir / "alert_creation_delay_top_alerts.csv" + ) + artifact_paths["summary_json"] = output_dir / "summary.json" + artifact_paths["analysis_md"] = ( + output_dir / "alert_creation_delay_analysis.md" + ) + + for resolution in resolutions: + write_bucket_csv(artifact_paths[resolution], bucket_summaries[resolution]) + + write_top_alerts_csv(artifact_paths["top_alerts"], top_delay_records) + + summary_payload = { + "alerts_path": str(alerts_path), + "input_format": input_format, + "bucket_time": args.bucket_time, + "resolutions": list(resolutions), + "processed_alerts": overall_summary.count, + "skipped_missing_timestamps": skipped_missing_timestamps, + "skipped_invalid_timestamps": skipped_invalid_timestamps, + "negative_delays": negative_count, + "zero_delays": zero_count, + "trend_start": trend_min.isoformat() if trend_min else None, + "trend_end": trend_max.isoformat() if trend_max else None, + "overall_delay_seconds": asdict(overall_summary), + "delay_bands": [ + { + "label": label, + "count": band_counts.get(label, 0), + "percentage": ( + band_counts.get(label, 0) / overall_summary.count * 100 + ), + } + for label, _, _ in DELAY_BANDS + ], + "top_delays": [asdict(item) for item in top_delay_records], + "artifacts": { + name: str(path) for name, path in artifact_paths.items() + }, + "bucket_counts": { + resolution: len(bucket_summaries[resolution]) + for resolution in resolutions + }, + } + with artifact_paths["summary_json"].open("w", encoding="utf-8") as handle: + json.dump(summary_payload, handle, indent=2) + handle.write("\n") + + write_markdown_report( + path=artifact_paths["analysis_md"], + alerts_path=alerts_path, + input_format=input_format, + bucket_time=args.bucket_time, + resolutions=resolutions, + overall_summary=overall_summary, + band_counts=band_counts, + top_delay_records=top_delay_records, + bucket_summaries=bucket_summaries, + artifact_paths=artifact_paths, + skipped_missing_timestamps=skipped_missing_timestamps, + skipped_invalid_timestamps=skipped_invalid_timestamps, + negative_count=negative_count, + zero_count=zero_count, + trend_min=trend_min, + trend_max=trend_max, + print_limit=args.print_limit, + top_buckets=args.top_buckets, + description_width=args.description_width, + ) + + print(f"Input: {alerts_path}") + print(f"Input format: {input_format}") + print(f"Trend bucket timestamp: {args.bucket_time} time") + print( + f"Valid alerts: {overall_summary.count:,}; skipped missing timestamps: " + f"{skipped_missing_timestamps:,}; skipped invalid timestamps: " + f"{skipped_invalid_timestamps:,}" + ) + if trend_min is not None and trend_max is not None: + print(f"Trend range: {trend_min.isoformat()} -> {trend_max.isoformat()}") + print( + f"Negative delays: {negative_count:,}; zero delays: {zero_count:,}" + ) + print_summary_stats(overall_summary) + print_delay_bands(band_counts, overall_summary.count) + print_top_alerts(top_delay_records, args.description_width) + + for resolution in resolutions: + csv_path = artifact_paths.get(resolution) + print_resolution_summary( + resolution=resolution, + rows=bucket_summaries[resolution], + print_limit=args.print_limit, + top_buckets=args.top_buckets, + csv_path=csv_path, + ) + + if output_dir is not None: + print(f"\nArtifacts written to: {output_dir}") + print(f"Summary JSON: {artifact_paths['summary_json']}") + print(f"Markdown report: {artifact_paths['analysis_md']}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/regex_coverage_report.py b/scripts/regex_coverage_report.py new file mode 100644 index 0000000000..144d5c8631 --- /dev/null +++ b/scripts/regex_coverage_report.py @@ -0,0 +1,1653 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +# ruff: noqa: E402 +""" +Offline coverage estimator for RegexGenerator output. + +This script reads accepted regexes from a Slips run output directory and +estimates how much of several reference populations they cover: + +- benign corpus stored by RegexGenerator +- malicious TI-derived strings +- observed traffic strings extracted from the same Slips run + +It writes a standalone HTML report and a JSON summary. +""" + +from __future__ import annotations + +import argparse +import json +import math +import random +import re +import signal +import sqlite3 +import sys +import time +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone +from html import escape +from pathlib import Path +from typing import Iterable +from urllib.parse import urlparse + +try: + import redis +except ImportError: # pragma: no cover - dependency should exist in runtime + redis = None + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from slips_files.core.database.redis_db.constants import Constants + +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import utils +from slips_files.core.database.sqlite_db.regex_generator_db import REGEX_TYPES +from modules.regex_generator.match_strength import ( + compute_match_strength, + measure_regex_specificity, +) + + +DOMAIN_LIKE_TYPES = ("dns_domain", "tls_sni", "certificate_cn") +TYPE_LABELS = { + "dns_domain": "DNS Domain", + "uri": "URI", + "filename": "Filename", + "tls_sni": "TLS SNI", + "certificate_cn": "Certificate CN", +} + + +@dataclass +class TIStats: + run_redis_port: int + run_redis_available: bool + ti_cache_port: int + ti_cache_db: int + ti_cache_available: bool + loaded_feeds: int + cache_domain_count: int + cache_ip_count: int + cache_ja3_count: int + cache_jarm_count: int + source_files_scanned: int + + +class ProgressTracker: + BAR_WIDTH = 24 + CLEAR_LINE = "\r\033[2K" + RESET = "\033[0m" + CYAN = "\033[36m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + MAGENTA = "\033[35m" + + def __init__(self, total_regexes: int, total_comparisons: int, mode: str): + self.total_regexes = max(1, total_regexes) + self.total_comparisons = max(1, total_comparisons) + self.mode = mode + self.regexes_done = 0 + self.comparisons_done = 0 + self.current_type = "-" + self.start_time = time.monotonic() + + def print_plan(self): + print( + f"🔬 Coverage work estimate: {self.total_regexes} regexes, " + f"{self.total_comparisons} planned regex/string comparisons " + "(not raw TI entries)", + flush=True, + ) + self._render() + + def advance(self, regex_type: str, regex: str, comparisons: int): + self.regexes_done += 1 + self.comparisons_done += comparisons + self.current_type = regex_type + self._render() + + def finish(self): + self.regexes_done = self.total_regexes + self.comparisons_done = self.total_comparisons + self._render(done=True) + print(flush=True) + + def _render(self, done: bool = False): + regex_ratio = min(1.0, self.regexes_done / self.total_regexes) + filled = int(regex_ratio * self.BAR_WIDTH) + bar = f"{self.GREEN}{'█' * filled}{self.YELLOW}{'░' * (self.BAR_WIDTH - filled)}{self.RESET}" + elapsed = max(0.001, time.monotonic() - self.start_time) + progress_ratio = min( + 1.0, self.comparisons_done / self.total_comparisons + ) + if done or progress_ratio >= 1.0: + eta_seconds = 0.0 + else: + eta_seconds = (elapsed / max(progress_ratio, 1e-9)) - elapsed + status = ( + f"{self.CLEAR_LINE}" + f"🧪 {self.MAGENTA}{self.mode}{self.RESET} " + f"{bar} " + f"{regex_ratio * 100:6.2f}% " + f"| regex {self.regexes_done}/{self.total_regexes} " + f"| cmp {self.comparisons_done:,}/{self.total_comparisons:,} " + f"| type {self.CYAN}{TYPE_LABELS.get(self.current_type, self.current_type)}{self.RESET} " + f"| ETA ⏳ {self._format_duration(eta_seconds)}" + ) + print(status, end="", flush=True) + + @staticmethod + def _format_duration(seconds: float) -> str: + total_seconds = max(0, int(seconds)) + hours, remainder = divmod(total_seconds, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + + +class _NullTimeout: + def __enter__(self): + return None + + def __exit__(self, exc_type, exc, exc_tb): + return False + + +class _SignalTimeout: + def __init__(self, timeout_seconds: float): + self.timeout_seconds = timeout_seconds + self._previous_handler = None + + def __enter__(self): + self._previous_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, self._handle_timeout) + signal.setitimer(signal.ITIMER_REAL, self.timeout_seconds) + return None + + def __exit__(self, exc_type, exc, exc_tb): + signal.setitimer(signal.ITIMER_REAL, 0) + if self._previous_handler is not None: + signal.signal(signal.SIGALRM, self._previous_handler) + return False + + @staticmethod + def _handle_timeout(signum, frame): + raise TimeoutError("regex population match timed out") + + +def timeout_context(timeout_seconds: float): + if timeout_seconds <= 0: + return _NullTimeout() + return _SignalTimeout(timeout_seconds) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate an offline RegexGenerator coverage report." + ) + parser.add_argument( + "--run-output-dir", + required=True, + help=( + "Slips run output directory containing regex_generator/*.sqlite, " + "or a direct regex store directory containing generated_regexes.sqlite " + "and benign_corpus.sqlite." + ), + ) + parser.add_argument( + "--redis-port", + type=int, + default=6379, + help="Redis port used by the Slips run. Default: 6379.", + ) + parser.add_argument( + "--ti-cache-port", + type=int, + default=6379, + help="Redis port of the shared Slips TI cache. Default: 6379.", + ) + parser.add_argument( + "--ti-cache-db", + type=int, + default=1, + help="Redis DB number for the shared Slips TI cache. Default: 1.", + ) + parser.add_argument( + "--output-html", + default="", + help="Path to output HTML report. Default: /regex_generator_coverage_report.html", + ) + parser.add_argument( + "--output-json", + default="", + help="Path to output JSON summary. Default: /regex_generator_coverage_report.json", + ) + parser.add_argument( + "--sample-limit", + type=int, + default=15, + help="Number of example strings to include per report section.", + ) + parser.add_argument( + "--top-regexes", + type=int, + default=20, + help="Number of top regexes to show per type.", + ) + parser.add_argument( + "--match-timeout-seconds", + type=float, + default=0.25, + help=( + "Maximum wall-clock seconds allowed for one regex against one " + "population of strings for one regex type before it is skipped. " + "The populations are: benign corpus values, malicious TI values, " + "observed traffic values, and the reference union of malicious+observed. " + "Set 0 to disable." + ), + ) + parser.add_argument( + "--max-population-size", + type=int, + default=10000, + help=( + "Maximum number of strings evaluated for each regex type inside each " + "population: benign corpus, malicious TI, observed traffic, and " + "reference union. This cap is applied after --sampling-ratio. " + "Larger populations are sampled deterministically. Set 0 to disable " + "the cap." + ), + ) + parser.add_argument( + "--sampling-ratio", + type=float, + default=0.1, + help=( + "Fraction of strings to evaluate from each regex-type population " + "before applying --max-population-size. This is applied separately " + "to benign corpus values, malicious TI values, observed traffic values, " + "and reference-union values. Use values in (0, 1]. Default: 0.1." + ), + ) + parser.add_argument( + "--full-scan", + action="store_true", + help=( + "Disable both --sampling-ratio and --max-population-size, and scan " + "all strings in all populations for every regex type: benign corpus, " + "malicious TI, observed traffic, and reference union." + ), + ) + parser.add_argument( + "--sampling-seed", + type=int, + default=1, + help="Deterministic seed used when sampling large populations.", + ) + return parser.parse_args() + + +def normalize_string(value: str) -> str: + return str(value or "").strip() + + +def normalize_domain(value: str) -> str: + value = normalize_string(value).rstrip(".").lower() + return value + + +def normalize_uri(value: str) -> str: + value = normalize_string(value) + if not value: + return "" + parsed = urlparse(value) + if parsed.scheme and parsed.netloc: + path = parsed.path or "/" + if parsed.query: + return f"{path}?{parsed.query}" + return path + return value + + +def normalize_filename(value: str) -> str: + value = normalize_string(value) + if not value: + return "" + value = value.split("/")[-1] + value = value.split("\\")[-1] + return value.strip() + + +def normalize_cn(value: str) -> str: + value = normalize_string(value) + if not value: + return "" + cn_match = re.search(r"(?:^|,)CN=([^,]+)", value) + if cn_match: + return cn_match.group(1).strip() + return value + + +def add_string(populations: dict[str, set[str]], regex_type: str, value: str): + if regex_type in DOMAIN_LIKE_TYPES: + normalized = normalize_domain(value) + elif regex_type == "uri": + normalized = normalize_uri(value) + elif regex_type == "filename": + normalized = normalize_filename(value) + else: + normalized = normalize_string(value) + + if normalized: + populations[regex_type].add(normalized) + + +def load_regexes(regex_db_path: Path) -> dict[str, list[dict]]: + regexes_by_type = defaultdict(list) + with sqlite3.connect(regex_db_path) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute( + """ + SELECT regex_type, regex, regex_hash, backend_alias, provider, model, + temperature, prompt_version, request_id, created_at + FROM generated_regexes + WHERE status = 'accepted' + ORDER BY created_at ASC + """ + ).fetchall() + + for row in rows: + regexes_by_type[row["regex_type"]].append( + { + "regex": row["regex"], + "regex_hash": row["regex_hash"], + "backend_alias": row["backend_alias"], + "provider": row["provider"], + "model": row["model"], + "temperature": row["temperature"], + "prompt_version": row["prompt_version"], + "request_id": row["request_id"], + "created_at": row["created_at"], + } + ) + + return regexes_by_type + + +def load_benign_corpus(benign_db_path: Path) -> dict[str, set[str]]: + populations = {regex_type: set() for regex_type in REGEX_TYPES} + with sqlite3.connect(benign_db_path) as conn: + for regex_type, value in conn.execute( + "SELECT regex_type, value FROM benign_strings" + ): + add_string(populations, regex_type, value) + return populations + + +def load_tranco_benign_populations( + ti_cache_port: int, + ti_cache_db: int, + limit: int, +) -> dict[str, set[str]]: + populations = {regex_type: set() for regex_type in REGEX_TYPES} + if redis is None: + return populations + + try: + cache_client = redis.Redis( + host="127.0.0.1", + port=ti_cache_port, + db=ti_cache_db, + decode_responses=True, + socket_connect_timeout=1, + socket_timeout=1, + ) + if limit <= 0: + return populations + tranco_domains = cache_client.zrange( + Constants.TRANCO_WHITELISTED_DOMAINS, 0, limit - 1 + ) + except Exception: + return populations + + for domain in tranco_domains: + domain = normalize_domain(domain) + if not domain: + continue + for regex_type in DOMAIN_LIKE_TYPES: + add_string(populations, regex_type, domain) + return populations + + +def parse_zeek_json_log(path: Path) -> Iterable[dict]: + with path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + yield json.loads(line) + except json.JSONDecodeError: + continue + + +def load_observed_populations(run_output_dir: Path) -> dict[str, set[str]]: + populations = {regex_type: set() for regex_type in REGEX_TYPES} + zeek_dir = run_output_dir / "zeek_files" + + dns_log = zeek_dir / "dns.log" + if dns_log.exists(): + for row in parse_zeek_json_log(dns_log): + add_string(populations, "dns_domain", row.get("query", "")) + + http_log = zeek_dir / "http.log" + if http_log.exists(): + for row in parse_zeek_json_log(http_log): + uri = row.get("uri", "") + add_string(populations, "uri", uri) + host = normalize_domain(row.get("host", "")) + if host: + add_string(populations, "dns_domain", host) + filename = filename_from_uri(uri) + if filename: + add_string(populations, "filename", filename) + + ssl_log = zeek_dir / "ssl.log" + if ssl_log.exists(): + for row in parse_zeek_json_log(ssl_log): + server_name = row.get("server_name", "") + add_string(populations, "tls_sni", server_name) + add_string(populations, "dns_domain", server_name) + + x509_log = zeek_dir / "x509.log" + if x509_log.exists(): + for row in parse_zeek_json_log(x509_log): + subject = row.get("certificate.subject", "") + cn = normalize_cn(subject) + add_string(populations, "certificate_cn", cn) + if utils.is_valid_domain(cn): + add_string(populations, "dns_domain", cn) + + files_log = zeek_dir / "files.log" + if files_log.exists(): + for row in parse_zeek_json_log(files_log): + filename = row.get("filename", "") + if filename: + add_string(populations, "filename", filename) + + if all(not values for values in populations.values()): + flow_db = run_output_dir / "flows.sqlite" + if flow_db.exists(): + load_observed_from_flows_sqlite(flow_db, populations) + + return populations + + +def load_observed_from_flows_sqlite( + flows_db_path: Path, populations: dict[str, set[str]] +): + with sqlite3.connect(flows_db_path) as conn: + rows = conn.execute("SELECT flow_type, flow FROM altflows") + for flow_type, flow_json in rows: + try: + flow = json.loads(flow_json) + except json.JSONDecodeError: + continue + + if flow_type == "dns": + add_string(populations, "dns_domain", flow.get("query", "")) + elif flow_type == "http": + uri = flow.get("uri", "") + add_string(populations, "uri", uri) + add_string(populations, "dns_domain", flow.get("host", "")) + filename = filename_from_uri(uri) + if filename: + add_string(populations, "filename", filename) + elif flow_type == "ssl": + add_string( + populations, + "tls_sni", + flow.get("server_name", flow.get("subject", "")), + ) + + +def merge_populations( + base: dict[str, set[str]], extra: dict[str, set[str]] +) -> dict[str, set[str]]: + for regex_type, values in extra.items(): + base.setdefault(regex_type, set()).update(values) + return base + + +def filename_from_uri(uri: str) -> str: + normalized = normalize_uri(uri) + if not normalized: + return "" + path = normalized.split("?", 1)[0] + filename = normalize_filename(path) + if "." not in filename: + return "" + return filename + + +def load_ti_populations( + run_redis_port: int, + ti_cache_port: int, + ti_cache_db: int, +) -> tuple[dict[str, set[str]], TIStats]: + populations = {regex_type: set() for regex_type in REGEX_TYPES} + config = ConfigParser() + loaded_feeds = 0 + cache_domain_count = 0 + cache_ip_count = 0 + cache_ja3_count = 0 + cache_jarm_count = 0 + run_redis_available = False + ti_cache_available = False + have_cached_domains = False + + if redis is not None: + try: + run_client = redis.Redis( + host="127.0.0.1", + port=run_redis_port, + decode_responses=True, + socket_connect_timeout=1, + socket_timeout=1, + ) + loaded = run_client.get("loaded_TI_files_number") + loaded_feeds = int(loaded or 0) + run_redis_available = True + except Exception: + run_redis_available = False + + try: + cache_client = redis.Redis( + host="127.0.0.1", + port=ti_cache_port, + db=ti_cache_db, + decode_responses=True, + socket_connect_timeout=1, + socket_timeout=1, + ) + redis_domains = cache_client.hkeys("IoC_domains") + cache_domain_count = len(redis_domains) + cache_ip_count = cache_client.hlen("IoC_ips") + cache_ja3_count = cache_client.hlen("IoC_JA3") + cache_jarm_count = cache_client.hlen("IoC_JARM") + ti_cache_available = True + for domain in redis_domains: + domain = normalize_domain(domain) + if not domain: + continue + for regex_type in DOMAIN_LIKE_TYPES: + add_string(populations, regex_type, domain) + add_string(populations, "dns_domain", domain) + have_cached_domains = bool(redis_domains) + except Exception: + ti_cache_available = False + + scanned_files = 0 + if not have_cached_domains: + for file_path in ti_source_files(config): + scanned_files += 1 + populate_ti_strings_from_file(file_path, populations) + else: + for file_path in ti_source_files(config): + scanned_files += 1 + populate_ti_strings_from_file( + file_path, + populations, + add_domains=False, + ) + + return populations, TIStats( + run_redis_port=run_redis_port, + run_redis_available=run_redis_available, + ti_cache_port=ti_cache_port, + ti_cache_db=ti_cache_db, + ti_cache_available=ti_cache_available, + loaded_feeds=loaded_feeds, + cache_domain_count=cache_domain_count, + cache_ip_count=cache_ip_count, + cache_ja3_count=cache_ja3_count, + cache_jarm_count=cache_jarm_count, + source_files_scanned=scanned_files, + ) + + +def ti_source_files(config: ConfigParser) -> Iterable[Path]: + candidates = [ + Path(config.local_ti_data_path()), + Path(config.remote_ti_data_path()), + ] + seen = set() + for base in candidates: + if not base.is_absolute(): + base = Path.cwd() / base + if not base.exists(): + continue + for path in sorted(base.rglob("*")): + if not path.is_file(): + continue + if path.name.startswith("."): + continue + if path.suffix.lower() in {".pyc", ".png", ".jpg", ".jpeg"}: + continue + resolved = str(path.resolve()) + if resolved in seen: + continue + seen.add(resolved) + yield path + + +def populate_ti_strings_from_file( + path: Path, + populations: dict[str, set[str]], + add_domains: bool = True, +) -> None: + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return + + for token in tokenize_ti_text(text): + add_ti_token(token, populations, add_domains=add_domains) + + +def tokenize_ti_text(text: str) -> Iterable[str]: + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#") or line.startswith(";"): + continue + + for token in re.split(r"[\s,\t;\"']+", line): + token = token.strip() + if token: + yield token + + +def add_ti_token( + token: str, + populations: dict[str, set[str]], + add_domains: bool = True, +) -> None: + token = token.strip().strip(",") + if not token: + return + + ioc_type = utils.detect_ioc_type(token) + if ioc_type == "domain" and add_domains: + domain = normalize_domain(token) + for regex_type in DOMAIN_LIKE_TYPES: + add_string(populations, regex_type, domain) + add_string(populations, "dns_domain", domain) + return + + if ioc_type != "url": + return + + parsed = urlparse(token) + domain = normalize_domain(parsed.hostname or "") + if domain and add_domains: + for regex_type in DOMAIN_LIKE_TYPES: + add_string(populations, regex_type, domain) + add_string(populations, "dns_domain", domain) + + uri = normalize_uri(token) + if uri: + add_string(populations, "uri", uri) + filename = filename_from_uri(uri) + if filename: + add_string(populations, "filename", filename) + + +def compile_regexes( + regexes_by_type: dict[str, list[dict]], +) -> dict[str, list[dict]]: + compiled_by_type = defaultdict(list) + for regex_type, regex_rows in regexes_by_type.items(): + for row in regex_rows: + try: + compiled = re.compile(row["regex"]) + except re.error: + continue + enriched = dict(row) + enriched["compiled"] = compiled + compiled_by_type[regex_type].append(enriched) + return compiled_by_type + + +def sample_population( + values: list[str], + max_population_size: int, + sampling_seed: int, + sampling_ratio: float, +) -> tuple[list[str], int]: + original_total = len(values) + if original_total == 0: + return values, original_total + + target_size = original_total + if 0 < sampling_ratio < 1: + target_size = max(1, int(original_total * sampling_ratio)) + + if max_population_size > 0: + target_size = min(target_size, max_population_size) + + if target_size >= original_total: + return values, original_total + + sampler = random.Random(f"{sampling_seed}:{original_total}") + sampled = sampler.sample(values, target_size) + sampled.sort() + return sampled, original_total + + +def mean_score(scores: list[float]) -> float | None: + if not scores: + return None + return sum(scores) / len(scores) + + +def stddev_score(scores: list[float]) -> float | None: + if not scores: + return None + avg = mean_score(scores) + if avg is None: + return None + variance = sum((score - avg) ** 2 for score in scores) / len(scores) + return math.sqrt(variance) + + +def build_score_stats( + scores_all: list[float], + matched_scores: list[float], + total_values: int, +) -> dict: + match_count = len(matched_scores) + return { + "total_evaluated": total_values, + "match_count": match_count, + "match_ratio": (match_count / total_values) if total_values else None, + "avg_all": mean_score(scores_all), + "std_all": stddev_score(scores_all), + "avg_match": mean_score(matched_scores), + "std_match": stddev_score(matched_scores), + "max": max(scores_all) if scores_all else None, + } + + +def compute_coverage( + compiled_regexes: dict[str, list[dict]], + benign_populations: dict[str, set[str]], + malicious_populations: dict[str, set[str]], + observed_populations: dict[str, set[str]], + match_timeout_seconds: float, + max_population_size: int, + sampling_seed: int, + sampling_ratio: float, + progress: ProgressTracker | None = None, +): + summary = {} + + for regex_type in REGEX_TYPES: + benign_values_all = sorted(benign_populations.get(regex_type, set())) + malicious_values_all = sorted( + malicious_populations.get(regex_type, set()) + ) + observed_values_all = sorted( + observed_populations.get(regex_type, set()) + ) + reference_union_all = sorted( + set(malicious_values_all).union(observed_values_all) + ) + + benign_values, benign_original_total = sample_population( + benign_values_all, + max_population_size, + sampling_seed, + sampling_ratio, + ) + malicious_values, malicious_original_total = sample_population( + malicious_values_all, + max_population_size, + sampling_seed, + sampling_ratio, + ) + observed_values, observed_original_total = sample_population( + observed_values_all, + max_population_size, + sampling_seed, + sampling_ratio, + ) + reference_union, reference_original_total = sample_population( + reference_union_all, + max_population_size, + sampling_seed, + sampling_ratio, + ) + + population_map = { + "benign": benign_values, + "malicious": malicious_values, + "observed": observed_values, + "reference_union": reference_union, + } + original_totals = { + "benign": benign_original_total, + "malicious": malicious_original_total, + "observed": observed_original_total, + "reference_union": reference_original_total, + } + regex_rows = compiled_regexes.get(regex_type, []) + + overall_matches = {name: set() for name in population_map} + population_timeout_counts = {name: 0 for name in population_map} + regex_details = [] + for row in regex_rows: + detail = { + "regex": row["regex"], + "request_id": row["request_id"], + "matches": {}, + "score_stats": {}, + "timed_out_populations": [], + "unique_reference_matches": 0, + "score": 0, + "quality_score": 0.0, + "strength_gap": 0.0, + } + compiled = row["compiled"] + regex_features = measure_regex_specificity(row["regex"]) + comparisons_for_regex = sum( + len(values) for values in population_map.values() + ) + for population_name, values in population_map.items(): + try: + with timeout_context(match_timeout_seconds): + matched = [] + scores_all = [] + matched_scores = [] + for value in values: + score = compute_match_strength( + compiled, + value, + regex_features, + ) + scores_all.append(score) + if score > 0: + matched.append(value) + matched_scores.append(score) + except TimeoutError: + matched = [] + scores_all = [] + matched_scores = [] + detail["timed_out_populations"].append(population_name) + population_timeout_counts[population_name] += 1 + detail["matches"][population_name] = matched + detail["score_stats"][population_name] = build_score_stats( + scores_all, + matched_scores, + len(values), + ) + overall_matches[population_name].update(matched) + + detail["unique_reference_matches"] = len( + set(detail["matches"]["reference_union"]) + ) + detail["score"] = len(detail["matches"]["reference_union"]) - len( + detail["matches"]["benign"] + ) + malicious_avg = ( + detail["score_stats"]["malicious"]["avg_all"] or 0.0 + ) + benign_avg = detail["score_stats"]["benign"]["avg_all"] or 0.0 + detail["strength_gap"] = malicious_avg - benign_avg + detail["quality_score"] = detail["strength_gap"] + regex_details.append(detail) + if progress is not None: + progress.advance( + regex_type, + row["regex"], + comparisons_for_regex, + ) + + regex_details.sort( + key=lambda item: ( + item["quality_score"], + item["score_stats"]["malicious"]["avg_all"] or 0.0, + item["unique_reference_matches"], + -len(item["matches"]["benign"]), + ), + reverse=True, + ) + + population_stats = {} + for population_name, values in population_map.items(): + total = len(values) + matched_values = sorted(overall_matches[population_name]) + unmatched_values = [ + value + for value in values + if value not in overall_matches[population_name] + ] + original_total = original_totals[population_name] + population_stats[population_name] = { + "total": total, + "original_total": original_total, + "sampled": total != original_total, + "matched": len(matched_values), + "coverage_ratio": ( + (len(matched_values) / total) if total else None + ), + "timeout_count": population_timeout_counts[population_name], + "matched_values": matched_values, + "unmatched_values": unmatched_values, + } + + summary[regex_type] = { + "regex_count": len(regex_rows), + "populations": population_stats, + "regex_details": regex_details, + } + + return summary + + +def build_report_payload( + run_output_dir: Path, + regex_db_path: Path, + benign_db_path: Path, + ti_stats: TIStats, + coverage_summary: dict, +): + totals = { + "accepted_regexes": sum( + details["regex_count"] for details in coverage_summary.values() + ), + "types_with_regexes": sum( + 1 + for details in coverage_summary.values() + if details["regex_count"] + ), + } + generated_at = datetime.now(timezone.utc).isoformat() + return { + "generated_at": generated_at, + "run_output_dir": str(run_output_dir), + "regex_db_path": str(regex_db_path), + "benign_db_path": str(benign_db_path), + "ti": { + "run_redis_port": ti_stats.run_redis_port, + "run_redis_available": ti_stats.run_redis_available, + "ti_cache_port": ti_stats.ti_cache_port, + "ti_cache_db": ti_stats.ti_cache_db, + "ti_cache_available": ti_stats.ti_cache_available, + "loaded_feeds": ti_stats.loaded_feeds, + "cache_domain_count": ti_stats.cache_domain_count, + "cache_ip_count": ti_stats.cache_ip_count, + "cache_ja3_count": ti_stats.cache_ja3_count, + "cache_jarm_count": ti_stats.cache_jarm_count, + "source_files_scanned": ti_stats.source_files_scanned, + }, + "totals": totals, + "types": coverage_summary, + } + + +def ratio_text(value: float | None) -> str: + if value is None: + return "n/a" + percentage = value * 100 + if percentage == 0: + return "0.0%" + + formatted = f"{percentage:.6f}".rstrip("0").rstrip(".") + if "." not in formatted: + formatted = f"{formatted}.0" + return f"{formatted}%" + + +def score_text(value: float | None) -> str: + if value is None: + return "n/a" + return f"{value:.2f}" + + +def avg_std_text(stats: dict) -> str: + avg = stats.get("avg_all") + std = stats.get("std_all") + if avg is None: + return "n/a" + if std is None: + return f"{avg:.2f}" + return f"{avg:.2f} ± {std:.2f}" + + +def matched_avg_std_text(stats: dict) -> str: + avg = stats.get("avg_match") + std = stats.get("std_match") + if avg is None: + return "n/a" + if std is None: + return f"{avg:.2f}" + return f"{avg:.2f} ± {std:.2f}" + + +def render_scatter_plot(regex_type: str, regex_rows: list[dict]) -> str: + points = [] + width = 520 + height = 360 + padding = 44 + inner_w = width - padding * 2 + inner_h = height - padding * 2 + usable_rows = 0 + for row in regex_rows: + benign_avg = row["score_stats"]["benign"]["avg_all"] + malicious_avg = row["score_stats"]["malicious"]["avg_all"] + if benign_avg is None and malicious_avg is None: + continue + usable_rows += 1 + x = padding + (benign_avg or 0.0) / 100.0 * inner_w + y = height - padding - (malicious_avg or 0.0) / 100.0 * inner_h + quality = row.get("quality_score", 0.0) + color = "#1e7a46" if quality >= 0 else "#a73f24" + radius = 3 if row["score_stats"]["malicious"]["match_count"] < 5 else 4 + title = ( + f"{row['regex']}\n" + f"malicious avg_all={score_text(malicious_avg)} std_all={score_text(row['score_stats']['malicious']['std_all'])} " + f"avg_match={score_text(row['score_stats']['malicious']['avg_match'])} matches={row['score_stats']['malicious']['match_count']}\n" + f"benign avg_all={score_text(benign_avg)} std_all={score_text(row['score_stats']['benign']['std_all'])} " + f"avg_match={score_text(row['score_stats']['benign']['avg_match'])} matches={row['score_stats']['benign']['match_count']}\n" + f"gap={score_text(row.get('strength_gap'))}" + ) + points.append( + f'' + f"{escape(title)}" + ) + + if usable_rows == 0: + return '

    No benign/malicious score data available for this type.

    ' + + return f""" +
    +

    Strength Scatter

    +

    Each point is one accepted regex. X uses the benign average score across all tested benign strings, with non-matches counted as 0. Y uses the malicious average score across all tested malicious strings, with non-matches counted as 0. The ideal area is upper-left.

    + + + + + + + Benign average score + Malicious average score + 0 + 100 + 0 + 100 + {''.join(points)} + +
    + """ + + +def render_html(report: dict, sample_limit: int, top_regexes: int) -> str: + rows = [] + for regex_type in REGEX_TYPES: + details = report["types"][regex_type] + populations = details["populations"] + rows.append( + f""" + + {escape(TYPE_LABELS[regex_type])} + {details['regex_count']} + {pop_text(populations['reference_union'])} + {pop_text(populations['malicious'])} + {pop_text(populations['observed'])} + {pop_text(populations['benign'])} + + """ + ) + + sections = [] + for regex_type in REGEX_TYPES: + details = report["types"][regex_type] + populations = details["populations"] + regex_rows = details["regex_details"][:top_regexes] + all_regex_rows = details["regex_details"] + + population_blocks = [] + for population_name in ( + "reference_union", + "malicious", + "observed", + "benign", + ): + stats = populations[population_name] + label = { + "reference_union": "Reference Union", + "malicious": "Malicious TI", + "observed": "Observed Traffic", + "benign": "Benign Corpus", + }[population_name] + population_blocks.append( + f""" +
    +

    {escape(label)}

    +

    {stats['matched']} matched out of {stats['total']} values

    +

    Coverage: {ratio_text(stats['coverage_ratio'])}

    +

    Sampled population: {str(stats['sampled']).lower()}{f", original total {stats['original_total']}" if stats['sampled'] else ""}

    +

    Timed-out regex checks: {stats['timeout_count']}

    +

    Matched samples: {sample_list(stats['matched_values'], sample_limit)}

    +

    Unmatched samples: {sample_list(stats['unmatched_values'], sample_limit)}

    +
    + """ + ) + + regex_table_rows = [] + for row in regex_rows: + regex_table_rows.append( + f""" + + {escape(row['regex'])} + {row['score_stats']['malicious']['match_count']} + {avg_std_text(row['score_stats']['malicious'])} + {matched_avg_std_text(row['score_stats']['malicious'])} + {row['score_stats']['benign']['match_count']} + {avg_std_text(row['score_stats']['benign'])} + {matched_avg_std_text(row['score_stats']['benign'])} + {score_text(row.get('strength_gap'))} + {len(row['matches']['reference_union'])} + {len(row['timed_out_populations'])} + + """ + ) + + sections.append( + f""" +
    +

    {escape(TYPE_LABELS[regex_type])}

    +
    + {''.join(population_blocks)} +
    + {render_scatter_plot(regex_type, all_regex_rows)} +

    Top Regexes By Malicious-vs-Benign Strength

    + + + + + + + + + + + + + + + + + {''.join(regex_table_rows) or ''} + +
    RegexMalicious MatchesMalicious All Avg ± StdMalicious Matched Avg ± StdBenign MatchesBenign All Avg ± StdBenign Matched Avg ± StdStrength GapReference UnionTimeouts
    No accepted regexes.
    +
    + """ + ) + + ti = report["ti"] + glossary = """ +
    +

    How To Read This

    +
    +
    +

    Accepted Regexes

    +

    + The number of regexes currently stored as accepted for that type. +

    +
    +
    +

    Reference Union

    +

    + The union of Malicious TI and Observed Traffic + for that type. It answers: how much of the combined malicious and seen-in-this-run + population is covered by the regex set. +

    +
    +
    +

    Malicious TI

    +

    + Strings derived from Slips threat-intelligence data. For domain-like types this mainly + comes from the TI cache. For URI and filename it may also come from parsed TI files. +

    +
    +
    +

    Observed

    +

    + Strings extracted from the selected run itself, using Zeek logs or flows.sqlite. + This is not necessarily malicious. It is the local seen population for that run. +

    +
    +
    +

    Benign Spillover

    +

    + Matches against the benign corpus. For domain-like types this benign side may also + include the Tranco top 1000 domains from the Slips cache. Lower is better. High benign + spillover means the regex set is too broad for that type. +

    +
    +
    +

    Coverage Numbers

    +

    + Values are shown as matched / total (percent). If a population was sampled, + the report says so explicitly and the percentage is over the sampled population, not the + full original population. +

    +
    +
    +

    Progress Bar Numbers

    +

    + In terminal output, regex means how many accepted regexes have been processed. + cmp means planned regex-versus-string match operations across the selected + populations. It is not the number of TI entries. The count grows because many regexes are + tested against many strings, often across multiple regex types. +

    +
    +
    +

    Timeouts

    +

    + Some regexes are expensive to evaluate. A timeout means the report skipped that regex for + that population instead of hanging forever. +

    +
    +
    +

    Top Regexes Score

    +

    + The top-regex ranking now uses strength_gap = malicious_avg_all - benign_avg_all. + Both averages are computed over all tested strings in that population, with non-matches counted as 0. + Higher is better because it means broader and/or stronger malicious matches with weaker benign matches. +

    +
    +
    +

    Match Strength

    +

    + Each regex/string match gets a score from 0 to 100 using the same + formula as the live RegexGenerator benign filter. The score rewards wider coverage, anchoring, + and specificity, and penalizes broad wildcard-heavy patterns. In the report, non-matches are + treated as score 0 when computing whole-population averages and standard deviations. +

    +
    +
    +

    Strength Scatter

    +

    + Each point is one regex. X is the average benign match score. Y is the average malicious match score. + The ideal region is upper-left: high malicious strength and low benign strength. +

    +
    +
    +
    + """ + return f""" + + + + Regex Coverage Report + + + +
    +
    +

    Regex Coverage Report

    +

    + Offline estimate of accepted RegexGenerator coverage against three reference populations: + benign corpus, TI-derived malicious strings, and observed traffic from the selected Slips run. + For domain-like types, the benign side also includes the Tranco top 1000 domains when available + in the Slips cache. +

    +
    +
    +

    Run

    +

    {escape(report['run_output_dir'])}

    +

    Generated at {escape(report['generated_at'])}

    +
    +
    +

    Regexes

    +

    {report['totals']['accepted_regexes']} accepted regexes

    +

    {report['totals']['types_with_regexes']} types currently populated

    +
    +
    +

    Threat Intelligence

    +

    Run Redis {ti['run_redis_port']}, TI cache {ti['ti_cache_port']}/{ti['ti_cache_db']}

    +

    Run Redis: {str(ti['run_redis_available']).lower()}, TI cache: {str(ti['ti_cache_available']).lower()}

    +

    Loaded feeds: {ti['loaded_feeds']}, cached domains: {ti['cache_domain_count']}, cached IPs: {ti['cache_ip_count']}, JA3: {ti['cache_ja3_count']}, JARM: {ti['cache_jarm_count']}

    +

    Supplemental TI files scanned for URL and filename extraction: {ti['source_files_scanned']}

    +
    +
    +

    Databases

    +

    Regex DB: {escape(report['regex_db_path'])}

    +

    Benign DB: {escape(report['benign_db_path'])}

    +
    +
    +
    + +

    Coverage by Type

    + + + + + + + + + + + + + {''.join(rows)} + +
    TypeAccepted RegexesReference UnionMalicious TIObservedBenign Spillover
    + + {glossary} + + {''.join(sections)} +
    + + +""" + + +def pop_text(stats: dict) -> str: + summary = ( + f"{stats['matched']}/{stats['total']} " + f"({ratio_text(stats['coverage_ratio'])})" + ) + if stats.get("sampled"): + return f"{summary}, sample of {stats['original_total']}" + return summary + + +def sample_list(values: list[str], limit: int) -> str: + if not values: + return "none" + values = values[:limit] + return ", ".join(f"{escape(value)}" for value in values) + + +def ensure_paths( + args: argparse.Namespace, +) -> tuple[Path, Path, Path, Path, Path]: + input_path = Path(args.run_output_dir).expanduser().resolve() + + store_dir_candidate = input_path + direct_regex_db_path = store_dir_candidate / "generated_regexes.sqlite" + direct_benign_db_path = store_dir_candidate / "benign_corpus.sqlite" + nested_regex_db_path = ( + input_path / "regex_generator" / "generated_regexes.sqlite" + ) + nested_benign_db_path = ( + input_path / "regex_generator" / "benign_corpus.sqlite" + ) + + if direct_regex_db_path.exists() and direct_benign_db_path.exists(): + run_output_dir = input_path + regex_db_path = direct_regex_db_path + benign_db_path = direct_benign_db_path + elif nested_regex_db_path.exists() and nested_benign_db_path.exists(): + run_output_dir = input_path + regex_db_path = nested_regex_db_path + benign_db_path = nested_benign_db_path + else: + raise FileNotFoundError( + "Could not find regex SQLite files. Expected either:\n" + f"- {direct_regex_db_path} and {direct_benign_db_path}\n" + f"- {nested_regex_db_path} and {nested_benign_db_path}" + ) + + output_html = ( + Path(args.output_html).expanduser().resolve() + if args.output_html + else run_output_dir / "regex_generator_coverage_report.html" + ) + output_json = ( + Path(args.output_json).expanduser().resolve() + if args.output_json + else run_output_dir / "regex_generator_coverage_report.json" + ) + output_html.parent.mkdir(parents=True, exist_ok=True) + output_json.parent.mkdir(parents=True, exist_ok=True) + return ( + run_output_dir, + regex_db_path, + benign_db_path, + output_html, + output_json, + ) + + +def main(): + args = parse_args() + if args.sampling_ratio <= 0 or args.sampling_ratio > 1: + raise ValueError( + "--sampling-ratio must be greater than 0 and less than or equal to 1" + ) + if args.full_scan: + args.max_population_size = 0 + args.sampling_ratio = 1.0 + config = ConfigParser() + tranco_top_benign_limit = config.tranco_top_benign_limit() + + ( + run_output_dir, + regex_db_path, + benign_db_path, + output_html, + output_json, + ) = ensure_paths(args) + + regexes_by_type = load_regexes(regex_db_path) + benign_populations = load_benign_corpus(benign_db_path) + benign_populations = merge_populations( + benign_populations, + load_tranco_benign_populations( + args.ti_cache_port, + args.ti_cache_db, + tranco_top_benign_limit, + ), + ) + observed_populations = load_observed_populations(run_output_dir) + malicious_populations, ti_stats = load_ti_populations( + args.redis_port, + args.ti_cache_port, + args.ti_cache_db, + ) + compiled_regexes = compile_regexes(regexes_by_type) + total_regexes = sum(len(rows) for rows in compiled_regexes.values()) + sampled_benign = { + regex_type: sample_population( + sorted(benign_populations.get(regex_type, set())), + args.max_population_size, + args.sampling_seed, + args.sampling_ratio, + )[0] + for regex_type in REGEX_TYPES + } + sampled_malicious = { + regex_type: sample_population( + sorted(malicious_populations.get(regex_type, set())), + args.max_population_size, + args.sampling_seed, + args.sampling_ratio, + )[0] + for regex_type in REGEX_TYPES + } + sampled_observed = { + regex_type: sample_population( + sorted(observed_populations.get(regex_type, set())), + args.max_population_size, + args.sampling_seed, + args.sampling_ratio, + )[0] + for regex_type in REGEX_TYPES + } + total_comparisons = 0 + for regex_type in REGEX_TYPES: + reference_union = set(sampled_malicious[regex_type]).union( + sampled_observed[regex_type] + ) + comparisons_per_regex = ( + len(sampled_benign[regex_type]) + + len(sampled_malicious[regex_type]) + + len(sampled_observed[regex_type]) + + len(reference_union) + ) + total_comparisons += comparisons_per_regex * len( + compiled_regexes.get(regex_type, []) + ) + + mode = "full scan" if args.full_scan else "sampled estimate" + progress = ProgressTracker(total_regexes, total_comparisons, mode) + print( + f"Starting coverage report in {mode} mode. " + f"match_timeout_seconds={args.match_timeout_seconds}, " + f"max_population_size={args.max_population_size}, " + f"sampling_ratio={args.sampling_ratio}", + flush=True, + ) + progress.print_plan() + coverage_summary = compute_coverage( + compiled_regexes, + benign_populations, + malicious_populations, + observed_populations, + args.match_timeout_seconds, + args.max_population_size, + args.sampling_seed, + args.sampling_ratio, + progress, + ) + progress.finish() + report = build_report_payload( + run_output_dir, + regex_db_path, + benign_db_path, + ti_stats, + coverage_summary, + ) + + output_html.write_text( + render_html(report, args.sample_limit, args.top_regexes), + encoding="utf-8", + ) + output_json.write_text(json.dumps(report, indent=2), encoding="utf-8") + + print(f"HTML report written to {output_html}") + print(f"JSON summary written to {output_json}") + + +if __name__ == "__main__": + main() diff --git a/scripts/regex_prune_benign_threshold.py b/scripts/regex_prune_benign_threshold.py new file mode 100755 index 0000000000..fe14cec516 --- /dev/null +++ b/scripts/regex_prune_benign_threshold.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +""" +Audit and optionally prune accepted regexes that exceed the benign threshold. + +This is meant for persistent regex stores where the benign corpus may have +grown over time. A regex accepted earlier can later become too strong against +the current benign corpus even though it passed at generation time. +""" + +from __future__ import annotations + +import argparse +import json +import re +import signal +import shutil +import sqlite3 +import sys +import time +import warnings +from collections import defaultdict +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.core.database.sqlite_db.regex_generator_db import REGEX_TYPES +from modules.regex_generator.match_strength import ( + compute_match_strength, + measure_regex_specificity, +) + + +@dataclass +class RegexAuditResult: + id: int + regex_type: str + regex: str + regex_hash: str + created_at: float + strongest_benign_score: float + strongest_benign_value: str + + +class _NullTimeout: + def __enter__(self): + return None + + def __exit__(self, exc_type, exc, exc_tb): + return False + + +class _SignalTimeout: + def __init__(self, timeout_seconds: float): + self.timeout_seconds = timeout_seconds + self._previous_handler = None + + def __enter__(self): + self._previous_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, self._handle_timeout) + signal.setitimer(signal.ITIMER_REAL, self.timeout_seconds) + return None + + def __exit__(self, exc_type, exc, exc_tb): + signal.setitimer(signal.ITIMER_REAL, 0) + if self._previous_handler is not None: + signal.signal(signal.SIGALRM, self._previous_handler) + return False + + @staticmethod + def _handle_timeout(signum, frame): + raise TimeoutError("regex benign scan timed out") + + +def timeout_context(timeout_seconds: float): + if timeout_seconds <= 0: + return _NullTimeout() + return _SignalTimeout(timeout_seconds) + + +class AuditProgressTracker: + BAR_WIDTH = 24 + + def __init__(self, total_regexes: int, totals_by_type: dict[str, int]): + self.total_regexes = max(1, total_regexes) + self.totals_by_type = dict(totals_by_type) + self.done_regexes = 0 + self.done_by_type = {regex_type: 0 for regex_type in totals_by_type} + self.current_type = "-" + self.comparisons_done = 0 + self.flagged_done = 0 + self.timed_out_done = 0 + self.started_at = time.monotonic() + self.last_render_at = 0.0 + self.enabled = sys.stderr.isatty() + + def start(self): + if not self.enabled: + return + print( + ( + "Auditing accepted regexes against the current benign corpus " + f"({self.total_regexes} regexes)" + ), + file=sys.stderr, + flush=True, + ) + self._render(force=True) + + def advance( + self, + regex_type: str, + comparisons: int, + flagged_increment: int = 0, + timed_out_increment: int = 0, + ): + self.done_regexes += 1 + self.current_type = regex_type + self.comparisons_done += comparisons + self.flagged_done += flagged_increment + self.timed_out_done += timed_out_increment + self.done_by_type[regex_type] = self.done_by_type.get(regex_type, 0) + 1 + self._render() + + def finish(self): + if not self.enabled: + return + self._render(force=True, done=True) + print(file=sys.stderr, flush=True) + + def _render(self, force: bool = False, done: bool = False): + if not self.enabled: + return + + now = time.monotonic() + if not force and not done and now - self.last_render_at < 0.1: + return + self.last_render_at = now + + ratio = min(1.0, self.done_regexes / self.total_regexes) + filled = int(ratio * self.BAR_WIDTH) + bar = "[" + ("=" * filled) + ("." * (self.BAR_WIDTH - filled)) + "]" + elapsed = max(0.001, now - self.started_at) + if done or ratio >= 1.0: + eta = 0.0 + else: + eta = (elapsed / max(ratio, 1e-9)) - elapsed + + type_done = self.done_by_type.get(self.current_type, 0) + type_total = self.totals_by_type.get(self.current_type, 0) + status = ( + "\r" + f"{bar} {ratio * 100:6.2f}% " + f"| regex {self.done_regexes}/{self.total_regexes} " + f"| type {self.current_type} {type_done}/{type_total} " + f"| flagged {self.flagged_done} " + f"| timed out {self.timed_out_done} " + f"| cmp {self.comparisons_done:,} " + f"| ETA {self._format_duration(eta)}" + ) + print(status, end="", file=sys.stderr, flush=True) + + @staticmethod + def _format_duration(seconds: float) -> str: + total_seconds = max(0, int(seconds)) + hours, remainder = divmod(total_seconds, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Audit accepted regexes against the current benign corpus and " + "optionally delete those whose strongest benign match meets or " + "exceeds the configured threshold." + ) + ) + parser.add_argument( + "--run-output-dir", + default="", + help=( + "Slips run output directory containing regex_generator/*.sqlite, " + "or a direct regex store directory containing generated_regexes.sqlite " + "and benign_corpus.sqlite." + ), + ) + parser.add_argument( + "--regex-db", + default="", + help="Path to generated_regexes.sqlite. Overrides --run-output-dir.", + ) + parser.add_argument( + "--benign-db", + default="", + help="Path to benign_corpus.sqlite. Overrides --run-output-dir.", + ) + parser.add_argument( + "--threshold", + type=float, + default=None, + help=( + "Benign match-strength threshold. Defaults to " + "regex_generator.benign_match_strength_threshold from config, " + "or 75 if unavailable." + ), + ) + parser.add_argument( + "--regex-type", + action="append", + choices=sorted(REGEX_TYPES), + help="Limit the audit to one or more regex types.", + ) + parser.add_argument( + "--match-timeout-seconds", + type=float, + default=None, + help=( + "Maximum wall-clock seconds allowed for one accepted regex to scan " + "the benign corpus for its regex type. Timed-out regexes are " + "skipped and never deleted. Defaults to " + "regex_generator.regex_validation_timeout_seconds from config, " + "or 2.0 if unavailable. Set 0 to disable." + ), + ) + parser.add_argument( + "--limit", + type=int, + default=20, + help="Maximum number of example rows to print per regex type.", + ) + parser.add_argument( + "--output-json", + default="", + help="Optional JSON output path for the audit summary.", + ) + parser.add_argument( + "--delete", + action="store_true", + help="Delete accepted regex rows that exceed the threshold.", + ) + parser.add_argument( + "--no-backup", + action="store_true", + help="Do not create a backup copy of generated_regexes.sqlite before deletion.", + ) + parser.add_argument( + "--vacuum", + action="store_true", + help="Run VACUUM on generated_regexes.sqlite after deletion.", + ) + return parser.parse_args() + + +def default_threshold() -> float: + try: + return float( + ConfigParser().regex_generator_benign_match_strength_threshold() + ) + except Exception: + return 75.0 + + +def default_match_timeout() -> float: + try: + return float(ConfigParser().regex_generator_regex_validation_timeout_seconds()) + except Exception: + return 2.0 + + +def resolve_paths(args: argparse.Namespace) -> tuple[Path, Path]: + if args.regex_db and args.benign_db: + return Path(args.regex_db).expanduser(), Path(args.benign_db).expanduser() + + if not args.run_output_dir: + raise SystemExit( + "Provide either --regex-db and --benign-db, or --run-output-dir." + ) + + base = Path(args.run_output_dir).expanduser() + direct_regex = base / "generated_regexes.sqlite" + direct_benign = base / "benign_corpus.sqlite" + nested_regex = base / "regex_generator" / "generated_regexes.sqlite" + nested_benign = base / "regex_generator" / "benign_corpus.sqlite" + + if direct_regex.exists() and direct_benign.exists(): + return direct_regex, direct_benign + if nested_regex.exists() and nested_benign.exists(): + return nested_regex, nested_benign + + raise SystemExit( + "Could not find regex DBs. Checked:\n" + f"- {direct_regex} and {direct_benign}\n" + f"- {nested_regex} and {nested_benign}" + ) + + +def load_benign_values(benign_db_path: Path) -> dict[str, list[str]]: + benign_values = {regex_type: [] for regex_type in REGEX_TYPES} + with sqlite3.connect(benign_db_path) as conn: + rows = conn.execute( + "SELECT regex_type, value FROM benign_strings ORDER BY id ASC" + ) + for regex_type, value in rows: + benign_values.setdefault(regex_type, []).append(str(value or "")) + return benign_values + + +def load_accepted_regexes( + regex_db_path: Path, regex_types: set[str] +) -> dict[str, list[dict]]: + accepted = defaultdict(list) + with sqlite3.connect(regex_db_path) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute( + """ + SELECT id, regex_type, regex, regex_hash, created_at + FROM generated_regexes + WHERE status = 'accepted' + ORDER BY created_at ASC, id ASC + """ + ).fetchall() + for row in rows: + regex_type = row["regex_type"] + if regex_type not in regex_types: + continue + accepted[regex_type].append(dict(row)) + return accepted + + +def audit_regex_type( + regex_rows: list[dict], + benign_values: list[str], + threshold: float, + match_timeout_seconds: float, + progress: AuditProgressTracker | None = None, +) -> tuple[list[RegexAuditResult], list[dict]]: + flagged = [] + timed_out = [] + for row in regex_rows: + comparisons_checked = 0 + flagged_increment = 0 + timed_out_increment = 0 + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + compiled = re.compile(row["regex"]) + except re.error: + if progress is not None: + progress.advance( + row["regex_type"], + comparisons=comparisons_checked, + flagged_increment=flagged_increment, + timed_out_increment=timed_out_increment, + ) + continue + + regex_features = measure_regex_specificity(row["regex"]) + best_score = 0.0 + best_value = "" + try: + with timeout_context(match_timeout_seconds): + for value in benign_values: + comparisons_checked += 1 + score = compute_match_strength(compiled, value, regex_features) + if score > best_score: + best_score = score + best_value = value + if best_score >= threshold: + flagged_increment = 1 + flagged.append( + RegexAuditResult( + id=int(row["id"]), + regex_type=row["regex_type"], + regex=row["regex"], + regex_hash=row["regex_hash"], + created_at=float(row["created_at"]), + strongest_benign_score=best_score, + strongest_benign_value=best_value, + ) + ) + break + except TimeoutError: + timed_out_increment = 1 + timed_out.append( + { + "id": int(row["id"]), + "regex_type": row["regex_type"], + "regex": row["regex"], + "regex_hash": row["regex_hash"], + "created_at": float(row["created_at"]), + "comparisons_checked": comparisons_checked, + } + ) + if progress is not None: + progress.advance( + row["regex_type"], + comparisons=comparisons_checked, + flagged_increment=flagged_increment, + timed_out_increment=timed_out_increment, + ) + flagged.sort( + key=lambda item: ( + item.regex_type, + item.strongest_benign_score, + item.created_at, + item.id, + ), + reverse=True, + ) + timed_out.sort( + key=lambda item: ( + item["regex_type"], + item["created_at"], + item["id"], + ), + reverse=True, + ) + return flagged, timed_out + + +def backup_regex_db(regex_db_path: Path) -> Path: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + backup_path = regex_db_path.with_suffix(regex_db_path.suffix + f".bak.{timestamp}") + shutil.copy2(regex_db_path, backup_path) + return backup_path + + +def delete_flagged_regexes( + regex_db_path: Path, flagged_results: list[RegexAuditResult], vacuum: bool +) -> int: + ids = [result.id for result in flagged_results] + if not ids: + return 0 + + placeholders = ",".join("?" for _ in ids) + with sqlite3.connect(regex_db_path) as conn: + cursor = conn.execute( + f"DELETE FROM generated_regexes WHERE id IN ({placeholders})", + ids, + ) + deleted = int(cursor.rowcount or 0) + conn.commit() + if vacuum: + conn.execute("VACUUM") + return deleted + + +def build_summary( + regex_db_path: Path, + benign_db_path: Path, + threshold: float, + regex_types: list[str], + accepted_by_type: dict[str, list[dict]], + flagged_by_type: dict[str, list[RegexAuditResult]], + timed_out_by_type: dict[str, list[dict]], + limit: int, + deleted: int, + backup_path: Path | None, + match_timeout_seconds: float, +) -> dict: + summary_types = {} + for regex_type in regex_types: + flagged_rows = flagged_by_type.get(regex_type, []) + timed_out_rows = timed_out_by_type.get(regex_type, []) + summary_types[regex_type] = { + "accepted_count": len(accepted_by_type.get(regex_type, [])), + "flagged_count": len(flagged_rows), + "timed_out_count": len(timed_out_rows), + "examples": [ + { + **asdict(result), + "created_at_iso": datetime.fromtimestamp( + result.created_at, tz=timezone.utc + ).isoformat(), + } + for result in flagged_rows[:limit] + ], + "timed_out_examples": [ + { + **row, + "created_at_iso": datetime.fromtimestamp( + row["created_at"], tz=timezone.utc + ).isoformat(), + } + for row in timed_out_rows[:limit] + ], + } + + return { + "generated_at": datetime.now(timezone.utc).isoformat(), + "regex_db_path": str(regex_db_path), + "benign_db_path": str(benign_db_path), + "threshold": threshold, + "match_timeout_seconds": match_timeout_seconds, + "regex_types": regex_types, + "deleted_count": deleted, + "backup_path": str(backup_path) if backup_path else "", + "totals": { + "accepted_count": sum( + len(accepted_by_type.get(regex_type, [])) + for regex_type in regex_types + ), + "flagged_count": sum( + len(flagged_by_type.get(regex_type, [])) + for regex_type in regex_types + ), + "timed_out_count": sum( + len(timed_out_by_type.get(regex_type, [])) + for regex_type in regex_types + ), + }, + "types": summary_types, + } + + +def print_summary(summary: dict, delete_mode: bool): + action = "Deleted" if delete_mode else "Flagged" + print( + f"Threshold: {summary['threshold']:.2f}\n" + f"Match timeout per regex: {summary['match_timeout_seconds']:.2f}s\n" + f"Regex DB: {summary['regex_db_path']}\n" + f"Benign DB: {summary['benign_db_path']}\n" + f"Accepted rows scanned: {summary['totals']['accepted_count']}\n" + f"{action} rows: {summary['totals']['flagged_count']}\n" + f"Timed-out rows skipped: {summary['totals']['timed_out_count']}" + ) + print( + "Accepted means rows currently stored in generated_regexes.sqlite " + "with status='accepted'." + ) + if delete_mode: + print( + "Deleted means accepted rows whose strongest benign match score " + "met or exceeded the threshold and were removed." + ) + else: + print( + "Flagged means accepted rows whose strongest benign match score " + "meets or exceeds the threshold against the current benign corpus." + ) + if summary.get("backup_path"): + print(f"Backup: {summary['backup_path']}") + + for regex_type in summary["regex_types"]: + row = summary["types"][regex_type] + print( + f"\n[{regex_type}] accepted={row['accepted_count']} " + f"flagged={row['flagged_count']} " + f"timed_out={row['timed_out_count']}" + ) + for example in row["examples"]: + print( + " " + f"score={example['strongest_benign_score']:.2f} " + f"value={example['strongest_benign_value']} " + f"created_at={example['created_at_iso']} " + f"regex={example['regex']}" + ) + for example in row["timed_out_examples"]: + print( + " " + "timed_out " + f"after_cmp={example['comparisons_checked']} " + f"created_at={example['created_at_iso']} " + f"regex={example['regex']}" + ) + + +def main(): + args = parse_args() + regex_db_path, benign_db_path = resolve_paths(args) + threshold = ( + float(args.threshold) if args.threshold is not None else default_threshold() + ) + match_timeout_seconds = ( + float(args.match_timeout_seconds) + if args.match_timeout_seconds is not None + else default_match_timeout() + ) + regex_types = sorted(set(args.regex_type or REGEX_TYPES)) + + benign_values = load_benign_values(benign_db_path) + accepted_by_type = load_accepted_regexes(regex_db_path, set(regex_types)) + progress = AuditProgressTracker( + total_regexes=sum( + len(accepted_by_type.get(regex_type, [])) for regex_type in regex_types + ), + totals_by_type={ + regex_type: len(accepted_by_type.get(regex_type, [])) + for regex_type in regex_types + }, + ) + progress.start() + flagged_by_type = {} + timed_out_by_type = {} + for regex_type in regex_types: + flagged_rows, timed_out_rows = audit_regex_type( + accepted_by_type.get(regex_type, []), + benign_values.get(regex_type, []), + threshold, + match_timeout_seconds, + progress=progress, + ) + flagged_by_type[regex_type] = flagged_rows + timed_out_by_type[regex_type] = timed_out_rows + progress.finish() + + backup_path = None + deleted = 0 + flagged_results = [ + result + for regex_type in regex_types + for result in flagged_by_type.get(regex_type, []) + ] + if args.delete and flagged_results: + if not args.no_backup: + backup_path = backup_regex_db(regex_db_path) + deleted = delete_flagged_regexes(regex_db_path, flagged_results, args.vacuum) + + summary = build_summary( + regex_db_path=regex_db_path, + benign_db_path=benign_db_path, + threshold=threshold, + regex_types=regex_types, + accepted_by_type=accepted_by_type, + flagged_by_type=flagged_by_type, + timed_out_by_type=timed_out_by_type, + limit=max(0, args.limit), + deleted=deleted, + backup_path=backup_path, + match_timeout_seconds=match_timeout_seconds, + ) + print_summary(summary, delete_mode=args.delete) + + if args.output_json: + output_path = Path(args.output_json).expanduser() + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(summary, indent=2), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/slips_files/common/abstracts/icore.py b/slips_files/common/abstracts/icore.py index 0b24861982..95780f9152 100644 --- a/slips_files/common/abstracts/icore.py +++ b/slips_files/common/abstracts/icore.py @@ -15,7 +15,7 @@ class ICore(IModule, Process): Interface for all Core files placed in slips_files/core/ """ - name = "" + name = "icore" description = "Short description of the core class purpose" authors = ["Name of the author creating the class"] diff --git a/slips_files/common/abstracts/imodule.py b/slips_files/common/abstracts/imodule.py index 72371883af..0421e46810 100644 --- a/slips_files/common/abstracts/imodule.py +++ b/slips_files/common/abstracts/imodule.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-only import json import os +import re import sys import traceback import warnings @@ -27,12 +28,47 @@ class IModule(ABC, Process): An interface for all slips modules """ + _SNAKE_CASE_NAME_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$") name = "imodule" description = "Template module" authors = ["Template Author"] # should be filled with the channels each module subscribes to channels = {} + def __init_subclass__(cls, **kwargs: object) -> None: + """ + Validate module metadata when subclasses are defined. + + Args: + **kwargs: Keyword arguments passed to parent class hooks. + + Returns: + None. + """ + super().__init_subclass__(**kwargs) + module_name = getattr(cls, "name", None) + if IModule._has_snake_case_module_name(module_name): + return + + raise RuntimeError( + f"{cls.__name__}.name must be snake_case, got " f"{module_name!r}." + ) + + @staticmethod + def _has_snake_case_module_name(module_name: object) -> bool: + """ + Check whether a module name follows the snake_case contract. + + Args: + module_name: Class-level module name attribute. + + Returns: + True if the module name is a snake_case string, False otherwise. + """ + return isinstance(module_name, str) and bool( + IModule._SNAKE_CASE_NAME_PATTERN.fullmatch(module_name) + ) + def __init__( self, logger: Output, diff --git a/slips_files/common/parsers/config_parser.py b/slips_files/common/parsers/config_parser.py index c06bdfea10..6bac125be2 100644 --- a/slips_files/common/parsers/config_parser.py +++ b/slips_files/common/parsers/config_parser.py @@ -14,6 +14,7 @@ from ipaddress import IPv4Network, IPv6Network, IPv4Address, IPv6Address from slips_files.common.parsers.arg_parser import ArgumentParser +from slips_files.common.input_type import InputType from slips_files.common.slips_utils import utils @@ -282,6 +283,16 @@ def online_whitelist_update_period(self): update_period = 604800 return update_period + def tranco_top_benign_limit(self): + limit = self.read_configuration( + "whitelists", "tranco_top_benign_limit", 1000 + ) + try: + limit = int(limit) + except ValueError: + limit = 1000 + return max(0, limit) + def popup_alerts(self): return self.read_configuration("detection", "popup_alerts", False) @@ -386,6 +397,34 @@ def disabled_detections(self) -> list: "DisabledAlerts", "disabled_detections", [] ) + def evidence_signal_default(self) -> str: + value = self.read_configuration( + "EvidenceSignals", "default_signal", "PAMP" + ) + if not isinstance(value, str): + return "PAMP" + value = value.strip().upper() + if value not in ("PAMP", "DAMP"): + return "PAMP" + return value + + def evidence_signal_overrides(self) -> dict: + overrides = self.read_configuration("EvidenceSignals", "overrides", {}) + if not isinstance(overrides, dict): + return {} + + sanitized = {} + for evidence_type, signal in overrides.items(): + if not isinstance(evidence_type, str): + continue + if not isinstance(signal, str): + continue + normalized_signal = signal.strip().upper() + if normalized_signal not in ("PAMP", "DAMP"): + continue + sanitized[evidence_type.strip().upper()] = normalized_signal + return sanitized + def get_tw_width(self) -> str: twid_width = self.get_tw_width_in_seconds() # timedelta puts it in the form of X days, hours:minutes:seconds @@ -909,6 +948,600 @@ def timeline_human_timestamp(self): "modules", "timeline_human_timestamp", False ) + def llm_enabled(self) -> bool: + value = self.read_configuration("llm_proxy", "enabled", False) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def llm_default_backend(self) -> str: + value = self.read_configuration("llm_proxy", "default_backend", "") + return str(value or "").strip() + + def llm_worker_threads(self) -> int: + value = self.read_configuration("llm_proxy", "worker_threads", 2) + try: + value = int(value) + except (TypeError, ValueError): + value = 2 + return max(1, value) + + def llm_queue_size(self) -> int: + value = self.read_configuration("llm_proxy", "queue_size", 100) + try: + value = int(value) + except (TypeError, ValueError): + value = 100 + return max(1, value) + + def llm_backends(self) -> dict: + backends = self.read_configuration("llm_proxy", "backends", {}) + return backends if isinstance(backends, dict) else {} + + def regex_generator_enabled(self) -> bool: + value = self.read_configuration("regex_generator", "enabled", False) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def regex_generator_generation_interval_seconds(self) -> float: + value = self.read_configuration( + "regex_generator", "generation_interval_seconds", 5 + ) + try: + value = float(value) + except (TypeError, ValueError): + value = 5 + return max(0.0, value) + + def regex_generator_create_log_file(self) -> bool: + value = self.read_configuration( + "regex_generator", "create_log_file", False + ) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def regex_generator_allowed_backends(self) -> list: + value = self.read_configuration( + "regex_generator", "allowed_backends", [] + ) + if not isinstance(value, list): + return [] + return [ + str(backend).strip() for backend in value if str(backend).strip() + ] + + def regex_generator_llm_temperature(self) -> float: + value = self.read_configuration( + "regex_generator", "llm_temperature", 1.2 + ) + try: + value = float(value) + except (TypeError, ValueError): + value = 1.2 + return max(0.0, value) + + def regex_generator_llm_max_tokens(self) -> int: + value = self.read_configuration( + "regex_generator", "llm_max_tokens", 80 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 80 + return max(1, value) + + def regex_generator_llm_response_timeout_seconds(self) -> int: + value = self.read_configuration( + "regex_generator", "llm_response_timeout_seconds", 90 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 90 + return max(0, value) + + def regex_generator_recent_history_size(self) -> int: + value = self.read_configuration( + "regex_generator", "recent_history_size", 0 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 0 + return max(0, value) + + def regex_generator_max_regex_length(self) -> int: + value = self.read_configuration( + "regex_generator", "max_regex_length", 180 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 180 + return max(1, value) + + def regex_generator_regex_validation_timeout_seconds(self) -> float: + value = self.read_configuration( + "regex_generator", "regex_validation_timeout_seconds", 2 + ) + try: + value = float(value) + except (TypeError, ValueError): + value = 2.0 + return max(0.0, value) + + def regex_generator_benign_match_strength_threshold(self) -> float: + value = self.read_configuration( + "regex_generator", "benign_match_strength_threshold", 75 + ) + try: + value = float(value) + except (TypeError, ValueError): + value = 75.0 + return max(0.0, min(100.0, value)) + + def regex_generator_type_weights(self) -> dict: + default_weights = { + "dns_domain": 1, + "uri": 1, + "filename": 1, + "tls_sni": 1, + "certificate_cn": 1, + } + value = self.read_configuration( + "regex_generator", "type_weights", default_weights + ) + if not isinstance(value, dict): + return default_weights + + sanitized_weights = {} + for regex_type, default_weight in default_weights.items(): + raw_weight = value.get(regex_type, default_weight) + try: + raw_weight = float(raw_weight) + except (TypeError, ValueError): + raw_weight = default_weight + sanitized_weights[regex_type] = max(0.0, raw_weight) + + if not any(sanitized_weights.values()): + return default_weights + return sanitized_weights + + def regex_generator_store_dir(self) -> str: + value = self.read_configuration( + "regex_generator", "store_dir", "output/regex_generator" + ) + if not isinstance(value, str) or not value.strip(): + return "output/regex_generator" + return value.strip() + + def regex_generator_persistent_store_dir(self) -> str: + value = self.read_configuration( + "regex_generator", + "persistent_store_dir", + "databases/regex_store", + ) + if not isinstance(value, str) or not value.strip(): + return "" + return value.strip() + + def regex_generator_store_rejected_regexes(self) -> bool: + value = self.read_configuration( + "regex_generator", "store_rejected_regexes", False + ) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def regex_generator_max_stored_rejected_regexes(self) -> int: + value = self.read_configuration( + "regex_generator", "max_stored_rejected_regexes", 10000 + ) + try: + value = int(value) + except (TypeError, ValueError): + return 10000 + return max(0, value) + + def regex_generator_seed_benign_samples(self) -> bool: + value = self.read_configuration( + "regex_generator", "seed_benign_samples", True + ) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def alert_summary_enabled(self) -> bool: + value = self.read_configuration("alert_summary", "enabled", False) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def alert_summary_allowed_backends(self) -> list: + value = self.read_configuration( + "alert_summary", "allowed_backends", [] + ) + if not isinstance(value, list): + return [] + return [ + str(backend).strip() for backend in value if str(backend).strip() + ] + + def alert_summary_llm_temperature(self) -> float: + value = self.read_configuration( + "alert_summary", "llm_temperature", 0.2 + ) + try: + value = float(value) + except (TypeError, ValueError): + value = 0.2 + return max(0.0, value) + + def alert_summary_llm_max_tokens(self) -> int: + value = self.read_configuration("alert_summary", "llm_max_tokens", 220) + try: + value = int(value) + except (TypeError, ValueError): + value = 220 + return max(1, value) + + def alert_summary_llm_response_timeout_seconds(self) -> int: + value = self.read_configuration( + "alert_summary", "llm_response_timeout_seconds", 120 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 120 + return max(0, value) + + def alert_summary_log_verbosity(self) -> int: + value = self.read_configuration("alert_summary", "log_verbosity", 2) + try: + value = int(value) + except (TypeError, ValueError): + value = 2 + return min(max(value, 0), 3) + + def alert_summary_history_enabled(self) -> bool: + value = self.read_configuration( + "alert_summary", "history_enabled", False + ) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def alert_summary_history_max_alerts(self) -> int: + value = self.read_configuration( + "alert_summary", "history_max_alerts", 3 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 3 + return max(0, value) + + def alert_summary_history_max_tokens(self) -> int: + value = self.read_configuration( + "alert_summary", "history_max_tokens", 700 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 700 + return max(0, value) + + def alert_summary_history_patterns_per_alert(self) -> int: + value = self.read_configuration( + "alert_summary", "history_patterns_per_alert", 2 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 2 + return max(0, value) + + def t_cell_enabled(self) -> bool: + value = self.read_configuration("t_cell", "enabled", True) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def t_cell_create_log_file(self) -> bool: + value = self.read_configuration("t_cell", "create_log_file", True) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def t_cell_log_colors(self) -> bool: + value = self.read_configuration("t_cell", "log_colors", True) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + + def t_cell_log_verbosity(self) -> int: + value = self.read_configuration("t_cell", "log_verbosity", 1) + if isinstance(value, bool): + return 1 + if isinstance(value, (int, float)): + value = int(value) + else: + normalized = str(value).strip().lower() + named_levels = { + "summary": 1, + "decision": 2, + "decisions": 2, + "debug": 3, + } + if normalized in named_levels: + value = named_levels[normalized] + else: + try: + value = int(normalized) + except (TypeError, ValueError): + value = 1 + return max(1, min(3, int(value))) + + def t_cell_decision_trace_mode(self) -> int: + value = self.read_configuration("t_cell", "decision_trace_mode", "off") + if isinstance(value, bool): + return 1 if value else 0 + if isinstance(value, (int, float)): + return max(0, min(2, int(value))) + + normalized = str(value).strip().lower() + named_levels = { + "off": 0, + "disabled": 0, + "none": 0, + "transitions": 1, + "transition": 1, + "state_changes": 1, + "changes": 1, + "all": 2, + "full": 2, + "debug": 2, + } + if normalized in named_levels: + return named_levels[normalized] + try: + return max(0, min(2, int(normalized))) + except (TypeError, ValueError): + return 0 + + def t_cell_decision_trace_file(self) -> str: + value = self.read_configuration( + "t_cell", "decision_trace_file", "t_cell_trace.jsonl" + ) + if not isinstance(value, str) or not value.strip(): + return "t_cell_trace.jsonl" + return value.strip() + + def t_cell_decision_trace_max_evidence(self) -> int: + value = self.read_configuration( + "t_cell", "decision_trace_max_evidence", 10 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 10 + return max(1, value) + + def t_cell_store_dir(self) -> str: + value = self.read_configuration("t_cell", "store_dir", "output/t_cell") + if not isinstance(value, str) or not value.strip(): + return "output/t_cell" + return value.strip() + + def t_cell_persistent_store_dir(self) -> str: + value = self.read_configuration("t_cell", "persistent_store_dir", "") + if not isinstance(value, str) or not value.strip(): + return "" + return value.strip() + + def t_cell_observation_retention_seconds(self) -> int: + value = self.read_configuration( + "t_cell", "observation_retention_seconds", 604800 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 604800 + return max(0, value) + + def t_cell_anergy_ttl_seconds(self) -> int: + value = self.read_configuration("t_cell", "anergy_ttl_seconds", 21600) + try: + value = int(value) + except (TypeError, ValueError): + value = 21600 + return max(0, value) + + def t_cell_related_lookback_seconds(self) -> int: + value = self.read_configuration( + "t_cell", "related_lookback_seconds", 3600 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 3600 + return max(1, value) + + def t_cell_related_pamps_saturation(self) -> float: + value = self.read_configuration( + "t_cell", "related_pamps_saturation", 5 + ) + try: + value = float(value) + except (TypeError, ValueError): + value = 5.0 + return max(0.01, value) + + def t_cell_danger_saturation(self) -> float: + value = self.read_configuration("t_cell", "danger_saturation", 2.5) + try: + value = float(value) + except (TypeError, ValueError): + value = 2.5 + return max(0.01, value) + + def t_cell_damp_danger_weight(self) -> float: + value = self.read_configuration("t_cell", "damp_danger_weight", 1.5) + try: + value = float(value) + except (TypeError, ValueError): + value = 1.5 + return max(0.0, value) + + def t_cell_co_stimulation_threshold(self) -> float: + value = self.read_configuration( + "t_cell", "co_stimulation_threshold", 0.65 + ) + try: + value = float(value) + except (TypeError, ValueError): + value = 0.65 + return max(0.0, min(1.0, value)) + + def t_cell_co_stimulation_weights(self) -> dict: + default_weights = { + "confidence": 0.35, + "related_pamps": 0.25, + "danger": 0.40, + } + value = self.read_configuration( + "t_cell", "co_stimulation_weights", default_weights + ) + if not isinstance(value, dict): + return default_weights + + sanitized_weights = {} + for weight_name, default_weight in default_weights.items(): + raw_weight = value.get(weight_name, default_weight) + try: + raw_weight = float(raw_weight) + except (TypeError, ValueError): + raw_weight = default_weight + sanitized_weights[weight_name] = max(0.0, raw_weight) + + if not any(sanitized_weights.values()): + return default_weights + return sanitized_weights + + def t_cell_priming_profiles(self) -> dict: + default_profiles = { + "PAMP": { + "strength": 1.0, + "co_stimulation_threshold_offset": 0.0, + "effector_threshold_offset": 0.0, + "memory_threshold_offset": 0.0, + "state_wait_timeout_factor": 1.0, + "effector_min_related_count_offset": 0, + "memory_min_related_count_offset": 0, + }, + "DAMP": { + "strength": 0.6, + "co_stimulation_threshold_offset": 0.15, + "effector_threshold_offset": 0.10, + "memory_threshold_offset": 0.05, + "state_wait_timeout_factor": 0.5, + "effector_min_related_count_offset": 1, + "memory_min_related_count_offset": 1, + }, + } + value = self.read_configuration( + "t_cell", "priming_profiles", default_profiles + ) + if not isinstance(value, dict): + return default_profiles + return value + + def t_cell_novelty_window_seconds(self) -> int: + value = self.read_configuration( + "t_cell", "novelty_window_seconds", 86400 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 86400 + return max(1, value) + + def t_cell_context_recent_window_seconds(self) -> int: + value = self.read_configuration( + "t_cell", "context_recent_window_seconds", 1800 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 1800 + return max(1, value) + + def t_cell_effector_threshold(self) -> float: + value = self.read_configuration("t_cell", "effector_threshold", 0.70) + try: + value = float(value) + except (TypeError, ValueError): + value = 0.70 + return max(0.0, min(1.0, value)) + + def t_cell_effector_min_related_count(self) -> int: + value = self.read_configuration( + "t_cell", "effector_min_related_count", 4 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 4 + return max(1, value) + + def t_cell_effector_cooldown_seconds(self) -> int: + value = self.read_configuration( + "t_cell", "effector_cooldown_seconds", 1800 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 1800 + return max(0, value) + + def t_cell_memory_threshold(self) -> float: + value = self.read_configuration("t_cell", "memory_threshold", 0.60) + try: + value = float(value) + except (TypeError, ValueError): + value = 0.60 + return max(0.0, min(1.0, value)) + + def t_cell_memory_trend_ratio_max(self) -> float: + value = self.read_configuration( + "t_cell", "memory_trend_ratio_max", 0.60 + ) + try: + value = float(value) + except (TypeError, ValueError): + value = 0.60 + return max(0.0, value) + + def t_cell_memory_min_related_count(self) -> int: + value = self.read_configuration( + "t_cell", "memory_min_related_count", 3 + ) + try: + value = int(value) + except (TypeError, ValueError): + value = 3 + return max(1, value) + + def t_cell_simulate_effector_without_blocking(self) -> bool: + value = self.read_configuration( + "t_cell", "simulate_effector_without_blocking", True + ) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("true", "1", "yes", "on") + def analysis_direction(self): """ Controls which traffic flows are processed and analyzed by SLIPS. @@ -970,11 +1603,32 @@ def mac_db_update_period(self): def delete_prev_db(self): return self.read_configuration("parameters", "deletePrevdb", True) - def default_rotation_interval(self): - default_rotation_interval = self.read_configuration( - "parameters", "default_rotation_interval", "1 day" + def rotation_period(self): + """ + Read the configured log rotation interval. + + Returns: + Sanitized interval string from `parameters.rotation_period` or the + legacy `parameters.default_rotation_interval` key. + """ + rotation_period = self.read_configuration( + "parameters", "rotation_period", "" ) - return utils.sanitize(default_rotation_interval) + if rotation_period in ("", None): + rotation_period = self.read_configuration( + "parameters", "default_rotation_interval", "1 day" + ) + return utils.sanitize(str(rotation_period)) + + def default_rotation_interval(self): + """ + Read the legacy Zeek rotation interval setting. + + Returns: + Sanitized interval string using the modern `rotation_period` + accessor with backward-compatible key lookup. + """ + return self.rotation_period() def parse_ip(self, ip: str): """converts the given IP address or CIDR to an obj""" @@ -1073,6 +1727,60 @@ def reading_flows_from_cyst(self): # param isn't used pass + def get_disabled_modules(self, input_type: str) -> list: + """ + Uses input type to enable leak detector only on pcaps + """ + to_ignore: List[str] = self.read_configuration( + "modules", "disable", ["template"] + ) + to_ignore = [mod.strip() for mod in to_ignore] + + # Ignore exporting alerts module if export_to is empty + export_to = self.export_to() + if "stix" not in export_to and "slack" not in export_to: + to_ignore.append("exporting_alerts") + + use_p2p = self.use_local_p2p() + if not (use_p2p and "-i" in sys.argv): + to_ignore.append("p2p_trust") + + use_global_p2p = self.use_global_p2p() + if not (use_global_p2p and ("-i" in sys.argv)): + to_ignore.append("fides") + to_ignore.append("iris") + + # ignore CESNET sharing module if send and receive are + # disabled in slips.yaml + send_to_warden = self.send_to_warden() + receive_from_warden = self.receive_from_warden() + + if not send_to_warden and not receive_from_warden: + to_ignore.append("cesnet") + + # don't run blocking module unless specified + if not ("-cb" in sys.argv or "-p" in sys.argv): + to_ignore.append("blocking") + to_ignore.append("arp_poisoner") + + # leak detector only works on pcap files + if input_type != InputType.PCAP: + to_ignore.append("leak_detector") + + if not self.reading_flows_from_cyst(): + to_ignore.append("cyst") + + if not self.llm_enabled(): + to_ignore.append("llm_proxy") + + if not self.regex_generator_enabled(): + to_ignore.append("regex_generator") + + if not self.t_cell_enabled(): + to_ignore.append("t_cell") + + return to_ignore + def get_cpu_profiler_enable(self): return self.read_configuration( "Profiling", "cpu_profiler_enable", False diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index f5d3f40102..5afc7b48b2 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -67,9 +67,11 @@ def __init__( self.conf = conf self.output_dir = output_dir self.redis_port = redis_port + self.main_pid = main_pid self.logger = logger self.printer = Printer(self.logger, self.name) - + self.regex_generator_storage = None + self.t_cell_storage = None # only the main process should ever flush the Redis DB. to avoid # children overwriting values set at the very start of slips if os.getpid() != main_pid: @@ -218,6 +220,22 @@ def get_p2p_trust_db_path(self) -> str: def print(self, *args, **kwargs): return self.printer.print(*args, **kwargs) + def reset_pending_llm_request_counts(self): + """Clear requester-level in-flight shared-LLM request counters.""" + return self.rdb.reset_pending_llm_request_counts() + + def increment_pending_llm_request_count(self, requester: str): + """Increment the in-flight shared-LLM request count for one requester.""" + return self.rdb.increment_pending_llm_request_count(requester) + + def decrement_pending_llm_request_count(self, requester: str): + """Decrement the in-flight shared-LLM request count for one requester.""" + return self.rdb.decrement_pending_llm_request_count(requester) + + def get_pending_llm_request_count(self, requester: str) -> int: + """Return the in-flight shared-LLM request count for one requester.""" + return self.rdb.get_pending_llm_request_count(requester) + @classmethod def read_configuration(cls): conf = ConfigParser() @@ -608,15 +626,12 @@ def get_field_separator(self, *args, **kwargs): def store_tranco_whitelisted_domains(self, *args, **kwargs): return self.rdb.store_tranco_whitelisted_domains(*args, **kwargs) + def get_tranco_top_domains(self, *args, **kwargs): + return self.rdb.get_tranco_top_domains(*args, **kwargs) + def is_whitelisted_tranco_domain(self, *args, **kwargs): return self.rdb.is_whitelisted_tranco_domain(*args, **kwargs) - def delete_tranco_whitelist(self, *args, **kwargs): - return self.rdb.delete_tranco_whitelist(*args, **kwargs) - - def is_tranco_whitelist_expired(self, *args, **kwargs): - return self.rdb.is_tranco_whitelist_expired(*args, **kwargs) - def get_ip_identification(self, *args, **kwargs): return self.rdb.get_ip_identification(*args, **kwargs) @@ -1060,6 +1075,12 @@ def incr_msgs_received_in_channel(self, *args, **kwargs): def get_enabled_modules(self, *args, **kwargs): return self.rdb.get_enabled_modules(*args, **kwargs) + def set_available_llm_backends(self, *args, **kwargs): + return self.rdb.set_available_llm_backends(*args, **kwargs) + + def get_available_llm_backends(self, *args, **kwargs): + return self.rdb.get_available_llm_backends(*args, **kwargs) + def get_msgs_received_at_runtime(self, *args, **kwargs): return self.rdb.get_msgs_received_at_runtime(*args, **kwargs) @@ -1114,6 +1135,9 @@ def get_altflow_from_uid(self, *args, **kwargs): def get_all_flows_in_profileid_twid(self, *args, **kwargs): return self.sqlite.get_all_flows_in_profileid_twid(*args, **kwargs) + def get_all_altflows_in_profileid_twid(self, *args, **kwargs): + return self.sqlite.get_all_altflows_in_profileid_twid(*args, **kwargs) + def get_all_flows_in_profileid(self, *args, **kwargs): return self.sqlite.get_all_flows_in_profileid(*args, **kwargs) @@ -1338,6 +1362,47 @@ def get_the_other_ip_version(self, *args, **kwargs): def get_separator(self): return self.rdb.separator + def _get_regex_generator_storage(self): + if self.regex_generator_storage is None: + from slips_files.core.database.sqlite_db.regex_generator_db import ( + RegexGeneratorStorage, + ) + + self.regex_generator_storage = RegexGeneratorStorage( + self.logger, + self.conf, + self.output_dir, + self.main_pid, + ) + return self.regex_generator_storage + + def get_generated_regexes(self, *args, **kwargs): + return self._get_regex_generator_storage().get_generated_regexes( + *args, **kwargs + ) + + def get_generated_regexes_count(self, *args, **kwargs): + return self._get_regex_generator_storage().get_generated_regexes_count( + *args, **kwargs + ) + + def _get_t_cell_storage(self): + if self.t_cell_storage is None: + from slips_files.core.database.sqlite_db.t_cell_db import ( + TCellStorage, + ) + + self.t_cell_storage = TCellStorage( + self.logger, + self.conf, + self.output_dir, + self.main_pid, + ) + return self.t_cell_storage + + def get_t_cell_storage(self): + return self._get_t_cell_storage() + def get_icmp_attack_info_to_single_host(self, *args, **kwargs): return self.rdb.get_icmp_attack_info_to_single_host(*args, **kwargs) @@ -1432,6 +1497,10 @@ def close_sqlite(self, *args, **kwargs): # when stopping the daemon using -S, slips doesn't start the sqlite db if self.sqlite: self.sqlite.close(*args, **kwargs) + if self.regex_generator_storage: + self.regex_generator_storage.close() + if self.t_cell_storage: + self.t_cell_storage.close() def close_all_dbs(self): self.rdb.r.close() diff --git a/slips_files/core/database/redis_db/alert_handler.py b/slips_files/core/database/redis_db/alert_handler.py index bca7e43b11..5022dc8b5a 100644 --- a/slips_files/core/database/redis_db/alert_handler.py +++ b/slips_files/core/database/redis_db/alert_handler.py @@ -18,6 +18,7 @@ ) from slips_files.core.structures.evidence import ( Evidence, + EvidenceSignal, EvidenceType, Victim, IoCType, @@ -37,6 +38,8 @@ class AlertHandler: default_ttl: int width: float disabled_detections: Any + default_evidence_signal: str + evidence_signal_overrides: Dict[str, str] publish: Callable[..., Any] zadd_but_keep_n_entries: Callable[..., Any] get_tw_start_time: Callable[..., Any] @@ -115,6 +118,19 @@ def is_detection_disabled(self, evidence_type: EvidenceType): """ return str(evidence_type) in self.disabled_detections + def _classify_evidence_signal( + self, evidence_type: EvidenceType + ) -> EvidenceSignal: + evidence_type_name = str(evidence_type).upper() + signal = self.evidence_signal_overrides.get( + evidence_type_name, + self.default_evidence_signal, + ) + try: + return EvidenceSignal[str(signal).upper()] + except KeyError: + return EvidenceSignal.PAMP + def set_flow_causing_evidence(self, uids: list, evidence_id): """ Used to be able to add the "malicious" tag to the flows that caused @@ -261,6 +277,9 @@ def set_evidence(self, evidence: Evidence): self.add_profile(str(evidence.profile), evidence.timestamp) # normalize confidence, should range from 0 to 1 evidence.confidence = min(evidence.confidence, 1) + evidence.evidence_signal = self._classify_evidence_signal( + evidence.evidence_type + ) # Ignore evidence if it's disabled in the configuration file if self.is_detection_disabled(evidence.evidence_type): diff --git a/slips_files/core/database/redis_db/constants.py b/slips_files/core/database/redis_db/constants.py index 24ef615f6a..b8d50bf7a6 100644 --- a/slips_files/core/database/redis_db/constants.py +++ b/slips_files/core/database/redis_db/constants.py @@ -118,6 +118,8 @@ class Constants: FLOWS_PER_MINUTE_MODULES = "flows_per_minute_modules" FLOWS_PER_MINUTE_LAST_LOGGED = "flows_per_minute_last_logged" FLOWS_PER_MINUTE_LOG_LOCK = "flows_per_minute_log_lock" + AVAILABLE_LLM_BACKENDS = "available_llm_backends" + PENDING_LLM_REQUESTS_BY_REQUESTER = "pending_llm_requests_by_requester" class Channels: @@ -126,3 +128,5 @@ class Channels: GIVE_TI = "get_modified_profiles_since" NEW_ZEEK_FIELDS_LINE = "new_zeek_fields_line" CONTROL_CHANNEL = "control_channel" + LLM_REQUEST = "llm_request" + LLM_RESPONSE = "llm_response" diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index 36df1b4026..e340eb8cf1 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -117,6 +117,8 @@ class RedisDB( "slips2fides", "iris_internal", "new_zeek_fields_line", + "llm_request", + "llm_response", } separator = "_" @@ -323,6 +325,8 @@ def _read_configuration(cls): # By default False. Meaning we don't DELETE the DB by default. cls.config_flush_db: bool = conf.delete_prev_db() cls.disabled_detections: List[str] = conf.disabled_detections() + cls.default_evidence_signal: str = conf.evidence_signal_default() + cls.evidence_signal_overrides: dict = conf.evidence_signal_overrides() cls.width = conf.get_tw_width_in_seconds() cls.client_ips: List[str] = conf.client_ips() @@ -762,6 +766,124 @@ def get_enabled_modules(self) -> List[str]: """ return self.r.hkeys(self.constants.PIDS) + @staticmethod + def _empty_available_llm_backends() -> dict: + return {"default_backend": "", "backends": {}} + + def set_available_llm_backends(self, registry: dict): + normalized = self._normalize_available_llm_backends_registry(registry) + self.r.set( + self.constants.AVAILABLE_LLM_BACKENDS, json.dumps(normalized) + ) + + def get_available_llm_backends(self) -> dict: + if registry := self.r.get(self.constants.AVAILABLE_LLM_BACKENDS): + try: + registry = json.loads(registry) + except json.JSONDecodeError: + return self._empty_available_llm_backends() + return self._normalize_available_llm_backends_registry(registry) + + return self._empty_available_llm_backends() + + def reset_pending_llm_request_counts(self): + """ + Clear requester-level in-flight LLM request counters. + + The shared LLM service owns this key and resets it on startup and + shutdown so stale counts from earlier runs do not block modules. + """ + self.r.delete(self.constants.PENDING_LLM_REQUESTS_BY_REQUESTER) + + def increment_pending_llm_request_count(self, requester: str): + """ + Increment the in-flight shared-LLM request count for one requester. + + :param requester: Caller module name. + :return: New counter value or 0 when requester is empty. + """ + requester = str(requester or "").strip() + if not requester: + return 0 + return self.r.hincrby( + self.constants.PENDING_LLM_REQUESTS_BY_REQUESTER, + requester, + 1, + ) + + def decrement_pending_llm_request_count(self, requester: str): + """ + Decrement the in-flight shared-LLM request count for one requester. + + :param requester: Caller module name. + :return: Updated non-negative counter value. + """ + requester = str(requester or "").strip() + if not requester: + return 0 + + key = self.constants.PENDING_LLM_REQUESTS_BY_REQUESTER + value = self.r.hincrby(key, requester, -1) + if value <= 0: + self.r.hdel(key, requester) + return 0 + return value + + def get_pending_llm_request_count(self, requester: str) -> int: + """ + Return the in-flight shared-LLM request count for one requester. + + :param requester: Caller module name. + :return: Non-negative count. + """ + requester = str(requester or "").strip() + if not requester: + return 0 + value = self.r.hget( + self.constants.PENDING_LLM_REQUESTS_BY_REQUESTER, + requester, + ) + try: + return max(0, int(value or 0)) + except (TypeError, ValueError): + return 0 + + def _normalize_available_llm_backends_registry( + self, registry: dict + ) -> dict: + if not isinstance(registry, dict): + return self._empty_available_llm_backends() + + backends = registry.get("backends") + if not isinstance(backends, dict): + backends = {} + + normalized_backends = {} + for alias, metadata in backends.items(): + if not isinstance(alias, str) or not alias.strip(): + continue + if not isinstance(metadata, dict): + continue + + provider = str(metadata.get("provider", "")).strip() + model = str(metadata.get("model", "")).strip() + if not provider or not model: + continue + + normalized_backends[alias.strip()] = { + "provider": provider, + "model": model, + } + + default_backend = str(registry.get("default_backend", "")).strip() + if default_backend not in normalized_backends: + default_backend = "" + + return { + "default_backend": default_backend, + "backends": normalized_backends, + } + def get_disabled_modules(self) -> List[str]: if disabled_modules := self.r.hget( self.constants.ANALYSIS, "disabled_modules" @@ -1179,35 +1301,47 @@ def get_field_separator(self): return self.separator def store_tranco_whitelisted_domains( - self, domains: List[str], ttl: Optional[int] = None - ): - """ - store whitelisted domains from tranco whitelist in the db + self, + domains: List[str], + limit: Optional[int] = None, + ) -> None: """ - # the reason we store tranco whitelisted domains in the cache db - # instead of the main db is, we don't want them cleared on every new - # instance of slips - self.rcache.sadd(self.constants.TRANCO_WHITELISTED_DOMAINS, *domains) - if ttl and ttl > 0: - self.rcache.expire( - self.constants.TRANCO_WHITELISTED_DOMAINS, int(ttl) - ) + Store ordered tranco domains in the db. - def is_tranco_whitelist_expired(self) -> bool: - """ - checks if tranco whitelist is expired based on Redis TTL - """ - ttl = self.rcache.ttl(self.constants.TRANCO_WHITELISTED_DOMAINS) - # -2: key does not exist, -1: no expire - return ttl <= 0 + Parameters: + domains: Ordered Tranco domains to store. they must be ordered. + limit: Optional maximum number of domains to store. + """ + if limit is not None: + if limit <= 0: + return + domains = domains[:limit] + + with self.rcache.pipeline() as pipe: + if domains: + pipe.delete(self.constants.TRANCO_WHITELISTED_DOMAINS) + pipe.zadd( + self.constants.TRANCO_WHITELISTED_DOMAINS, + {domain: rank for rank, domain in enumerate(domains)}, + ) + pipe.execute() - def is_whitelisted_tranco_domain(self, domain): - return self.rcache.sismember( - self.constants.TRANCO_WHITELISTED_DOMAINS, domain + def get_tranco_top_domains(self, limit: Optional[int] = None) -> List[str]: + end = -1 if limit is None or limit <= 0 else limit - 1 + return ( + self.rcache.zrange( + self.constants.TRANCO_WHITELISTED_DOMAINS, 0, end + ) + or [] ) - def delete_tranco_whitelist(self): - return self.rcache.delete(self.constants.TRANCO_WHITELISTED_DOMAINS) + def is_whitelisted_tranco_domain(self, domain: str) -> bool: + return ( + self.rcache.zscore( + self.constants.TRANCO_WHITELISTED_DOMAINS, domain + ) + is not None + ) def get_asn_info(self, ip: str) -> Optional[Dict[str, str]]: """ diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index 7c08268d52..d8ccad3330 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -18,6 +18,7 @@ import redis import validators +from slips_files.common.slips_utils import utils from slips_files.core.structures.flow_attributes import Role @@ -1051,7 +1052,7 @@ def add_profile(self, profileid, starttime, confidence=0.05) -> bool: self.zadd_but_keep_n_entries( self.constants.PROFILES, - {str(profileid): float(starttime)}, + {str(profileid): self._get_profile_start_score(starttime)}, 2000, ) @@ -1084,6 +1085,13 @@ def add_profile(self, profileid, starttime, confidence=0.05) -> bool: self.print(inst, 0, 1) return False + @staticmethod + def _get_profile_start_score(starttime) -> float: + try: + return float(utils.convert_ts_format(starttime, "unixtimestamp")) + except Exception: + return float(starttime) + def set_module_label_for_profile(self, profileid, module, label): """ Set a module label for a profile. diff --git a/slips_files/core/database/sqlite_db/database.py b/slips_files/core/database/sqlite_db/database.py index 339498595d..6967684d24 100644 --- a/slips_files/core/database/sqlite_db/database.py +++ b/slips_files/core/database/sqlite_db/database.py @@ -3,7 +3,6 @@ from datetime import datetime from typing import List, Dict import os.path -import sqlite3 import json import csv from dataclasses import asdict @@ -111,6 +110,26 @@ def get_all_flows_in_profileid_twid(self, profileid, twid): res[uid] = json.loads(flow) return res + def get_all_altflows_in_profileid_twid(self, profileid, twid): + condition = f'profileid = "{profileid}" ' f'AND twid = "{twid}"' + altflows: list = self.select("altflows", condition=condition) + if not altflows: + return [] + + rows = [] + for altflow in altflows: + rows.append( + { + "uid": altflow[0], + "flow": json.loads(altflow[1]), + "label": altflow[2], + "profileid": altflow[3], + "twid": altflow[4], + "flow_type": altflow[5], + } + ) + return rows + def get_all_flows_in_profileid(self, profileid) -> Dict[str, dict]: """ Return a list of all the flows in this profileid diff --git a/slips_files/core/database/sqlite_db/regex_generator_db.py b/slips_files/core/database/sqlite_db/regex_generator_db.py new file mode 100644 index 0000000000..6fe16edc27 --- /dev/null +++ b/slips_files/core/database/sqlite_db/regex_generator_db.py @@ -0,0 +1,791 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import hashlib +import os +from pathlib import Path +from time import time +from typing import Dict, Iterable, List + +from pybloom_live import ScalableBloomFilter + +from slips_files.common.abstracts.isqlite import ISQLite +from slips_files.common.output_paths import ( + get_this_filepath_inside_permanent_dir, +) +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.printer import Printer +from slips_files.common.slips_utils import utils +from slips_files.core.output import Output + +REGEX_TYPES = ( + "dns_domain", + "uri", + "filename", + "tls_sni", + "certificate_cn", +) +DEFAULT_REGEX_GENERATOR_STORE_DIR = "output/regex_generator" +DEFAULT_BENIGN_SEED_SAMPLES = { + "dns_domain": [ + "google.com", + "microsoft.com", + "github.com", + "cloudflare.com", + "ubuntu.com", + ], + "uri": [ + "/", + "/index.html", + "/favicon.ico", + "/api/v1/health", + "/login", + ], + "filename": [ + "document.pdf", + "invoice-2024.xlsx", + "photo.jpg", + "notes.txt", + "setup.exe", + ], + "tls_sni": [ + "www.google.com", + "api.github.com", + "login.microsoftonline.com", + "cdn.cloudflare.com", + "packages.ubuntu.com", + ], + "certificate_cn": [ + "www.google.com", + "github.com", + "login.microsoftonline.com", + "letsencrypt.org", + "updates.ubuntu.com", + ], +} +WHITELIST_COMPATIBLE_REGEX_TYPES = ( + "dns_domain", + "tls_sni", + "certificate_cn", +) + + +def _make_sha256(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +class _BaseRegexSQLiteDB(ISQLite): + name = "BaseRegexSQLiteDB" + + def __init__(self, logger: Output, db_path: str, main_pid: int): + self.printer = Printer(logger, self.name) + self.db_path = db_path + self._init_db_file() + super().__init__(self.name.lower(), main_pid, db_path) + self.init_tables() + + def _init_db_file(self): + db_file = Path(self.db_path) + db_file.parent.mkdir(parents=True, exist_ok=True) + if not db_file.exists(): + db_file.touch() + os.chmod(db_file, 0o777) + + +class BenignCorpusSQLiteDB(_BaseRegexSQLiteDB): + name = "BenignCorpusSQLiteDB" + + def init_tables(self): + self.create_table( + "benign_strings", + "id INTEGER PRIMARY KEY, regex_type TEXT NOT NULL, value TEXT NOT NULL, " + "value_hash TEXT NOT NULL UNIQUE, source TEXT NOT NULL, created_at REAL NOT NULL", + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_benign_strings_type_hash " + "ON benign_strings (regex_type, value_hash)" + ) + + def is_empty(self) -> bool: + return self.get_count("benign_strings") == 0 + + def insert_benign_string( + self, + regex_type: str, + value: str, + source: str, + created_at: float | None = None, + ) -> bool: + created_at = created_at or time() + value_hash = _make_sha256(f"{regex_type}\0{value}") + cursor = self.execute( + "INSERT OR IGNORE INTO benign_strings " + "(regex_type, value, value_hash, source, created_at) " + "VALUES (?, ?, ?, ?, ?)", + (regex_type, value, value_hash, source, created_at), + ) + return bool(cursor and cursor.rowcount) + + def seed_strings(self, seed_samples: Dict[str, Iterable[str]], source: str): + for regex_type, values in seed_samples.items(): + for value in values: + self.insert_benign_string(regex_type, value, source) + + def get_examples(self, regex_type: str, limit: int = 5) -> List[str]: + rows = self.select( + "benign_strings", + columns="value", + condition="regex_type = ?", + params=(regex_type,), + order_by="id ASC", + ) + rows = rows or [] + return [row[0] for row in rows[:limit]] + + def iter_values(self, regex_type: str): + cursor = self.execute( + "SELECT value FROM benign_strings WHERE regex_type = ? ORDER BY id ASC", + (regex_type,), + ) + if not cursor: + return + + while True: + row = self.fetchone(cursor) + if row is None: + break + yield row[0] + + +class GeneratedRegexSQLiteDB(_BaseRegexSQLiteDB): + name = "GeneratedRegexSQLiteDB" + + def init_tables(self): + self.create_table( + "generated_regexes", + "id INTEGER PRIMARY KEY, regex_type TEXT NOT NULL, regex TEXT NOT NULL, " + "regex_hash TEXT NOT NULL UNIQUE, status TEXT NOT NULL, " + "rejection_reason TEXT, matched_benign_value TEXT, backend_alias TEXT, " + "provider TEXT, model TEXT, temperature REAL, prompt_version TEXT, " + "request_id TEXT, created_at REAL NOT NULL", + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_generated_regexes_status_type_created " + "ON generated_regexes (status, regex_type, created_at)" + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_generated_regexes_type_created " + "ON generated_regexes (regex_type, created_at)" + ) + + @staticmethod + def _row_to_dict(row) -> dict: + return { + "id": row[0], + "regex_type": row[1], + "regex": row[2], + "regex_hash": row[3], + "status": row[4], + "rejection_reason": row[5], + "matched_benign_value": row[6], + "backend_alias": row[7], + "provider": row[8], + "model": row[9], + "temperature": row[10], + "prompt_version": row[11], + "request_id": row[12], + "created_at": row[13], + } + + def get_by_hash(self, regex_hash: str) -> dict | None: + row = self.select( + "generated_regexes", + condition="regex_hash = ?", + params=(regex_hash,), + limit=1, + ) + if not row: + return None + return self._row_to_dict(row) + + def insert_generated_regex(self, record: dict): + self.execute( + "INSERT OR IGNORE INTO generated_regexes " + "(regex_type, regex, regex_hash, status, rejection_reason, " + "matched_benign_value, backend_alias, provider, model, temperature, " + "prompt_version, request_id, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + record["regex_type"], + record["regex"], + record["regex_hash"], + record["status"], + record.get("rejection_reason"), + record.get("matched_benign_value"), + record.get("backend_alias"), + record.get("provider"), + record.get("model"), + record.get("temperature"), + record.get("prompt_version"), + record.get("request_id"), + record.get("created_at") or time(), + ), + ) + + def get_recent_history(self, regex_type: str, limit: int) -> List[dict]: + rows = self.select( + "generated_regexes", + condition="regex_type = ?", + params=(regex_type,), + order_by="created_at DESC", + ) + rows = rows or [] + return [self._row_to_dict(row) for row in rows[:limit]] + + def get_generated_regexes( + self, + regex_type: str | None = None, + limit: int | None = None, + status: str = "accepted", + ) -> List[dict]: + condition_parts = [] + params = [] + if status: + condition_parts.append("status = ?") + params.append(status) + if regex_type: + condition_parts.append("regex_type = ?") + params.append(regex_type) + + condition = " AND ".join(condition_parts) if condition_parts else None + rows = self.select( + "generated_regexes", + condition=condition, + params=tuple(params), + order_by="created_at DESC", + ) + rows = rows or [] + if limit is not None: + rows = rows[:limit] + return [self._row_to_dict(row) for row in rows] + + def get_generated_regexes_count( + self, + regex_type: str | None = None, + status: str = "accepted", + ) -> int: + condition_parts = [] + params = [] + if status: + condition_parts.append("status = ?") + params.append(status) + if regex_type: + condition_parts.append("regex_type = ?") + params.append(regex_type) + + condition = " AND ".join(condition_parts) if condition_parts else None + row = self.select( + "generated_regexes", + columns="COUNT(*)", + condition=condition, + params=tuple(params), + limit=1, + ) + return row[0] if row else 0 + + def iter_regex_hashes(self, status: str | None = None): + query = "SELECT regex_hash FROM generated_regexes" + params = () + if status: + query += " WHERE status = ?" + params = (status,) + query += " ORDER BY id ASC" + cursor = self.execute( + query, + params, + ) + if not cursor: + return + + while True: + row = self.fetchone(cursor) + if row is None: + break + yield row[0] + + def prune_rejected_regexes(self, max_records: int): + if max_records <= 0: + return + + count = self.get_generated_regexes_count(status="rejected") + excess = count - max_records + if excess <= 0: + return + + self.execute( + "DELETE FROM generated_regexes WHERE id IN (" + "SELECT id FROM generated_regexes " + "WHERE status = 'rejected' " + "ORDER BY created_at ASC, id ASC LIMIT ?" + ")", + (excess,), + ) + + +class RegexGeneratorStorage: + def __init__( + self, + logger: Output, + conf, + output_dir: str, + main_pid: int, + db=None, + ): + self.logger = logger + self.conf = conf + self.output_dir = output_dir + self.main_pid = main_pid + self.db = db + self.store_dir = self._resolve_store_dir() + self.store_rejected_regexes = self._read_store_rejected_regexes() + self.max_stored_rejected_regexes = ( + self._read_max_stored_rejected_regexes() + ) + self.seed_benign_samples = self._read_seed_benign_samples() + self.enable_local_whitelist = self._read_enable_local_whitelist() + self.local_whitelist_path = self._read_local_whitelist_path() + self.tranco_top_benign_limit = self._read_tranco_top_benign_limit() + self.benign_db = BenignCorpusSQLiteDB( + self.logger, + str(Path(self.store_dir) / "benign_corpus.sqlite"), + self.main_pid, + ) + self.generated_db = GeneratedRegexSQLiteDB( + self.logger, + str(Path(self.store_dir) / "generated_regexes.sqlite"), + self.main_pid, + ) + if self.seed_benign_samples and self.benign_db.is_empty(): + self.seed_default_benign_samples() + self._import_local_whitelist_into_benign_corpus() + self._import_tranco_top_domains_into_benign_corpus() + self.bloom_filters = self._build_bloom_filters() + self.generated_regex_filter = self._build_generated_regex_filter() + self.rejected_regex_filter = self._build_rejected_regex_filter() + + def _resolve_store_dir(self) -> str: + raw_store_dir, is_persistent = self._read_store_dir() + store_dir = self._normalize_store_dir(raw_store_dir, is_persistent) + store_dir.mkdir(parents=True, exist_ok=True) + return str(store_dir) + + def _normalize_store_dir( + self, raw_store_dir: str, is_persistent: bool = False + ) -> Path: + store_dir = Path(raw_store_dir).expanduser() + if store_dir.is_absolute(): + return store_dir + + relative_parts = list(store_dir.parts) + while relative_parts and relative_parts[0] == ".": + relative_parts = relative_parts[1:] + if is_persistent: + if not relative_parts: + relative_parts = ["databases", "regex_store"] + return Path( + get_this_filepath_inside_permanent_dir( + os.path.join(*relative_parts) + ) + ) + if relative_parts and relative_parts[0] == "output": + relative_parts = relative_parts[1:] + if not relative_parts: + relative_parts = ["regex_generator"] + + return Path(self.output_dir).expanduser().joinpath(*relative_parts) + + def _read_store_dir(self) -> tuple[str, bool]: + persistent_value = self._read_string_config( + "regex_generator_persistent_store_dir" + ) + if persistent_value: + return persistent_value, True + + value = self._read_string_config("regex_generator_store_dir") + if value: + return value, False + + parser = ConfigParser() + persistent_getter = getattr( + parser, "regex_generator_persistent_store_dir", None + ) + if callable(persistent_getter): + try: + persistent_value = persistent_getter() + except TypeError: + persistent_value = None + if isinstance(persistent_value, str) and persistent_value.strip(): + return persistent_value.strip(), True + + parser_getter = getattr(parser, "regex_generator_store_dir", None) + if callable(parser_getter): + try: + value = parser_getter() + except TypeError: + value = None + if isinstance(value, str) and value.strip(): + return value.strip(), False + return DEFAULT_REGEX_GENERATOR_STORE_DIR, False + + def _read_seed_benign_samples(self) -> bool: + value = self._read_bool_config("regex_generator_seed_benign_samples") + if value is not None: + return value + + parser = ConfigParser() + parser_getter = getattr(parser, "regex_generator_seed_benign_samples", None) + if callable(parser_getter): + try: + value = parser_getter() + except TypeError: + value = None + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in ("true", "1", "yes", "on") + return True + + def _read_store_rejected_regexes(self) -> bool: + value = self._read_bool_config("regex_generator_store_rejected_regexes") + if value is not None: + return value + + parser = ConfigParser() + parser_getter = getattr( + parser, "regex_generator_store_rejected_regexes", None + ) + if callable(parser_getter): + try: + value = parser_getter() + except TypeError: + value = None + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in ("true", "1", "yes", "on") + return False + + def _read_max_stored_rejected_regexes(self) -> int: + value = self._read_int_config("regex_generator_max_stored_rejected_regexes") + if value is not None: + return max(0, value) + + parser = ConfigParser() + parser_getter = getattr( + parser, "regex_generator_max_stored_rejected_regexes", None + ) + if callable(parser_getter): + try: + value = parser_getter() + except TypeError: + value = None + if isinstance(value, int): + return max(0, value) + if isinstance(value, str): + try: + return max(0, int(value.strip())) + except ValueError: + pass + return 10000 + + def _read_enable_local_whitelist(self) -> bool: + value = self._read_bool_config("enable_local_whitelist") + if value is not None: + return value + + parser = ConfigParser() + parser_getter = getattr(parser, "enable_local_whitelist", None) + if callable(parser_getter): + try: + value = parser_getter() + except TypeError: + value = None + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in ("true", "1", "yes", "on") + return True + + def _read_local_whitelist_path(self) -> str: + value = self._read_string_config("local_whitelist_path") + if value: + return value + + parser = ConfigParser() + parser_getter = getattr(parser, "local_whitelist_path", None) + if callable(parser_getter): + try: + value = parser_getter() + except TypeError: + value = None + if isinstance(value, str) and value.strip(): + return value.strip() + return "config/whitelist.conf" + + def _read_tranco_top_benign_limit(self) -> int: + value = self._read_int_config("tranco_top_benign_limit") + if value is not None: + return max(0, value) + + parser = ConfigParser() + parser_getter = getattr(parser, "tranco_top_benign_limit", None) + if callable(parser_getter): + try: + value = parser_getter() + except TypeError: + value = None + if isinstance(value, int): + return max(0, value) + if isinstance(value, str): + try: + return max(0, int(value.strip())) + except ValueError: + pass + return 1000 + + def _read_string_config(self, method_name: str) -> str | None: + getter = getattr(self.conf, method_name, None) + if not callable(getter): + return None + try: + value = getter() + except TypeError: + return None + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + def _read_bool_config(self, method_name: str) -> bool | None: + getter = getattr(self.conf, method_name, None) + if not callable(getter): + return None + try: + value = getter() + except TypeError: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in ("true", "1", "yes", "on") + return None + + def _read_int_config(self, method_name: str) -> int | None: + getter = getattr(self.conf, method_name, None) + if not callable(getter): + return None + try: + value = getter() + except TypeError: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + def seed_default_benign_samples(self): + self.benign_db.seed_strings( + DEFAULT_BENIGN_SEED_SAMPLES, + source="seed_v1", + ) + + def _import_local_whitelist_into_benign_corpus(self): + if not self.enable_local_whitelist: + return + + whitelist_path = Path(self.local_whitelist_path).expanduser() + if not whitelist_path.is_absolute(): + whitelist_path = Path(os.getcwd()) / whitelist_path + if not whitelist_path.exists(): + return + + for domain in self._iter_whitelist_domains(whitelist_path): + hostname = utils.extract_hostname(domain) + values = {domain} + if hostname: + values.add(hostname) + + for regex_type in WHITELIST_COMPATIBLE_REGEX_TYPES: + for value in values: + self.benign_db.insert_benign_string( + regex_type, + value, + source=f"local_whitelist:{whitelist_path}", + ) + + def _import_tranco_top_domains_into_benign_corpus(self): + if self.db is None or self.tranco_top_benign_limit <= 0: + return + + getter = getattr(self.db, "get_tranco_top_domains", None) + if not callable(getter): + return + + try: + domains = getter(limit=self.tranco_top_benign_limit) or [] + except TypeError: + domains = getter() or [] + + for domain in domains[: self.tranco_top_benign_limit]: + domain = str(domain or "").strip().lower() + if not utils.is_valid_domain(domain): + continue + + values = {domain} + hostname = utils.extract_hostname(domain) + if hostname: + values.add(hostname) + + for regex_type in WHITELIST_COMPATIBLE_REGEX_TYPES: + for value in values: + self.benign_db.insert_benign_string( + regex_type, + value, + source="tranco_top_1000", + ) + + @staticmethod + def _iter_whitelist_domains(whitelist_path: Path): + with open(whitelist_path, encoding="utf-8") as whitelist: + for raw_line in whitelist: + if ( + not raw_line + or raw_line.startswith(";") + or raw_line.startswith("#") + or raw_line.startswith('"IoCType"') + ): + continue + + line = raw_line.replace("\n", "").replace(" ", "") + parts = line.split(",") + if len(parts) < 4: + continue + if parts[0].lower() != "domain": + continue + + domain = parts[1].strip().lower() + if not utils.is_valid_domain(domain): + continue + yield domain + + def _build_bloom_filters(self) -> dict: + bloom_filters = {} + for regex_type in REGEX_TYPES: + bloom = ScalableBloomFilter( + mode=ScalableBloomFilter.SMALL_SET_GROWTH, + error_rate=0.001, + ) + for value in self.benign_db.iter_values(regex_type): + bloom.add(value) + bloom_filters[regex_type] = bloom + return bloom_filters + + def _build_generated_regex_filter(self): + bloom = ScalableBloomFilter( + mode=ScalableBloomFilter.SMALL_SET_GROWTH, + error_rate=0.001, + ) + for regex_hash in self.generated_db.iter_regex_hashes(): + bloom.add(regex_hash) + return bloom + + def _build_rejected_regex_filter(self): + return ScalableBloomFilter( + mode=ScalableBloomFilter.SMALL_SET_GROWTH, + error_rate=0.001, + ) + + def get_benign_examples(self, regex_type: str, limit: int = 5) -> List[str]: + return self.benign_db.get_examples(regex_type, limit) + + def iter_benign_strings(self, regex_type: str): + yield from self.benign_db.iter_values(regex_type) + + def add_benign_strings( + self, + regex_type: str, + values: Iterable[str], + source: str, + ) -> int: + inserted = 0 + bloom = self.bloom_filters.get(regex_type) + for value in values: + normalized = str(value or "").strip() + if not normalized: + continue + added = self.benign_db.insert_benign_string( + regex_type, + normalized, + source=source, + ) + if added: + inserted += 1 + if bloom is not None: + bloom.add(normalized) + return inserted + + def get_recent_history(self, regex_type: str, limit: int) -> List[dict]: + return self.generated_db.get_recent_history(regex_type, limit) + + def get_generated_regexes( + self, + regex_type: str | None = None, + limit: int | None = None, + status: str = "accepted", + ) -> List[dict]: + return self.generated_db.get_generated_regexes( + regex_type=regex_type, + limit=limit, + status=status, + ) + + def get_generated_regexes_count( + self, + regex_type: str | None = None, + status: str = "accepted", + ) -> int: + return self.generated_db.get_generated_regexes_count( + regex_type=regex_type, + status=status, + ) + + def get_existing_generated_regex(self, regex_hash: str) -> dict | None: + return self.generated_db.get_by_hash(regex_hash) + + def might_have_generated_regex(self, regex_hash: str) -> bool: + return ( + regex_hash in self.generated_regex_filter + or regex_hash in self.rejected_regex_filter + ) + + def was_rejected_in_current_run(self, regex_hash: str) -> bool: + return regex_hash in self.rejected_regex_filter + + def store_generated_regex(self, record: dict): + regex_hash = record["regex_hash"] + status = record.get("status", "") + + if status == "rejected": + self.rejected_regex_filter.add(regex_hash) + if not self.store_rejected_regexes: + return + + self.generated_db.insert_generated_regex(record) + self.generated_regex_filter.add(regex_hash) + if status == "rejected" and self.max_stored_rejected_regexes > 0: + self.generated_db.prune_rejected_regexes( + self.max_stored_rejected_regexes + ) + self.generated_regex_filter = self._build_generated_regex_filter() + + def close(self): + self.benign_db.close() + self.generated_db.close() diff --git a/slips_files/core/database/sqlite_db/t_cell_db.py b/slips_files/core/database/sqlite_db/t_cell_db.py new file mode 100644 index 0000000000..f053d1af5c --- /dev/null +++ b/slips_files/core/database/sqlite_db/t_cell_db.py @@ -0,0 +1,662 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +import os +from pathlib import Path +from time import time + +from slips_files.common.abstracts.isqlite import ISQLite +from slips_files.common.output_paths import ( + get_this_filepath_inside_permanent_dir, +) +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.printer import Printer +from slips_files.core.output import Output + +DEFAULT_T_CELL_STORE_DIR = "output/t_cell" + + +class _BaseTCellSQLiteDB(ISQLite): + name = "BaseTCellSQLiteDB" + + def __init__(self, logger: Output, db_path: str, main_pid: int): + self.printer = Printer(logger, self.name) + self.db_path = db_path + self._init_db_file() + super().__init__(self.name.lower(), main_pid, db_path) + self.init_tables() + + def _init_db_file(self): + db_file = Path(self.db_path) + db_file.parent.mkdir(parents=True, exist_ok=True) + if not db_file.exists(): + db_file.touch() + os.chmod(db_file, 0o777) + + @staticmethod + def _loads(value: str, fallback): + try: + return json.loads(value) + except (TypeError, ValueError): + return fallback + + +class TCellSQLiteDB(_BaseTCellSQLiteDB): + name = "TCellSQLiteDB" + + def init_tables(self): + self.create_table( + "observations", + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + "evidence_id TEXT NOT NULL, " + "evidence_type TEXT NOT NULL, " + "evidence_signal TEXT NOT NULL, " + "profile_ip TEXT NOT NULL, " + "timewindow_number INTEGER NOT NULL, " + "timestamp TEXT NOT NULL, " + "observed_at REAL NOT NULL, " + "confidence REAL NOT NULL, " + "threat_level TEXT NOT NULL, " + "threat_level_value REAL NOT NULL, " + "interface TEXT, " + "uid_json TEXT NOT NULL, " + "antigen_count INTEGER NOT NULL, " + "antigens_json TEXT NOT NULL, " + "matched_regexes_json TEXT NOT NULL, " + "raw_evidence_json TEXT NOT NULL", + ) + self.create_table( + "cells", + "cell_key TEXT PRIMARY KEY, " + "profile_ip TEXT NOT NULL, " + "regex_type TEXT NOT NULL, " + "antigen_value TEXT NOT NULL, " + "state INTEGER NOT NULL, " + "state_name TEXT NOT NULL, " + "matched_regex_hash TEXT, " + "matched_regex TEXT, " + "matched_value TEXT, " + "anergic_until REAL, " + "effector_cooldown_until REAL, " + "last_observation_id INTEGER, " + "last_evidence_id TEXT, " + "last_transition_at REAL, " + "last_co_stimulation REAL, " + "last_effector_score REAL, " + "last_memory_score REAL, " + "context_json TEXT NOT NULL, " + "created_at REAL NOT NULL, " + "updated_at REAL NOT NULL", + ) + self.create_table( + "transitions", + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + "cell_key TEXT NOT NULL, " + "profile_ip TEXT NOT NULL, " + "regex_type TEXT NOT NULL, " + "antigen_value TEXT NOT NULL, " + "evidence_id TEXT NOT NULL, " + "observation_id INTEGER, " + "from_state INTEGER, " + "to_state INTEGER, " + "reason TEXT NOT NULL, " + "matched_regex_hash TEXT, " + "matched_regex TEXT, " + "matched_value TEXT, " + "scores_json TEXT NOT NULL, " + "created_at REAL NOT NULL", + ) + self.create_table( + "memories", + "cell_key TEXT PRIMARY KEY, " + "profile_ip TEXT NOT NULL, " + "regex_type TEXT NOT NULL, " + "antigen_value TEXT NOT NULL, " + "regex_hash TEXT NOT NULL, " + "regex TEXT NOT NULL, " + "matched_value TEXT NOT NULL, " + "context_json TEXT NOT NULL, " + "created_at REAL NOT NULL, " + "updated_at REAL NOT NULL", + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_tcell_observations_profile_time " + "ON observations (profile_ip, observed_at)" + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_tcell_observations_signal_time " + "ON observations (evidence_signal, observed_at)" + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_tcell_cells_profile_type " + "ON cells (profile_ip, regex_type)" + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_tcell_cells_regex_hash " + "ON cells (matched_regex_hash)" + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_tcell_transitions_cell_time " + "ON transitions (cell_key, created_at)" + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_tcell_transitions_regex_time " + "ON transitions (matched_regex_hash, profile_ip, created_at)" + ) + self.execute( + "CREATE INDEX IF NOT EXISTS idx_tcell_memories_regex_hash " + "ON memories (regex_hash)" + ) + + def insert_observation(self, record: dict) -> int: + cursor = self.execute( + "INSERT INTO observations (" + "evidence_id, evidence_type, evidence_signal, profile_ip, " + "timewindow_number, timestamp, observed_at, confidence, " + "threat_level, threat_level_value, interface, uid_json, " + "antigen_count, antigens_json, matched_regexes_json, raw_evidence_json" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + record["evidence_id"], + record["evidence_type"], + record["evidence_signal"], + record["profile_ip"], + record["timewindow_number"], + record["timestamp"], + record["observed_at"], + record["confidence"], + record["threat_level"], + record["threat_level_value"], + record.get("interface"), + json.dumps(record.get("uids", [])), + int(record.get("antigen_count", 0)), + json.dumps(record.get("antigens", [])), + json.dumps(record.get("matched_regexes", [])), + json.dumps(record.get("raw_evidence", {})), + ), + ) + return cursor.lastrowid if cursor else 0 + + def update_observation_matches( + self, observation_id: int, matched_regexes: list[dict] + ): + self.execute( + "UPDATE observations SET matched_regexes_json = ? WHERE id = ?", + (json.dumps(matched_regexes or []), observation_id), + ) + + @staticmethod + def _row_to_observation(row) -> dict: + return { + "id": row[0], + "evidence_id": row[1], + "evidence_type": row[2], + "evidence_signal": row[3], + "profile_ip": row[4], + "timewindow_number": row[5], + "timestamp": row[6], + "observed_at": row[7], + "confidence": row[8], + "threat_level": row[9], + "threat_level_value": row[10], + "interface": row[11], + "uids": _BaseTCellSQLiteDB._loads(row[12], []), + "antigen_count": row[13], + "antigens": _BaseTCellSQLiteDB._loads(row[14], []), + "matched_regexes": _BaseTCellSQLiteDB._loads(row[15], []), + "raw_evidence": _BaseTCellSQLiteDB._loads(row[16], {}), + } + + def get_observation(self, observation_id: int) -> dict | None: + row = self.select( + "observations", + condition="id = ?", + params=(observation_id,), + limit=1, + ) + if not row: + return None + return self._row_to_observation(row) + + def get_recent_observations( + self, + profile_ip: str, + since_ts: float, + until_ts: float | None = None, + evidence_signal: str | None = None, + ) -> list[dict]: + condition_parts = ["profile_ip = ?", "observed_at >= ?"] + params = [profile_ip, since_ts] + if until_ts is not None: + condition_parts.append("observed_at < ?") + params.append(until_ts) + if evidence_signal: + condition_parts.append("evidence_signal = ?") + params.append(evidence_signal) + + rows = self.select( + "observations", + condition=" AND ".join(condition_parts), + params=tuple(params), + order_by="observed_at DESC, id DESC", + ) + rows = rows or [] + return [self._row_to_observation(row) for row in rows] + + def prune_observations(self, created_before: float): + self.execute( + "DELETE FROM observations WHERE observed_at < ?", (created_before,) + ) + + @staticmethod + def _row_to_cell(row) -> dict: + return { + "cell_key": row[0], + "profile_ip": row[1], + "regex_type": row[2], + "antigen_value": row[3], + "state": row[4], + "state_name": row[5], + "matched_regex_hash": row[6], + "matched_regex": row[7], + "matched_value": row[8], + "anergic_until": row[9], + "effector_cooldown_until": row[10], + "last_observation_id": row[11], + "last_evidence_id": row[12], + "last_transition_at": row[13], + "last_co_stimulation": row[14], + "last_effector_score": row[15], + "last_memory_score": row[16], + "context": _BaseTCellSQLiteDB._loads(row[17], {}), + "created_at": row[18], + "updated_at": row[19], + } + + def get_cell(self, cell_key: str) -> dict | None: + row = self.select( + "cells", + condition="cell_key = ?", + params=(cell_key,), + limit=1, + ) + if not row: + return None + return self._row_to_cell(row) + + def get_all_cells(self) -> list[dict]: + rows = self.select("cells", order_by="updated_at DESC") or [] + return [self._row_to_cell(row) for row in rows] + + def get_cells_for_profile_states( + self, profile_ip: str, states: list[int] | tuple[int, ...] + ) -> list[dict]: + normalized_states = [ + int(state) for state in (states or []) if state is not None + ] + if not normalized_states: + return [] + + placeholders = ", ".join("?" for _ in normalized_states) + rows = self.select( + "cells", + condition=( + f"profile_ip = ? AND state IN ({placeholders})" + ), + params=(profile_ip, *normalized_states), + order_by="updated_at DESC, created_at DESC", + ) + rows = rows or [] + return [self._row_to_cell(row) for row in rows] + + def upsert_cell(self, record: dict): + self.execute( + "INSERT OR REPLACE INTO cells (" + "cell_key, profile_ip, regex_type, antigen_value, state, state_name, " + "matched_regex_hash, matched_regex, matched_value, anergic_until, " + "effector_cooldown_until, last_observation_id, last_evidence_id, " + "last_transition_at, last_co_stimulation, last_effector_score, " + "last_memory_score, context_json, created_at, updated_at" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + record["cell_key"], + record["profile_ip"], + record["regex_type"], + record["antigen_value"], + record["state"], + record["state_name"], + record.get("matched_regex_hash"), + record.get("matched_regex"), + record.get("matched_value"), + record.get("anergic_until"), + record.get("effector_cooldown_until"), + record.get("last_observation_id"), + record.get("last_evidence_id"), + record.get("last_transition_at"), + record.get("last_co_stimulation"), + record.get("last_effector_score"), + record.get("last_memory_score"), + json.dumps(record.get("context", {})), + record["created_at"], + record["updated_at"], + ), + ) + + @staticmethod + def _row_to_transition(row) -> dict: + return { + "id": row[0], + "cell_key": row[1], + "profile_ip": row[2], + "regex_type": row[3], + "antigen_value": row[4], + "evidence_id": row[5], + "observation_id": row[6], + "from_state": row[7], + "to_state": row[8], + "reason": row[9], + "matched_regex_hash": row[10], + "matched_regex": row[11], + "matched_value": row[12], + "scores": _BaseTCellSQLiteDB._loads(row[13], {}), + "created_at": row[14], + } + + def insert_transition(self, record: dict) -> int: + cursor = self.execute( + "INSERT INTO transitions (" + "cell_key, profile_ip, regex_type, antigen_value, evidence_id, " + "observation_id, from_state, to_state, reason, matched_regex_hash, " + "matched_regex, matched_value, scores_json, created_at" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + record["cell_key"], + record["profile_ip"], + record["regex_type"], + record["antigen_value"], + record["evidence_id"], + record.get("observation_id"), + record.get("from_state"), + record.get("to_state"), + record["reason"], + record.get("matched_regex_hash"), + record.get("matched_regex"), + record.get("matched_value"), + json.dumps(record.get("scores", {})), + record.get("created_at") or time(), + ), + ) + return cursor.lastrowid if cursor else 0 + + def get_transitions(self, cell_key: str | None = None) -> list[dict]: + condition = None + params = () + if cell_key: + condition = "cell_key = ?" + params = (cell_key,) + rows = self.select( + "transitions", + condition=condition, + params=params, + order_by="created_at ASC, id ASC", + ) + rows = rows or [] + return [self._row_to_transition(row) for row in rows] + + def has_recent_regex_activity( + self, + profile_ip: str, + regex_hash: str, + since_ts: float, + exclude_observation_ids: list[int] | tuple[int, ...] | set[int] | None = None, + exclude_observation_id: int | None = None, + ) -> bool: + condition = ( + "profile_ip = ? AND matched_regex_hash = ? AND created_at >= ?" + ) + params = [profile_ip, regex_hash, since_ts] + excluded_ids = set() + for value in exclude_observation_ids or []: + try: + excluded_ids.add(int(value)) + except (TypeError, ValueError): + continue + if exclude_observation_id is not None: + try: + excluded_ids.add(int(exclude_observation_id)) + except (TypeError, ValueError): + pass + if excluded_ids: + placeholders = ",".join("?" for _ in excluded_ids) + condition += ( + " AND (observation_id IS NULL OR observation_id NOT IN (" + + placeholders + + "))" + ) + params.extend(sorted(excluded_ids)) + row = self.select( + "transitions", + columns="id", + condition=condition, + params=tuple(params), + limit=1, + ) + return bool(row) + + @staticmethod + def _row_to_memory(row) -> dict: + return { + "cell_key": row[0], + "profile_ip": row[1], + "regex_type": row[2], + "antigen_value": row[3], + "regex_hash": row[4], + "regex": row[5], + "matched_value": row[6], + "context": _BaseTCellSQLiteDB._loads(row[7], {}), + "created_at": row[8], + "updated_at": row[9], + } + + def upsert_memory(self, record: dict): + self.execute( + "INSERT OR REPLACE INTO memories (" + "cell_key, profile_ip, regex_type, antigen_value, regex_hash, regex, " + "matched_value, context_json, created_at, updated_at" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + record["cell_key"], + record["profile_ip"], + record["regex_type"], + record["antigen_value"], + record["regex_hash"], + record["regex"], + record["matched_value"], + json.dumps(record.get("context", {})), + record["created_at"], + record["updated_at"], + ), + ) + + def has_memory_for_regex(self, regex_hash: str) -> bool: + row = self.select( + "memories", + columns="cell_key", + condition="regex_hash = ?", + params=(regex_hash,), + limit=1, + ) + return bool(row) + + def get_memories(self) -> list[dict]: + rows = self.select("memories", order_by="updated_at DESC") or [] + return [self._row_to_memory(row) for row in rows] + + +class TCellStorage: + def __init__( + self, + logger: Output, + conf, + output_dir: str, + main_pid: int, + ): + self.logger = logger + self.conf = conf + self.output_dir = output_dir + self.main_pid = main_pid + self.store_dir = self._resolve_store_dir() + self.db = TCellSQLiteDB( + self.logger, + str(Path(self.store_dir) / "t_cell.sqlite"), + self.main_pid, + ) + + def _resolve_store_dir(self) -> str: + raw_store_dir, is_persistent = self._read_store_dir() + store_dir = self._normalize_store_dir(raw_store_dir, is_persistent) + store_dir.mkdir(parents=True, exist_ok=True) + return str(store_dir) + + def _normalize_store_dir( + self, raw_store_dir: str, is_persistent: bool = False + ) -> Path: + store_dir = Path(raw_store_dir).expanduser() + if store_dir.is_absolute(): + return store_dir + + relative_parts = list(store_dir.parts) + while relative_parts and relative_parts[0] == ".": + relative_parts = relative_parts[1:] + if is_persistent: + if not relative_parts: + relative_parts = ["t_cell"] + return Path( + get_this_filepath_inside_permanent_dir( + os.path.join(*relative_parts) + ) + ) + if relative_parts and relative_parts[0] == "output": + relative_parts = relative_parts[1:] + if not relative_parts: + relative_parts = ["t_cell"] + return Path(self.output_dir).expanduser().joinpath(*relative_parts) + + def _read_store_dir(self) -> tuple[str, bool]: + persistent_value = self._read_string_config( + "t_cell_persistent_store_dir" + ) + if persistent_value: + return persistent_value, True + + value = self._read_string_config("t_cell_store_dir") + if value: + return value, False + + parser = ConfigParser() + persistent_getter = getattr(parser, "t_cell_persistent_store_dir", None) + if callable(persistent_getter): + try: + persistent_value = persistent_getter() + except TypeError: + persistent_value = None + if isinstance(persistent_value, str) and persistent_value.strip(): + return persistent_value.strip(), True + + parser_getter = getattr(parser, "t_cell_store_dir", None) + if callable(parser_getter): + try: + value = parser_getter() + except TypeError: + value = None + if isinstance(value, str) and value.strip(): + return value.strip(), False + return DEFAULT_T_CELL_STORE_DIR, False + + def _read_string_config(self, method_name: str) -> str | None: + getter = getattr(self.conf, method_name, None) + if not callable(getter): + return None + try: + value = getter() + except TypeError: + return None + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + def insert_observation(self, record: dict) -> int: + return self.db.insert_observation(record) + + def get_observation(self, observation_id: int) -> dict | None: + return self.db.get_observation(observation_id) + + def update_observation_matches( + self, observation_id: int, matched_regexes: list[dict] + ): + self.db.update_observation_matches(observation_id, matched_regexes) + + def get_recent_observations( + self, + profile_ip: str, + since_ts: float, + until_ts: float | None = None, + evidence_signal: str | None = None, + ) -> list[dict]: + return self.db.get_recent_observations( + profile_ip, + since_ts, + until_ts=until_ts, + evidence_signal=evidence_signal, + ) + + def prune_observations(self, created_before: float): + self.db.prune_observations(created_before) + + def get_cell(self, cell_key: str) -> dict | None: + return self.db.get_cell(cell_key) + + def get_all_cells(self) -> list[dict]: + return self.db.get_all_cells() + + def get_cells_for_profile_states( + self, profile_ip: str, states: list[int] | tuple[int, ...] + ) -> list[dict]: + return self.db.get_cells_for_profile_states(profile_ip, states) + + def upsert_cell(self, record: dict): + self.db.upsert_cell(record) + + def insert_transition(self, record: dict) -> int: + return self.db.insert_transition(record) + + def get_transitions(self, cell_key: str | None = None) -> list[dict]: + return self.db.get_transitions(cell_key) + + def has_recent_regex_activity( + self, + profile_ip: str, + regex_hash: str, + since_ts: float, + exclude_observation_ids: list[int] | tuple[int, ...] | set[int] | None = None, + exclude_observation_id: int | None = None, + ) -> bool: + return self.db.has_recent_regex_activity( + profile_ip, + regex_hash, + since_ts, + exclude_observation_ids=exclude_observation_ids, + exclude_observation_id=exclude_observation_id, + ) + + def upsert_memory(self, record: dict): + self.db.upsert_memory(record) + + def has_memory_for_regex(self, regex_hash: str) -> bool: + return self.db.has_memory_for_regex(regex_hash) + + def get_memories(self) -> list[dict]: + return self.db.get_memories() + + def close(self): + self.db.close() diff --git a/slips_files/core/evidence_handler.py b/slips_files/core/evidence_handler.py index bd79dff769..55b55103b3 100644 --- a/slips_files/core/evidence_handler.py +++ b/slips_files/core/evidence_handler.py @@ -20,21 +20,28 @@ # Contact: eldraco@gmail.com, sebastian.garcia@agents.fel.cvut.cz, # stratosphere@aic.fel.cvut.cz -import threading +import json import multiprocessing -from typing import List +import threading import time +from typing import List from multiprocessing import Process +from slips_files.common.idmefv2 import IDMEFv2 +from slips_files.common.abstracts.icore import ICore from slips_files.common.output_paths import get_alerts_path_inside_output_dir +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import IS_IN_A_DOCKER_CONTAINER, utils from slips_files.common.style import ( green, ) -from slips_files.common.parsers.config_parser import ConfigParser -from slips_files.common.slips_utils import utils from slips_files.core.evidence_logger import EvidenceLogger -from slips_files.common.abstracts.icore import ICore +from slips_files.core.helpers.notify import Notify +from slips_files.core.structures.alerts import Alert +from slips_files.core.text_formatters.evidence_formatter import ( + EvidenceFormatter, +) from slips_files.core.evidence_handler_worker import EvidenceHandlerWorker @@ -44,9 +51,24 @@ # Evidence Process class EvidenceHandler(ICore): name = "evidence_handler" + is_evidence_done_by_others = ( + EvidenceHandlerWorker.is_evidence_done_by_others + ) + is_filtered_evidence = EvidenceHandlerWorker.is_filtered_evidence + get_threat_level = EvidenceHandlerWorker.get_threat_level + send_to_exporting_module = ( + EvidenceHandlerWorker.send_to_exporting_module + ) + is_blocking_modules_supported = ( + EvidenceHandlerWorker.is_blocking_modules_supported + ) + show_popup = EvidenceHandlerWorker.show_popup def init(self): self.read_configuration() + self.idmefv2 = IDMEFv2(self.logger, self.db) + self.formatter = EvidenceFormatter(self.db, self.args) + self.is_running_non_stop = self.db.is_running_non_stop() # to keep track of the number of generated evidence self.db.init_evidence_number() # thats just a tmp value, this variable will be set and used when @@ -77,6 +99,17 @@ def init(self): ) utils.start_thread(self.logger_thread, self.db) + conf = ConfigParser() + self.exporting_modules_enabled = ( + conf.export_to() or conf.send_to_warden() + ) + if self.popup_alerts: + self.notify = Notify() + if self.notify.bin_found: + self.notify.setup_notifications() + else: + self.popup_alerts = False + def subscribe_to_channels(self): self.c1 = self.db.subscribe("evidence_added") self.c2 = self.db.subscribe("new_blame") @@ -97,6 +130,113 @@ def read_configuration(self): 2, 0, ) + self.GID = conf.get_GID() + self.UID = conf.get_UID() + + self.popup_alerts = conf.popup_alerts() + # In docker, disable alerts no matter what slips.yaml says + if IS_IN_A_DOCKER_CONTAINER: + self.popup_alerts = False + + def handle_unable_to_log(self, failed_log, error=None): + self.print(f"Error logging evidence/alert: {error}. {failed_log}.") + + def add_alert_to_json_log_file(self, alert: Alert): + """ + Add a new alert/event line to our alerts.json file in json format. + """ + idmef_alert: dict = self.idmefv2.convert_to_idmef_alert(alert) + if not idmef_alert: + self.handle_unable_to_log(alert, "Can't convert to IDMEF alert") + return + + to_log = { + "to_log": idmef_alert, + "where": "alerts.json", + } + self.evidence_logger_q.put(to_log) + + def add_evidence_to_json_log_file( + self, + evidence, + accumulated_threat_level: float = 0, + ): + """ + Add a new evidence line to our alerts.json file in json format. + """ + idmef_evidence: dict = self.idmefv2.convert_to_idmef_event(evidence) + if not idmef_evidence: + self.handle_unable_to_log( + evidence, "Can't convert to IDMEF evidence" + ) + return + + try: + idmef_evidence.update( + { + "Note": json.dumps( + { + # this is all the uids of the flows that cause + # this evidence + "uids": evidence.uid, + "accumulated_threat_level": accumulated_threat_level, + "threat_level": str(evidence.threat_level), + "evidence_signal": str( + evidence.evidence_signal + ), + "timewindow": evidence.timewindow.number, + } + ) + } + ) + + to_log = { + "to_log": idmef_evidence, + "where": "alerts.json", + } + + self.evidence_logger_q.put(to_log) + + except KeyboardInterrupt: + return True + except Exception as e: + self.handle_unable_to_log(evidence, e) + + def add_to_log_file(self, data: str): + """ + Add a new evidence line to the alerts.log and other log files if + logging is enabled. + """ + to_log = {"to_log": data, "where": "alerts.log"} + self.evidence_logger_q.put(to_log) + + def log_alert(self, alert: Alert, blocked=False): + """ + constructs the alert descript ion from the given alert and logs it + to alerts.log and alerts.json + :param blocked: bool. if the ip was blocked by the blocking module, + we should say so in alerts.log, if not, we should say that + we generated an alert + """ + now = utils.get_human_readable_datetime() + + alert_description = ( + f"{alert.last_flow_datetime}: " f"Src IP {alert.profile.ip:26}. " + ) + if blocked: + # Add to log files that this srcip is being blocked + alert_description += "Is blocked " + else: + alert_description += "Generated an alert " + + alert_description += ( + f"given enough evidence on timewindow " + f"{alert.timewindow.number}. (real time {now})" + ) + # log to alerts.log + self.add_to_log_file(alert_description) + # log to alerts.json + self.add_alert_to_json_log_file(alert) def shutdown_gracefully(self): self.stop_evidence_workers() @@ -121,7 +261,16 @@ def stop_evidence_workers(self): for process in self.evidence_worker_child_processes: try: - process.join() + process.join(timeout=5) + if process.is_alive(): + self.print( + f"Evidence worker {process.pid} did not stop in time. " + "Killing it.", + 0, + 1, + ) + process.kill() + process.join(timeout=1) except (OSError, ChildProcessError): pass diff --git a/slips_files/core/evidence_handler_worker.py b/slips_files/core/evidence_handler_worker.py index b2b7dac44e..980cf928b0 100644 --- a/slips_files/core/evidence_handler_worker.py +++ b/slips_files/core/evidence_handler_worker.py @@ -409,6 +409,22 @@ def pre_main(self): def should_stop(self) -> bool: return False + def shutdown_gracefully(self): + """ + Release queue handles so the worker does not hang during process + finalization while waiting on queue feeder threads. + """ + for q in (self.evidence_queue, self.evidence_logger_q): + try: + q.cancel_join_thread() + except (AttributeError, OSError, ValueError): + pass + + try: + q.close() + except (AttributeError, OSError, ValueError): + pass + def handle_evidence_added_message(self, msg: dict): evidence = json.loads(msg["data"]) try: diff --git a/slips_files/core/input_profilers/zeek.py b/slips_files/core/input_profilers/zeek.py index c854523bd2..06b4505fa9 100644 --- a/slips_files/core/input_profilers/zeek.py +++ b/slips_files/core/input_profilers/zeek.py @@ -87,20 +87,14 @@ class Zeek: def remove_subsuffix(self, file_name: str) -> str: """ - turns any x.log.y to x.log - turns any x..log to x.log - """ + Normalize Zeek filenames to their base ``.log`` form. - # is it something like notice.13:00:00-14:00:00.log? + This collapses rotated logs such as ``notice.13:00:00-14:00:00.log`` + and labeled logs such as ``notice.log.labeled``. + """ splitted_filename = file_name.split(".") - if len(splitted_filename) == 3: - if splitted_filename[-1] == "log": - return splitted_filename[0] + ".log" - - # its something.log - elif len(splitted_filename) == 2 and ".log" in file_name: - return file_name.split(".log")[0] + ".log" - + if len(splitted_filename) > 1 and "log" in splitted_filename[1:]: + return splitted_filename[0] + ".log" return file_name def get_file_type(self, new_line: dict) -> str: diff --git a/slips_files/core/structures/evidence.py b/slips_files/core/structures/evidence.py index beaf4701ac..55eaff8640 100644 --- a/slips_files/core/structures/evidence.py +++ b/slips_files/core/structures/evidence.py @@ -85,6 +85,7 @@ class EvidenceType(Enum): BAD_SMTP_LOGIN = auto() SMTP_LOGIN_BRUTEFORCE = auto() MALICIOUS_SSL_CERT = auto() + ANOMALOUS_FLOW = auto() MALICIOUS_FLOW = auto() SUSPICIOUS_USER_AGENT = auto() EMPTY_CONNECTIONS = auto() @@ -146,6 +147,14 @@ def __str__(self): return self.name.lower() +class EvidenceSignal(Enum): + PAMP = "PAMP" + DAMP = "DAMP" + + def __str__(self): + return self.name + + class Proto(Enum): TCP = "tcp" UDP = "udp" @@ -311,6 +320,7 @@ class Evidence: ) }, ) + evidence_signal: EvidenceSignal = field(default=EvidenceSignal.PAMP) def __post_init__(self): if not isinstance(self.uid, list) or not all( @@ -339,6 +349,7 @@ def __str__(self): f" ID: {self.id},\n" f" Confidence: {self.confidence},\n" f" Related ID: {self.rel_id}\n" + f" Evidence Signal: {self.evidence_signal}\n" f")" ) @@ -349,6 +360,13 @@ def dict_to_evidence(evidence: dict) -> Evidence: :param evidence: Dictionary with evidence details. returns an instance of the Evidence class. """ + try: + evidence_signal = EvidenceSignal[ + str(evidence.get("evidence_signal", "PAMP")).upper() + ] + except KeyError: + evidence_signal = EvidenceSignal.PAMP + evidence_attributes = { "evidence_type": EvidenceType[evidence["evidence_type"]], "description": evidence["description"], @@ -379,6 +397,7 @@ def dict_to_evidence(evidence: dict) -> Evidence: "rel_id": evidence["rel_id"], "confidence": evidence["confidence"], "method": Method[evidence["method"].upper()], + "evidence_signal": evidence_signal, } return Evidence(**evidence_attributes) diff --git a/slips_files/logs_analysis/analyze_incidents.py b/slips_files/logs_analysis/analyze_incidents.py new file mode 100644 index 0000000000..3cfae92bd5 --- /dev/null +++ b/slips_files/logs_analysis/analyze_incidents.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path +import glob + +# --- Colors --- +RESET = "\033[0m" +BOLD = "\033[1m" +CYAN = "\033[96m" +GREEN = "\033[92m" +YELLOW = "\033[93m" +RED = "\033[91m" +MAGENTA = "\033[95m" +GRAY = "\033[90m" +BLUE = "\033[94m" + +def usage(): + print(f"Usage: {sys.argv[0]} [--debug]") + print(" : 'incident' or 'event'") + sys.exit(1) + +def load_jsonl(path): + """Yield parsed JSON objects (line-delimited JSON).""" + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + try: + yield json.loads(line) + except json.JSONDecodeError: + continue + +def load_zeek_log(path): + """Load a Zeek log file (JSON or TSV) and index by UID or UIDs.""" + flows = {} + headers = set() + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + + flow = None + + # JSON log (modern Zeek format) + if line.startswith("{"): + try: + flow = json.loads(line) + except json.JSONDecodeError: + continue + else: + # TSV fallback + if line.startswith("#fields"): + headers.update(line.split()[1:]) + continue + parts = line.split("\t") + if headers and len(parts) == len(headers): + flow = dict(zip(list(headers), parts)) + + if not flow: + continue + + headers.update(flow.keys()) + + # Normalize UID handling + uids = [] + if "uid" in flow: + uids = [flow["uid"]] + elif "uids" in flow and isinstance(flow["uids"], list): + uids = flow["uids"] + + for uid in uids: + if uid: + flows.setdefault(uid.strip(), []).append(flow) + except Exception as e: + print(f"{RED}Error parsing {path}:{RESET} {e}") + return flows, sorted(headers) + +def parse_note_uids(note_str): + """Extract UIDs from the Note JSON string inside an Event.""" + if not note_str: + return [] + try: + note = json.loads(note_str) + if isinstance(note, str): + note = json.loads(note) + if isinstance(note, dict): + return note.get("uids", []) + except Exception: + pass + return [] + +def show_event(event, log_files, logs_data, debug=False): + """Show an event and all matching Zeek flows with all columns per log type.""" + eid = event.get("ID") + desc = event.get("Description", "").replace("\n", " ").strip() + sev = event.get("Severity", "Unknown") + src_ips = ", ".join(sv.get("IP") for sv in event.get("Source", []) if sv.get("IP")) + uids = parse_note_uids(event.get("Note", "{}")) + + sev_color = {"Low": GREEN, "Medium": YELLOW, "High": RED, "Critical": MAGENTA}.get(sev, RESET) + + print(f"{BOLD}{CYAN}Event:{RESET} {eid}") + print(f" {BOLD}Severity:{RESET} {sev_color}{sev}{RESET}") + print(f" {BOLD}Source IP(s):{RESET} {src_ips}") + print(f" {BOLD}Description:{RESET} {desc}") + print(f" {BOLD}UIDs from Note:{RESET} {uids if uids else '(none)'}") + + if not uids: + print(f" {GRAY}(no flow UIDs in Note){RESET}") + print(f"{GRAY}{'-'*120}{RESET}") + return + + # --- Search all Zeek logs for these UIDs --- + matched = [] + for lf in log_files: + for uid in uids: + if uid in logs_data[lf]["flows"]: + for row in logs_data[lf]["flows"][uid]: + matched.append((row, Path(lf).name)) + + if not matched: + print(f" {GRAY}(no matching flows found in Zeek logs){RESET}") + print(f"{GRAY}{'-'*120}{RESET}") + return + + print(f"\n {BOLD}{MAGENTA}Flows found:{RESET} {len(matched)}") + print(f"{GRAY}{'-'*120}{RESET}") + + # Group by log file + by_file = {} + for flow, fname in matched: + by_file.setdefault(fname, []).append(flow) + + for fname, flows in by_file.items(): + print(f"{BOLD}{BLUE}{fname}:{RESET}") + all_fields = sorted({k for f in flows for k in f.keys()}) + widths = {k: len(k) for k in all_fields} + for f in flows: + for k in all_fields: + val = str(f.get(k, "-")) + if len(val) > widths[k]: + widths[k] = min(len(val), 80) # avoid super wide columns + + # Header + header_line = " " + " ".join(f"{CYAN}{BOLD}{h.ljust(widths[h])}{RESET}" for h in all_fields) + print(header_line) + print(" " + "-" * (len(header_line) - 2)) + + # Rows + for f in sorted(flows, key=lambda x: float(x.get("ts", 0)) if "ts" in x else 0): + row = " " + " ".join( + str(f.get(h, "-"))[:widths[h]].ljust(widths[h]) for h in all_fields + ) + print(row) + print(f"{GRAY}{'-'*120}{RESET}") + +def main(): + if len(sys.argv) < 5: + usage() + + alerts_file = Path(sys.argv[1]) + mode = sys.argv[2].lower() + target_id = sys.argv[3] + zeek_folder = Path(sys.argv[4]) + debug = "--debug" in sys.argv + + if mode not in ("incident", "event"): + usage() + + if not alerts_file.exists(): + sys.exit(f"{RED}Alerts file not found:{RESET} {alerts_file}") + if not zeek_folder.exists(): + sys.exit(f"{RED}Zeek folder not found:{RESET} {zeek_folder}") + + # --- Load alerts --- + incidents, events = [], [] + for obj in load_jsonl(alerts_file): + if obj.get("Status") == "Incident": + incidents.append(obj) + elif obj.get("Status") == "Event": + events.append(obj) + + # --- Load all .log files (recursively) --- + log_files = sorted(glob.glob(str(zeek_folder / "**" / "*.log"), recursive=True)) + if not log_files: + sys.exit(f"{RED}No .log files found in folder:{RESET} {zeek_folder}") + + logs_data = {} + for lf in log_files: + flows, headers = load_zeek_log(lf) + logs_data[lf] = {"flows": flows, "headers": headers} + if debug: + print(f"{GRAY}Loaded {len(flows)} UIDs from {Path(lf).name}{RESET}") + if headers: + print(f" {BOLD}{BLUE}Columns ({len(headers)}):{RESET} {', '.join(sorted(headers))}\n") + + # --- Main logic --- + if mode == "incident": + incident = next((i for i in incidents if i.get("ID") == target_id), None) + if not incident: + sys.exit(f"{RED}Incident {target_id} not found.{RESET}") + + correl_ids = set(incident.get("CorrelID", [])) + related_events = [e for e in events if e.get("ID") in correl_ids] + + print(f"\n{BOLD}{CYAN}Incident:{RESET} {target_id}") + print(f"{GRAY}{'-'*120}{RESET}") + + if not related_events: + print(f"{YELLOW}(No related events found){RESET}") + return + + for ev in related_events: + show_event(ev, log_files, logs_data, debug=debug) + + elif mode == "event": + event = next((e for e in events if e.get("ID") == target_id), None) + if not event: + sys.exit(f"{RED}Event {target_id} not found.{RESET}") + + print(f"\n{BOLD}{CYAN}Analyzing single Event:{RESET} {target_id}") + print(f"{GRAY}{'-'*120}{RESET}") + show_event(event, log_files, logs_data, debug=debug) + +if __name__ == "__main__": + main() diff --git a/tests/integration/test_fides/fides_config.yaml b/tests/integration/test_fides/fides_config.yaml index 00618fd6fb..0178f37579 100644 --- a/tests/integration/test_fides/fides_config.yaml +++ b/tests/integration/test_fides/fides_config.yaml @@ -353,7 +353,7 @@ DisabledAlerts: # MULTIPLE_RECONNECTION_ATTEMPTS, CONNECTION_TO_MULTIPLE_PORTS, HIGH_ENTROPY_DNS_ANSWER, # INVALID_DNS_RESOLUTION, PORT_0_CONNECTION, MALICIOUS_JA3, MALICIOUS_JA3S, # DATA_UPLOAD, BAD_SMTP_LOGIN, SMTP_LOGIN_BRUTEFORCE, MALICIOUS_SSL_CERT, - # MALICIOUS_FLOW, SUSPICIOUS_USER_AGENT, EMPTY_CONNECTIONS, INCOMPATIBLE_USER_AGENT, + # ANOMALOUS_FLOW, MALICIOUS_FLOW, SUSPICIOUS_USER_AGENT, EMPTY_CONNECTIONS, INCOMPATIBLE_USER_AGENT, # EXECUTABLE_MIME_TYPE, MULTIPLE_USER_AGENT, HTTP_TRAFFIC, MALICIOUS_JARM, # NETWORK_GPS_LOCATION_LEAKED, ICMP_TIMESTAMP_SCAN, ICMP_ADDRESS_SCAN, # ICMP_ADDRESS_MASK_SCAN, DHCP_SCAN, MALICIOUS_IP_FROM_P2P_NETWORK, P2P_REPORT, diff --git a/tests/module_factory.py b/tests/module_factory.py index d434261551..a2e77dc313 100644 --- a/tests/module_factory.py +++ b/tests/module_factory.py @@ -71,6 +71,13 @@ def create_db_manager_obj( conf = Mock() conf.delete_prev_db = Mock(return_value=False) conf.disabled_detections = Mock(return_value=[]) + conf.evidence_signal_default = Mock(return_value="PAMP") + conf.evidence_signal_overrides = Mock( + return_value={ + "ANOMALOUS_FLOW": "DAMP", + "MALICIOUS_FLOW": "DAMP", + } + ) conf.get_tw_width_as_float = Mock(return_value=3600.0) conf.get_tw_width_in_seconds = Mock(return_value=3600) conf.get_args = Mock(return_value=Mock(killall=False)) @@ -78,6 +85,20 @@ def create_db_manager_obj( conf.use_local_p2p = Mock(return_value=False) conf.permanent_dir = Mock(return_value="permanent") conf.width = Mock(return_value=3600) + conf.regex_generator_store_dir = Mock( + return_value=os.path.join(output_dir, "regex_generator") + ) + conf.regex_generator_persistent_store_dir = Mock(return_value="") + conf.regex_generator_store_rejected_regexes = Mock(return_value=False) + conf.regex_generator_max_stored_rejected_regexes = Mock( + return_value=10000 + ) + conf.regex_generator_seed_benign_samples = Mock(return_value=True) + conf.t_cell_store_dir = Mock( + return_value=os.path.join(output_dir, "t_cell") + ) + conf.t_cell_persistent_store_dir = Mock(return_value="") + conf.tranco_top_benign_limit = Mock(return_value=1000) with ( # to prevent config/redis.conf from being overwritten @@ -152,6 +173,250 @@ def create_http_analyzer_obj(self, mock_db): http_analyzer.print = Mock() return http_analyzer + @patch(MODULE_DB_MANAGER, name="mock_db") + def create_llm_obj(self, mock_db): + from modules.llm_proxy.llm_proxy import LLMProxy + + conf = Mock() + conf.llm_enabled = Mock(return_value=True) + conf.llm_default_backend = Mock(return_value="local_qwen") + conf.llm_worker_threads = Mock(return_value=1) + conf.llm_queue_size = Mock(return_value=10) + conf.llm_backends = Mock( + return_value={ + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b", + "base_url": "http://127.0.0.1:11434", + "timeout": 60, + } + } + ) + + llm = LLMProxy( + logger=self.logger, + output_dir="dummy_output_dir", + redis_port=6379, + termination_event=Mock(), + slips_args=Mock(), + conf=conf, + ppid=Mock(), + bloom_filters_manager=Mock(), + ) + llm.db.channels.LLM_REQUEST = "llm_request" + llm.db.channels.LLM_RESPONSE = "llm_response" + llm.channels = {"llm_request": llm.c1} + llm.db.reset_pending_llm_request_counts = Mock() + llm.db.increment_pending_llm_request_count = Mock(return_value=1) + llm.db.decrement_pending_llm_request_count = Mock(return_value=0) + llm.db.get_pid_of = Mock(return_value=None) + llm.print = Mock() + return llm + + @patch(MODULE_DB_MANAGER, name="mock_db") + def create_regex_generator_obj( + self, mock_db, store_dir="dummy_output_dir/regex_generator" + ): + from modules.regex_generator.regex_generator import RegexGenerator + + conf = Mock() + conf.regex_generator_enabled = Mock(return_value=True) + conf.regex_generator_create_log_file = Mock(return_value=False) + conf.regex_generator_generation_interval_seconds = Mock(return_value=5) + conf.regex_generator_allowed_backends = Mock( + return_value=["local_qwen"] + ) + conf.regex_generator_llm_temperature = Mock(return_value=1.2) + conf.regex_generator_llm_max_tokens = Mock(return_value=80) + conf.regex_generator_llm_response_timeout_seconds = Mock( + return_value=90 + ) + conf.regex_generator_recent_history_size = Mock(return_value=0) + conf.regex_generator_max_regex_length = Mock(return_value=180) + conf.regex_generator_regex_validation_timeout_seconds = Mock( + return_value=2 + ) + conf.regex_generator_benign_match_strength_threshold = Mock( + return_value=75 + ) + conf.regex_generator_type_weights = Mock( + return_value={ + "dns_domain": 1, + "uri": 1, + "filename": 1, + "tls_sni": 1, + "certificate_cn": 1, + } + ) + conf.regex_generator_store_dir = Mock(return_value=store_dir) + conf.regex_generator_persistent_store_dir = Mock(return_value="") + conf.regex_generator_store_rejected_regexes = Mock(return_value=False) + conf.regex_generator_max_stored_rejected_regexes = Mock( + return_value=10000 + ) + conf.regex_generator_seed_benign_samples = Mock(return_value=True) + conf.tranco_top_benign_limit = Mock(return_value=1000) + conf.rotation = Mock(return_value=True) + conf.default_rotation_interval = Mock(return_value="1day") + conf.rotation_period = Mock(return_value="1day") + + regex_generator = RegexGenerator( + logger=self.logger, + output_dir="dummy_output_dir", + redis_port=6379, + termination_event=Mock(), + slips_args=Mock(), + conf=conf, + ppid=12345, + bloom_filters_manager=Mock(), + ) + regex_generator.db.channels.LLM_REQUEST = "llm_request" + regex_generator.db.channels.LLM_RESPONSE = "llm_response" + regex_generator.channels = { + "llm_response": regex_generator.c_llm, + "tw_closed": regex_generator.c_tw_closed, + } + regex_generator.print = Mock() + return regex_generator + + @patch(MODULE_DB_MANAGER, name="mock_db") + def create_alert_summary_obj(self, mock_db): + from modules.alert_summary.alert_summary import AlertSummary + + conf = Mock() + conf.alert_summary_enabled = Mock(return_value=True) + conf.alert_summary_allowed_backends = Mock(return_value=["local_qwen"]) + conf.alert_summary_log_verbosity = Mock(return_value=2) + conf.alert_summary_llm_temperature = Mock(return_value=0.2) + conf.alert_summary_llm_max_tokens = Mock(return_value=220) + conf.alert_summary_llm_response_timeout_seconds = Mock( + return_value=120 + ) + conf.alert_summary_history_enabled = Mock(return_value=True) + conf.alert_summary_history_max_alerts = Mock(return_value=3) + conf.alert_summary_history_max_tokens = Mock(return_value=700) + conf.alert_summary_history_patterns_per_alert = Mock(return_value=2) + + args = Mock() + args.is_slips_started_by_an_update = False + + alert_summary = AlertSummary( + logger=self.logger, + output_dir="dummy_output_dir", + redis_port=6379, + termination_event=Mock(), + slips_args=args, + conf=conf, + ppid=12345, + bloom_filters_manager=Mock(), + ) + alert_summary.db.channels.LLM_REQUEST = "llm_request" + alert_summary.db.channels.LLM_RESPONSE = "llm_response" + alert_summary.subscribe_to_channels() + alert_summary.db.get_available_llm_backends = Mock( + return_value={ + "default_backend": "local_qwen", + "backends": { + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b", + } + }, + } + ) + alert_summary.db.get_pending_llm_request_count = Mock(return_value=0) + alert_summary.db.get_twid_evidence = Mock(return_value={}) + alert_summary.db.get_hostname_from_profile = Mock(return_value="") + alert_summary.db.get_pid_of = Mock(return_value=None) + alert_summary.print = Mock() + return alert_summary + + @patch(MODULE_DB_MANAGER, name="mock_db") + def create_t_cell_obj(self, mock_db): + from modules.t_cell.t_cell import TCell + + conf = Mock() + conf.t_cell_enabled = Mock(return_value=True) + conf.t_cell_create_log_file = Mock(return_value=True) + conf.t_cell_log_colors = Mock(return_value=True) + conf.t_cell_log_verbosity = Mock(return_value=1) + conf.t_cell_decision_trace_mode = Mock(return_value=0) + conf.t_cell_decision_trace_file = Mock( + return_value="t_cell_trace.jsonl" + ) + conf.t_cell_decision_trace_max_evidence = Mock(return_value=10) + conf.get_tw_width_in_seconds = Mock(return_value=3600.0) + conf.t_cell_store_dir = Mock(return_value="dummy_output_dir/t_cell") + conf.t_cell_persistent_store_dir = Mock(return_value="") + conf.t_cell_observation_retention_seconds = Mock(return_value=604800) + conf.t_cell_anergy_ttl_seconds = Mock(return_value=21600) + conf.t_cell_related_lookback_seconds = Mock(return_value=3600) + conf.t_cell_related_pamps_saturation = Mock(return_value=5.0) + conf.t_cell_danger_saturation = Mock(return_value=2.5) + conf.t_cell_damp_danger_weight = Mock(return_value=1.5) + conf.t_cell_co_stimulation_threshold = Mock(return_value=0.65) + conf.t_cell_co_stimulation_weights = Mock( + return_value={ + "confidence": 0.35, + "related_pamps": 0.25, + "danger": 0.40, + } + ) + conf.t_cell_priming_profiles = Mock( + return_value={ + "PAMP": { + "strength": 1.0, + "co_stimulation_threshold_offset": 0.0, + "effector_threshold_offset": 0.0, + "memory_threshold_offset": 0.0, + "state_wait_timeout_factor": 1.0, + "effector_min_related_count_offset": 0, + "memory_min_related_count_offset": 0, + }, + "DAMP": { + "strength": 0.6, + "co_stimulation_threshold_offset": 0.15, + "effector_threshold_offset": 0.10, + "memory_threshold_offset": 0.05, + "state_wait_timeout_factor": 0.5, + "effector_min_related_count_offset": 1, + "memory_min_related_count_offset": 1, + }, + } + ) + conf.t_cell_novelty_window_seconds = Mock(return_value=86400) + conf.t_cell_context_recent_window_seconds = Mock(return_value=1800) + conf.t_cell_effector_threshold = Mock(return_value=0.70) + conf.t_cell_effector_min_related_count = Mock(return_value=4) + conf.t_cell_effector_cooldown_seconds = Mock(return_value=1800) + conf.t_cell_memory_threshold = Mock(return_value=0.60) + conf.t_cell_memory_trend_ratio_max = Mock(return_value=0.60) + conf.t_cell_memory_min_related_count = Mock(return_value=3) + conf.t_cell_simulate_effector_without_blocking = Mock( + return_value=True + ) + + args = Mock() + args.interface = None + args.access_point = False + + t_cell = TCell( + logger=self.logger, + output_dir="dummy_output_dir", + redis_port=6379, + termination_event=Mock(), + slips_args=args, + conf=conf, + ppid=12345, + bloom_filters_manager=Mock(), + ) + t_cell.db.get_generated_regexes.return_value = [] + t_cell.db.get_altflow_from_uid.return_value = {} + t_cell.db.get_pid_of.return_value = None + t_cell.db.publish = Mock() + t_cell.print = Mock() + return t_cell + @patch(MODULE_DB_MANAGER, name="mock_db") def create_fides_obj(self, mock_db): from modules.fides.fides import FidesModule @@ -1135,6 +1400,11 @@ def create_alert_handler_obj(self): alert_handler.constants = Constants() alert_handler.default_ttl = 3600 alert_handler.extended_ttl = 3600 + alert_handler.default_evidence_signal = "PAMP" + alert_handler.evidence_signal_overrides = { + "ANOMALOUS_FLOW": "DAMP", + "MALICIOUS_FLOW": "DAMP", + } alert_handler.set_profileid_field = Mock() return alert_handler diff --git a/tests/unit/managers/test_process_manager.py b/tests/unit/managers/test_process_manager.py index b15d0e921a..7c4b89b921 100644 --- a/tests/unit/managers/test_process_manager.py +++ b/tests/unit/managers/test_process_manager.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-only import json import signal +import ast import pytest from unittest.mock import Mock, call, patch from managers.process_manager import ProcessManager @@ -127,6 +128,62 @@ def test_is_ignored_module(module_name, modules_to_ignore, expected): assert process_manager.is_disabled_module(module_name) == expected +def test_get_disabled_modules_uses_disabled_module_helpers() -> None: + """Test disabled modules are returned from the dedicated helper methods.""" + process_manager = ModuleFactory().create_process_manager_obj() + process_manager.get_user_disabled_modules = Mock(return_value=["template"]) + process_manager.get_runtime_disabled_modules = Mock( + return_value=["blocking"] + ) + + disabled_modules = process_manager.get_disabled_modules() + + assert disabled_modules == (["template"], ["blocking"]) + process_manager.get_user_disabled_modules.assert_called_once_with() + process_manager.get_runtime_disabled_modules.assert_called_once_with() + + +@pytest.mark.parametrize( + "configured_modules, expected_modules", + [ + ([" template ", "custom_module"], ["template", "custom_module"]), + ([], []), + ], +) +def test_get_user_disabled_modules( + configured_modules: list, expected_modules: list +) -> None: + """Test user-disabled modules are read from config and stripped.""" + process_manager = ModuleFactory().create_process_manager_obj() + process_manager.main.conf.read_configuration.reset_mock() + process_manager.main.conf.read_configuration.side_effect = None + process_manager.main.conf.read_configuration.return_value = ( + configured_modules + ) + + disabled_modules = process_manager.get_user_disabled_modules() + + assert disabled_modules == expected_modules + process_manager.main.conf.read_configuration.assert_called_once_with( + "modules", "disable", ["template"] + ) + + +def test_get_slips_disabled_modules_recomputes_runtime_disabled_modules() -> ( + None +): + """Test runtime-disabled modules are recomputed from current settings.""" + process_manager = ModuleFactory().create_process_manager_obj() + process_manager.main.args.clearblocking = False + process_manager.main.args.blocking = False + process_manager.slips_disabled_modules = ["blocking", "runtime_module"] + + disabled_modules = process_manager.get_runtime_disabled_modules() + + assert disabled_modules.count("blocking") == 1 + assert "runtime_module" not in disabled_modules + + @pytest.mark.parametrize( "input_type, argv, export_to, expected_user, expected_slips", [ @@ -238,10 +295,13 @@ def test_print_disabled_modules(): process_manager.user_disabled_modules = ["Module1", "Module2"] with patch.object(process_manager.main, "print") as mock_print: process_manager.print_disabled_modules() - mock_print.assert_called_once_with( - "Disabled Modules: " "['Module1', 'Module2']", 1, 0 + printed_modules = ast.literal_eval( + mock_print.call_args.args[0].removeprefix("Disabled Modules: ") ) + assert set(printed_modules) == {"Module1", "Module2"} + mock_print.assert_called_once() + @pytest.mark.parametrize( "pending_modules, expected_print_calls", @@ -273,15 +333,15 @@ def test_warn_about_pending_modules(pending_modules, expected_print_calls): "blocking_enabled, exporting_alerts_disabled, " "expected_kill_first, expected_kill_last", [ # Testcase1: blocking enabled, exporting_alerts enabled - (True, False, [1, 2], [3, 4, 5]), + (True, False, [1, 2, 6, 7], [3, 4, 5]), # Testcase2: blocking disabled, exporting_alerts enabled - (False, False, [1, 2, 4], [3, 5]), + (False, False, [1, 2, 4, 6, 7], [3, 5]), # Testcase3: blocking enabled, exporting_alerts disabled - (True, True, [1, 2, 5], [3, 4]), + (True, True, [1, 2, 5, 6, 7], [3, 4]), # Testcase4: blocking disabled, exporting_alerts disabled - (False, True, [1, 2, 4, 5], [3]), + (False, True, [1, 2, 4, 5, 6, 7], [3]), # Testcase5: All enabled, some PIDs are None - (True, False, [1, 2], [3, 4, 5]), + (True, False, [1, 2, 6, 7], [3, 4, 5]), ], ) def test_get_hitlist_in_order( @@ -297,12 +357,16 @@ def test_get_hitlist_in_order( Mock(pid=3, name="evidence_handler"), Mock(pid=4, name="blocking"), Mock(pid=5, name="exporting_alerts"), + Mock(pid=6, name="alert_summary"), + Mock(pid=7, name="LLM"), ] process_manager.main.db.get_pid_of = lambda x: { "evidence_handler": 3, "blocking": 4, "exporting_alerts": 5, + "alert_summary": 6, + "LLM": 7, }.get(x) process_manager.main.args.blocking = blocking_enabled process_manager.main.db.get_disabled_modules = lambda: ( diff --git a/tests/unit/managers/test_redis_manager.py b/tests/unit/managers/test_redis_manager.py index 5200be0a09..55183ce2a5 100644 --- a/tests/unit/managers/test_redis_manager.py +++ b/tests/unit/managers/test_redis_manager.py @@ -882,6 +882,7 @@ def test_close_open_redis_servers_interactive(mock_db): # Mocking user choosing server #1 with ( patch("builtins.input", return_value="1"), + patch("sys.stdin.isatty", return_value=True), patch.object( redis_manager, "print_open_redis_servers", diff --git a/tests/unit/modules/alert_summary/test_alert_summary.py b/tests/unit/modules/alert_summary/test_alert_summary.py new file mode 100644 index 0000000000..ead16e998b --- /dev/null +++ b/tests/unit/modules/alert_summary/test_alert_summary.py @@ -0,0 +1,824 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +import time + +from modules.alert_summary.alert_summary import ( + PROMPT_VERSION, + REDUCTION_SYSTEM_PROMPT, + SYSTEM_PROMPT, +) +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import utils +from slips_files.core.structures.alerts import Alert +from slips_files.core.structures.evidence import ( + Attacker, + Direction, + Evidence, + EvidenceType, + IoCType, + ProfileID, + ThreatLevel, + TimeWindow, +) +from tests.module_factory import ModuleFactory + + +def _build_evidence(): + return Evidence( + evidence_type=EvidenceType.CONNECTION_WITHOUT_DNS, + description="Connection to 203.0.113.10 without a preceding DNS lookup.", + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value="192.168.1.25", + ), + threat_level=ThreatLevel.MEDIUM, + profile=ProfileID("192.168.1.25"), + timewindow=TimeWindow(7), + uid=["uid-1", "uid-2"], + timestamp="2026/04/28 10:00:00.000000+0000", + confidence=0.8, + dst_port=443, + ) + + +def _build_alert(evidence): + return Alert( + profile=ProfileID("192.168.1.25"), + timewindow=TimeWindow( + 7, + start_time="2026-04-28T09:00:00+00:00", + end_time="2026-04-28T10:00:00+00:00", + ), + last_evidence=evidence, + accumulated_threat_level=1.2, + last_flow_datetime="2026/04/28 10:00:00.000000+0000", + correl_id=[evidence.id], + ) + + +def test_alert_summary_config_defaults(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = {} + + assert parser.alert_summary_enabled() is False + assert parser.alert_summary_allowed_backends() == [] + assert parser.alert_summary_log_verbosity() == 2 + assert parser.alert_summary_llm_temperature() == 0.2 + assert parser.alert_summary_llm_max_tokens() == 220 + assert parser.alert_summary_llm_response_timeout_seconds() == 120 + assert parser.alert_summary_history_enabled() is False + assert parser.alert_summary_history_max_alerts() == 3 + assert parser.alert_summary_history_max_tokens() == 700 + assert parser.alert_summary_history_patterns_per_alert() == 2 + + +def test_alert_summary_config_sanitization(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = { + "alert_summary": { + "enabled": "true", + "allowed_backends": "local_qwen", + "log_verbosity": 99, + "llm_temperature": "bad", + "llm_max_tokens": "bad", + "llm_response_timeout_seconds": -10, + "history_enabled": "true", + "history_max_alerts": "bad", + "history_max_tokens": -10, + "history_patterns_per_alert": "bad", + } + } + + assert parser.alert_summary_enabled() is True + assert parser.alert_summary_allowed_backends() == [] + assert parser.alert_summary_log_verbosity() == 3 + assert parser.alert_summary_llm_temperature() == 0.2 + assert parser.alert_summary_llm_max_tokens() == 220 + assert parser.alert_summary_llm_response_timeout_seconds() == 0 + assert parser.alert_summary_history_enabled() is True + assert parser.alert_summary_history_max_alerts() == 3 + assert parser.alert_summary_history_max_tokens() == 0 + assert parser.alert_summary_history_patterns_per_alert() == 2 + + +def test_build_prompt_messages_uses_incident_metadata_and_digest(): + alert_summary = ModuleFactory().create_alert_summary_obj() + evidence = _build_evidence() + alert = _build_alert(evidence) + + messages = alert_summary._build_prompt_messages( + alert, + ["10:00 | Connection to 203.0.113.10 without a preceding DNS lookup."], + 1, + 1, + 0, + ) + + assert messages[0]["content"] == SYSTEM_PROMPT + assert ( + "Treat informational (`info`) evidence as context only" + in messages[0]["content"] + ) + assert "INCIDENT METADATA:" in messages[1]["content"] + assert "CURRENT ALERT EVIDENCE DIGEST:" in messages[1]["content"] + assert "Grouped Evidence Patterns: 1" in messages[1]["content"] + assert ( + "Describe the current alert using only details from CURRENT ALERT EVIDENCE DIGEST." + in messages[1]["content"] + ) + assert ( + "Do not present historical-only details as part of the current alert." + in messages[1]["content"] + ) + assert ( + "Weigh evidence according to threat level." in messages[1]["content"] + ) + assert ( + "Treat informational (`info`) evidence as context only" + in messages[1]["content"] + ) + assert "Prompt version: alert-summary-v4" in messages[1]["content"] + assert "RECENT ALERT HISTORY" not in messages[1]["content"] + + +def test_build_reduction_messages_preserves_info_threat_level_guidance(): + alert_summary = ModuleFactory().create_alert_summary_obj() + evidence = _build_evidence() + alert = _build_alert(evidence) + alert_summary.active_job = { + "alert": alert, + "evidences": [evidence], + "grouped_item_count": 1, + } + + messages = alert_summary._build_reduction_messages( + alert, + ["10:00 | Connection to 203.0.113.10 without a preceding DNS lookup."], + 1, + 1, + 1, + 1, + ) + + assert messages[0]["content"] == REDUCTION_SYSTEM_PROMPT + assert ( + "Informational (`info`) evidence is context only" + in messages[0]["content"] + ) + assert ( + "Preserve threat-level distinctions and keep informational (`info`) evidence as context only." + in messages[1]["content"] + ) + + +def test_build_prompt_messages_includes_recent_history_for_same_profile(): + alert_summary = ModuleFactory().create_alert_summary_obj() + prior_alert = _build_alert(_build_evidence()) + prior_alert.timewindow = TimeWindow( + 6, + start_time="2026-04-28T08:00:00+00:00", + end_time="2026-04-28T09:00:00+00:00", + ) + alert_summary._remember_alert_summary( + prior_alert, + "Earlier scanning activity suggests reconnaissance.", + [ + "08:10 | Horizontal port scan to port 443/TCP", + "08:15 | Repeated unknown-port traffic to 198.51.100.10", + ], + ) + + current_alert = _build_alert(_build_evidence()) + messages = alert_summary._build_prompt_messages( + current_alert, + ["10:00 | Connection to 203.0.113.10 without a preceding DNS lookup."], + 1, + 1, + 0, + ) + + assert "RECENT ALERT HISTORY" in messages[1]["content"] + assert "HISTORICAL PROGRESSION ONLY" in messages[1]["content"] + assert ( + "do not restate ports, IPs, destinations, or behaviors from this history as current-alert facts" + in messages[1]["content"] + ) + assert "Horizontal port scan to port 443/TCP" in messages[1]["content"] + assert "historical patterns:" in messages[1]["content"] + assert ( + "Earlier scanning activity suggests reconnaissance." + not in messages[1]["content"] + ) + assert ( + "continuation, escalation, repetition, diversification, or a different pattern" + in messages[1]["content"] + ) + assert ( + "recurrence raises, lowers, or does not materially change confidence and urgency" + in messages[1]["content"] + ) + + +def test_build_prompt_messages_excludes_prior_summary_text_from_history(): + alert_summary = ModuleFactory().create_alert_summary_obj() + prior_alert = _build_alert(_build_evidence()) + alert_summary._remember_alert_summary( + prior_alert, + "Prior summary claimed a single connection to 192.168.1.255 on port 137.", + ["08:10 | Horizontal port scan to port 443/TCP"], + ) + + current_alert = _build_alert(_build_evidence()) + messages = alert_summary._build_prompt_messages( + current_alert, + [ + "10:00 | Malicious JA3 to 95.46.8.65 over 443/TCP " + "(2x similar, severities: high=2)" + ], + 2, + 1, + 0, + ) + + assert "port 137" not in messages[1]["content"] + assert ( + "Prior summary claimed a single connection" + not in messages[1]["content"] + ) + + +def test_analyze_recent_history_counts_repeated_pattern_overlap(): + alert_summary = ModuleFactory().create_alert_summary_obj() + prior_alert = _build_alert(_build_evidence()) + prior_alert.timewindow = TimeWindow(6) + alert_summary._remember_alert_summary( + prior_alert, + "Earlier scan summary.", + [ + "09:15 | Connection to 203.0.113.10 without a preceding DNS lookup.", + "09:20 | Horizontal port scan to port 443/TCP", + ], + ) + alert = _build_alert(_build_evidence()) + + history_analysis = alert_summary._analyze_recent_history( + alert, + alert_summary._get_recent_alert_history(alert), + [ + "10:00 | Connection to 203.0.113.10 without a preceding DNS lookup.", + "10:05 | Connection to 203.0.113.11 without a preceding DNS lookup.", + ], + ) + + assert history_analysis["prior_alert_count"] == 1 + assert history_analysis["matching_alert_count"] == 1 + assert history_analysis["repeated_pattern_count"] >= 1 + assert history_analysis["same_timewindow_alert_count"] == 0 + + +def test_remember_alert_summary_keeps_only_recent_entries_per_profile(): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.history_max_alerts = 2 + + first_alert = _build_alert(_build_evidence()) + first_alert.timewindow = TimeWindow(5) + second_alert = _build_alert(_build_evidence()) + second_alert.timewindow = TimeWindow(6) + third_alert = _build_alert(_build_evidence()) + third_alert.timewindow = TimeWindow(7) + + alert_summary._remember_alert_summary( + first_alert, + "first summary", + ["05:00 | first pattern"], + ) + alert_summary._remember_alert_summary( + second_alert, + "second summary", + ["06:00 | second pattern"], + ) + alert_summary._remember_alert_summary( + third_alert, + "third summary", + ["07:00 | third pattern"], + ) + + stored_history = list( + alert_summary.alert_history_by_profile[str(third_alert.profile)] + ) + + assert len(stored_history) == 2 + assert stored_history[0]["summary_text"] == "second summary" + assert stored_history[1]["summary_text"] == "third summary" + + +def test_build_grouped_evidence_items_merges_similar_descriptions(): + alert_summary = ModuleFactory().create_alert_summary_obj() + first_evidence = _build_evidence() + second_evidence = _build_evidence() + second_evidence.id = "evidence-2" + second_evidence.description = ( + "Connection to 203.0.113.11 without a preceding DNS lookup." + ) + second_evidence.timestamp = "2026/04/28 10:05:00.000000+0000" + + grouped_items = alert_summary._build_grouped_evidence_items( + [first_evidence, second_evidence] + ) + + assert len(grouped_items) == 1 + assert "2x similar" in grouped_items[0] + assert "203.0.113.10" in grouped_items[0] + assert "203.0.113.11" in grouped_items[0] + + +def test_split_text_to_budget_preserves_content_without_truncation(): + alert_summary = ModuleFactory().create_alert_summary_obj() + text = " ".join( + [ + "Repeated outbound HTTPS session to 203.0.113.10 after no DNS lookup." + for _ in range(20) + ] + ) + + parts = alert_summary._split_text_to_budget(text, 40) + + assert len(parts) > 1 + assert "..." not in " ".join(parts) + assert "Repeated outbound HTTPS session" in parts[0] + assert "no DNS lookup" in parts[-1] + + +def test_advance_active_job_starts_reduction_when_final_prompt_is_too_large( + mocker, +): + alert_summary = ModuleFactory().create_alert_summary_obj() + evidence = _build_evidence() + alert = _build_alert(evidence) + alert_summary.active_job = { + "alert": alert, + "evidences": [evidence], + "backend": "local_qwen", + "grouped_item_count": 2, + "current_items": [ + "10:00 | repeated connection one", + "10:05 | repeated connection two", + ], + "reduction_layer": 0, + "current_chunks": [], + "completed_chunk_summaries": [], + } + + def _messages_fit(messages, _budget): + return messages[0]["content"] == REDUCTION_SYSTEM_PROMPT + + mocker.patch.object( + alert_summary, "_messages_fit", side_effect=_messages_fit + ) + mocker.patch.object( + alert_summary, + "_chunk_items_for_reduction", + return_value=[ + ["10:00 | repeated connection one"], + ["10:05 | repeated connection two"], + ], + ) + + alert_summary._advance_active_job() + + channel, payload = alert_summary.db.publish.call_args.args + request = json.loads(payload) + + assert channel == "llm_request" + assert request["metadata"]["chunk_index"] == 1 + assert request["metadata"]["chunk_count"] == 2 + assert alert_summary.pending_request["phase"] == "reduction" + + +def test_reduction_response_advances_to_final_summary(mocker): + alert_summary = ModuleFactory().create_alert_summary_obj() + evidence = _build_evidence() + alert = _build_alert(evidence) + alert_summary.active_job = { + "alert": alert, + "evidences": [evidence], + "backend": "local_qwen", + "grouped_item_count": 2, + "current_items": [ + "10:00 | repeated connection one", + "10:05 | repeated connection two", + ], + "reduction_layer": 0, + "current_chunks": [ + ["10:00 | repeated connection one"], + ["10:05 | repeated connection two"], + ], + "completed_chunk_summaries": ["first digest"], + } + alert_summary.pending_request = { + "request_id": "req-1", + "backend": "local_qwen", + "alert": alert, + "evidences": [evidence], + "phase": "reduction", + "sent_at": time.time(), + "metadata": { + "reduction_layer": 1, + "chunk_index": 2, + "chunk_count": 2, + }, + } + mocker.patch.object(alert_summary, "_messages_fit", return_value=True) + + alert_summary._finalize_request( + { + "request_id": "req-1", + "success": True, + "text": "second digest", + } + ) + + channel, payload = alert_summary.db.publish.call_args.args + request = json.loads(payload) + + assert channel == "llm_request" + assert request["metadata"]["reduction_layer"] == 1 + assert alert_summary.active_job["current_items"] == [ + "first digest", + "second digest", + ] + assert alert_summary.active_job["reduction_layer"] == 1 + assert alert_summary.pending_request["phase"] == "final_summary" + + +def test_handle_pending_response_writes_one_paragraph_summary( + tmp_path, mocker +): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.parent_output_dir = str(tmp_path) + alert_summary.summary_log_path = str( + tmp_path / "alerts" / "alerts-summary.log" + ) + alert_summary.operation_log_path = str( + tmp_path / "llm_proxy-summary" / "alert_summary.log" + ) + mocker.patch( + "modules.alert_summary.alert_summary.utils.drop_root_privs_permanently" + ) + alert_summary.pre_main() + + evidence = _build_evidence() + alert = _build_alert(evidence) + alert_summary.active_job = { + "alert": alert, + "evidences": [evidence], + "backend": "local_qwen", + "grouped_item_count": 1, + "current_items": [ + "10:00 | Connection to 203.0.113.10 without a preceding DNS lookup." + ], + "reduction_layer": 0, + "current_chunks": [], + "completed_chunk_summaries": [], + } + alert_summary.pending_request = { + "request_id": "req-1", + "backend": "local_qwen", + "alert": alert, + "evidences": [evidence], + "phase": "final_summary", + "sent_at": time.time(), + "metadata": {"prompt_version": PROMPT_VERSION}, + } + alert_summary.get_msg = lambda _channel: { + "data": json.dumps( + { + "request_id": "req-1", + "success": True, + "text": "Likely true positive.\n\nRepeated outbound behavior suggests beaconing.", + } + ) + } + + alert_summary._handle_pending_response() + + with open(alert_summary.summary_log_path, "r", encoding="utf-8") as handle: + content = handle.read() + with open( + alert_summary.operation_log_path, "r", encoding="utf-8" + ) as handle: + operation_content = handle.read() + + assert ( + "Likely true positive. Repeated outbound behavior suggests beaconing." + in content + ) + assert "\n\n" not in content + assert "phase=final_summary" in operation_content + assert alert_summary.pending_request is None + assert alert_summary.active_job is None + alert_summary.shutdown_gracefully() + + +def test_handle_pending_response_does_not_timeout_during_shutdown( + tmp_path, mocker +): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.parent_output_dir = str(tmp_path) + alert_summary.summary_log_path = str( + tmp_path / "alerts" / "alerts-summary.log" + ) + alert_summary.operation_log_path = str( + tmp_path / "llm_proxy-summary" / "alert_summary.log" + ) + mocker.patch( + "modules.alert_summary.alert_summary.utils.drop_root_privs_permanently" + ) + mocker.patch( + "modules.alert_summary.alert_summary.time.time", return_value=200 + ) + alert_summary.pre_main() + + evidence = _build_evidence() + alert = _build_alert(evidence) + alert_summary.active_job = { + "alert": alert, + "evidences": [evidence], + "backend": "local_qwen", + "grouped_item_count": 1, + "current_items": [ + "10:00 | Connection to 203.0.113.10 without a preceding DNS lookup." + ], + "reduction_layer": 0, + "current_chunks": [], + "completed_chunk_summaries": [], + } + alert_summary.pending_request = { + "request_id": "req-shutdown", + "backend": "local_qwen", + "alert": alert, + "evidences": [evidence], + "phase": "final_summary", + "sent_at": 0, + "metadata": {"prompt_version": PROMPT_VERSION}, + } + alert_summary.termination_event.is_set.return_value = True + alert_summary.get_msg = lambda _channel: None + + alert_summary._handle_pending_response() + + with open(alert_summary.summary_log_path, "r", encoding="utf-8") as handle: + content = handle.read() + with open( + alert_summary.operation_log_path, "r", encoding="utf-8" + ) as handle: + operation_content = handle.read() + + assert content == "" + assert alert_summary.pending_request is not None + assert "keeping alert_summary alive" in operation_content + alert_summary.shutdown_gracefully() + + +def test_main_flushes_pending_alerts_without_backend_on_shutdown( + tmp_path, mocker +): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.parent_output_dir = str(tmp_path) + alert_summary.summary_log_path = str( + tmp_path / "alerts" / "alerts-summary.log" + ) + alert_summary.operation_log_path = str( + tmp_path / "llm_proxy-summary" / "alert_summary.log" + ) + mocker.patch( + "modules.alert_summary.alert_summary.utils.drop_root_privs_permanently" + ) + alert_summary.pre_main() + + evidence = _build_evidence() + alert = _build_alert(evidence) + alert_summary.pending_alerts.append( + {"alert": alert, "evidences": [evidence]} + ) + alert_summary.db.get_available_llm_backends.return_value = { + "default_backend": "", + "backends": {}, + } + alert_summary.termination_event.is_set.return_value = True + alert_summary._queue_new_alert = lambda: None + + alert_summary.main() + + with open(alert_summary.summary_log_path, "r", encoding="utf-8") as handle: + content = handle.read() + + assert ( + "LLM summary unavailable (No runtime-ready LLM backend available.)." + in content + ) + assert "Local heuristic summary:" in content + assert not alert_summary.pending_alerts + alert_summary.shutdown_gracefully() + + +def test_should_stop_waits_for_pending_alerts_during_shutdown(): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.pending_alerts.append( + {"alert": _build_alert(_build_evidence())} + ) + alert_summary.termination_event.is_set.return_value = True + + assert alert_summary.should_stop() is False + + +def test_should_stop_ignores_stale_llm_response_channel_after_work_finishes(): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.channel_tracker = alert_summary.init_channel_tracker() + alert_summary.termination_event.is_set.return_value = True + alert_summary.channel_tracker["llm_response"]["msg_received"] = True + alert_summary.channel_tracker["new_alert"]["msg_received"] = False + + assert alert_summary.should_stop() is True + + +def test_should_stop_waits_for_pending_shared_llm_request_count(): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.channel_tracker = alert_summary.init_channel_tracker() + alert_summary.termination_event.is_set.return_value = True + alert_summary.channel_tracker["new_alert"]["msg_received"] = False + alert_summary.db.get_pending_llm_request_count.return_value = 2 + + assert alert_summary.should_stop() is False + + +def test_should_stop_waits_for_alive_evidence_handler_on_shutdown(mocker): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.channel_tracker = alert_summary.init_channel_tracker() + alert_summary.termination_event.is_set.return_value = True + alert_summary.channel_tracker["new_alert"]["msg_received"] = False + alert_summary.db.get_pid_of.return_value = 43210 + mocker.patch.object( + alert_summary, + "_is_process_alive", + return_value=True, + ) + + assert alert_summary.should_stop() is False + + +def test_get_alert_evidence_handles_mixed_timestamp_types_without_crashing(): + alert_summary = ModuleFactory().create_alert_summary_obj() + first_evidence = _build_evidence() + second_evidence = _build_evidence() + second_evidence.id = "evidence-2" + second_evidence.timestamp = 1714299000.0 + alert = _build_alert(first_evidence) + alert.correl_id = [first_evidence.id, second_evidence.id] + + first_payload = utils.to_dict(first_evidence) + second_payload = utils.to_dict(second_evidence) + second_payload["timestamp"] = 1714299000.0 + alert_summary.db.get_twid_evidence.return_value = { + first_evidence.id: json.dumps(first_payload), + second_evidence.id: json.dumps(second_payload), + } + + evidences = alert_summary._get_alert_evidence(alert) + + assert len(evidences) == 2 + assert {evidence.id for evidence in evidences} == { + first_evidence.id, + second_evidence.id, + } + + +def test_fallback_summary_contains_local_heuristic_context(): + alert_summary = ModuleFactory().create_alert_summary_obj() + evidence = _build_evidence() + alert = _build_alert(evidence) + + summary = alert_summary._build_fallback_summary( + alert, + [evidence], + "LLM request timed out.", + ) + + assert "LLM summary unavailable (LLM request timed out.)." in summary + assert "Local heuristic summary:" in summary + assert ( + "Connection to 203.0.113.10 without a preceding DNS lookup." in summary + ) + + +def test_fallback_summary_mentions_recent_history_when_available(): + alert_summary = ModuleFactory().create_alert_summary_obj() + prior_alert = _build_alert(_build_evidence()) + prior_alert.timewindow = TimeWindow(6) + alert_summary._remember_alert_summary( + prior_alert, + "Earlier scan summary.", + ["09:15 | Horizontal port scan to port 443/TCP"], + ) + alert = _build_alert(_build_evidence()) + + summary = alert_summary._build_fallback_summary( + alert, + [_build_evidence()], + "LLM request timed out.", + ) + + assert ( + "Recent related alert history for this source includes 1 prior summarized alerts" + in summary + ) + assert "Horizontal port scan to port 443/TCP" in summary + + +def test_fallback_summary_raises_risk_when_same_pattern_repeats_across_history(): + alert_summary = ModuleFactory().create_alert_summary_obj() + prior_alert_one = _build_alert(_build_evidence()) + prior_alert_one.timewindow = TimeWindow(5) + prior_alert_one.accumulated_threat_level = 6.0 + prior_alert_two = _build_alert(_build_evidence()) + prior_alert_two.timewindow = TimeWindow(6) + prior_alert_two.accumulated_threat_level = 7.0 + current_evidence = _build_evidence() + alert = _build_alert(current_evidence) + alert.accumulated_threat_level = 6.5 + alert.confidence = 0.55 + + alert_summary._remember_alert_summary( + prior_alert_one, + "Earlier repeated DNSless connection alert.", + ["09:00 | Connection to 203.0.113.10 without a preceding DNS lookup."], + ) + alert_summary._remember_alert_summary( + prior_alert_two, + "Another repeated DNSless connection alert.", + ["09:30 | Connection to 203.0.113.11 without a preceding DNS lookup."], + ) + + summary = alert_summary._build_fallback_summary( + alert, + [current_evidence], + "LLM request timed out.", + ) + + assert "repeated current-pattern matches" in summary + assert "increasingly like a likely true positive" in summary + assert "operational risk appears high" in summary + + +def test_operation_log_respects_configured_verbosity(tmp_path, mocker): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.log_verbosity = 1 + alert_summary.parent_output_dir = str(tmp_path) + alert_summary.summary_log_path = str( + tmp_path / "alerts" / "alerts-summary.log" + ) + alert_summary.operation_log_path = str( + tmp_path / "llm_proxy-summary" / "alert_summary.log" + ) + mocker.patch( + "modules.alert_summary.alert_summary.utils.drop_root_privs_permanently" + ) + + alert_summary.pre_main() + alert_summary._log_operation("summary line", verbosity=1) + alert_summary._log_operation("debug line", verbosity=3) + + with open( + alert_summary.operation_log_path, "r", encoding="utf-8" + ) as handle: + content = handle.read() + + assert "summary line" in content + assert "debug line" not in content + alert_summary.shutdown_gracefully() + + +def test_shutdown_gracefully_logs_stop_message(tmp_path, mocker): + alert_summary = ModuleFactory().create_alert_summary_obj() + alert_summary.parent_output_dir = str(tmp_path) + alert_summary.summary_log_path = str( + tmp_path / "alerts" / "alerts-summary.log" + ) + alert_summary.operation_log_path = str( + tmp_path / "llm_proxy-summary" / "alert_summary.log" + ) + mocker.patch( + "modules.alert_summary.alert_summary.utils.drop_root_privs_permanently" + ) + + alert_summary.pre_main() + alert_summary.shutdown_gracefully() + + with open( + alert_summary.operation_log_path, "r", encoding="utf-8" + ) as handle: + content = handle.read() + + assert "AlertSummary module stopped." in content diff --git a/tests/unit/modules/anomaly_detection_https/test_anomaly_detection_https.py b/tests/unit/modules/anomaly_detection_https/test_anomaly_detection_https.py index ccbf1f16bf..2c57d2b1bb 100644 --- a/tests/unit/modules/anomaly_detection_https/test_anomaly_detection_https.py +++ b/tests/unit/modules/anomaly_detection_https/test_anomaly_detection_https.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2026 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only - from unittest.mock import Mock, patch from modules.anomaly_detection_https.anomaly_detection_https import ( AnomalyDetectionHTTPS, ) +from slips_files.core.structures.evidence import EvidenceType def make_https_anomaly_conf(): @@ -70,3 +70,79 @@ def test_https_anomaly_module_is_instantiable_and_subscribes_to_new_ssl( db.subscribe.assert_called_once_with("new_ssl") assert module.channels == {"new_ssl": "ssl_channel"} + + + +def _create_https_anomaly_module(tmp_path): + conf = Mock() + conf.https_anomaly_training_hours = Mock(return_value=24) + conf.https_anomaly_hourly_zscore_thr = Mock(return_value=3.0) + conf.https_anomaly_flow_zscore_thr = Mock(return_value=3.5) + conf.https_anomaly_adapt_score_thr = Mock(return_value=2.0) + conf.https_anomaly_baseline_alpha = Mock(return_value=0.1) + conf.https_anomaly_drift_alpha = Mock(return_value=0.05) + conf.https_anomaly_suspicious_alpha = Mock(return_value=0.005) + conf.https_anomaly_min_baseline_points = Mock(return_value=6) + conf.https_anomaly_max_small_flow_anomalies = Mock(return_value=1) + conf.https_anomaly_ja3_min_variants_per_server = Mock(return_value=3) + conf.https_anomaly_use_adwin_drift = Mock(return_value=False) + conf.https_anomaly_adwin_delta = Mock(return_value=0.002) + conf.https_anomaly_adwin_clock = Mock(return_value=32) + conf.https_anomaly_adwin_grace_period = Mock(return_value=10) + conf.https_anomaly_adwin_min_window_length = Mock(return_value=5) + conf.https_anomaly_log_verbosity = Mock(return_value=0) + + db = Mock() + db.subscribe.return_value = Mock() + db.client_setname = Mock() + db.set_evidence = Mock() + db.close_sqlite = Mock() + + args = Mock() + args.interface = None + args.access_point = False + + with ( + patch( + "slips_files.common.abstracts.imodule.DBManager", + return_value=db, + ), + patch( + "modules.anomaly_detection_https.anomaly_detection_https.ConfigParser", + return_value=conf, + ), + ): + module = AnomalyDetectionHTTPS( + logger=Mock(), + output_dir=str(tmp_path), + redis_port=6379, + termination_event=Mock(), + slips_args=args, + conf=conf, + ppid=12345, + bloom_filters_manager=Mock(), + ) + module.print = Mock() + return module, db + + +def test_emit_anomaly_evidence_uses_anomalous_flow_type(tmp_path): + module, db = _create_https_anomaly_module(tmp_path) + + module.emit_anomaly_evidence( + profileid="profile_192.168.1.20", + twid_number=7, + traffic_ts=1_700_000_000.0, + uid="uid-1", + confidence={"level": "high", "score": 0.8}, + reasons=[{"feature": "new_server", "value": "bad.example.com"}], + kind="flow", + server="bad.example.com", + sni="bad.example.com", + daddr="93.184.216.34", + ) + + evidence = db.set_evidence.call_args.args[0] + assert evidence.evidence_type == EvidenceType.ANOMALOUS_FLOW + assert evidence.description.startswith("HTTPS anomaly:") + assert evidence.profile.ip == "192.168.1.20" diff --git a/tests/unit/modules/llm/test_anthropic_backend_mixin.py b/tests/unit/modules/llm/test_anthropic_backend_mixin.py new file mode 100644 index 0000000000..274d9c6281 --- /dev/null +++ b/tests/unit/modules/llm/test_anthropic_backend_mixin.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + +from unittest.mock import patch + +from modules.llm_proxy.anthropic_backend_mixin import MixinAnthropicBackend +from modules.llm_proxy.llm_backend_config import LLMBackendConfig +from tests.module_factory import ModuleFactory + + +def test_anthropic_backend_mixin_moves_system_messages() -> None: + llm = ModuleFactory().create_llm_obj() + config = LLMBackendConfig.from_dict( + "claude_default", + { + "provider": "anthropic", + "model": "claude-sonnet-4-5", + "api_key": "secret", + }, + ) + backend = MixinAnthropicBackend(config) + with patch.object(backend, "_request_json") as mock_request: + mock_request.return_value = { + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "anthropic answer"}], + "usage": {"input_tokens": 3, "output_tokens": 4}, + } + response = backend.generate( + { + "messages": [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hello"}, + ], + "model": None, + "temperature": 0.2, + "max_tokens": 128, + } + ) + + sent_payload = mock_request.call_args.args[2] + assert llm.name == "llm_proxy" + assert sent_payload["system"] == "be terse" + assert sent_payload["messages"] == [{"role": "user", "content": "hello"}] + assert response["text"] == "anthropic answer" diff --git a/tests/unit/modules/llm/test_llm.py b/tests/unit/modules/llm/test_llm.py new file mode 100644 index 0000000000..2e25c9ee0a --- /dev/null +++ b/tests/unit/modules/llm/test_llm.py @@ -0,0 +1,256 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + +import json +from unittest.mock import Mock, patch + +import pytest + +from modules.llm_proxy.llm_backend_config import LLMBackendConfig +from tests.module_factory import ModuleFactory + + +def test_prepare_request_uses_default_backend_and_prompt(): + llm = ModuleFactory().create_llm_obj() + + request = llm._prepare_request( + {"request_id": "req-1", "prompt": "summarize this"} + ) + + assert request["backend"] == "local_qwen" + assert request["messages"] == [ + {"role": "user", "content": "summarize this"} + ] + + +def test_get_available_backends_registry_has_runtime_ready_backends(): + llm = ModuleFactory().create_llm_obj() + + assert llm._get_available_backends_registry() == { + "default_backend": "local_qwen", + "backends": { + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b", + } + }, + } + + +def test_get_available_backends_registry_blanks_invalid_default(): + llm = ModuleFactory().create_llm_obj() + llm.default_backend = "missing_backend" + + assert llm._get_available_backends_registry() == { + "default_backend": "", + "backends": { + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b", + } + }, + } + + +def test_pre_main_publishes_runtime_ready_registry(): + llm = ModuleFactory().create_llm_obj() + + llm.pre_main() + + llm.db.reset_pending_llm_request_counts.assert_called_once() + llm.db.set_available_llm_backends.assert_called_once_with( + { + "default_backend": "local_qwen", + "backends": { + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b", + } + }, + } + ) + + +def test_pre_main_publishes_empty_registry_when_disabled(): + llm = ModuleFactory().create_llm_obj() + llm.enabled = False + + assert llm.pre_main() is True + llm.db.reset_pending_llm_request_counts.assert_called_once() + llm.db.set_available_llm_backends.assert_called_once_with( + { + "default_backend": "", + "backends": {}, + } + ) + + +def test_pre_main_publishes_empty_registry_when_no_valid_backends(): + llm = ModuleFactory().create_llm_obj() + llm.backends = {} + + assert llm.pre_main() is True + llm.db.reset_pending_llm_request_counts.assert_called_once() + llm.db.set_available_llm_backends.assert_called_once_with( + { + "default_backend": "", + "backends": {}, + } + ) + + +def test_handle_request_publishes_success_response(): + llm = ModuleFactory().create_llm_obj() + llm.backends = { + "local_qwen": Mock( + generate=Mock( + return_value={ + "text": "analysis result", + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + }, + "provider": "ollama", + "model": "qwen2.5:3b", + } + ) + ) + } + + llm._handle_request( + { + "request_id": "req-2", + "requester": "HTTP Analyzer", + "prompt": "analyze this flow", + "metadata": {"uid": "C1"}, + } + ) + + channel, payload = llm.db.publish.call_args.args + response = json.loads(payload) + assert channel == "llm_response" + assert response["success"] is True + assert response["request_id"] == "req-2" + assert response["text"] == "analysis result" + assert response["metadata"] == {"uid": "C1"} + llm.db.decrement_pending_llm_request_count.assert_called_once_with( + "HTTP Analyzer" + ) + + +def test_handle_request_publishes_error_for_unknown_backend(): + llm = ModuleFactory().create_llm_obj() + + llm._handle_request( + { + "request_id": "req-3", + "backend": "missing_backend", + "prompt": "hello", + } + ) + + channel, payload = llm.db.publish.call_args.args + response = json.loads(payload) + assert channel == "llm_response" + assert response["success"] is False + assert "Unknown LLM backend" in response["error"] + llm.db.decrement_pending_llm_request_count.assert_called_once_with("") + + +def test_enqueue_request_increments_requester_pending_count(): + llm = ModuleFactory().create_llm_obj() + + llm._enqueue_request( + { + "data": json.dumps( + { + "request_id": "req-enqueue", + "requester": "alert_summary", + "prompt": "hello", + } + ) + } + ) + + llm.db.increment_pending_llm_request_count.assert_called_once_with( + "alert_summary" + ) + + +def test_llm_backend_pool_size_scales_with_worker_threads(): + llm = ModuleFactory().create_llm_obj() + llm.worker_threads = 3 + config = LLMBackendConfig.from_dict( + "local_qwen", + { + "provider": "ollama", + "model": "qwen2.5:3b", + "base_url": "http://127.0.0.1:11434", + }, + ) + + with patch( + "modules.llm_proxy.llm_backend.urllib3.PoolManager" + ) as mock_pool: + llm._create_backend(config) + + assert mock_pool.call_args.kwargs["maxsize"] == 6 + + +@pytest.mark.parametrize( + ("termination_requested", "msg_received", "expected"), + [ + (False, False, False), + (True, True, False), + (True, False, True), + ], +) +def test_should_stop_uses_base_module_stop_conditions( + termination_requested, + msg_received, + expected, +): + llm = ModuleFactory().create_llm_obj() + llm.termination_event.is_set.return_value = termination_requested + llm.channel_tracker = { + "llm_request": { + "msg_received": msg_received, + } + } + + assert llm.should_stop() is expected + + +def test_shutdown_gracefully_clears_available_backend_registry(): + llm = ModuleFactory().create_llm_obj() + + assert llm.shutdown_gracefully() is True + llm.db.reset_pending_llm_request_counts.assert_called_once() + llm.db.set_available_llm_backends.assert_called_once_with( + { + "default_backend": "", + "backends": {}, + } + ) + + +def test_pre_main_creates_module_specific_llm_log(tmp_path, mocker): + llm = ModuleFactory().create_llm_obj() + llm.parent_output_dir = str(tmp_path) + llm.output_dir = str(tmp_path / llm.name) + llm.operation_log_path = llm.get_module_specific_output_path( + "llm_proxy.log" + ) + mocker.patch( + "modules.llm_proxy.llm_proxy.utils.drop_root_privs_permanently" + ) + + llm.pre_main() + llm.shutdown_gracefully() + + with open(llm.operation_log_path, "r", encoding="utf-8") as handle: + content = handle.read() + + assert "LLM module ready." in content + assert "LLM module stopped." in content diff --git a/tests/unit/modules/llm/test_llm_backend.py b/tests/unit/modules/llm/test_llm_backend.py new file mode 100644 index 0000000000..53bbb4c560 --- /dev/null +++ b/tests/unit/modules/llm/test_llm_backend.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + +from unittest.mock import Mock + +import urllib3 + +from modules.llm_proxy.llm_backend import LLMBackend +from modules.llm_proxy.llm_backend_config import LLMBackendConfig +from tests.module_factory import ModuleFactory + + +def test_backend_request_json_uses_explicit_connect_and_read_timeouts() -> ( + None +): + llm = ModuleFactory().create_llm_obj() + config = LLMBackendConfig.from_dict( + "local_qwen", + { + "provider": "ollama", + "model": "qwen2.5:3b", + "base_url": "http://127.0.0.1:11434", + "timeout": 42, + }, + ) + backend = LLMBackend(config) + backend.http = Mock() + backend.http.request.return_value = Mock( + status=200, + data=b'{"message": {"content": "ok"}}', + ) + + backend._request_json("POST", "http://127.0.0.1:11434/api/chat", {}) + + timeout = backend.http.request.call_args.kwargs["timeout"] + assert llm.name == "llm_proxy" + assert isinstance(timeout, urllib3.Timeout) + assert timeout.connect_timeout == 42 + assert timeout.read_timeout == 42 + + +def test_backend_build_url_avoids_duplicate_v1_prefix() -> None: + llm = ModuleFactory().create_llm_obj() + config = LLMBackendConfig.from_dict( + "openai_default", + { + "provider": "openai", + "model": "gpt-4o-mini", + "base_url": "https://api.openai.com/v1", + "api_key": "secret", + }, + ) + backend = LLMBackend(config) + + url = backend._build_url("/v1/messages") + + assert llm.name == "llm_proxy" + assert url == "https://api.openai.com/v1/messages" diff --git a/tests/unit/modules/llm/test_llm_backend_config.py b/tests/unit/modules/llm/test_llm_backend_config.py new file mode 100644 index 0000000000..c57ef859a1 --- /dev/null +++ b/tests/unit/modules/llm/test_llm_backend_config.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + +from typing import Any + +import pytest + +from modules.llm_proxy.llm_backend_config import LLMBackendConfig +from modules.llm_proxy.llm_errors import LLMConfigurationError +from tests.module_factory import ModuleFactory + + +def test_backend_config_reads_api_key_from_env(mocker: Any) -> None: + llm = ModuleFactory().create_llm_obj() + mocker.patch.dict("os.environ", {"OPENAI_API_KEY": "secret-key"}) + + config = LLMBackendConfig.from_dict( + "openai_default", + { + "provider": "openai", + "model": "gpt-4o-mini", + "api_key_env": "OPENAI_API_KEY", + }, + ) + + assert llm.name == "llm_proxy" + assert config.api_key == "secret-key" + assert config.base_url == "https://api.openai.com/v1" + + +@pytest.mark.parametrize( + ("alias", "data", "expected_error"), + [ + ("bad_backend", [], "must be a mapping"), + ( + "bad_provider", + {"provider": "invalid", "model": "model"}, + "unsupported provider", + ), + ("missing_model", {"provider": "ollama"}, "missing a model"), + ], +) +def test_backend_config_rejects_invalid_config( + alias: str, + data: Any, + expected_error: str, +) -> None: + llm = ModuleFactory().create_llm_obj() + + with pytest.raises(LLMConfigurationError, match=expected_error): + LLMBackendConfig.from_dict(alias, data) + + assert llm.name == "llm_proxy" diff --git a/tests/unit/modules/llm/test_llm_errors.py b/tests/unit/modules/llm/test_llm_errors.py new file mode 100644 index 0000000000..4917126079 --- /dev/null +++ b/tests/unit/modules/llm/test_llm_errors.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + +import pytest + +from modules.llm_proxy.llm_errors import LLMConfigurationError, LLMRequestError +from tests.module_factory import ModuleFactory + + +@pytest.mark.parametrize( + "error_cls", + [ + LLMConfigurationError, + LLMRequestError, + ], +) +def test_llm_errors_are_exceptions(error_cls: type[Exception]) -> None: + llm = ModuleFactory().create_llm_obj() + + error = error_cls("failed") + + assert llm.name == "llm_proxy" + assert isinstance(error, Exception) diff --git a/tests/unit/modules/llm/test_ollama_backend_mixin.py b/tests/unit/modules/llm/test_ollama_backend_mixin.py new file mode 100644 index 0000000000..38dc75c537 --- /dev/null +++ b/tests/unit/modules/llm/test_ollama_backend_mixin.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + +from unittest.mock import Mock + +from modules.llm_proxy.llm_backend_config import LLMBackendConfig +from modules.llm_proxy.ollama_backend_mixin import MixinOllamaBackend +from tests.module_factory import ModuleFactory + + +def test_ollama_backend_mixin_parses_response() -> None: + llm = ModuleFactory().create_llm_obj() + config = LLMBackendConfig.from_dict( + "local_qwen", + { + "provider": "ollama", + "model": "qwen2.5:3b", + "base_url": "http://127.0.0.1:11434", + }, + ) + backend = MixinOllamaBackend(config) + backend._request_json = Mock( + return_value={ + "model": "qwen2.5:3b", + "message": {"content": "ollama answer"}, + "prompt_eval_count": 9, + "eval_count": 11, + } + ) + + response = backend.generate( + { + "messages": [{"role": "user", "content": "Hello"}], + "model": None, + "temperature": None, + "max_tokens": None, + } + ) + + assert llm.name == "llm_proxy" + assert response["text"] == "ollama answer" + assert response["usage"]["input_tokens"] == 9 + assert response["usage"]["output_tokens"] == 11 + assert response["usage"]["total_tokens"] == 20 diff --git a/tests/unit/modules/llm/test_openai_backend_mixin.py b/tests/unit/modules/llm/test_openai_backend_mixin.py new file mode 100644 index 0000000000..581e5ef130 --- /dev/null +++ b/tests/unit/modules/llm/test_openai_backend_mixin.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + +from unittest.mock import Mock + +import pytest + +from modules.llm_proxy.llm_backend_config import LLMBackendConfig +from modules.llm_proxy.llm_errors import LLMRequestError +from modules.llm_proxy.openai_backend_mixin import MixinOpenAIBackend +from tests.module_factory import ModuleFactory + + +def test_openai_backend_mixin_parses_chat_completion_response() -> None: + llm = ModuleFactory().create_llm_obj() + config = LLMBackendConfig.from_dict( + "openai_default", + { + "provider": "openai", + "model": "gpt-4o-mini", + "api_key": "secret", + }, + ) + backend = MixinOpenAIBackend(config) + backend._request_json = Mock( + return_value={ + "model": "gpt-4o-mini", + "choices": [ + { + "message": { + "content": "final answer", + } + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 7, + "total_tokens": 19, + }, + } + ) + + response = backend.generate( + { + "messages": [{"role": "user", "content": "Hello"}], + "model": None, + "temperature": None, + "max_tokens": None, + } + ) + + assert llm.name == "llm_proxy" + assert response["text"] == "final answer" + assert response["usage"]["total_tokens"] == 19 + + +def test_openai_backend_mixin_rejects_empty_choices() -> None: + llm = ModuleFactory().create_llm_obj() + config = LLMBackendConfig.from_dict( + "openai_default", + { + "provider": "openai", + "model": "gpt-4o-mini", + "api_key": "secret", + }, + ) + backend = MixinOpenAIBackend(config) + backend._request_json = Mock( + return_value={"model": "gpt-4o-mini", "choices": []} + ) + + with pytest.raises(LLMRequestError, match="returned no choices"): + backend.generate( + { + "messages": [{"role": "user", "content": "Hello"}], + "model": None, + "temperature": None, + "max_tokens": None, + } + ) + + assert llm.name == "llm_proxy" diff --git a/tests/unit/modules/regex_generator/test_log_rotator.py b/tests/unit/modules/regex_generator/test_log_rotator.py new file mode 100644 index 0000000000..3eb17c3afc --- /dev/null +++ b/tests/unit/modules/regex_generator/test_log_rotator.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import time + +import pytest + +from modules.regex_generator.log_rotator import LogRotator + + +@pytest.mark.parametrize( + ("rotation_period", "expected_seconds"), + [ + (30, 30), + (0, 1), + ("2hr", 7200), + ("3 days", 259200), + ("bad", 86400), + ], +) +def test_parse_rotation_period_seconds( + rotation_period: object, expected_seconds: int +) -> None: + """ + Check conversion of supported and invalid rotation period values. + + Parameters: + rotation_period: Rotation period value to parse. + expected_seconds: Expected parsed value in seconds. + + Returns: + None + """ + assert ( + LogRotator.parse_rotation_period_seconds(rotation_period) + == expected_seconds + ) + + +def test_rotate_log_file_if_needed_rotates_non_empty_log_file( + tmp_path, +) -> None: + """ + Check that rotation moves a stale non-empty log out of the active path. + + Parameters: + tmp_path: Temporary pytest path fixture. + + Returns: + None + """ + output_dir = tmp_path / "output" + log_file_path = output_dir / "regex_generator.log" + log_rotator = LogRotator( + str(output_dir), + str(log_file_path), + create_log_file=True, + enable_log_rotation=True, + log_rotation_period=1, + ) + log_rotator.init_log_file() + + with open(log_file_path, "a", encoding="utf-8") as log_file: + log_file.write("old line\n") + log_rotator.last_log_rotation_time = time.time() - 10 + + log_rotator.rotate_log_file_if_needed() + + rotated_logs = list(output_dir.glob("regex_generator.log.*")) + assert rotated_logs + with open(log_file_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + assert "old line" not in log_contents diff --git a/tests/unit/modules/regex_generator/test_match_strength.py b/tests/unit/modules/regex_generator/test_match_strength.py new file mode 100644 index 0000000000..5314b088dc --- /dev/null +++ b/tests/unit/modules/regex_generator/test_match_strength.py @@ -0,0 +1,55 @@ +import re + +import pytest + +from modules.regex_generator.match_strength import ( + compute_match_strength, + measure_regex_specificity, +) +from tests.module_factory import ModuleFactory + + +@pytest.mark.parametrize( + "regex_text, expected_key", + [ + (r"evil\.example\.com", "specificity_ratio"), + (r".*example.*", "wildcard_penalty"), + ], +) +def test_measure_regex_specificity_returns_feature_scores( + regex_text: str, expected_key: str +) -> None: + """Verify regex specificity metrics include normalized feature scores. + + Parameters: + regex_text: Regex text to inspect. + expected_key: Expected feature key in the metrics. + + Return value: + None. + """ + module_factory = ModuleFactory() + + features = measure_regex_specificity(regex_text) + + assert module_factory + assert expected_key in features + assert 0.0 <= features[expected_key] <= 1.0 + + +def test_compute_match_strength_scores_full_match_above_partial() -> None: + """Verify full regex matches score higher than partial matches. + + Return value: + None. + """ + module_factory = ModuleFactory() + compiled_regex = re.compile(r"evil\.example\.com") + + full_score = compute_match_strength(compiled_regex, "evil.example.com") + partial_score = compute_match_strength( + compiled_regex, "prefix-evil.example.com" + ) + + assert module_factory + assert full_score > partial_score diff --git a/tests/unit/modules/regex_generator/test_regex_generator.py b/tests/unit/modules/regex_generator/test_regex_generator.py new file mode 100644 index 0000000000..f923950070 --- /dev/null +++ b/tests/unit/modules/regex_generator/test_regex_generator.py @@ -0,0 +1,1115 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +import re +import time +from unittest.mock import Mock + +from modules.regex_generator.regex_generator import ( + PROMPT_VERSION, + RegexGenerator, + SYSTEM_PROMPT, + TYPE_PROMPTS, +) +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.core.database.sqlite_db.regex_generator_db import ( + REGEX_TYPES, + RegexGeneratorStorage, +) +from tests.module_factory import ModuleFactory + + +def _build_storage_conf( + store_dir: str, + persistent_store_dir: str = "", + seed_benign_samples: bool = True, + store_rejected_regexes: bool = False, + max_stored_rejected_regexes: int = 10000, + enable_local_whitelist: bool = True, + local_whitelist_path: str = "config/whitelist.conf", + tranco_top_benign_limit: int = 1000, +): + conf = Mock() + conf.regex_generator_store_dir = Mock(return_value=store_dir) + conf.regex_generator_persistent_store_dir = Mock( + return_value=persistent_store_dir + ) + conf.regex_generator_seed_benign_samples = Mock( + return_value=seed_benign_samples + ) + conf.regex_generator_store_rejected_regexes = Mock( + return_value=store_rejected_regexes + ) + conf.regex_generator_max_stored_rejected_regexes = Mock( + return_value=max_stored_rejected_regexes + ) + conf.enable_local_whitelist = Mock(return_value=enable_local_whitelist) + conf.local_whitelist_path = Mock(return_value=local_whitelist_path) + conf.tranco_top_benign_limit = Mock(return_value=tranco_top_benign_limit) + return conf + + +def test_regex_generator_config_defaults(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = {} + + assert parser.rotation_period() == "1 day" + assert parser.default_rotation_interval() == "1 day" + assert parser.regex_generator_enabled() is False + assert parser.regex_generator_create_log_file() is False + assert parser.regex_generator_generation_interval_seconds() == 5 + assert parser.regex_generator_allowed_backends() == [] + assert parser.regex_generator_llm_temperature() == 1.2 + assert parser.regex_generator_llm_max_tokens() == 80 + assert parser.regex_generator_llm_response_timeout_seconds() == 90 + assert parser.regex_generator_recent_history_size() == 0 + assert parser.regex_generator_max_regex_length() == 180 + assert parser.regex_generator_regex_validation_timeout_seconds() == 2 + assert parser.regex_generator_benign_match_strength_threshold() == 75 + assert parser.regex_generator_store_dir() == "output/regex_generator" + assert ( + parser.regex_generator_persistent_store_dir() + == "databases/regex_store" + ) + assert parser.regex_generator_store_rejected_regexes() is False + assert parser.regex_generator_max_stored_rejected_regexes() == 10000 + assert parser.regex_generator_seed_benign_samples() is True + assert parser.tranco_top_benign_limit() == 1000 + + +def test_regex_generator_config_sanitization(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = { + "parameters": { + "rotation_period": " 30min; ", + }, + "regex_generator": { + "generation_interval_seconds": "bad", + "create_log_file": "true", + "allowed_backends": "local_qwen", + "llm_temperature": "bad", + "llm_max_tokens": "bad", + "llm_response_timeout_seconds": 0, + "recent_history_size": -2, + "max_regex_length": "bad", + "regex_validation_timeout_seconds": "bad", + "benign_match_strength_threshold": "bad", + "type_weights": { + "dns_domain": 0, + "uri": 0, + "filename": 0, + "tls_sni": 0, + "certificate_cn": 0, + }, + "store_dir": "", + "persistent_store_dir": " /tmp/regex-db ", + "store_rejected_regexes": "true", + "max_stored_rejected_regexes": "bad", + "seed_benign_samples": "false", + }, + } + + assert parser.rotation_period() == "30min" + assert parser.default_rotation_interval() == "30min" + assert parser.regex_generator_generation_interval_seconds() == 5 + assert parser.regex_generator_create_log_file() is True + assert parser.regex_generator_allowed_backends() == [] + assert parser.regex_generator_llm_temperature() == 1.2 + assert parser.regex_generator_llm_max_tokens() == 80 + assert parser.regex_generator_llm_response_timeout_seconds() == 0 + assert parser.regex_generator_recent_history_size() == 0 + assert parser.regex_generator_max_regex_length() == 180 + assert parser.regex_generator_regex_validation_timeout_seconds() == 2 + assert parser.regex_generator_benign_match_strength_threshold() == 75 + assert parser.regex_generator_type_weights() == { + "dns_domain": 1, + "uri": 1, + "filename": 1, + "tls_sni": 1, + "certificate_cn": 1, + } + assert parser.regex_generator_store_dir() == "output/regex_generator" + assert parser.regex_generator_persistent_store_dir() == "/tmp/regex-db" + assert parser.regex_generator_store_rejected_regexes() is True + assert parser.regex_generator_max_stored_rejected_regexes() == 10000 + assert parser.regex_generator_seed_benign_samples() is False + assert parser.tranco_top_benign_limit() == 1000 + + +def test_rotation_period_falls_back_to_legacy_key(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = { + "parameters": { + "default_rotation_interval": "2hr", + } + } + + assert parser.rotation_period() == "2hr" + assert parser.default_rotation_interval() == "2hr" + + +def test_regex_generator_generation_interval_allows_zero(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = { + "regex_generator": { + "generation_interval_seconds": 0, + } + } + + assert parser.regex_generator_generation_interval_seconds() == 0 + + +def test_choose_regex_type_honors_weights(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.type_weights = { + "dns_domain": 1, + "uri": 0, + "filename": 0, + "tls_sni": 0, + "certificate_cn": 0, + } + + assert regex_generator._choose_regex_type() == "dns_domain" + + +def test_select_backend_prefers_allowed_backends(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.allowed_backends = ["local_qwen", "openai_default"] + + backend = regex_generator._select_backend( + { + "default_backend": "openai_default", + "backends": { + "openai_default": { + "provider": "openai", + "model": "gpt-4o-mini", + }, + "local_qwen": {"provider": "ollama", "model": "qwen2.5:3b"}, + }, + } + ) + + assert backend == "local_qwen" + + +def test_select_backend_falls_back_to_default_backend(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.allowed_backends = [] + + backend = regex_generator._select_backend( + { + "default_backend": "local_qwen", + "backends": { + "local_qwen": {"provider": "ollama", "model": "qwen2.5:3b"}, + }, + } + ) + + assert backend == "local_qwen" + + +def test_main_waits_when_no_runtime_ready_backend(tmp_path, mocker): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + mocker.patch( + "modules.regex_generator.regex_generator.utils.drop_root_privs_permanently" + ) + mocker.patch("modules.regex_generator.regex_generator.time.sleep") + regex_generator.pre_main() + regex_generator.db.get_available_llm_backends = Mock( + return_value={"default_backend": "", "backends": {}} + ) + regex_generator.next_generation_at = 0 + + regex_generator.main() + + regex_generator.db.publish.assert_not_called() + assert regex_generator.next_generation_at > 0 + regex_generator.shutdown_gracefully() + + +def test_create_log_file_writes_progress_log(tmp_path, mocker): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.output_dir = str(tmp_path / "output") + regex_generator.log_rotator.log_file_path = str( + tmp_path / "output" / "regex_generator.log" + ) + regex_generator.log_rotator.create_log_file = True + mocker.patch( + "modules.regex_generator.regex_generator.utils.drop_root_privs_permanently" + ) + + regex_generator.pre_main() + regex_generator.log_detail("test log line") + + with open( + regex_generator.log_rotator.log_file_path, "r", encoding="utf-8" + ) as log_file: + log_contents = log_file.read() + + assert "RegexGenerator module ready." in log_contents + assert "test log line" in log_contents + regex_generator.shutdown_gracefully() + + +def test_log_file_rotates_with_global_rotation_settings(tmp_path, mocker): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + output_dir = tmp_path / "output" + regex_generator.output_dir = str(output_dir) + regex_generator.log_rotator.log_file_path = str( + output_dir / "regex_generator.log" + ) + regex_generator.log_rotator.create_log_file = True + regex_generator.log_rotator.enable_log_rotation = True + regex_generator.log_rotator.log_rotation_period = 1 + regex_generator.log_rotator.last_log_rotation_time = time.time() - 10 + mocker.patch( + "modules.regex_generator.regex_generator.utils.drop_root_privs_permanently" + ) + + regex_generator.pre_main() + with open( + regex_generator.log_rotator.log_file_path, "a", encoding="utf-8" + ) as log_file: + log_file.write("old line\n") + regex_generator.log_rotator.last_log_rotation_time = time.time() - 10 + + regex_generator.log_detail("new line") + + rotated_logs = list(output_dir.glob("regex_generator.log.*")) + assert rotated_logs + with open( + regex_generator.log_rotator.log_file_path, "r", encoding="utf-8" + ) as log_file: + log_contents = log_file.read() + assert "new line" in log_contents + regex_generator.shutdown_gracefully() + + +def test_clean_host_tw_imports_runtime_benign_strings(tmp_path, mocker): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + mocker.patch( + "modules.regex_generator.regex_generator.utils.drop_root_privs_permanently" + ) + regex_generator.pre_main() + regex_generator.get_msg = Mock( + side_effect=[ + {"data": "profile_192.168.1.10_timewindow7"}, + ] + ) + regex_generator.db.get_all_host_ips = Mock(return_value=["192.168.1.10"]) + regex_generator.db.get_profileid_twid_alerts = Mock(return_value={}) + regex_generator.db.get_twid_evidence = Mock(return_value={}) + regex_generator.db.get_all_altflows_in_profileid_twid = Mock( + return_value=[ + { + "flow_type": "dns", + "flow": {"query": "printer.example.org"}, + }, + { + "flow_type": "http", + "flow": { + "host": "updates.example.org", + "uri": "/downloads/setup.msi", + }, + }, + { + "flow_type": "ssl", + "flow": { + "server_name": "api.github.com", + "subject": "C=US,O=GitHub,CN=github.com", + }, + }, + ] + ) + + regex_generator._handle_one_tw_closed_message() + + assert "printer.example.org" in set( + regex_generator.storage.iter_benign_strings("dns_domain") + ) + assert "updates.example.org" in set( + regex_generator.storage.iter_benign_strings("dns_domain") + ) + assert "setup.msi" in set( + regex_generator.storage.iter_benign_strings("filename") + ) + assert "api.github.com" in set( + regex_generator.storage.iter_benign_strings("tls_sni") + ) + assert "github.com" in set( + regex_generator.storage.iter_benign_strings("certificate_cn") + ) + regex_generator.shutdown_gracefully() + + +def test_dirty_host_tw_does_not_import_runtime_benign_strings( + tmp_path, mocker +): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + mocker.patch( + "modules.regex_generator.regex_generator.utils.drop_root_privs_permanently" + ) + regex_generator.pre_main() + before_dns = set(regex_generator.storage.iter_benign_strings("dns_domain")) + regex_generator.get_msg = Mock( + side_effect=[ + {"data": "profile_192.168.1.10_timewindow8"}, + ] + ) + regex_generator.db.get_all_host_ips = Mock(return_value=["192.168.1.10"]) + regex_generator.db.get_profileid_twid_alerts = Mock( + return_value={"alert-1": ["ev-1"]} + ) + regex_generator.db.get_twid_evidence = Mock( + return_value={"ev-1": json.dumps({"evidence_type": "MALICIOUS_FLOW"})} + ) + regex_generator.db.get_all_altflows_in_profileid_twid = Mock( + return_value=[ + { + "flow_type": "dns", + "flow": {"query": "should-not-be-added.example"}, + }, + ] + ) + + regex_generator._handle_one_tw_closed_message() + + after_dns = set(regex_generator.storage.iter_benign_strings("dns_domain")) + assert after_dns == before_dns + regex_generator.shutdown_gracefully() + + +def test_count_anomaly_evidence_counts_anomalous_flow(): + count = RegexGenerator._count_anomaly_evidence( + { + "ev-1": {"evidence_type": "ANOMALOUS_FLOW", "description": ""}, + "ev-2": {"evidence_type": "SSH_SUCCESSFUL", "description": ""}, + } + ) + + assert count == 1 + + +def test_build_prompt_messages_uses_type_specific_prompt(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = Mock() + + messages = regex_generator._build_prompt_messages("dns_domain", "nonce-1") + + assert messages[0]["content"] == SYSTEM_PROMPT + assert TYPE_PROMPTS["dns_domain"] in messages[1]["content"] + assert PROMPT_VERSION in messages[1]["content"] + regex_generator.storage.get_recent_history.assert_not_called() + regex_generator.storage.get_benign_examples.assert_not_called() + + +def test_send_generation_request_publishes_expected_payload(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = Mock() + + regex_generator._send_generation_request("dns_domain", "local_qwen") + + channel, payload = regex_generator.db.publish.call_args.args + request = json.loads(payload) + assert channel == "llm_request" + assert request["backend"] == "local_qwen" + assert request["temperature"] == 1.2 + assert request["max_tokens"] == 80 + assert request["metadata"]["regex_type"] == "dns_domain" + assert request["metadata"]["prompt_version"] == PROMPT_VERSION + assert request["request_id"].startswith("regex_generator-") + + +def test_handle_pending_response_matches_by_request_id(tmp_path, mocker): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.pending_request = { + "request_id": "req-1", + "regex_type": "dns_domain", + "backend": "local_qwen", + "sent_at": time.time(), + } + regex_generator._finalize_request = Mock() + regex_generator.get_msg = Mock( + return_value={ + "data": json.dumps( + { + "request_id": "other-req", + "success": True, + "text": "^abc$", + } + ) + } + ) + + regex_generator._handle_pending_response(time.time()) + + regex_generator._finalize_request.assert_not_called() + assert regex_generator.pending_request["request_id"] == "req-1" + + +def test_handle_pending_response_keeps_waiting_after_soft_timeout( + tmp_path, mocker +): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + mocker.patch("modules.regex_generator.regex_generator.time.sleep") + regex_generator.pending_request = { + "request_id": "req-1", + "regex_type": "dns_domain", + "backend": "local_qwen", + "sent_at": time.time() - 120, + "last_warning_at": 0.0, + } + regex_generator.get_msg = Mock(return_value=None) + + regex_generator._handle_pending_response(time.time()) + + assert regex_generator.pending_request["request_id"] == "req-1" + assert regex_generator.pending_request["last_warning_at"] > 0 + regex_generator.print.assert_called() + + +def test_finalize_request_drops_malformed_llm_response_without_error_logging( + tmp_path, +): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.pending_request = { + "request_id": "req-1", + "regex_type": "dns_domain", + "backend": "local_qwen", + } + regex_generator.log_detail = Mock() + regex_generator._validate_and_store_regex = Mock() + + regex_generator._finalize_request( + { + "request_id": "req-1", + "success": True, + "text": "not json", + } + ) + + regex_generator.print.assert_not_called() + regex_generator.log_detail.assert_not_called() + regex_generator._validate_and_store_regex.assert_not_called() + + +def test_extract_regex_from_llm_text_rejects_invalid_payloads(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + + assert regex_generator._extract_regex_from_llm_text("not json") == ( + "", + "invalid_response", + ) + assert regex_generator._extract_regex_from_llm_text( + '{"rationale":"x"}' + ) == ( + "", + "missing_regex", + ) + + +def test_extract_regex_from_llm_text_accepts_raw_regex_line(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + + regex, error = regex_generator._extract_regex_from_llm_text( + r"^xqz[a-z0-9]{8,12}\.invalid$" + ) + + assert error is None + assert regex == r"^xqz[a-z0-9]{8,12}\.invalid$" + + +def test_extract_regex_from_llm_text_accepts_fenced_json(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + + regex, error = regex_generator._extract_regex_from_llm_text( + '```json\n{"regex":"^abc$","rationale":"ok"}\n```' + ) + + assert error is None + assert regex == "^abc$" + + +def test_extract_regex_from_llm_text_accepts_embedded_json_object(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + + regex, error = regex_generator._extract_regex_from_llm_text( + 'Here is the result: {"regex":"^abc$","rationale":"ok"}' + ) + + assert error is None + assert regex == "^abc$" + + +def test_validate_regex_rejects_unsupported_or_too_broad_patterns(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + + assert regex_generator._validate_regex(".*") == "regex_too_broad" + assert ( + regex_generator._validate_regex("(?<=abc)def") + == "unsupported_lookbehind" + ) + assert ( + regex_generator._validate_regex(r"^(abc)\1$") + == "unsupported_backreference" + ) + assert regex_generator._validate_regex(r"^(.*a)+$") == "nested_wildcards" + + +def test_validate_and_store_regex_rejects_duplicate_exact(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = Mock() + regex_generator.storage.might_have_generated_regex.return_value = True + regex_generator.storage.get_existing_generated_regex.return_value = { + "regex": "^dup$" + } + + regex_generator._validate_and_store_regex( + { + "regex_type": "dns_domain", + "regex": "^dup$", + "regex_hash": regex_generator._hash_regex("^dup$"), + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-1", + "created_at": time.time(), + } + ) + + regex_generator.storage.store_generated_regex.assert_not_called() + + +def test_validate_and_store_regex_rejects_validation_timeout(tmp_path): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = Mock() + regex_generator.storage.might_have_generated_regex.return_value = False + regex_generator._find_strong_benign_match = Mock( + side_effect=TimeoutError("timed out") + ) + + regex_generator._validate_and_store_regex( + { + "regex_type": "dns_domain", + "regex": r"^slow-example$", + "regex_hash": regex_generator._hash_regex(r"^slow-example$"), + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-timeout", + "created_at": time.time(), + } + ) + + stored = regex_generator.storage.store_generated_regex.call_args.args[0] + assert stored["status"] == "rejected" + assert stored["rejection_reason"] == "regex_validation_timeout" + + +def test_benign_seeding_initializes_all_types(tmp_path): + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf(str(tmp_path / "regex_generator")), + "dummy_output_dir", + 12345, + ) + + for regex_type in REGEX_TYPES: + assert storage.get_benign_examples(regex_type, limit=1) + + storage.close() + + +def test_storage_resolves_relative_store_dir_inside_run_output_dir(tmp_path): + output_dir = tmp_path / "slips_run_output" + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf("output/regex_generator"), + str(output_dir), + 12345, + ) + + assert storage.store_dir == str(output_dir / "regex_generator") + storage.close() + + +def test_storage_prefers_persistent_store_dir_when_configured(tmp_path): + output_dir = tmp_path / "slips_run_output" + persistent_dir = tmp_path / "persistent_regex_generator" + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf( + "output/regex_generator", + persistent_store_dir=str(persistent_dir), + ), + str(output_dir), + 12345, + ) + + assert storage.store_dir == str(persistent_dir) + storage.close() + + +def test_storage_resolves_relative_persistent_store_dir_inside_permanent_dir( + tmp_path, monkeypatch +): + output_dir = tmp_path / "slips_run_output" + permanent_dir = tmp_path / "permanent" + monkeypatch.setattr( + "slips_files.core.database.sqlite_db.regex_generator_db." + "get_this_filepath_inside_permanent_dir", + lambda filename: str(permanent_dir / filename), + ) + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf( + "output/regex_generator", + persistent_store_dir="databases/regex_store", + ), + str(output_dir), + 12345, + ) + + assert storage.store_dir == str(permanent_dir / "databases/regex_store") + storage.close() + + +def test_storage_imports_whitelist_domains_into_matching_regex_types(tmp_path): + whitelist_path = tmp_path / "whitelist.conf" + whitelist_path.write_text( + "\n".join( + [ + "; comment", + "domain,example.com,both,alerts", + "domain,api.github.com,both,alerts", + "ip,1.2.3.4,both,alerts", + ] + ), + encoding="utf-8", + ) + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf( + str(tmp_path / "regex_generator"), + local_whitelist_path=str(whitelist_path), + ), + "dummy_output_dir", + 12345, + ) + + assert "example.com" in storage.get_benign_examples( + "dns_domain", limit=100 + ) + assert "example.com" in storage.get_benign_examples("tls_sni", limit=100) + assert "example.com" in storage.get_benign_examples( + "certificate_cn", limit=100 + ) + assert "github.com" in storage.get_benign_examples("dns_domain", limit=100) + assert "/index.html" in storage.get_benign_examples("uri", limit=100) + storage.close() + + +def test_storage_skips_whitelist_import_when_disabled(tmp_path): + whitelist_path = tmp_path / "whitelist.conf" + whitelist_path.write_text( + "domain,example.com,both,alerts\n", + encoding="utf-8", + ) + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf( + str(tmp_path / "regex_generator"), + enable_local_whitelist=False, + local_whitelist_path=str(whitelist_path), + ), + "dummy_output_dir", + 12345, + ) + + assert "example.com" not in storage.get_benign_examples( + "dns_domain", limit=100 + ) + storage.close() + + +def test_storage_imports_tranco_top_domains_into_matching_regex_types( + tmp_path, +): + db = Mock() + db.get_tranco_top_domains = Mock( + return_value=["google.com", "github.com", "microsoft.com"] + ) + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf(str(tmp_path / "regex_generator")), + "dummy_output_dir", + 12345, + db=db, + ) + + assert "google.com" in storage.get_benign_examples("dns_domain", limit=200) + assert "github.com" in storage.get_benign_examples("tls_sni", limit=200) + assert "microsoft.com" in storage.get_benign_examples( + "certificate_cn", limit=200 + ) + db.get_tranco_top_domains.assert_called_once_with(limit=1000) + storage.close() + + +def test_storage_skips_tranco_import_when_limit_is_zero(tmp_path): + db = Mock() + db.get_tranco_top_domains = Mock(return_value=["tranco-only-example.test"]) + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf( + str(tmp_path / "regex_generator"), + seed_benign_samples=False, + tranco_top_benign_limit=0, + ), + "dummy_output_dir", + 12345, + db=db, + ) + + assert "tranco-only-example.test" not in storage.get_benign_examples( + "dns_domain", limit=200 + ) + db.get_tranco_top_domains.assert_not_called() + storage.close() + + +def test_benign_corpus_scan_rejects_matching_regex(tmp_path): + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf( + str(tmp_path / "regex_generator"), + store_rejected_regexes=True, + ), + "dummy_output_dir", + 12345, + ) + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = storage + + regex_generator._validate_and_store_regex( + { + "regex_type": "dns_domain", + "regex": r"^google\.com$", + "regex_hash": regex_generator._hash_regex(r"^google\.com$"), + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-1", + "created_at": time.time(), + } + ) + + rejected = storage.get_generated_regexes( + regex_type="dns_domain", + status="rejected", + ) + assert rejected[0]["rejection_reason"] == "matched_benign_data_too_strong" + assert rejected[0]["matched_benign_value"] == "google.com" + storage.close() + + +def test_benign_corpus_scan_rejects_regex_matching_whitelist_domain(tmp_path): + whitelist_path = tmp_path / "whitelist.conf" + whitelist_path.write_text( + "domain,example.com,both,alerts\n", + encoding="utf-8", + ) + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf( + str(tmp_path / "regex_generator"), + store_rejected_regexes=True, + local_whitelist_path=str(whitelist_path), + ), + "dummy_output_dir", + 12345, + ) + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = storage + + regex_generator._validate_and_store_regex( + { + "regex_type": "dns_domain", + "regex": r"^example\.com$", + "regex_hash": regex_generator._hash_regex(r"^example\.com$"), + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-whitelist", + "created_at": time.time(), + } + ) + + rejected = storage.get_generated_regexes( + regex_type="dns_domain", + status="rejected", + ) + assert rejected[0]["rejection_reason"] == "matched_benign_data_too_strong" + assert rejected[0]["matched_benign_value"] == "example.com" + storage.close() + + +def test_partial_benign_match_can_be_accepted_below_strength_threshold( + tmp_path, +): + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf(str(tmp_path / "regex_generator")), + "dummy_output_dir", + 12345, + ) + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = storage + regex_generator.benign_match_strength_threshold = 80 + + regex_generator._validate_and_store_regex( + { + "regex_type": "dns_domain", + "regex": r"google", + "regex_hash": regex_generator._hash_regex(r"google"), + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-weak-benign", + "created_at": time.time(), + } + ) + + accepted = storage.get_generated_regexes(regex_type="dns_domain") + assert accepted + assert accepted[0]["regex"] == r"google" + storage.close() + + +def test_match_strength_scores_full_specific_match_higher_than_partial_match( + tmp_path, +): + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + full_score = regex_generator._compute_match_strength( + re.compile(r"^google\.com$"), + "google.com", + regex_generator._measure_regex_specificity(r"^google\.com$"), + ) + partial_score = regex_generator._compute_match_strength( + re.compile(r"google"), + "google.com", + regex_generator._measure_regex_specificity(r"google"), + ) + + assert full_score > partial_score + + +def test_rejected_regexes_are_not_persisted_by_default(tmp_path): + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf(str(tmp_path / "regex_generator")), + "dummy_output_dir", + 12345, + ) + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = storage + + regex_generator._validate_and_store_regex( + { + "regex_type": "dns_domain", + "regex": r"^google\.com$", + "regex_hash": regex_generator._hash_regex(r"^google\.com$"), + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-default-reject", + "created_at": time.time(), + } + ) + + assert storage.get_generated_regexes(status="rejected") == [] + assert storage.was_rejected_in_current_run( + regex_generator._hash_regex(r"^google\.com$") + ) + storage.close() + + +def test_stored_rejected_regexes_are_pruned_to_max_size(tmp_path): + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf( + str(tmp_path / "regex_generator"), + store_rejected_regexes=True, + max_stored_rejected_regexes=1, + ), + "dummy_output_dir", + 12345, + ) + + storage.store_generated_regex( + { + "regex_type": "dns_domain", + "regex": "^first$", + "regex_hash": "hash-first", + "status": "rejected", + "rejection_reason": "invalid_regex_syntax", + "matched_benign_value": None, + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-first", + "created_at": 1.0, + } + ) + storage.store_generated_regex( + { + "regex_type": "dns_domain", + "regex": "^second$", + "regex_hash": "hash-second", + "status": "rejected", + "rejection_reason": "invalid_regex_syntax", + "matched_benign_value": None, + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-second", + "created_at": 2.0, + } + ) + + rejected = storage.get_generated_regexes(status="rejected") + assert [row["regex"] for row in rejected] == ["^second$"] + storage.close() + + +def test_validate_and_store_regex_accepts_non_matching_regex(tmp_path): + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf(str(tmp_path / "regex_generator")), + "dummy_output_dir", + 12345, + ) + regex_generator = ModuleFactory().create_regex_generator_obj( + store_dir=str(tmp_path / "regex_generator") + ) + regex_generator.storage = storage + + regex_generator._validate_and_store_regex( + { + "regex_type": "dns_domain", + "regex": r"^xqz[a-z0-9]{8,12}\.invalid$", + "regex_hash": regex_generator._hash_regex( + r"^xqz[a-z0-9]{8,12}\.invalid$" + ), + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-2", + "created_at": time.time(), + } + ) + + accepted = storage.get_generated_regexes( + regex_type="dns_domain", + status="accepted", + ) + assert accepted[0]["regex"] == r"^xqz[a-z0-9]{8,12}\.invalid$" + storage.close() + + +def test_storage_generated_regex_bloom_filter_tracks_inserted_hash(tmp_path): + storage = RegexGeneratorStorage( + Mock(), + _build_storage_conf(str(tmp_path / "regex_generator")), + "dummy_output_dir", + 12345, + ) + + record = { + "regex_type": "dns_domain", + "regex": r"^xqz[a-z0-9]{8,12}\.invalid$", + "regex_hash": "hash-1", + "status": "accepted", + "rejection_reason": None, + "matched_benign_value": None, + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": PROMPT_VERSION, + "request_id": "req-3", + "created_at": time.time(), + } + + assert storage.might_have_generated_regex("hash-1") is False + storage.store_generated_regex(record) + assert storage.might_have_generated_regex("hash-1") is True + storage.close() diff --git a/tests/unit/modules/t_cell/test_analyze_t_cell.py b/tests/unit/modules/t_cell/test_analyze_t_cell.py new file mode 100644 index 0000000000..b33d409240 --- /dev/null +++ b/tests/unit/modules/t_cell/test_analyze_t_cell.py @@ -0,0 +1,605 @@ +# SPDX-FileCopyrightText: 2026 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +from pathlib import Path +from unittest.mock import Mock + +from modules.t_cell.analyze_t_cell import build_report_payload, render_html +from slips_files.core.database.sqlite_db.t_cell_db import TCellStorage + + +def _build_storage(run_dir: Path) -> TCellStorage: + conf = Mock() + conf.t_cell_store_dir = Mock(return_value="output/t_cell") + conf.t_cell_persistent_store_dir = Mock(return_value="") + return TCellStorage(Mock(), conf, str(run_dir), 12345) + + +def _raw_evidence( + evidence_id: str, + evidence_type: str, + signal: str, + related_profile_ip: str, + attacker_ip: str, + victim_ip: str, + description: str, +) -> dict: + return { + "evidence_type": evidence_type, + "description": description, + "attacker": { + "direction": "SRC", + "ioc_type": "IP", + "value": attacker_ip, + }, + "victim": { + "direction": "DST", + "ioc_type": "IP", + "value": victim_ip, + }, + "profile": {"ip": related_profile_ip}, + "timewindow": {"number": 1}, + "uid": [], + "timestamp": "2026/03/21 09:22:37.000000+0000", + "interface": "eno1", + "id": evidence_id, + "confidence": 1.0, + "threat_level": "HIGH", + "evidence_signal": signal, + } + + +def test_build_report_payload_and_html(tmp_path): + run_dir = tmp_path / "run-output" + (run_dir / "metadata").mkdir(parents=True) + module_dir = run_dir / "t_cell" + module_dir.mkdir(parents=True) + storage = _build_storage(run_dir) + + damp_observation_id = storage.insert_observation( + { + "evidence_id": "damp-1", + "evidence_type": "HTTP_TRAFFIC", + "evidence_signal": "DAMP", + "profile_ip": "2001:db8::5", + "timewindow_number": 1, + "timestamp": "2026/03/21 09:22:37.000000+0000", + "observed_at": 1000.0, + "confidence": 0.9, + "threat_level": "medium", + "threat_level_value": 0.5, + "interface": "eno1", + "uids": ["uid-damp-1"], + "antigen_count": 2, + "antigens": [ + {"regex_type": "dns_domain", "value": "rdap.db.ripe.net"}, + {"regex_type": "uri", "value": "/ip/5.161.194.92"}, + ], + "matched_regexes": [], + "raw_evidence": _raw_evidence( + "damp-1", + "HTTP_TRAFFIC", + "DAMP", + "2001:db8::5", + "2001:db8::5", + "2001:67c:2e8:22::c100:697", + "RDAP lookup over HTTP", + ), + } + ) + + pamp_observation_id = storage.insert_observation( + { + "evidence_id": "pamp-1", + "evidence_type": "THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN", + "evidence_signal": "PAMP", + "profile_ip": "203.0.113.90", + "timewindow_number": 2, + "timestamp": "2026/03/21 09:23:37.000000+0000", + "observed_at": 2000.0, + "confidence": 1.0, + "threat_level": "high", + "threat_level_value": 0.8, + "interface": "eno1", + "uids": ["uid-pamp-1"], + "antigen_count": 1, + "antigens": [ + {"regex_type": "dns_domain", "value": "bad.example.com"} + ], + "matched_regexes": [ + { + "regex_type": "dns_domain", + "value": "bad.example.com", + "regex_hash": "regex-hash-1", + "regex": r"^bad\.example\.com$", + "created_at": 1990.0, + "specificity": 1.0, + } + ], + "raw_evidence": _raw_evidence( + "pamp-1", + "THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN", + "PAMP", + "147.32.80.37", + "203.0.113.90", + "147.32.80.37", + "Known malicious domain", + ), + } + ) + + cell_key = "203.0.113.90|dns_domain|bad.example.com" + storage.upsert_cell( + { + "cell_key": cell_key, + "profile_ip": "203.0.113.90", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "state": 5, + "state_name": "5 - memory", + "matched_regex_hash": "regex-hash-1", + "matched_regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "anergic_until": None, + "effector_cooldown_until": None, + "last_observation_id": pamp_observation_id, + "last_evidence_id": "pamp-1", + "last_transition_at": 2000.3, + "last_co_stimulation": 0.91, + "last_effector_score": 0.33, + "last_memory_score": 0.78, + "context": { + "novelty_score": 0, + "recent_pressure": 0.42, + "priming_signal": "PAMP", + "priming_label": "pamp-primed", + "priming_strength": 1.0, + "priming_profile": { + "signal": "PAMP", + "label": "pamp-primed", + "strength": 1.0, + "co_stimulation_threshold": 0.65, + "effector_threshold": 0.70, + "memory_threshold": 0.60, + "state_wait_timeout_seconds": 3600.0, + "effector_min_related_count": 4, + "memory_min_related_count": 3, + }, + }, + "created_at": 2000.0, + "updated_at": 2000.3, + } + ) + storage.upsert_cell( + { + "cell_key": "192.168.1.121|tls_sni|arpanet-network.com", + "profile_ip": "192.168.1.121", + "regex_type": "tls_sni", + "antigen_value": "arpanet-network.com", + "state": 3, + "state_name": "3 - activated", + "matched_regex_hash": "regex-hash-2", + "matched_regex": r"arpanet-network\.com$", + "matched_value": "arpanet-network.com", + "anergic_until": None, + "effector_cooldown_until": None, + "last_observation_id": pamp_observation_id, + "last_evidence_id": "pamp-1", + "last_transition_at": 2000.4, + "last_co_stimulation": 1.0, + "last_effector_score": 0.70, + "last_memory_score": 0.40, + "context": { + "waiting_for": "context", + "waiting_since": 2000.4, + "wait_deadline": 2060.4, + }, + "created_at": 2000.4, + "updated_at": 2000.4, + } + ) + storage.insert_transition( + { + "cell_key": cell_key, + "profile_ip": "203.0.113.90", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "evidence_id": "pamp-1", + "observation_id": pamp_observation_id, + "from_state": 0, + "to_state": 1, + "reason": "antigen_recognized", + "matched_regex_hash": "regex-hash-1", + "matched_regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "scores": { + "regex_specificity": 1.0, + "priming_signal": "PAMP", + "priming_label": "pamp-primed", + "priming_strength": 1.0, + "co_stimulation_threshold": 0.65, + "effector_threshold": 0.70, + "memory_threshold": 0.60, + "state_wait_timeout_seconds": 3600.0, + "effector_min_related_count": 4, + "memory_min_related_count": 3, + }, + "created_at": 2000.1, + } + ) + storage.insert_transition( + { + "cell_key": cell_key, + "profile_ip": "203.0.113.90", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "evidence_id": "pamp-1", + "observation_id": pamp_observation_id, + "from_state": 1, + "to_state": 3, + "reason": "co_stimulation_threshold_met", + "matched_regex_hash": "regex-hash-1", + "matched_regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "scores": {"value": 0.91, "threshold": 0.65}, + "created_at": 2000.2, + } + ) + storage.insert_transition( + { + "cell_key": cell_key, + "profile_ip": "203.0.113.90", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "evidence_id": "pamp-1", + "observation_id": pamp_observation_id, + "from_state": 3, + "to_state": 5, + "reason": "context_memory", + "matched_regex_hash": "regex-hash-1", + "matched_regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "scores": {"memory_score": 0.78, "memory_threshold": 0.60}, + "created_at": 2000.3, + } + ) + storage.upsert_memory( + { + "cell_key": cell_key, + "profile_ip": "203.0.113.90", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "regex_hash": "regex-hash-1", + "regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "context": {"memory_score": 0.78, "recent_pressure": 0.42}, + "created_at": 2000.3, + "updated_at": 2000.3, + } + ) + + (run_dir / "metadata" / "slips.yaml").write_text( + "\n".join( + [ + "t_cell:", + " enabled: true", + " log_verbosity: 3", + " decision_trace_mode: transitions", + " co_stimulation_threshold: 0.65", + " effector_threshold: 0.70", + " memory_threshold: 0.60", + ] + ), + encoding="utf-8", + ) + (module_dir / "t_cell.log").write_text( + "\n".join( + [ + "T Cell module ready.", + "2026/03/21 09:22:37.597262 | action=antigens_extracted | evidence=HTTP_TRAFFIC | eid=damp-1 | signal=DAMP | profile=2001:db8::5 | responsible=2001:db8::5 | target=2001:67c:2e8:22::c100:697 | antigens=dns_domain:rdap.db.ripe.net, uri:/ip/5.161.194.92", + "2026/03/21 09:22:37.607926 | action=damp_reverification | evidence=HTTP_TRAFFIC | eid=damp-1 | signal=DAMP | profile=2001:db8::5 | responsible=2001:db8::5 | target=2001:67c:2e8:22::c100:697 | reevaluated_cells=0", + "2026/03/21 09:23:37.607926 | action=memory_stored | state=5 - memory | evidence=THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN | eid=pamp-1 | signal=PAMP | profile=147.32.80.37 | responsible=203.0.113.90 | target=147.32.80.37 | cell=203.0.113.90|dns_domain|bad.example.com | regex=regex-hash-1 | value=bad.example.com", + ] + ), + encoding="utf-8", + ) + (module_dir / "t_cell_trace.jsonl").write_text( + "\n".join( + [ + json.dumps( + { + "ts": "2026/03/21 09:23:37.200000+0000", + "stage": "co_stimulation", + "action": "co_stimulation_threshold_met", + "cell_key": cell_key, + "from_state": "1 - antigen-recognized", + "to_state": "3 - activated", + "responsible_ip": "203.0.113.90", + "candidate": { + "regex_type": "dns_domain", + "value": "bad.example.com", + }, + "current_evidence": { + "observation_id": pamp_observation_id, + "evidence_id": "pamp-1", + "evidence_type": "THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN", + "signal": "PAMP", + "confidence": 1.0, + "threat_level": "high", + "threat_level_value": 0.8, + "danger_contribution": 0.8, + }, + "formula": { + "value": 0.91, + "threshold": 0.65, + "priming": { + "signal": "PAMP", + "label": "pamp-primed", + "strength": 1.0, + "co_stimulation_threshold": 0.65, + "effector_threshold": 0.70, + "memory_threshold": 0.60, + "state_wait_timeout_seconds": 3600.0, + "effector_min_related_count": 4, + "memory_min_related_count": 3, + }, + "components": { + "confidence": { + "value": 1.0, + "weighted": 0.35, + }, + "related_pamps": { + "count": 1, + "saturation": 5, + "score": 0.2, + "weighted": 0.05, + "contributors": [ + { + "observation_id": damp_observation_id, + "evidence_id": "damp-1", + "evidence_type": "HTTP_TRAFFIC", + "signal": "DAMP", + "confidence": 0.9, + "threat_level": "medium", + "threat_level_value": 0.5, + "danger_contribution": 0.45, + "relations": ["same_antigen"], + } + ], + }, + "danger": { + "score": 0.51, + "weighted": 0.204, + "pamp_score": 0.32, + "damp_score": 0.18, + "damp_weight": 1.5, + "danger_saturation": 2.5, + "pamp_contributors": [], + "damp_contributors": [ + { + "observation_id": damp_observation_id, + "evidence_id": "damp-1", + "evidence_type": "HTTP_TRAFFIC", + "signal": "DAMP", + "confidence": 0.9, + "threat_level": "medium", + "threat_level_value": 0.5, + "danger_contribution": 0.45, + } + ], + }, + }, + }, + } + ), + json.dumps( + { + "ts": "2026/03/21 09:23:37.300000+0000", + "stage": "context", + "action": "context_memory", + "cell_key": cell_key, + "from_state": "3 - activated", + "to_state": "5 - memory", + "responsible_ip": "203.0.113.90", + "candidate": { + "regex_type": "dns_domain", + "value": "bad.example.com", + }, + "current_evidence": { + "observation_id": pamp_observation_id, + "evidence_id": "pamp-1", + "evidence_type": "THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN", + "signal": "PAMP", + "confidence": 1.0, + "threat_level": "high", + "threat_level_value": 0.8, + "danger_contribution": 0.8, + }, + "formula": { + "effector_score": 0.33, + "effector_threshold": 0.70, + "memory_score": 0.78, + "memory_threshold": 0.60, + "priming": { + "signal": "PAMP", + "label": "pamp-primed", + "strength": 1.0, + "co_stimulation_threshold": 0.65, + "effector_threshold": 0.70, + "memory_threshold": 0.60, + "state_wait_timeout_seconds": 3600.0, + "effector_min_related_count": 4, + "memory_min_related_count": 3, + }, + "decision": {"effector": False, "memory": True}, + "components": { + "novelty": { + "score": 0, + "has_memory_for_regex": True, + "has_recent_regex_activity": True, + }, + "recent_related": { + "count": 1, + "saturation": 5, + "score": 0.2, + "contributors": [], + }, + "recent_pressure": { + "combined_score": 0.42, + "pamp_score": 0.32, + "damp_score": 0.10, + "pamp_total_raw": 0.8, + "damp_total_raw": 0.25, + "pamp_contributors": [], + "damp_contributors": [], + }, + "previous_pressure": { + "combined_score": 0.70, + "pamp_score": 0.52, + "damp_score": 0.18, + "pamp_total_raw": 1.3, + "damp_total_raw": 0.45, + "pamp_contributors": [], + "damp_contributors": [], + }, + "trend_ratio": 0.6, + "decrease_score": 0.4, + "familiarity_score": 1.0, + "stability_score": 0.33, + }, + }, + } + ), + ] + ), + encoding="utf-8", + ) + + payload = build_report_payload(run_dir, max_observations=50, max_log_lines=50, max_trace_rows=50) + + assert payload["totals"]["observations"] == 2 + assert payload["totals"]["signals"] == {"DAMP": 1, "PAMP": 1} + assert payload["totals"]["transitions"] == 3 + assert payload["totals"]["memories"] == 1 + assert payload["cell_states"]["5 - memory"] == 1 + assert payload["cell_states"]["3 - activated"] == 1 + assert payload["sources"]["trace_enabled"] is True + assert payload["trace"]["total_rows"] == 2 + assert payload["recent_observations"][0]["category"] == "PAMP with regex match" + assert any( + row["waiting_label"] == "waiting for context" + for row in payload["recent_cells"] + ) + assert any( + row["category"] == "DAMP with extracted antigens" + for row in payload["recent_observations"] + ) + assert payload["top_responsible_ips"][0]["label"] == "2001:db8::5" + + html = render_html(payload) + + assert "T Cell Report" in html + assert "T Cell Run Report" in html + assert "Run Findings" in html + assert "Quick Summary" in html + assert "Decision Trace" in html + assert "Decision Reference" in html + assert "T Cell Histories" in html + assert "T Cell State Machine" in html + assert "accepted regex match" in html + assert "no accepted regex match" in html + assert "weaker priming profile" in html + assert "co-stimulation below threshold" in html + assert "no co-stimulation timeout" in html + assert "current cells:" in html + assert "Module Log Tail" not in html + assert "data-sortable-table='recent-observations'" in html + assert "data-sortable-table='recent-transitions'" in html + assert "data-sortable-table='recent-cells'" in html + assert "data-default-sort-column='4'" in html + assert "Default order groups rows by T cell" in html + assert "Click a column header to sort." in html + assert html.index("Recent Observations") < html.index("Run configuration snapshot") + assert "co_stimulation_threshold_met" in html + assert "context_memory" in html + assert "bad.example.com" in html + assert "DAMP with extracted antigens" in html + assert "PAMP with regex match" in html + assert "waiting for context" in html + assert "clamp01(x) = max(0, min(1, x))" in html + assert "Recognition & Priming: 0 -> 1 setup" in html + assert "Co-Stimulation: 1 -> 3 activation" in html + assert "Context Effector: 3 -> 4 containment" in html + assert "Context Memory: 3 -> 5 storage" in html + assert "Hover or focus a node to see where that term comes from." in html + assert "related_pamp_score" in html + assert "novelty_score" in html + assert "recent_pressure / max(previous_pressure, 0.01)" in html + assert "Rule-Based Decisions" in html + assert "signal_specific_priming" in html + assert "effector = (novelty_score > 0)" in html + assert "priming=pamp-primed" in html + assert "Effective profile:" in html + assert "data-report-tab=\"histories\"" in html + assert "History Index" in html + assert "State transition" in html + assert "Decision trace" in html + assert "passed: 0.910 >= 0.650" in html + assert "effector=no (0.330 / 0.700) | memory=yes (0.780 / 0.600)" in html + assert "Chronological lifecycle view for each cell" in html + assert "3 - activated (waiting for context)" not in html + + storage.close() + + +def test_build_report_payload_resolves_persistent_db_and_module_output( + tmp_path, +): + """Report builder should read persistent DBs and module output paths.""" + run_dir = tmp_path / "run-output" + permanent_store = tmp_path / "permanent" / "t_cell" + module_dir = run_dir / "t_cell" + (run_dir / "metadata").mkdir(parents=True) + (module_dir / "audit").mkdir(parents=True) + + conf = Mock() + conf.t_cell_store_dir = Mock(return_value="output/t_cell") + conf.t_cell_persistent_store_dir = Mock(return_value=str(permanent_store)) + storage = TCellStorage(Mock(), conf, str(run_dir), 12345) + storage.close() + + (run_dir / "metadata" / "slips-sebas.yaml").write_text( + "\n".join( + [ + "parameters:", + f" permanent_dir: {tmp_path / 'permanent'}", + "t_cell:", + " persistent_store_dir: t_cell", + " decision_trace_file: audit/t_cell_trace.jsonl", + ] + ), + encoding="utf-8", + ) + (module_dir / "t_cell.log").write_text( + "T Cell module ready.\n", + encoding="utf-8", + ) + (module_dir / "audit" / "t_cell_trace.jsonl").write_text( + "", + encoding="utf-8", + ) + + payload = build_report_payload(run_dir) + + assert payload["sources"]["db_path"] == str( + permanent_store / "t_cell.sqlite" + ) + assert payload["sources"]["log_path"] == str(module_dir / "t_cell.log") + assert payload["sources"]["trace_path"] == str( + module_dir / "audit" / "t_cell_trace.jsonl" + ) + assert payload["sources"]["metadata_path"] == str( + run_dir / "metadata" / "slips-sebas.yaml" + ) + assert payload["sources"]["log_present"] is True + assert payload["sources"]["trace_enabled"] is True diff --git a/tests/unit/modules/t_cell/test_t_cell.py b/tests/unit/modules/t_cell/test_t_cell.py new file mode 100644 index 0000000000..1019472ad8 --- /dev/null +++ b/tests/unit/modules/t_cell/test_t_cell.py @@ -0,0 +1,1411 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json +from pathlib import Path +from unittest.mock import Mock, patch + +from modules.t_cell.t_cell import ( + STATE_ACTIVATED, + STATE_ANERGIC, + STATE_ANTIGEN_RECOGNIZED, + STATE_EFFECTOR, + STATE_MATURE, + STATE_MEMORY, + AntigenCandidate, + RegexMatch, +) +from slips_files.common.slips_utils import utils +from slips_files.core.database.sqlite_db.t_cell_db import TCellStorage +from slips_files.core.structures.evidence import ( + Attacker, + Direction, + Evidence, + EvidenceSignal, + EvidenceType, + IoCType, + Method, + ProfileID, + Proto, + ThreatLevel, + TimeWindow, + Victim, +) +from tests.module_factory import ModuleFactory + +TEST_TS = utils.convert_ts_format(1700000000, utils.alerts_format) + + +def _build_storage(tmp_path): + conf = Mock() + conf.t_cell_store_dir = Mock(return_value="output/t_cell") + conf.t_cell_persistent_store_dir = Mock(return_value="") + return TCellStorage(Mock(), conf, str(tmp_path), 12345) + + +def _prepare_t_cell( + tmp_path, + log_verbosity: int = 3, + trace_mode: int = 0, + trace_max_evidence: int = 10, +): + t_cell = ModuleFactory().create_t_cell_obj() + t_cell.output_dir = str(tmp_path) + t_cell.log_file_path = str(tmp_path / "t_cell.log") + t_cell.trace_file_path = str(tmp_path / "t_cell_trace.jsonl") + storage = _build_storage(tmp_path) + t_cell.db.get_t_cell_storage.return_value = storage + with patch("modules.t_cell.t_cell.utils.drop_root_privs_permanently"): + assert t_cell.pre_main() is False + t_cell.log_verbosity = log_verbosity + t_cell.decision_trace_mode = trace_mode + t_cell.decision_trace_max_evidence = trace_max_evidence + t_cell._init_trace_file() + return t_cell, storage + + +def test_t_cell_uses_lowercase_underscore_output_dir(): + """T Cell should use a lowercase underscore module output directory.""" + t_cell = ModuleFactory().create_t_cell_obj() + + assert t_cell.output_dir == str(Path("dummy_output_dir") / "t_cell") + assert t_cell.log_file_path == str( + Path("dummy_output_dir") / "t_cell" / "t_cell.log" + ) + + +def _build_evidence( + evidence_id: str, + signal: EvidenceSignal = EvidenceSignal.PAMP, + attacker=None, + victim=None, + uids=None, + profile_ip: str = "10.0.0.50", + threat_level: ThreatLevel = ThreatLevel.HIGH, + confidence: float = 1.0, +): + attacker = attacker or Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value=profile_ip, + ) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, + description="test evidence", + attacker=attacker, + victim=victim, + threat_level=threat_level, + profile=ProfileID(ip=profile_ip), + timewindow=TimeWindow(number=1), + uid=uids or ["uid-1"], + timestamp=TEST_TS, + proto=Proto.TCP, + dst_port=443, + method=Method.HEURISTIC, + id=evidence_id, + confidence=confidence, + ) + evidence.evidence_signal = signal + return evidence + + +def _message_for(evidence: Evidence) -> dict: + return {"data": json.dumps(utils.to_dict(evidence))} + + +def _process_evidence_at(t_cell, evidence: Evidence, ts: float): + with patch("modules.t_cell.t_cell.time.time", return_value=ts): + t_cell._process_evidence_message(_message_for(evidence)) + + +def _read_trace_entries(trace_path): + with open(trace_path, encoding="utf-8") as trace_file: + return [ + json.loads(line) + for line in trace_file + if line.strip() + ] + + +def _insert_observation( + storage, + evidence_id: str, + profile_ip: str, + antigens: list[dict], + observed_at: float, + confidence: float, + threat_level_value: float, + threat_level: str = "high", + matched_regexes: list[dict] | None = None, + evidence_signal: str = "PAMP", +): + return storage.insert_observation( + { + "evidence_id": evidence_id, + "evidence_type": "THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN", + "evidence_signal": evidence_signal, + "profile_ip": profile_ip, + "timewindow_number": 1, + "timestamp": TEST_TS, + "observed_at": observed_at, + "confidence": confidence, + "threat_level": threat_level, + "threat_level_value": threat_level_value, + "interface": "default", + "uids": [f"{evidence_id}-uid"], + "antigen_count": len(antigens), + "antigens": antigens, + "matched_regexes": matched_regexes or [], + "raw_evidence": {}, + } + ) + + +def _seed_recent_related_observations( + storage, + profile_ip: str, + antigen: AntigenCandidate, + fixed_now: float, + count: int, + confidence: float = 1.0, + threat_level_value: float = 0.8, + age_seconds: int = 300, +): + for index in range(count): + _insert_observation( + storage=storage, + evidence_id=f"hist-recent-{index}", + profile_ip=profile_ip, + antigens=[antigen.as_dict()], + observed_at=fixed_now - age_seconds - index, + confidence=confidence, + threat_level_value=threat_level_value, + ) + + +def _accepted_domain_regex(regex_hash: str = "regex-hash") -> list[dict]: + return [ + { + "regex_type": "dns_domain", + "regex": r"^bad\.example\.com$", + "regex_hash": regex_hash, + "created_at": 10, + } + ] + + +def test_extract_antigen_candidates_from_entities_and_altflows(tmp_path): + t_cell, _ = _prepare_t_cell(tmp_path) + attacker = Attacker( + direction=Direction.SRC, + ioc_type=IoCType.URL, + value="https://download.bad.example.com/payload/run.exe?stage=2", + ) + victim = Victim( + direction=Direction.DST, + ioc_type=IoCType.DOMAIN, + value="victim.bad.example.com", + SNI="sni.bad.example.com", + ) + evidence = _build_evidence( + "extract-1", + attacker=attacker, + victim=victim, + uids=["dns-1", "http-1", "ssl-1"], + ) + t_cell.db.get_altflow_from_uid.side_effect = lambda uid: { + "dns-1": {"type_": "dns", "query": "dns.bad.example.com"}, + "http-1": { + "type_": "http", + "host": "http.bad.example.com", + "uri": "/dropper/setup.exe", + }, + "ssl-1": { + "type_": "ssl", + "server_name": "tls.bad.example.com", + "subject": "C=US,O=Test,CN=cn.bad.example.com", + }, + }[uid] + + extracted = { + (item.regex_type, item.value) + for item in t_cell._extract_antigen_candidates(evidence) + } + + assert ("dns_domain", "download.bad.example.com") in extracted + assert ("dns_domain", "victim.bad.example.com") in extracted + assert ("dns_domain", "dns.bad.example.com") in extracted + assert ("dns_domain", "http.bad.example.com") in extracted + assert ("uri", "/payload/run.exe?stage=2") in extracted + assert ("uri", "/dropper/setup.exe") in extracted + assert ("filename", "run.exe") in extracted + assert ("filename", "setup.exe") in extracted + assert ("tls_sni", "sni.bad.example.com") in extracted + assert ("tls_sni", "tls.bad.example.com") in extracted + assert ("certificate_cn", "cn.bad.example.com") in extracted + + +def test_t_cell_stores_damp_evidence_and_checks_waiting_cells(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + evidence = _build_evidence("damp-1", signal=EvidenceSignal.DAMP) + + with patch("modules.t_cell.t_cell.time.time", return_value=2000.0): + t_cell._process_evidence_message(_message_for(evidence)) + + observations = storage.get_recent_observations(evidence.profile.ip, 0) + assert len(observations) == 1 + assert observations[0]["evidence_signal"] == "DAMP" + assert storage.get_all_cells() == [] + t_cell.db.publish.assert_not_called() + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + assert "damp_reverification" in log_contents + assert "reevaluated_cells=0" in log_contents + assert "signal=DAMP" in log_contents + + +def test_t_cell_antigen_log_includes_evidence_signal(tmp_path): + t_cell, _ = _prepare_t_cell(tmp_path) + evidence = _build_evidence( + "damp-antigen-1", + signal=EvidenceSignal.DAMP, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.URL, + value="https://download.bad.example.com/payload/run.exe", + ), + ) + + with patch("modules.t_cell.t_cell.time.time", return_value=2050.0): + t_cell._process_evidence_message(_message_for(evidence)) + + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + log_lines = log_file.read().splitlines() + + antigen_line = next( + line for line in log_lines if "action=antigens_extracted" in line + ) + assert "signal=DAMP" in antigen_line + + +def test_t_cell_damp_can_prime_antigen_recognized_cell(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + evidence = _build_evidence( + "damp-prime-1", + signal=EvidenceSignal.DAMP, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.DOMAIN, + value="bad.example.com", + ), + ) + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "damp-prime-regex" + ) + + with patch("modules.t_cell.t_cell.time.time", return_value=2100.0): + t_cell._process_evidence_message(_message_for(evidence)) + + cell = storage.get_all_cells()[0] + assert cell["state"] == STATE_ANTIGEN_RECOGNIZED + assert cell["context"]["waiting_for"] == "co_stimulation" + assert cell["context"]["priming_signal"] == "DAMP" + assert cell["context"]["priming_label"] == "damp-primed" + assert cell["context"]["priming_strength"] == 0.6 + assert ( + cell["context"]["priming_profile"]["co_stimulation_threshold"] + > t_cell.co_stimulation_threshold + ) + assert ( + cell["context"]["priming_profile"]["state_wait_timeout_seconds"] + < t_cell.state_wait_timeout_seconds + ) + transition = storage.get_transitions(cell["cell_key"])[0] + assert transition["reason"] == "antigen_recognized" + assert transition["scores"]["priming_signal"] == "DAMP" + assert transition["scores"]["co_stimulation_threshold"] == 0.8 + assert transition["scores"]["effector_threshold"] == 0.8 + assert transition["scores"]["memory_threshold"] == 0.65 + + +def test_t_cell_damp_no_regex_match_becomes_anergic(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + evidence = _build_evidence( + "damp-no-match-1", + signal=EvidenceSignal.DAMP, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.DOMAIN, + value="unknown.example.com", + ), + ) + t_cell.db.get_generated_regexes.return_value = [] + + with patch("modules.t_cell.t_cell.time.time", return_value=2150.0): + t_cell._process_evidence_message(_message_for(evidence)) + + cell = storage.get_all_cells()[0] + assert cell["state"] == STATE_ANERGIC + assert cell["anergic_until"] == 2150.0 + t_cell.anergy_ttl_seconds + transitions = storage.get_transitions(cell["cell_key"]) + assert len(transitions) == 1 + assert transitions[0]["reason"] == "no_regex_match" + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + assert "action=no_regex_match" in log_contents + assert "state=2 - anergic" in log_contents + + +def test_t_cell_damp_priming_uses_stricter_co_stimulation_threshold(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + fixed_now = 2_200.0 + profile_ip = "10.0.0.77" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "damp-threshold-regex" + ) + + damp_prime = _build_evidence( + "damp-threshold-1", + signal=EvidenceSignal.DAMP, + profile_ip=profile_ip, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.DOMAIN, + value="bad.example.com", + ), + ) + low_costim = _build_evidence( + "damp-threshold-2", + profile_ip=profile_ip, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.DOMAIN, + value="bad.example.com", + ), + confidence=0.5, + ) + high_costim = _build_evidence( + "damp-threshold-3", + profile_ip=profile_ip, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.DOMAIN, + value="bad.example.com", + ), + confidence=0.7, + ) + + _seed_recent_related_observations( + storage, + profile_ip, + antigen, + fixed_now, + count=4, + ) + + _process_evidence_at(t_cell, damp_prime, fixed_now) + _process_evidence_at(t_cell, low_costim, fixed_now + 1) + + cell = storage.get_all_cells()[0] + assert cell["state"] == STATE_ANTIGEN_RECOGNIZED + assert abs(cell["last_co_stimulation"] - 0.775) < 1e-9 + assert cell["context"]["co_stimulation"]["threshold"] == 0.8 + + _process_evidence_at(t_cell, high_costim, fixed_now + 2) + + cell = storage.get_all_cells()[0] + transitions = storage.get_transitions(cell["cell_key"]) + assert cell["state"] == STATE_ACTIVATED + assert any( + transition["reason"] == "co_stimulation_threshold_met" + and transition["evidence_id"] == high_costim.id + for transition in transitions + ) + + +def test_t_cell_skips_pamp_without_antigens(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + evidence = _build_evidence("no-antigen-1") + + with patch("modules.t_cell.t_cell.time.time", return_value=3000.0): + t_cell._process_evidence_message(_message_for(evidence)) + + assert storage.get_all_cells() == [] + assert t_cell.db.publish.call_count == 0 + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + assert "no_antigen_extracted" in log_file.read() + + +def test_t_cell_no_match_becomes_anergic_and_expires(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + evidence = _build_evidence("anergy-1", uids=["http-1"]) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "http", + "host": "bad.example.com", + "uri": "/setup.exe", + } + t_cell.db.get_generated_regexes.return_value = [] + + with patch("modules.t_cell.t_cell.time.time", return_value=4000.0): + t_cell._process_evidence_message(_message_for(evidence)) + + cell = storage.get_all_cells()[0] + assert cell["state"] == STATE_ANERGIC + assert cell["anergic_until"] == 4000.0 + t_cell.anergy_ttl_seconds + + evidence2 = _build_evidence("anergy-2", uids=["http-1"]) + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex() + with patch( + "modules.t_cell.t_cell.time.time", + return_value=4000.0 + t_cell.anergy_ttl_seconds + 1, + ): + t_cell._process_evidence_message(_message_for(evidence2)) + + cell = storage.get_all_cells()[0] + transitions = [ + transition["reason"] + for transition in storage.get_transitions(cell["cell_key"]) + ] + assert "anergy_expired" in transitions + assert cell["state"] == STATE_ANTIGEN_RECOGNIZED + + +def test_t_cell_co_stimulation_times_out_after_one_tw(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path, log_verbosity=2) + t_cell.state_wait_timeout_seconds = 100.0 + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "timeout-regex" + ) + + first = _build_evidence( + "costim-timeout-1", + uids=["dns-1"], + threat_level=ThreatLevel.LOW, + confidence=0.1, + ) + second = _build_evidence( + "costim-timeout-2", + uids=["dns-1"], + threat_level=ThreatLevel.LOW, + confidence=0.1, + ) + + with patch("modules.t_cell.t_cell.time.time", return_value=5_000.0): + t_cell._process_evidence_message(_message_for(first)) + with patch("modules.t_cell.t_cell.time.time", return_value=5_101.0): + t_cell._process_evidence_message(_message_for(second)) + + cell = storage.get_all_cells()[0] + transitions = [ + transition["reason"] + for transition in storage.get_transitions(cell["cell_key"]) + ] + assert "co_stimulation_timeout" in transitions + assert cell["state"] == STATE_ANERGIC + assert cell["anergic_until"] == 5_101.0 + t_cell.anergy_ttl_seconds + + +def test_find_best_regex_match_prefers_specificity_and_newest(tmp_path): + t_cell, _ = _prepare_t_cell(tmp_path) + t_cell.db.get_generated_regexes.return_value = [ + { + "regex_type": "dns_domain", + "regex": r"example\.com$", + "regex_hash": "broad", + "created_at": 1, + }, + { + "regex_type": "dns_domain", + "regex": r"^bad\.example\.com$", + "regex_hash": "specific-old", + "created_at": 2, + }, + { + "regex_type": "dns_domain", + "regex": r"^bad\.example\.com$", + "regex_hash": "specific-new", + "created_at": 3, + }, + ] + + match = t_cell._find_best_regex_match( + AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + ) + + assert match.regex_hash == "specific-new" + assert match.regex == r"^bad\.example\.com$" + + +def test_t_cell_effector_publishes_blocking_and_respects_cooldown(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + fixed_now = 10_000.0 + profile_ip = "10.0.0.60" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_1 = _build_evidence( + "effector-1", profile_ip=profile_ip, uids=["dns-1"] + ) + evidence_2 = _build_evidence( + "effector-2", profile_ip=profile_ip, uids=["dns-1"] + ) + evidence_3 = _build_evidence( + "effector-3", profile_ip=profile_ip, uids=["dns-1"] + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "live-effector" + ) + t_cell.db.get_pid_of.side_effect = lambda name: 123 if name == "Blocking" else None + _seed_recent_related_observations( + storage, profile_ip, antigen, fixed_now, count=4 + ) + + _process_evidence_at(t_cell, evidence_1, fixed_now) + _process_evidence_at(t_cell, evidence_2, fixed_now + 1) + _process_evidence_at(t_cell, evidence_3, fixed_now + 2) + + assert t_cell.db.publish.call_count == 1 + channel, payload = t_cell.db.publish.call_args.args + assert channel == "new_blocking" + assert json.loads(payload) == { + "ip": profile_ip, + "block": True, + "tw": 1, + "interface": None, + } + + cell = storage.get_all_cells()[0] + assert cell["state"] == STATE_EFFECTOR + match = RegexMatch( + regex_type="dns_domain", + value="bad.example.com", + regex_hash="live-effector", + regex=r"^bad\.example\.com$", + created_at=10, + specificity=10.0, + ) + with patch("modules.t_cell.t_cell.time.time", return_value=fixed_now + 3): + t_cell._apply_effector( + cell, + evidence_3, + match, + {"effector_score": 0.95}, + fixed_now + 3, + profile_ip, + ) + + assert t_cell.db.publish.call_count == 1 + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + assert "effector_cooldown" in log_file.read() + + +def test_t_cell_simulates_effector_without_blocking_modules(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + fixed_now = 11_000.0 + profile_ip = "10.0.0.61" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_1 = _build_evidence( + "simulate-1", profile_ip=profile_ip, uids=["dns-1"] + ) + evidence_2 = _build_evidence( + "simulate-2", profile_ip=profile_ip, uids=["dns-1"] + ) + evidence_3 = _build_evidence( + "simulate-3", profile_ip=profile_ip, uids=["dns-1"] + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "sim-effector" + ) + t_cell.db.get_pid_of.return_value = None + _seed_recent_related_observations( + storage, profile_ip, antigen, fixed_now, count=4 + ) + + _process_evidence_at(t_cell, evidence_1, fixed_now) + _process_evidence_at(t_cell, evidence_2, fixed_now + 1) + _process_evidence_at(t_cell, evidence_3, fixed_now + 2) + + assert t_cell.db.publish.call_count == 0 + assert storage.get_all_cells()[0]["state"] == STATE_EFFECTOR + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + assert "effector_simulated" in log_file.read() + + +def test_t_cell_moves_to_memory_and_stores_context(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + fixed_now = 12_000.0 + profile_ip = "10.0.0.62" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_1 = _build_evidence( + "memory-1", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.MEDIUM, + confidence=0.5, + ) + evidence_2 = _build_evidence( + "memory-2", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.MEDIUM, + confidence=0.5, + ) + evidence_3 = _build_evidence( + "memory-3", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.MEDIUM, + confidence=0.5, + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "memory-regex" + ) + t_cell.db.get_pid_of.return_value = None + + for index in range(5): + _insert_observation( + storage=storage, + evidence_id=f"hist-old-{index}", + profile_ip=profile_ip, + antigens=[antigen.as_dict()], + observed_at=fixed_now - 2400 - index, + confidence=1.0, + threat_level_value=0.8, + ) + for index in range(3): + _insert_observation( + storage=storage, + evidence_id=f"hist-new-{index}", + profile_ip=profile_ip, + antigens=[antigen.as_dict()], + observed_at=fixed_now - 300 - index, + confidence=0.5, + threat_level_value=0.5, + threat_level="medium", + ) + storage.upsert_memory( + { + "cell_key": "old-memory-cell", + "profile_ip": "10.0.0.1", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "regex_hash": "memory-regex", + "regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "context": {"seeded": True}, + "created_at": fixed_now - 100, + "updated_at": fixed_now - 100, + } + ) + + _process_evidence_at(t_cell, evidence_1, fixed_now) + _process_evidence_at(t_cell, evidence_2, fixed_now + 1) + _process_evidence_at(t_cell, evidence_3, fixed_now + 2) + + cell = storage.get_all_cells()[0] + memories = storage.get_memories() + assert cell["state"] == STATE_MEMORY + assert any(memory["cell_key"] == cell["cell_key"] for memory in memories) + + +def test_t_cell_does_not_repeat_memory_events_for_same_cell(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path, log_verbosity=3) + fixed_now = 12_500.0 + profile_ip = "10.0.0.66" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_1 = _build_evidence( + "memory-repeat-1", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.MEDIUM, + confidence=0.5, + ) + evidence_2 = _build_evidence( + "memory-repeat-2", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.MEDIUM, + confidence=0.5, + ) + evidence_3 = _build_evidence( + "memory-repeat-3", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.MEDIUM, + confidence=0.5, + ) + evidence_4 = _build_evidence( + "memory-repeat-4", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.MEDIUM, + confidence=0.5, + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "memory-repeat-regex" + ) + t_cell.db.get_pid_of.return_value = None + + for index in range(5): + _insert_observation( + storage=storage, + evidence_id=f"hist-old-repeat-{index}", + profile_ip=profile_ip, + antigens=[antigen.as_dict()], + observed_at=fixed_now - 2400 - index, + confidence=1.0, + threat_level_value=0.8, + ) + for index in range(3): + _insert_observation( + storage=storage, + evidence_id=f"hist-new-repeat-{index}", + profile_ip=profile_ip, + antigens=[antigen.as_dict()], + observed_at=fixed_now - 300 - index, + confidence=0.5, + threat_level_value=0.5, + threat_level="medium", + ) + storage.upsert_memory( + { + "cell_key": "seeded-memory-cell", + "profile_ip": "10.0.0.1", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "regex_hash": "memory-repeat-regex", + "regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "context": {"seeded": True}, + "created_at": fixed_now - 100, + "updated_at": fixed_now - 100, + } + ) + + _process_evidence_at(t_cell, evidence_1, fixed_now) + _process_evidence_at(t_cell, evidence_2, fixed_now + 1) + _process_evidence_at(t_cell, evidence_3, fixed_now + 2) + _process_evidence_at(t_cell, evidence_4, fixed_now + 10) + + cell = storage.get_all_cells()[0] + transitions = storage.get_transitions(cell["cell_key"]) + assert cell["state"] == STATE_MEMORY + assert sum( + 1 for transition in transitions if transition["reason"] == "context_memory" + ) == 1 + + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + + assert log_contents.count("action=memory_stored") == 1 + assert "action=memory_retained" in log_contents + + +def test_t_cell_context_times_out_after_one_tw(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path, log_verbosity=2) + t_cell.state_wait_timeout_seconds = 100.0 + profile_ip = "10.0.0.63" + evidence_1 = _build_evidence( + "context-timeout-1", profile_ip=profile_ip, uids=["dns-1"] + ) + evidence_2 = _build_evidence( + "context-timeout-2", profile_ip=profile_ip, uids=["dns-1"] + ) + evidence_3 = _build_evidence( + "context-timeout-3", profile_ip=profile_ip, uids=["dns-1"] + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "context-timeout-regex" + ) + for index in range(4): + _insert_observation( + storage=storage, + evidence_id=f"danger-{index}", + profile_ip=profile_ip, + antigens=[ + { + "regex_type": "dns_domain", + "value": f"other-{index}.example.com", + } + ], + observed_at=5_800.0 - index, + confidence=1.0, + threat_level_value=0.8, + ) + + _process_evidence_at(t_cell, evidence_1, 6_000.0) + _process_evidence_at(t_cell, evidence_2, 6_001.0) + _process_evidence_at(t_cell, evidence_3, 6_102.0) + + cell = storage.get_all_cells()[0] + transitions = [ + transition["reason"] + for transition in storage.get_transitions(cell["cell_key"]) + ] + assert "context_timeout" in transitions + assert cell["state"] == STATE_MATURE + + +def test_t_cell_damp_observations_raise_co_stimulation(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + fixed_now = 14_000.0 + profile_ip = "10.0.0.64" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_pamp = _build_evidence( + "damp-costim-1", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.MEDIUM, + confidence=0.7, + ) + evidence_damp = _build_evidence( + "damp-costim-2", + signal=EvidenceSignal.DAMP, + profile_ip=profile_ip, + threat_level=ThreatLevel.CRITICAL, + confidence=1.0, + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "damp-costim-regex" + ) + t_cell.db.get_pid_of.return_value = None + _seed_recent_related_observations( + storage, + profile_ip, + antigen, + fixed_now, + count=2, + confidence=0.5, + threat_level_value=0.5, + ) + _insert_observation( + storage=storage, + evidence_id="damp-pressure-1", + profile_ip=profile_ip, + antigens=[], + observed_at=fixed_now - 30, + confidence=1.0, + threat_level_value=1.0, + threat_level="critical", + evidence_signal="DAMP", + ) + + _process_evidence_at(t_cell, evidence_pamp, fixed_now) + _process_evidence_at(t_cell, evidence_damp, fixed_now + 10) + + cell = storage.get_all_cells()[0] + transitions = storage.get_transitions(cell["cell_key"]) + assert cell["state"] == STATE_ACTIVATED + assert any( + transition["reason"] == "co_stimulation_threshold_met" + and transition["evidence_id"] == evidence_damp.id + and transition["scores"]["damp_danger_score"] > 0 + for transition in transitions + ) + assert t_cell.db.publish.call_count == 0 + + +def test_t_cell_damp_observations_raise_context_pressure(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path) + fixed_now = 15_000.0 + profile_ip = "10.0.0.65" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_1 = _build_evidence( + "damp-context-1", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.LOW, + confidence=1.0, + ) + evidence_2 = _build_evidence( + "damp-context-2", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.LOW, + confidence=1.0, + ) + evidence_3 = _build_evidence( + "damp-context-3", + signal=EvidenceSignal.DAMP, + profile_ip=profile_ip, + threat_level=ThreatLevel.CRITICAL, + confidence=1.0, + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "damp-context-regex" + ) + t_cell.db.get_pid_of.side_effect = ( + lambda name: 123 if name == "Blocking" else None + ) + _seed_recent_related_observations( + storage, + profile_ip, + antigen, + fixed_now, + count=4, + confidence=1.0, + threat_level_value=0.2, + age_seconds=120, + ) + _insert_observation( + storage=storage, + evidence_id="damp-pressure-2", + profile_ip=profile_ip, + antigens=[], + observed_at=fixed_now - 20, + confidence=1.0, + threat_level_value=1.0, + threat_level="critical", + evidence_signal="DAMP", + ) + + _process_evidence_at(t_cell, evidence_1, fixed_now) + _process_evidence_at(t_cell, evidence_2, fixed_now + 1) + _process_evidence_at(t_cell, evidence_3, fixed_now + 10) + + cell = storage.get_all_cells()[0] + transitions = storage.get_transitions(cell["cell_key"]) + assert cell["state"] == STATE_EFFECTOR + assert any( + transition["reason"] == "context_effector" + and transition["evidence_id"] == evidence_3.id + and transition["scores"]["recent_damp_pressure"] > 0 + for transition in transitions + ) + assert t_cell.db.publish.call_count == 1 + + +def test_t_cell_summary_log_hides_waiting_for_co_stimulation(tmp_path): + t_cell, _ = _prepare_t_cell(tmp_path, log_verbosity=1) + evidence = _build_evidence("pending-1", uids=["dns-1"]) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "pending-regex" + ) + + with patch("modules.t_cell.t_cell.time.time", return_value=13_000.0): + t_cell._process_evidence_message(_message_for(evidence)) + + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + + assert "action=antigen_recognized" in log_contents + assert "waiting_for_co_stimulation" not in log_contents + + +def test_t_cell_decision_log_explains_waiting_for_co_stimulation(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path, log_verbosity=2) + evidence = _build_evidence("pending-2", uids=["dns-1"]) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "pending-regex" + ) + + with patch("modules.t_cell.t_cell.time.time", return_value=13_500.0): + t_cell._process_evidence_message(_message_for(evidence)) + + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + + cell = storage.get_all_cells()[0] + assert cell["context"]["waiting_for"] == "co_stimulation" + assert "waiting_for_co_stimulation" in log_contents + assert "waiting=waiting for co-stimulation" in log_contents + assert "consumed_observation_id=" in log_contents + assert "consumed_evidence_id=pending-2" in log_contents + + +def test_t_cell_damp_reverifies_waiting_co_stimulation_cells(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path, log_verbosity=2) + fixed_now = 14_500.0 + profile_ip = "10.0.0.80" + evidence_pamp = _build_evidence( + "damp-reverify-costim-pamp", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.LOW, + confidence=1.0, + ) + evidence_damp = _build_evidence( + "damp-reverify-costim-damp", + signal=EvidenceSignal.DAMP, + profile_ip=profile_ip, + threat_level=ThreatLevel.CRITICAL, + confidence=1.0, + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "damp-reverify-costim-regex" + ) + _insert_observation( + storage=storage, + evidence_id="seed-damp-1", + profile_ip=profile_ip, + antigens=[], + observed_at=fixed_now - 20, + confidence=1.0, + threat_level_value=1.0, + threat_level="critical", + evidence_signal="DAMP", + ) + + with patch("modules.t_cell.t_cell.time.time", return_value=fixed_now): + t_cell._process_evidence_message(_message_for(evidence_pamp)) + + first_cell = storage.get_all_cells()[0] + assert first_cell["state"] == STATE_ANTIGEN_RECOGNIZED + assert first_cell["context"]["waiting_for"] == "co_stimulation" + + with patch("modules.t_cell.t_cell.time.time", return_value=fixed_now + 10): + t_cell._process_evidence_message(_message_for(evidence_damp)) + + cell = storage.get_all_cells()[0] + transitions = storage.get_transitions(cell["cell_key"]) + assert cell["state"] == STATE_ACTIVATED + assert cell["context"]["waiting_for"] == "context" + assert any( + transition["reason"] == "co_stimulation_threshold_met" + and transition["evidence_id"] == evidence_damp.id + for transition in transitions + ) + + +def test_t_cell_damp_reverifies_waiting_context_cells(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path, log_verbosity=2) + fixed_now = 14_800.0 + profile_ip = "10.0.0.81" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_pamp_1 = _build_evidence( + "damp-reverify-context-pamp", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.LOW, + confidence=1.0, + ) + evidence_pamp_2 = _build_evidence( + "damp-reverify-context-pamp-2", + profile_ip=profile_ip, + uids=["dns-1"], + threat_level=ThreatLevel.LOW, + confidence=1.0, + ) + evidence_damp = _build_evidence( + "damp-reverify-context-damp", + signal=EvidenceSignal.DAMP, + profile_ip=profile_ip, + threat_level=ThreatLevel.CRITICAL, + confidence=1.0, + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "damp-reverify-context-regex" + ) + t_cell.db.get_pid_of.side_effect = ( + lambda name: 123 if name == "Blocking" else None + ) + _seed_recent_related_observations( + storage, + profile_ip, + antigen, + fixed_now, + count=5, + confidence=1.0, + threat_level_value=0.1, + age_seconds=120, + ) + + _process_evidence_at(t_cell, evidence_pamp_1, fixed_now) + + first_cell = storage.get_all_cells()[0] + assert first_cell["state"] == STATE_ANTIGEN_RECOGNIZED + assert first_cell["context"]["waiting_for"] == "co_stimulation" + + _process_evidence_at(t_cell, evidence_pamp_2, fixed_now + 1) + activated_cell = storage.get_all_cells()[0] + assert activated_cell["state"] == STATE_ACTIVATED + assert activated_cell["context"]["waiting_for"] == "context" + + _process_evidence_at(t_cell, evidence_damp, fixed_now + 10) + + cell = storage.get_all_cells()[0] + transitions = storage.get_transitions(cell["cell_key"]) + assert cell["state"] == STATE_EFFECTOR + assert "waiting_for" not in cell["context"] + assert any( + transition["reason"] == "context_effector" + and transition["evidence_id"] == evidence_damp.id + for transition in transitions + ) + assert t_cell.db.publish.call_count == 1 + + +def test_t_cell_log_file_contains_color_codes(tmp_path): + t_cell, _ = _prepare_t_cell(tmp_path) + evidence = _build_evidence("log-1") + + t_cell._log_event( + action="test_log", + state=STATE_EFFECTOR, + evidence=evidence, + metrics={"score": 0.95}, + verbosity=3, + ) + + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + + assert "\033[" in log_contents + assert "4 - effector" in log_contents + + +def test_t_cell_uses_responsible_attacker_ip_for_cell_and_blocking(tmp_path): + t_cell, storage = _prepare_t_cell(tmp_path, log_verbosity=3) + fixed_now = 16_000.0 + responsible_ip = "138.68.100.107" + related_profile_ip = "147.32.80.37" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_1 = _build_evidence( + "responsible-ip-1", + profile_ip=related_profile_ip, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value=responsible_ip, + ), + victim=Victim( + direction=Direction.DST, + ioc_type=IoCType.IP, + value=related_profile_ip, + ), + uids=["dns-1"], + ) + evidence_2 = _build_evidence( + "responsible-ip-2", + profile_ip=related_profile_ip, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value=responsible_ip, + ), + victim=Victim( + direction=Direction.DST, + ioc_type=IoCType.IP, + value=related_profile_ip, + ), + uids=["dns-1"], + ) + evidence_3 = _build_evidence( + "responsible-ip-3", + profile_ip=related_profile_ip, + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value=responsible_ip, + ), + victim=Victim( + direction=Direction.DST, + ioc_type=IoCType.IP, + value=related_profile_ip, + ), + uids=["dns-1"], + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "responsible-ip-regex" + ) + t_cell.db.get_pid_of.side_effect = ( + lambda name: 123 if name == "Blocking" else None + ) + _seed_recent_related_observations( + storage, responsible_ip, antigen, fixed_now, count=4 + ) + + _process_evidence_at(t_cell, evidence_1, fixed_now) + _process_evidence_at(t_cell, evidence_2, fixed_now + 1) + _process_evidence_at(t_cell, evidence_3, fixed_now + 2) + + cell = storage.get_all_cells()[0] + assert cell["cell_key"].startswith(f"{responsible_ip}|") + assert cell["profile_ip"] == responsible_ip + + channel, payload = t_cell.db.publish.call_args.args + assert channel == "new_blocking" + assert json.loads(payload) == { + "ip": responsible_ip, + "block": True, + "tw": 1, + "interface": None, + } + + with open(t_cell.log_file_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + + assert f"profile={related_profile_ip}" in log_contents + assert f"responsible={responsible_ip}" in log_contents + assert f"target={related_profile_ip}" in log_contents + + +def test_t_cell_falls_back_to_src_side_ip_when_attacker_is_not_ip(tmp_path): + t_cell, _ = _prepare_t_cell(tmp_path) + evidence = _build_evidence( + "src-fallback-1", + profile_ip="203.0.113.50", + attacker=Attacker( + direction=Direction.DST, + ioc_type=IoCType.DOMAIN, + value="bad.example.com", + ), + victim=Victim( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value="10.0.0.50", + ), + ) + + assert t_cell._get_responsible_ip(evidence) == "10.0.0.50" + + +def test_t_cell_transition_trace_lists_contributing_evidence(tmp_path): + t_cell, storage = _prepare_t_cell( + tmp_path, trace_mode=1, trace_max_evidence=10 + ) + fixed_now = 17_000.0 + profile_ip = "10.0.0.67" + antigen = AntigenCandidate(regex_type="dns_domain", value="bad.example.com") + evidence_1 = _build_evidence( + "trace-transition-1", + profile_ip=profile_ip, + uids=["dns-1"], + ) + evidence_2 = _build_evidence( + "trace-transition-2", + profile_ip=profile_ip, + uids=["dns-1"], + ) + evidence_3 = _build_evidence( + "trace-transition-3", + profile_ip=profile_ip, + uids=["dns-1"], + ) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "trace-transition-regex" + ) + t_cell.db.get_pid_of.side_effect = ( + lambda name: 123 if name == "Blocking" else None + ) + _seed_recent_related_observations( + storage, profile_ip, antigen, fixed_now, count=4 + ) + + _process_evidence_at(t_cell, evidence_1, fixed_now) + _process_evidence_at(t_cell, evidence_2, fixed_now + 1) + _process_evidence_at(t_cell, evidence_3, fixed_now + 2) + + entries = _read_trace_entries(t_cell.trace_file_path) + actions = [entry["action"] for entry in entries] + + assert "co_stimulation_threshold_met" in actions + assert "context_effector" in actions + + co_stim_entry = next( + entry + for entry in entries + if entry["action"] == "co_stimulation_threshold_met" + ) + assert co_stim_entry["formula"]["components"]["related_pamps"]["count"] == 4 + related_ids = { + item["evidence_id"] + for item in co_stim_entry["formula"]["components"]["related_pamps"][ + "contributors" + ] + } + assert "hist-recent-0" in related_ids + pamp_danger_ids = { + item["evidence_id"] + for item in co_stim_entry["formula"]["components"]["danger"][ + "pamp_contributors" + ] + } + assert evidence_2.id in pamp_danger_ids + assert evidence_1.id not in pamp_danger_ids + + +def test_t_cell_all_trace_includes_waiting_evaluations(tmp_path): + t_cell, _ = _prepare_t_cell(tmp_path, trace_mode=2) + evidence = _build_evidence("trace-wait-1", uids=["dns-1"]) + t_cell.db.get_altflow_from_uid.return_value = { + "type_": "dns", + "query": "bad.example.com", + } + t_cell.db.get_generated_regexes.return_value = _accepted_domain_regex( + "trace-wait-regex" + ) + + with patch("modules.t_cell.t_cell.time.time", return_value=18_000.0): + t_cell._process_evidence_message(_message_for(evidence)) + + entries = _read_trace_entries(t_cell.trace_file_path) + assert entries == [] + + +def test_t_cell_trace_file_is_forced_inside_output_dir(tmp_path): + t_cell = ModuleFactory().create_t_cell_obj() + t_cell.output_dir = str(tmp_path / "selected-output") + t_cell.conf.t_cell_decision_trace_file.return_value = ( + "/tmp/escape/outside_trace.jsonl" + ) + + t_cell.read_configuration() + + assert t_cell.trace_file_path == str( + tmp_path / "selected-output" / "outside_trace.jsonl" + ) diff --git a/tests/unit/modules/t_cell/test_t_cell_db.py b/tests/unit/modules/t_cell/test_t_cell_db.py new file mode 100644 index 0000000000..996cc5a1bd --- /dev/null +++ b/tests/unit/modules/t_cell/test_t_cell_db.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +from unittest.mock import Mock + +from slips_files.core.database.sqlite_db.t_cell_db import TCellStorage + + +def _build_storage(tmp_path, persistent_store_dir: str = ""): + conf = Mock() + conf.t_cell_store_dir = Mock(return_value="output/t_cell") + conf.t_cell_persistent_store_dir = Mock( + return_value=persistent_store_dir + ) + return TCellStorage(Mock(), conf, str(tmp_path), 12345) + + +def test_t_cell_storage_uses_persistent_store_dir_when_configured(tmp_path): + persistent_dir = tmp_path / "persistent-store" + storage = _build_storage(tmp_path, persistent_store_dir=str(persistent_dir)) + + assert storage.store_dir == str(persistent_dir) + assert storage.db.db_path == str(persistent_dir / "t_cell.sqlite") + storage.close() + + +def test_t_cell_storage_resolves_relative_persistent_store_dir_inside_permanent_dir( + tmp_path, monkeypatch +): + permanent_dir = tmp_path / "permanent" + monkeypatch.setattr( + "slips_files.core.database.sqlite_db.t_cell_db." + "get_this_filepath_inside_permanent_dir", + lambda filename: str(permanent_dir / filename), + ) + storage = _build_storage(tmp_path, persistent_store_dir="t_cell") + + assert storage.store_dir == str(permanent_dir / "t_cell") + assert storage.db.db_path == str( + permanent_dir / "t_cell" / "t_cell.sqlite" + ) + storage.close() + + +def test_t_cell_storage_crud_and_pruning(tmp_path): + storage = _build_storage(tmp_path) + observation_id = storage.insert_observation( + { + "evidence_id": "obs-1", + "evidence_type": "THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN", + "evidence_signal": "PAMP", + "profile_ip": "10.0.0.50", + "timewindow_number": 1, + "timestamp": "2023/11/14 22:13:20.000000+0000", + "observed_at": 100.0, + "confidence": 0.9, + "threat_level": "high", + "threat_level_value": 0.8, + "interface": "default", + "uids": ["uid-1"], + "antigen_count": 1, + "antigens": [{"regex_type": "dns_domain", "value": "bad.example.com"}], + "matched_regexes": [], + "raw_evidence": {"id": "obs-1"}, + } + ) + + storage.update_observation_matches( + observation_id, + [ + { + "regex_type": "dns_domain", + "value": "bad.example.com", + "regex_hash": "hash-1", + "regex": r"^bad\.example\.com$", + } + ], + ) + storage.upsert_cell( + { + "cell_key": "10.0.0.50|dns_domain|bad.example.com", + "profile_ip": "10.0.0.50", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "state": 4, + "state_name": "4 - effector", + "matched_regex_hash": "hash-1", + "matched_regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "anergic_until": None, + "effector_cooldown_until": 500.0, + "last_observation_id": observation_id, + "last_evidence_id": "obs-1", + "last_transition_at": 100.0, + "last_co_stimulation": 0.9, + "last_effector_score": 0.95, + "last_memory_score": 0.1, + "context": {"state": "effector"}, + "created_at": 100.0, + "updated_at": 100.0, + } + ) + storage.insert_transition( + { + "cell_key": "10.0.0.50|dns_domain|bad.example.com", + "profile_ip": "10.0.0.50", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "evidence_id": "obs-1", + "observation_id": observation_id, + "from_state": 3, + "to_state": 4, + "reason": "context_effector", + "matched_regex_hash": "hash-1", + "matched_regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "scores": {"effector_score": 0.95}, + "created_at": 100.0, + } + ) + storage.upsert_memory( + { + "cell_key": "10.0.0.50|dns_domain|bad.example.com", + "profile_ip": "10.0.0.50", + "regex_type": "dns_domain", + "antigen_value": "bad.example.com", + "regex_hash": "hash-1", + "regex": r"^bad\.example\.com$", + "matched_value": "bad.example.com", + "context": {"memory_score": 0.7}, + "created_at": 100.0, + "updated_at": 100.0, + } + ) + + observation = storage.get_observation(observation_id) + cells = storage.get_all_cells() + transitions = storage.get_transitions() + memories = storage.get_memories() + + assert observation["matched_regexes"][0]["regex_hash"] == "hash-1" + assert cells[0]["state"] == 4 + assert transitions[0]["reason"] == "context_effector" + assert memories[0]["regex_hash"] == "hash-1" + assert storage.has_recent_regex_activity( + "10.0.0.50", "hash-1", since_ts=50.0 + ) + assert storage.has_memory_for_regex("hash-1") is True + + storage.prune_observations(101.0) + assert storage.get_recent_observations("10.0.0.50", 0) == [] diff --git a/tests/unit/modules/update_manager/test_update_file_manager.py b/tests/unit/modules/update_manager/test_update_file_manager.py index be7197c059..3ad0737274 100644 --- a/tests/unit/modules/update_manager/test_update_file_manager.py +++ b/tests/unit/modules/update_manager/test_update_file_manager.py @@ -291,6 +291,26 @@ def test_check_if_update_online_whitelist_not_updated(): update_manager.db.set_ti_feed_info.assert_not_called() +def test_update_online_whitelist_stores_ordered_valid_tranco_rows(): + update_manager = ModuleFactory().create_update_manager_obj() + update_manager.online_whitelist_update_period = 86400 + lines = [ + "1,Example.com", + "2,google.com", + "3,github.com", + "4,google.com", + "bad-line", + "5,localhost", + ] + update_manager.responses["tranco_whitelist"] = Mock(text="\n".join(lines)) + + update_manager._update_online_whitelist() + + update_manager.db.store_tranco_whitelisted_domains.assert_called_once_with( + ["example.com", "google.com", "github.com", "google.com"] + ) + + @pytest.mark.parametrize( "headers, expected_last_modified", [ diff --git a/tests/unit/scripts/test_analyze_alert_creation_delay.py b/tests/unit/scripts/test_analyze_alert_creation_delay.py new file mode 100644 index 0000000000..2c2d76986f --- /dev/null +++ b/tests/unit/scripts/test_analyze_alert_creation_delay.py @@ -0,0 +1,80 @@ +from datetime import datetime + +import pytest + +from scripts.analyze_alert_creation_delay import ( + build_summary, + delay_band_label, + truncate_datetime, +) +from tests.module_factory import ModuleFactory + + +@pytest.mark.parametrize( + "resolution, expected", + [ + ("day", datetime(2026, 6, 23)), + ("hour", datetime(2026, 6, 23, 12)), + ("minute", datetime(2026, 6, 23, 12, 34)), + ], +) +def test_truncate_datetime_returns_expected_resolution( + resolution: str, expected: datetime +) -> None: + """Verify timestamps are truncated at the requested resolution. + + Parameters: + resolution: Resolution name to apply. + expected: Expected truncated timestamp. + + Return value: + None. + """ + module_factory = ModuleFactory() + value = datetime(2026, 6, 23, 12, 34, 56, 789) + + assert module_factory + assert truncate_datetime(value, resolution) == expected + + +@pytest.mark.parametrize( + "delay_seconds, expected", + [ + (-0.1, "negative"), + (0.5, "0s-1s"), + (60.0, "1m-5m"), + (86400.0, ">=1d"), + ], +) +def test_delay_band_label_classifies_boundaries( + delay_seconds: float, expected: str +) -> None: + """Verify delay band labels include boundary values correctly. + + Parameters: + delay_seconds: Delay value to classify. + expected: Expected delay band label. + + Return value: + None. + """ + module_factory = ModuleFactory() + + assert module_factory + assert delay_band_label(delay_seconds) == expected + + +def test_build_summary_computes_distribution_stats() -> None: + """Verify alert delay summary statistics are computed from values. + + Return value: + None. + """ + module_factory = ModuleFactory() + + summary = build_summary([1.0, 2.0, 3.0]) + + assert module_factory + assert summary.count == 3 + assert summary.mean_seconds == 2.0 + assert summary.p50_seconds == 2.0 diff --git a/tests/unit/scripts/test_regex_coverage_report.py b/tests/unit/scripts/test_regex_coverage_report.py new file mode 100644 index 0000000000..2f18d1c6d2 --- /dev/null +++ b/tests/unit/scripts/test_regex_coverage_report.py @@ -0,0 +1,71 @@ +import pytest + +from scripts.regex_coverage_report import ( + filename_from_uri, + normalize_domain, + ratio_text, +) +from tests.module_factory import ModuleFactory + + +@pytest.mark.parametrize( + "domain, expected", + [ + ("Example.COM.", "example.com"), + (" sub.example.org ", "sub.example.org"), + ], +) +def test_normalize_domain_strips_dot_and_lowercases( + domain: str, expected: str +) -> None: + """Verify domain normalization trims common formatting noise. + + Parameters: + domain: Domain value to normalize. + expected: Expected normalized domain. + + Return value: + None. + """ + module_factory = ModuleFactory() + + assert module_factory + assert normalize_domain(domain) == expected + + +@pytest.mark.parametrize( + "uri, expected", + [ + ("https://example.com/download/file.exe?x=1", "file.exe"), + ("/path/no_extension", ""), + ], +) +def test_filename_from_uri_extracts_only_file_names( + uri: str, expected: str +) -> None: + """Verify filenames are extracted only when an extension is present. + + Parameters: + uri: URI or path to inspect. + expected: Expected filename value. + + Return value: + None. + """ + module_factory = ModuleFactory() + + assert module_factory + assert filename_from_uri(uri) == expected + + +def test_ratio_text_formats_none_and_percentages() -> None: + """Verify ratio formatting handles empty and numeric values. + + Return value: + None. + """ + module_factory = ModuleFactory() + + assert module_factory + assert ratio_text(None) == "n/a" + assert ratio_text(0.125) == "12.5%" diff --git a/tests/unit/scripts/test_regex_prune_benign_threshold.py b/tests/unit/scripts/test_regex_prune_benign_threshold.py new file mode 100644 index 0000000000..887a94b71c --- /dev/null +++ b/tests/unit/scripts/test_regex_prune_benign_threshold.py @@ -0,0 +1,44 @@ +from pathlib import Path + +from scripts.regex_prune_benign_threshold import ( + RegexAuditResult, + build_summary, +) +from tests.module_factory import ModuleFactory + + +def test_build_summary_counts_flagged_and_timed_out_regexes() -> None: + """Verify prune audit summaries aggregate per-type counts. + + Return value: + None. + """ + module_factory = ModuleFactory() + flagged = RegexAuditResult( + id=1, + regex_type="domain", + regex="evil\\.example", + regex_hash="abc", + created_at=0.0, + strongest_benign_score=88.0, + strongest_benign_value="evil.example", + ) + + summary = build_summary( + regex_db_path=Path("regex.sqlite"), + benign_db_path=Path("benign.sqlite"), + threshold=75.0, + regex_types=["domain"], + accepted_by_type={"domain": [{"id": 1}]}, + flagged_by_type={"domain": [flagged]}, + timed_out_by_type={"domain": [{"id": 2, "created_at": 0.0}]}, + limit=5, + deleted=0, + backup_path=None, + match_timeout_seconds=0.5, + ) + + assert module_factory + assert summary["totals"]["accepted_count"] == 1 + assert summary["totals"]["flagged_count"] == 1 + assert summary["types"]["domain"]["timed_out_count"] == 1 diff --git a/tests/unit/slips_files/common/abstracts/test_imodule.py b/tests/unit/slips_files/common/abstracts/test_imodule.py index 80d4ef9b8b..b6232991f1 100644 --- a/tests/unit/slips_files/common/abstracts/test_imodule.py +++ b/tests/unit/slips_files/common/abstracts/test_imodule.py @@ -2,7 +2,10 @@ # SPDX-License-Identifier: GPL-2.0-only import json +import pytest + from tests.module_factory import ModuleFactory +from slips_files.common.abstracts.imodule import IModule from slips_files.common.slips_utils import utils @@ -12,6 +15,54 @@ def test_imodule_exposes_slips_version(): assert ip_info.slips_version == utils.get_slips_version() +@pytest.mark.parametrize( + "module_name", + ["imodule", "http_analyzer", "p2p_trust", "module1"], +) +def test_imodule_accepts_snake_case_name(module_name): + """Ensure IModule subclasses can define snake_case names.""" + module_factory = ModuleFactory() + + class SnakeCaseNameModule(IModule): + """Test module with a snake_case name.""" + + name = module_name + + assert module_factory.logger is not None + assert SnakeCaseNameModule.name == module_name + + +@pytest.mark.parametrize( + "module_name", + [ + "RegexGenerator", + "T Cell", + "module-name", + "module__name", + "module_", + "_module", + "1module", + "", + None, + ], +) +def test_imodule_rejects_non_snake_case_name(module_name): + """Ensure IModule subclasses reject names that are not snake_case.""" + module_factory = ModuleFactory() + + with pytest.raises( + RuntimeError, + match="NonSnakeCaseNameModule.name must be snake_case", + ): + + class NonSnakeCaseNameModule(IModule): + """Test module with a non-snake-case name.""" + + name = module_name + + assert module_factory.logger is not None + + def test_get_msg_discards_messages_with_different_version(): ip_info = ModuleFactory().create_ip_info_obj() ip_info.channels = {"new_ip": "channel_obj"} diff --git a/tests/unit/slips_files/common/test_config_parser.py b/tests/unit/slips_files/common/test_config_parser.py new file mode 100644 index 0000000000..641d55336c --- /dev/null +++ b/tests/unit/slips_files/common/test_config_parser.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.input_type import InputType + + +def test_evidence_signal_default_falls_back_to_pamp(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = {"EvidenceSignals": {"default_signal": "invalid"}} + + assert parser.evidence_signal_default() == "PAMP" + + +def test_evidence_signal_overrides_sanitizes_values(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = { + "EvidenceSignals": { + "overrides": { + "anomalous_flow": "DAMP", + "malicious_flow": "damp", + "ssh_successful": "PAMP", + "bad_type": "invalid", + 123: "DAMP", + } + } + } + + assert parser.evidence_signal_overrides() == { + "ANOMALOUS_FLOW": "DAMP", + "MALICIOUS_FLOW": "DAMP", + "SSH_SUCCESSFUL": "PAMP", + } + + +def test_t_cell_config_defaults(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = {} + + assert parser.t_cell_enabled() is True + assert parser.t_cell_create_log_file() is True + assert parser.t_cell_log_colors() is True + assert parser.t_cell_log_verbosity() == 1 + assert parser.t_cell_decision_trace_mode() == 0 + assert parser.t_cell_decision_trace_file() == "t_cell_trace.jsonl" + assert parser.t_cell_decision_trace_max_evidence() == 10 + assert parser.t_cell_store_dir() == "output/t_cell" + assert parser.t_cell_persistent_store_dir() == "" + assert parser.t_cell_observation_retention_seconds() == 604800 + assert parser.t_cell_anergy_ttl_seconds() == 21600 + assert parser.t_cell_related_lookback_seconds() == 3600 + assert parser.t_cell_related_pamps_saturation() == 5 + assert parser.t_cell_danger_saturation() == 2.5 + assert parser.t_cell_damp_danger_weight() == 1.5 + assert parser.t_cell_co_stimulation_threshold() == 0.65 + assert parser.t_cell_co_stimulation_weights() == { + "confidence": 0.35, + "related_pamps": 0.25, + "danger": 0.40, + } + assert parser.t_cell_novelty_window_seconds() == 86400 + assert parser.t_cell_context_recent_window_seconds() == 1800 + assert parser.t_cell_effector_threshold() == 0.70 + assert parser.t_cell_effector_min_related_count() == 4 + assert parser.t_cell_effector_cooldown_seconds() == 1800 + assert parser.t_cell_memory_threshold() == 0.60 + assert parser.t_cell_memory_trend_ratio_max() == 0.60 + assert parser.t_cell_memory_min_related_count() == 3 + assert parser.t_cell_simulate_effector_without_blocking() is True + + +def test_t_cell_config_sanitization(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = { + "t_cell": { + "enabled": "true", + "create_log_file": "false", + "log_colors": "false", + "log_verbosity": "debug", + "decision_trace_mode": "all", + "decision_trace_file": " ", + "decision_trace_max_evidence": "bad", + "store_dir": "", + "persistent_store_dir": " /tmp/tcell ", + "observation_retention_seconds": "bad", + "anergy_ttl_seconds": -2, + "related_lookback_seconds": "bad", + "related_pamps_saturation": "bad", + "danger_saturation": 0, + "damp_danger_weight": -5, + "co_stimulation_threshold": "bad", + "co_stimulation_weights": { + "confidence": 0, + "related_pamps": 0, + "danger": 0, + }, + "novelty_window_seconds": "bad", + "context_recent_window_seconds": 0, + "effector_threshold": 2, + "effector_min_related_count": "bad", + "effector_cooldown_seconds": "bad", + "memory_threshold": "bad", + "memory_trend_ratio_max": "bad", + "memory_min_related_count": "bad", + "simulate_effector_without_blocking": "false", + } + } + + assert parser.t_cell_enabled() is True + assert parser.t_cell_create_log_file() is False + assert parser.t_cell_log_colors() is False + assert parser.t_cell_log_verbosity() == 3 + assert parser.t_cell_decision_trace_mode() == 2 + assert parser.t_cell_decision_trace_file() == "t_cell_trace.jsonl" + assert parser.t_cell_decision_trace_max_evidence() == 10 + assert parser.t_cell_store_dir() == "output/t_cell" + assert parser.t_cell_persistent_store_dir() == "/tmp/tcell" + assert parser.t_cell_observation_retention_seconds() == 604800 + assert parser.t_cell_anergy_ttl_seconds() == 0 + assert parser.t_cell_related_lookback_seconds() == 3600 + assert parser.t_cell_related_pamps_saturation() == 5 + assert parser.t_cell_danger_saturation() == 0.01 + assert parser.t_cell_damp_danger_weight() == 0.0 + assert parser.t_cell_co_stimulation_threshold() == 0.65 + assert parser.t_cell_co_stimulation_weights() == { + "confidence": 0.35, + "related_pamps": 0.25, + "danger": 0.40, + } + assert parser.t_cell_novelty_window_seconds() == 86400 + assert parser.t_cell_context_recent_window_seconds() == 1 + assert parser.t_cell_effector_threshold() == 1.0 + assert parser.t_cell_effector_min_related_count() == 4 + assert parser.t_cell_effector_cooldown_seconds() == 1800 + assert parser.t_cell_memory_threshold() == 0.60 + assert parser.t_cell_memory_trend_ratio_max() == 0.60 + assert parser.t_cell_memory_min_related_count() == 3 + assert parser.t_cell_simulate_effector_without_blocking() is False + + +def test_get_disabled_modules_tracks_t_cell_enablement(): + parser = ConfigParser.__new__(ConfigParser) + parser.config = { + "modules": {"disable": ["template"]}, + "llm_proxy": {"enabled": True}, + "regex_generator": {"enabled": True}, + "t_cell": {"enabled": False}, + } + + disabled = parser.get_disabled_modules(InputType.PCAP) + assert "t_cell" in disabled + + parser.config["t_cell"]["enabled"] = True + disabled = parser.get_disabled_modules(InputType.PCAP) + assert "t_cell" not in disabled diff --git a/tests/unit/slips_files/core/database/redis_db/test_alert_handler.py b/tests/unit/slips_files/core/database/redis_db/test_alert_handler.py index e941fb0142..b89c7aeece 100644 --- a/tests/unit/slips_files/core/database/redis_db/test_alert_handler.py +++ b/tests/unit/slips_files/core/database/redis_db/test_alert_handler.py @@ -12,6 +12,7 @@ ProfileID, TimeWindow, Evidence, + EvidenceSignal, EvidenceType, Attacker, Direction, @@ -296,7 +297,21 @@ def test_init_evidence_number(initial_value, expected_value): (None, True, False), # whitelisted → ignored ], ) -def test_set_evidence(evidence_exists, whitelisted, expected): +@pytest.mark.parametrize( + "evidence_type, expected_signal", + [ + (EvidenceType.ANOMALOUS_FLOW, EvidenceSignal.DAMP), + (EvidenceType.SSH_SUCCESSFUL, EvidenceSignal.PAMP), + (EvidenceType.MALICIOUS_FLOW, EvidenceSignal.DAMP), + ], +) +def test_set_evidence( + evidence_exists, + whitelisted, + expected, + evidence_type, + expected_signal, +): db = ModuleFactory().create_alert_handler_obj() db.add_profile = Mock() @@ -331,7 +346,7 @@ def test_set_evidence(evidence_exists, whitelisted, expected): ) evidence = Evidence( - evidence_type=EvidenceType.SSH_SUCCESSFUL, + evidence_type=evidence_type, attacker=attacker, victim=victim, threat_level=ThreatLevel.INFO, @@ -346,11 +361,16 @@ def test_set_evidence(evidence_exists, whitelisted, expected): result = db.set_evidence(evidence) assert result is expected + assert evidence.evidence_signal == expected_signal if expected: db.r.hset.assert_called_once() db.r.incr.assert_called_once_with(db.constants.NUMBER_OF_EVIDENCE) db.publish.assert_called_once() + stored_evidence = json.loads(db.r.hset.call_args.args[2]) + published_evidence = json.loads(db.publish.call_args.args[1]) + assert stored_evidence["evidence_signal"] == expected_signal.name + assert published_evidence["evidence_signal"] == expected_signal.name else: db.r.hset.assert_not_called() db.publish.assert_not_called() diff --git a/tests/unit/slips_files/core/database/redis_db/test_profile_handler.py b/tests/unit/slips_files/core/database/redis_db/test_profile_handler.py index 553ad64897..ae5d8602b7 100644 --- a/tests/unit/slips_files/core/database/redis_db/test_profile_handler.py +++ b/tests/unit/slips_files/core/database/redis_db/test_profile_handler.py @@ -4,6 +4,7 @@ from unittest.mock import patch, MagicMock, call, Mock import json from tests.module_factory import ModuleFactory +from slips_files.common.slips_utils import utils from slips_files.core.structures.flow_attributes import Role from slips_files.core.flows.zeek import HTTP, DNS from unittest.mock import ANY @@ -2051,6 +2052,32 @@ def test_add_profile_existing_profile(): handler.update_threat_level.assert_not_called() +def test_add_profile_accepts_alerts_format_timestamp(): + handler = ModuleFactory().create_profile_handler_obj() + + handler.set_new_ip = MagicMock() + handler.publish = MagicMock() + + profileid = f"profile{handler.separator}1" + starttime = utils.convert_ts_format(1678886400.0, utils.alerts_format) + + handler.r.zscore.return_value = None + + pipe = MagicMock() + handler.r.pipeline = MagicMock(return_value=pipe) + pipe.__enter__ = MagicMock(return_value=pipe) + pipe.__exit__ = MagicMock(return_value=False) + + result = handler.add_profile(profileid, starttime) + + assert result is True + handler.zadd_but_keep_n_entries.assert_called_once_with( + handler.constants.PROFILES, + {profileid: 1678886400.0}, + 2000, + ) + + def test_mark_profile_as_dhcp_profile_not_exist(): handler = ModuleFactory().create_profile_handler_obj() diff --git a/tests/unit/slips_files/core/database/test_database.py b/tests/unit/slips_files/core/database/test_database.py index 5b298340c6..c9f3992721 100644 --- a/tests/unit/slips_files/core/database/test_database.py +++ b/tests/unit/slips_files/core/database/test_database.py @@ -79,6 +79,102 @@ def test_subscribe(): assert isinstance(db.subscribe("new_flow"), redis.client.PubSub) +def test_get_available_llm_backends_returns_empty_dict_when_unset(): + db = ModuleFactory().create_db_manager_obj(6379, flush_db=True) + db.r.delete(db.rdb.constants.AVAILABLE_LLM_BACKENDS) + + assert db.get_available_llm_backends() == { + "default_backend": "", + "backends": {}, + } + + +def test_set_and_get_available_llm_backends(): + db = ModuleFactory().create_db_manager_obj(6379, flush_db=True) + + db.set_available_llm_backends( + { + "default_backend": "local_qwen", + "backends": { + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b", + }, + "openai_default": { + "provider": "openai", + "model": "gpt-4o-mini", + }, + }, + } + ) + + assert db.get_available_llm_backends() == { + "default_backend": "local_qwen", + "backends": { + "local_qwen": { + "provider": "ollama", + "model": "qwen2.5:3b", + }, + "openai_default": { + "provider": "openai", + "model": "gpt-4o-mini", + }, + }, + } + + +def test_get_generated_regexes_and_count(tmp_path): + db = ModuleFactory().create_db_manager_obj( + 6379, + output_dir=str(tmp_path / "output"), + flush_db=True, + ) + db.conf.regex_generator_store_dir = Mock( + return_value=str(tmp_path / "regex_generator") + ) + db.conf.regex_generator_seed_benign_samples = Mock(return_value=False) + + storage = db._get_regex_generator_storage() + storage.store_generated_regex( + { + "regex_type": "dns_domain", + "regex": r"^xqz[a-z0-9]{8,12}\.invalid$", + "regex_hash": "hash-1", + "status": "accepted", + "rejection_reason": None, + "matched_benign_value": None, + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": "regex-generator-v1", + "request_id": "req-1", + "created_at": 1.0, + } + ) + + regexes = db.get_generated_regexes("dns_domain") + assert regexes == [ + { + "id": regexes[0]["id"], + "regex_type": "dns_domain", + "regex": r"^xqz[a-z0-9]{8,12}\.invalid$", + "regex_hash": "hash-1", + "status": "accepted", + "rejection_reason": None, + "matched_benign_value": None, + "backend_alias": "local_qwen", + "provider": "ollama", + "model": "qwen2.5:3b", + "temperature": 1.2, + "prompt_version": "regex-generator-v1", + "request_id": "req-1", + "created_at": 1.0, + } + ] + assert db.get_generated_regexes_count("dns_domain") == 1 + + def test_profile_moddule_labels(): """tests set and get_profile_module_label""" db = ModuleFactory().create_db_manager_obj(6379, flush_db=True) @@ -212,6 +308,21 @@ def test_store_official_dns_server(): assert db.is_official_dns_server("not-an-ip") is False +def test_tranco_whitelist_stores_ordered_domains_with_limit() -> None: + """Test Tranco whitelist storage preserves order and supports limits.""" + db = ModuleFactory().create_db_manager_obj(6379, flush_db=True) + + db.store_tranco_whitelisted_domains( + ["example.com", "google.com", "github.com"], + limit=2, + ) + + assert db.get_tranco_top_domains() == ["example.com", "google.com"] + assert db.get_tranco_top_domains(limit=1) == ["example.com"] + assert db.is_whitelisted_tranco_domain("google.com") is True + assert db.is_whitelisted_tranco_domain("github.com") is False + + def test_setup_config_file_uses_isolated_path_and_preserves_save( tmp_path, monkeypatch ): diff --git a/tests/unit/slips_files/core/structures/test_evidence.py b/tests/unit/slips_files/core/structures/test_evidence.py index 6b79a46dda..14265de21e 100644 --- a/tests/unit/slips_files/core/structures/test_evidence.py +++ b/tests/unit/slips_files/core/structures/test_evidence.py @@ -8,6 +8,7 @@ Attacker, Direction, Evidence, + EvidenceSignal, EvidenceType, IoCType, ProfileID, @@ -110,6 +111,7 @@ def test_evidence_post_init( assert evidence.dst_port == port assert evidence.id == id assert evidence.confidence == confidence + assert evidence.evidence_signal == EvidenceSignal.PAMP def test_evidence_post_init_invalid_uid(): @@ -258,6 +260,69 @@ def test_evidence_to_dict( assert evidence_dict["dst_port"] == port assert evidence_dict["id"] == id assert evidence_dict["confidence"] == confidence + assert evidence_dict["evidence_signal"] == EvidenceSignal.PAMP.name + + +@pytest.mark.parametrize( + "raw_signal, expected_signal", + [ + ("DAMP", EvidenceSignal.DAMP), + ("damp", EvidenceSignal.DAMP), + ("unknown", EvidenceSignal.PAMP), + (None, EvidenceSignal.PAMP), + ], +) +def test_dict_to_evidence_signal(raw_signal, expected_signal): + from slips_files.core.structures.evidence import dict_to_evidence + + evidence_dict = { + "evidence_type": "ARP_SCAN", + "description": "ARP scan detected", + "interface": "eth0", + "attacker": { + "direction": "SRC", + "ioc_type": "IP", + "value": "192.168.1.1", + "profile": "", + "TI": None, + "AS": None, + "rDNS": None, + "SNI": None, + "DNS_resolution": None, + "queries": None, + "CNAME": None, + }, + "threat_level": "info", + "victim": { + "direction": "DST", + "ioc_type": "IP", + "value": "8.8.8.8", + "TI": None, + "AS": None, + "rDNS": None, + "SNI": None, + "DNS_resolution": None, + "queries": None, + "CNAME": None, + }, + "profile": {"ip": "192.168.1.1"}, + "timewindow": {"number": 1}, + "uid": ["uid-1"], + "timestamp": "2023/10/26 10:10:10.000000+0000", + "proto": "TCP", + "dst_port": 80, + "src_port": 12345, + "id": "d243119b-2aae-4d7a-8ea1-edf3c6e72f4a", + "rel_id": None, + "confidence": 0.8, + "method": "heuristic", + } + if raw_signal is not None: + evidence_dict["evidence_signal"] = raw_signal + + evidence = dict_to_evidence(evidence_dict) + + assert evidence.evidence_signal == expected_signal def test_validate_timestamp(): diff --git a/tests/unit/slips_files/core/test_evidence_handler.py b/tests/unit/slips_files/core/test_evidence_handler.py index 070f1636bd..ec20e17bac 100644 --- a/tests/unit/slips_files/core/test_evidence_handler.py +++ b/tests/unit/slips_files/core/test_evidence_handler.py @@ -1,9 +1,25 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only -from unittest.mock import Mock, call, patch +from datetime import datetime +import json +import pytest +from unittest.mock import Mock, patch, call + +from slips_files.core.structures.alerts import Alert from slips_files.core.evidence_handler import DEFAULT_EVIDENCE_HANDLER_WORKERS +from slips_files.core.structures.evidence import ( + Evidence, + ProfileID, + EvidenceSignal, + EvidenceType, + TimeWindow, + Attacker, + IoCType, + Direction, + ThreatLevel, +) from tests.module_factory import ModuleFactory @@ -30,8 +46,11 @@ def test_stop_evidence_workers(): handler = ModuleFactory().create_evidence_handler_obj() process_1 = Mock() process_2 = Mock() + process_1.is_alive.return_value = False + process_2.is_alive.return_value = True handler.evidence_worker_child_processes = [process_1, process_2] handler.evidence_worker_queue = Mock() + handler.print = Mock() handler.stop_evidence_workers() @@ -39,8 +58,11 @@ def test_stop_evidence_workers(): call("stop"), call("stop"), ] - process_1.join.assert_called_once() - process_2.join.assert_called_once() + process_1.join.assert_called_once_with(timeout=5) + process_1.kill.assert_not_called() + process_2.join.assert_has_calls([call(timeout=5), call(timeout=1)]) + process_2.kill.assert_called_once() + handler.print.assert_called_once() @patch("slips_files.core.evidence_handler.EvidenceHandlerWorker") @@ -150,3 +172,327 @@ def get_msg(channel): } ), ] + + +@pytest.mark.parametrize( + "all_uids, timewindow, accumulated_threat_level", + [ + (["uid1", "uid2"], 1, 0.5), + ([], 10, 1.0), + ], +) +def test_add_alert_to_json_log_file( + all_uids, timewindow, accumulated_threat_level +): + mock_file = Mock() + alert = Alert( + profile=ProfileID("192.168.1.20"), + timewindow=TimeWindow( + timewindow, + start_time="2024-10-04T18:46:50+03:00", + end_time="2024-10-04T19:46:50+03:00", + ), + last_evidence=Evidence( + evidence_type=EvidenceType.ARP_SCAN, + description="ARP scan detected", + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value="192.168.1.20", + ), + threat_level=ThreatLevel.INFO, + profile=ProfileID("192.168.1.20"), + timewindow=TimeWindow(timewindow), + uid=all_uids, + timestamp="1728417813.8868346", + ), + accumulated_threat_level=accumulated_threat_level, + last_flow_datetime="2024/10/04 15:45:30.123456+0000", + ) + evidence_handler = ModuleFactory().create_evidence_handler_obj() + evidence_handler.jsonfile = mock_file + evidence_handler.idmefv2.convert_to_idmef_alert = Mock( + return_value="alert_in_idmef_format" + ) + evidence_handler.evidence_logger_q.put = Mock() + + evidence_handler.add_alert_to_json_log_file(alert) + evidence_handler.evidence_logger_q.put.assert_called_once_with( + { + "to_log": "alert_in_idmef_format", + "where": "alerts.json", + } + ) + + +def test_add_evidence_to_json_log_file_includes_evidence_signal(): + evidence_handler = ModuleFactory().create_evidence_handler_obj() + evidence_handler.idmefv2.convert_to_idmef_event = Mock( + return_value={"Category": "Intrusion.Detection"} + ) + evidence_handler.evidence_logger_q.put = Mock() + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_FLOW, + description="Anomalous HTTPS flow", + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value="192.168.1.20", + ), + threat_level=ThreatLevel.HIGH, + profile=ProfileID("192.168.1.20"), + timewindow=TimeWindow(1), + uid=["uid-1"], + timestamp="2024/10/04 15:45:30.123456+0000", + evidence_signal=EvidenceSignal.DAMP, + ) + + evidence_handler.add_evidence_to_json_log_file( + evidence, accumulated_threat_level=1.2 + ) + + evidence_handler.evidence_logger_q.put.assert_called_once() + logged_event = evidence_handler.evidence_logger_q.put.call_args.args[0] + note = json.loads(logged_event["to_log"]["Note"]) + assert logged_event["where"] == "alerts.json" + assert note["evidence_signal"] == "DAMP" + assert note["threat_level"] == "high" + assert note["timewindow"] == 1 + + +def test_show_popup(): + evidence_handler = ModuleFactory().create_evidence_handler_obj() + evidence_handler.notify = Mock() + alert = Mock(spec=Alert) + evidence_handler.formatter.get_printable_alert = Mock( + return_value="alert_time_desc" + ) + + evidence_handler.show_popup(alert) + + evidence_handler.notify.show_popup.assert_called_once_with( + "alert_time_desc" + ) + + +def test_send_to_exporting_module(): + evidence_handler = ModuleFactory().create_evidence_handler_obj() + tw_evidence = { + "evidence1": Evidence( + evidence_type=EvidenceType.ARP_SCAN, + description="ARP scan detected", + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value="192.168.1.1", + ), + threat_level=ThreatLevel.MEDIUM, + profile=ProfileID(ip="192.168.1.1"), + timewindow=TimeWindow(number=1), + uid=["uid1"], + timestamp="2023/04/01 10:00:00.000000+0000", + ), + "evidence2": Evidence( + evidence_type=EvidenceType.DNS_WITHOUT_CONNECTION, + description="DNS query without connection", + attacker=Attacker( + direction=Direction.SRC, + ioc_type=IoCType.IP, + value="192.168.1.2", + ), + threat_level=ThreatLevel.LOW, + profile=ProfileID(ip="192.168.1.2"), + timewindow=TimeWindow(number=1), + uid=["uid2"], + timestamp="2023/04/01 10:01:00.000000+0000", + ), + } + + evidence_handler.exporting_modules_enabled = True + evidence_handler.db.publish = Mock() + evidence_handler.send_to_exporting_module(tw_evidence) + assert evidence_handler.db.publish.call_count == 2 + + +@pytest.mark.parametrize( + "sys_argv, running_non_stop, expected_result", + [ + # testcase 1: running non stop with -p enabled + (["-i", "-p"], True, True), + # testcase 2: custom flows but the module is disabled + (["-i", "-im"], False, False), + # testcase 3: -i not in sys.argv and not running non stop + ([], False, False), + ], +) +def test_is_blocking_module_supported( + sys_argv, running_non_stop, expected_result +): + evidence_handler = ModuleFactory().create_evidence_handler_obj() + evidence_handler.is_running_non_stop = running_non_stop + + with patch("sys.argv", sys_argv): + result = evidence_handler.is_blocking_modules_supported() + assert result == expected_result + + +@pytest.mark.parametrize( + "evidence, past_evidence_ids, expected_result", + [ + # testcase1: Evidence not filtered + ( + Evidence( + evidence_type=EvidenceType.ARP_SCAN, + description="", + attacker=Attacker( + direction="SRC", + ioc_type=IoCType.IP, + value="192.168.1.1", + ), + threat_level=ThreatLevel.INFO, + profile=ProfileID("192.168.1.1"), + timewindow=TimeWindow(1), + uid=[], + timestamp=datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f%z"), + id="1", + ), + [], + False, + ), + # testcase2: Evidence filtered (part of past alert) + ( + Evidence( + evidence_type=EvidenceType.ARP_SCAN, + description="", + attacker=Attacker( + direction="SRC", + ioc_type=IoCType.IP, + value="192.168.1.1", + ), + threat_level=ThreatLevel.INFO, + profile=ProfileID("192.168.1.1"), + timewindow=TimeWindow(1), + uid=[], + timestamp=datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f%z"), + id="2", + ), + ["2"], + True, + ), + # testcase3: Evidence filtered (evidence that wasnt done by the given + # profileid) + ( + Evidence( + evidence_type=EvidenceType.ARP_SCAN, + description="", + attacker=Attacker( + direction="DST", + ioc_type=IoCType.IP, + value="192.168.1.1", + ), + threat_level=ThreatLevel.INFO, + profile=ProfileID("192.168.1.1"), + timewindow=TimeWindow(1), + uid=[], + timestamp=datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f%z"), + id="3", + ), + [], + True, + ), + ], +) +def test_is_filtered_evidence(evidence, past_evidence_ids, expected_result): + evidence_handler = ModuleFactory().create_evidence_handler_obj() + result = evidence_handler.is_filtered_evidence(evidence, past_evidence_ids) + assert result == expected_result + + +@pytest.mark.parametrize( + "evidence, expected_result", + [ # Testcase1: Attacker direction is SRC + (Mock(attacker=Mock(direction="SRC")), False), + # Testcase2: Attacker direction is DST + (Mock(attacker=Mock(direction="DST")), True), + ], +) +def test_is_evidence_done_by_others(evidence, expected_result): + evidence_handler = ModuleFactory().create_evidence_handler_obj() + result = evidence_handler.is_evidence_done_by_others(evidence) + assert result == expected_result + + +@pytest.mark.parametrize( + "confidence, threat_level, expected_output", + [ + # Testcase 1: Low threat level, confidence 0.5 + (0.5, ThreatLevel.LOW, 0.1), + # Testcase 2: Medium threat level, full confidence + (1.0, ThreatLevel.MEDIUM, 0.5), + # Testcase 3: High threat level, confidence 0.8 + (0.8, ThreatLevel.HIGH, 0.64), + # Testcase 4: Critical threat level, confidence 0.3 + (0.3, ThreatLevel.CRITICAL, 0.3), + # Testcase 5: Info threat level, zero confidence + (0.0, ThreatLevel.INFO, 0.0), + ], +) +def test_get_threat_level(confidence, threat_level, expected_output): + evidence_handler = ModuleFactory().create_evidence_handler_obj() + evidence = Mock(spec=Evidence) + evidence.confidence = confidence + evidence.threat_level = threat_level + with patch.object(evidence_handler, "print") as mock_print: + result = evidence_handler.get_threat_level(evidence) + + assert pytest.approx(result, abs=1e-6) == expected_output + mock_print.assert_called_once_with( + f"\t\tWeighted Threat Level: {result}", 3, 0 + ) + + +@pytest.mark.parametrize( + "ip, twid, flow_datetime, " "accumulated_threat_level, blocked", + [ + # testcase1: IP blocked by blocking module + ( + "192.168.1.100", + 1, + "2023/10/26 10:10:10", + 0.8, + True, + ), + # testcase2: IP not blocked by blocking module + ( + "10.0.0.100", + 2, + "2023/10/26 11:11:11", + 1.0, + False, + ), + ], +) +def test_log_alert( + ip, + twid, + flow_datetime, + accumulated_threat_level, + blocked, +): + evidence_handler = ModuleFactory().create_evidence_handler_obj() + evidence_handler.width = 300 + evidence_handler.add_alert_to_json_log_file = Mock() + evidence_handler.add_to_log_file = Mock() + alert = Alert( + profile=ProfileID(ip), + timewindow=TimeWindow(twid), + last_evidence=Mock(), + accumulated_threat_level=accumulated_threat_level, + last_flow_datetime=flow_datetime, + ) + evidence_handler.log_alert(alert, blocked=blocked) + + evidence_handler.add_alert_to_json_log_file.assert_called_once() + assert flow_datetime in evidence_handler.add_to_log_file.call_args[0][0] + assert str(twid) in evidence_handler.add_to_log_file.call_args[0][0] diff --git a/tests/unit/slips_files/logs_analysis/test_analyze_incidents.py b/tests/unit/slips_files/logs_analysis/test_analyze_incidents.py new file mode 100644 index 0000000000..6979dcbcbc --- /dev/null +++ b/tests/unit/slips_files/logs_analysis/test_analyze_incidents.py @@ -0,0 +1,58 @@ +import json +from pathlib import Path + +import pytest + +from slips_files.logs_analysis.analyze_incidents import ( + load_jsonl, + parse_note_uids, +) +from tests.module_factory import ModuleFactory + + +@pytest.mark.parametrize( + "note, expected", + [ + (json.dumps({"uids": ["C1", "C2"]}), ["C1", "C2"]), + ("not-json", []), + ], +) +def test_parse_note_uids_returns_note_uid_list( + note: str, expected: list[str] +) -> None: + """Verify Event Note UID extraction handles valid and invalid JSON. + + Parameters: + note: Event Note value to parse. + expected: Expected UID list. + + Return value: + None. + """ + module_factory = ModuleFactory() + + assert module_factory + assert parse_note_uids(note) == expected + + +def test_load_jsonl_skips_comments_and_invalid_lines(tmp_path: Path) -> None: + """Verify JSONL loader yields valid objects and ignores bad lines. + + Parameters: + tmp_path: Pytest temporary directory fixture. + + Return value: + None. + """ + module_factory = ModuleFactory() + alerts_path = tmp_path / "alerts.jsonl" + alerts_path.write_text( + '# comment\n{"ID": "event-1"}\nnot-json\n\n{"ID": "event-2"}\n', + encoding="utf-8", + ) + + assert module_factory + assert list(load_jsonl(alerts_path)) == [ + {"ID": "event-1"}, + {"ID": "event-2"}, + ]