diff --git a/Pipfile b/Pipfile index 791d137e..d8d2dc98 100644 --- a/Pipfile +++ b/Pipfile @@ -4,35 +4,28 @@ verify_ssl = true name = "pypi" [packages] -celery = {version = "==5.4.0", extras = ["sqs"]} cryptography = "==46.0.7" -boto3 = "==1.36.14" -django = "==5.2.13" +django = "==5.2.14" djangorestframework = "==3.16.1" django-filter = "==25.1" django-countries = "==7.6.1" django-cors-headers = "==4.7.0" django-csp = "==3.8" -django-storages = {version = "==1.14.6", extras = ["s3"]} pyotp = "==2.9.0" -python-dotenv = "==1.0.1" psycopg2-binary = "==2.9.9" redis = {version = "==5.2.1", extras = ["hiredis"]} regex = "==2024.11.6" requests = "==2.33.1" -gunicorn = "==23.0.0" -uvicorn-worker = "==0.2.0" pyjwt = "==2.12.1" psutil = "==7.0.0" google-auth = "==2.48.0" -google-cloud-bigquery = "==3.38.0" +google-cloud-secret-manager = "==2.27.0" +google-crc32c = "==1.8.0" tink = {version = "==1.13.0", extras = ["gcpkms"]} cachetools = "==6.2.6" [dev-packages] -celery-types = "==0.23.0" black = "==24.8.0" -boto3-stubs = {version = "==1.38.39", extras = ["essential"]} pytest = "==8.3.3" pytest-cov = "==5.0.0" pytest-env = "==0.8.1" diff --git a/Pipfile.lock b/Pipfile.lock index c4bd13c9..46b56258 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f13eca1325fddf9cd61dcf42ba3f7b60dee86b663b7d4d004979795be3a7ceb1" + "sha256": "e6df2afb03370d56a82bd85b7d69ab1ed1850841050580930b88ff579764e6ee" }, "pipfile-spec": 6, "requires": { @@ -24,14 +24,6 @@ "markers": "python_version >= '3.10'", "version": "==2.4.0" }, - "amqp": { - "hashes": [ - "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", - "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432" - ], - "markers": "python_version >= '3.6'", - "version": "==5.3.1" - }, "asgiref": { "hashes": [ "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", @@ -42,35 +34,10 @@ }, "bazel-runfiles": { "hashes": [ - "sha256:66fb0f221e72ad904086eda6b208e873c76a2f5511ea308e7eae449534abc202" - ], - "markers": "python_version >= '3.7'", - "version": "==1.9.0" - }, - "billiard": { - "hashes": [ - "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", - "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f" + "sha256:cc8c6390587d0de162a0826f5e8a1537ac3f5f0ebb0cf5e76328da059ab4ec01" ], "markers": "python_version >= '3.7'", - "version": "==4.2.4" - }, - "boto3": { - "hashes": [ - "sha256:4b0b8dd593b95f32a5a761dee65094423fbd06a4ad09f26b2e6c80493139569f", - "sha256:e2dab15944c3f517c88850d60b07f2f6fd3bc69aa51c47670e4f45d62a8c41fd" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==1.36.14" - }, - "botocore": { - "hashes": [ - "sha256:4a63bcef7ecf6146fd3a61dc4f9b33b7473b49bdaf1770e9aaca6eee0c9eab62", - "sha256:4e3f19913887a58502e71ef8d696fe7eaa54de7813ff73390cd5883f837dfa6e" - ], - "markers": "python_version >= '3.8'", - "version": "==1.36.26" + "version": "==2.0.0" }, "cachetools": { "hashes": [ @@ -81,24 +48,13 @@ "markers": "python_version >= '3.9'", "version": "==6.2.6" }, - "celery": { - "extras": [ - "sqs" - ], - "hashes": [ - "sha256:369631eb580cf8c51a82721ec538684994f8277637edde2dfc0dacd73ed97f64", - "sha256:504a19140e8d3029d5acad88330c541d4c3f64c789d85f94756762d8bca7e706" - ], - "markers": "python_version >= '3.8'", - "version": "==5.4.0" - }, "certifi": { "hashes": [ - "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", - "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" + "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", + "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580" ], "markers": "python_version >= '3.7'", - "version": "==2026.2.25" + "version": "==2026.4.22" }, "cffi": { "hashes": [ @@ -187,7 +143,7 @@ "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf" ], - "markers": "python_full_version >= '3.9' and platform_python_implementation != 'PyPy'", + "markers": "python_version >= '3.9'", "version": "==2.0.0" }, "charset-normalizer": { @@ -325,37 +281,6 @@ "markers": "python_version >= '3.7'", "version": "==3.4.7" }, - "click": { - "hashes": [ - "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", - "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d" - ], - "markers": "python_version >= '3.10'", - "version": "==8.3.2" - }, - "click-didyoumean": { - "hashes": [ - "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", - "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c" - ], - "markers": "python_full_version >= '3.6.2'", - "version": "==0.3.1" - }, - "click-plugins": { - "hashes": [ - "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", - "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261" - ], - "version": "==1.1.1.2" - }, - "click-repl": { - "hashes": [ - "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", - "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812" - ], - "markers": "python_version >= '3.6'", - "version": "==0.3.0" - }, "cryptography": { "hashes": [ "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65", @@ -414,12 +339,12 @@ }, "django": { "hashes": [ - "sha256:5788fce61da23788a8ce6f02583765ab060d396720924789f97fa42119d37f7a", - "sha256:a31589db5188d074c63f0945c3888fad104627dfcc236fb2b97f71f89da33bc4" + "sha256:58a63ba841662e5c686b57ba1fec52ddd68c0b93bd96ac3029d55728f00bf8a2", + "sha256:6f712143bd3064310d1f50fac859c3e9a274bdcfc9595339853be7779297fc76" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==5.2.13" + "version": "==5.2.14" }, "django-cors-headers": { "hashes": [ @@ -455,17 +380,6 @@ "markers": "python_version >= '3.9'", "version": "==25.1" }, - "django-storages": { - "extras": [ - "s3" - ], - "hashes": [ - "sha256:11b7b6200e1cb5ffcd9962bd3673a39c7d6a6109e8096f0e03d46fab3d3aabd9", - "sha256:7a25ce8f4214f69ac9c7ce87e2603887f7ae99326c316bc8d2d75375e09341c9" - ], - "markers": "python_version >= '3.7'", - "version": "==1.14.6" - }, "djangorestframework": { "hashes": [ "sha256:166809528b1aced0a17dc66c24492af18049f2c9420dbd0be29422029cfc3ff7", @@ -495,30 +409,22 @@ "markers": "python_version >= '3.8'", "version": "==2.48.0" }, - "google-cloud-bigquery": { + "google-cloud-kms": { "hashes": [ - "sha256:8afcb7116f5eac849097a344eb8bfda78b7cfaae128e60e019193dd483873520", - "sha256:e06e93ff7b245b239945ef59cb59616057598d369edac457ebf292bd61984da6" + "sha256:672fdc594b928b0415c22e41f3d67c854e940a8a5917e8ff16a6566096b12407", + "sha256:a58a40d8d37129805db5be1353a397be09614d691433ea246b9eed63efdc5df3" ], - "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==3.38.0" + "version": "==3.13.0" }, - "google-cloud-core": { + "google-cloud-secret-manager": { "hashes": [ - "sha256:3dc94bdec9d05a31d9f355045ed0f369fbc0d8c665076c734f065d729800f811", - "sha256:ea62cdf502c20e3e14be8a32c05ed02113d7bef454e40ff3fab6fe1ec9f1f4e7" + "sha256:6af864c252bd3c11db7bb02b80cb0b14a8c9a33fc7ec4d6f245f33d8ce1f7cd1", + "sha256:e5540bece65a3ad720146f3b438973faf9315109b3ffa012a58711843047a3dc" ], + "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==2.5.1" - }, - "google-cloud-kms": { - "hashes": [ - "sha256:93ae05ff3a3de49ced4f234a5da0d118cf29b5dfcb9f790f85ccdaeab03eeadc", - "sha256:c9445aac1e35a9f76efb4a7d57c9a2aa7b73034b6afec339ba0862fe4993ff25" - ], - "markers": "python_version >= '3.7'", - "version": "==3.12.0" + "version": "==2.27.0" }, "google-crc32c": { "hashes": [ @@ -556,27 +462,20 @@ "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c" ], + "index": "pypi", "markers": "python_version >= '3.9'", "version": "==1.8.0" }, - "google-resumable-media": { - "hashes": [ - "sha256:82b6d8ccd11765268cdd2a2123f417ec806b8eef3000a9a38dfe3033da5fb220", - "sha256:f3354a182ebd193ae3f42e3ef95e6c9b10f128320de23ac7637236713b1acd70" - ], - "markers": "python_version >= '3.9'", - "version": "==2.8.2" - }, "googleapis-common-protos": { "extras": [ "grpc" ], "hashes": [ - "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", - "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5" + "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", + "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed" ], "markers": "python_version >= '3.9'", - "version": "==1.74.0" + "version": "==1.75.0" }, "grpc-google-iam-v1": { "hashes": [ @@ -661,23 +560,6 @@ "markers": "python_version >= '3.9'", "version": "==1.80.0" }, - "gunicorn": { - "hashes": [ - "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", - "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==23.0.0" - }, - "h11": { - "hashes": [ - "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", - "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86" - ], - "markers": "python_version >= '3.8'", - "version": "==0.16.0" - }, "hiredis": { "hashes": [ "sha256:002fc0201b9af1cc8960e27cdc501ad1f8cdd6dbadb2091c6ddbd4e5ace6cb77", @@ -791,54 +673,19 @@ }, "idna": { "hashes": [ - "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", - "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902" + "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242", + "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3" ], "markers": "python_version >= '3.8'", - "version": "==3.11" - }, - "jmespath": { - "hashes": [ - "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", - "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64" - ], - "markers": "python_version >= '3.9'", - "version": "==1.1.0" - }, - "kombu": { - "extras": [ - "sqs" - ], - "hashes": [ - "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", - "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93" - ], - "markers": "python_version >= '3.9'", - "version": "==5.6.2" - }, - "packaging": { - "hashes": [ - "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", - "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" - ], - "markers": "python_version >= '3.8'", - "version": "==26.0" - }, - "prompt-toolkit": { - "hashes": [ - "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", - "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955" - ], - "markers": "python_version >= '3.8'", - "version": "==3.0.52" + "version": "==3.13" }, "proto-plus": { "hashes": [ - "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718", - "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24" + "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9", + "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8" ], - "markers": "python_version >= '3.9'", - "version": "==1.27.2" + "markers": "python_version >= '3.10'", + "version": "==1.28.0" }, "protobuf": { "hashes": [ @@ -973,58 +820,9 @@ "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" ], - "markers": "implementation_name != 'PyPy'", + "markers": "python_version >= '3.10'", "version": "==3.0" }, - "pycurl": { - "hashes": [ - "sha256:00824c3c64d5e935d0325e6a000bb0242622cd763d1e202c1a8df455bc58da58", - "sha256:0263e94d2ea3cd25b5fcf96527f521dc29b8a73a9b19e71223b6e2452f5cb35a", - "sha256:09ac9a855273a77f7c6a72f05be430646c47a298d403a75876c2188ea5d77534", - "sha256:09e426cf5e61e5b37c30903734204f75023baa6e4bfe574e037f12e53ec3f0ba", - "sha256:0bcfeaf766372ec40e3830d4ad6ebc3064e8a0bd145214d4291d17ed6017f3e3", - "sha256:150b1d2e98bcf760878b3891e6f2f937a4b2a90c559ffc6add65aeb5844bc66a", - "sha256:2101c343f425545d8329004971d3549a1c553fb2b027c23ddfffebdd7b3fb309", - "sha256:255cd5510bf6dc82bf00e08711e3d8f0ed317ef5fa2f70d70d15192bdfe48593", - "sha256:26fcc5b982fdc7c49ab9dd84910e986ef2631f1b57e02ecefd5393366f7acbe6", - "sha256:38492a48bd51252b70d6008b2f38a62b8ddb55ed7d4c905717eca290c94ef125", - "sha256:3d19b4a94c2e20976cbc7773e5398bae19e3bc65a0894fcd01d8477bfc5fdb2f", - "sha256:3d9d718c366983bc65b149f0947f016929b4001f604f2523e6de5a49907f4474", - "sha256:3ff1afa5e0a0cb6d64703c476a106474bec335958d6d445b02a6be3da4c4acfb", - "sha256:40c83986431e9e685d04f2a0c12efafbf7decfc358b3421aeaecec97458d9663", - "sha256:435f140afa59eb0cece57e6364348977af768add452d49a8547a26fc90dc2bfa", - "sha256:54cffa56a5ead14915f1f0d3e8fabf6fcea3056a8e5f971fc1caebb9715bd9b1", - "sha256:5e39d6503d5a62dcd73f5de40042b20111f829fea0057fed95f38f06f367b14c", - "sha256:5ed0cab4e05558ba112111585ae2a85627ab48d6185547fc4ba8ec8e2cfecd0d", - "sha256:5fa5afd86d86e156051c7d7d6de34f6c03a7a64e4794ac0d72379bee8fcafafd", - "sha256:614eff94e9ef4b90540fdcc6d520fcf5f9b7560e8a1cdd5e8a3644f12cefbc74", - "sha256:6157ac0daf30ffc28446c54ebbd073c8c11b45cba9c45256619e218a07a31312", - "sha256:69f16d6709f3305da1043ee9edb5e1715212b75011779d58c4bd8fc9ad9fc88a", - "sha256:752a5bbbd1b148b61d69be718e2c1c89a1b16de27a21e6382d28c8d2a33e9bf1", - "sha256:788df7ab964f03894c616e742e088f1bf209a54465f61541ba77901a9e1cd701", - "sha256:824a785d01500da47fe6935dbadc4b0a14577405c25d01327a94717578dd2b1d", - "sha256:83c217e0c7aea28bbd9512e27fb6f9ef9b9f398dff57ebe223ecaa043bcd08fa", - "sha256:87140865a8661cedcd2d057e43ddc3a7c6b2c5919e0f47a0aff111d125733951", - "sha256:8d39cc233c7c3303072758b088ad7ccde0a5dcb30ce752f2f8a31428a07faac1", - "sha256:8eab3a83670d83966c7a0df4fa02cf2272499966066bfaa810444c427cb653b0", - "sha256:94a3c02381b62b2f0498f4ea3a3bd8b96ae2a6a905fcfd198547805894793a5b", - "sha256:9903eabe2578143487d2387947d28323e12df49e142ffb0e3b5709fc6c942c47", - "sha256:998335e6b69691c097e3a74214901c71773e0a7a8d3d0cc87c60d6969eb8b584", - "sha256:9d43013002eab2fd6d0dcc671cd1e9149e2fc1c56d5e796fad94d076d6cb69ef", - "sha256:bdb72c53445b1c09315940c81a14a3635dfde6710aea703976ce9617209830e7", - "sha256:bfe49668ba0a7fba183cf0e189bb8da515793da8f383d8d157ac8647143dc516", - "sha256:c21c3262244f7b2af27636bc1052cb861a70df933b8bc188a7fb8fe3c895b608", - "sha256:cd69340f5a49d6e4f67723db6ea4459c554d2210c3dd4da99950d6d405ddafb4", - "sha256:d5d7bc6e01d74548e93d5b9b3e38c3a91b6f54a36f2fe13407cac035b57f0c1f", - "sha256:e79f90c0af413933dedc79a1717f49076f08b32ff4a42776d6c38ba37125717e", - "sha256:f17264adc95a7f92a148d17c55eee885da640cbd98cf9169e643029d79bb18c7", - "sha256:f463a7f7c12b322b4f72af908f863b4da1f71dae344dd75f50d30c33ff0f197a", - "sha256:fb97f8f3f7754a5a00da450248de79d83e34938db04486efd26db81526dc25b4", - "sha256:ff50a3d787c3d059f330d7cb7325b40416bcb0f43f5006b396a6e61871df0ebf" - ], - "markers": "python_version >= '3.5'", - "version": "==7.45.7" - }, "pyjwt": { "hashes": [ "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", @@ -1043,23 +841,6 @@ "markers": "python_version >= '3.7'", "version": "==2.9.0" }, - "python-dateutil": { - "hashes": [ - "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", - "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.9.0.post0" - }, - "python-dotenv": { - "hashes": [ - "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca", - "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==1.0.1" - }, "redis": { "extras": [ "hiredis" @@ -1189,22 +970,6 @@ "markers": "python_version >= '3.6' and python_version < '4'", "version": "==4.9.1" }, - "s3transfer": { - "hashes": [ - "sha256:ca855bdeb885174b5ffa95b9913622459d4ad8e331fc98eb01e6d5eb6a30655d", - "sha256:edae4977e3a122445660c7c114bba949f9d191bae3b34a096f18a1c8c354527a" - ], - "markers": "python_version >= '3.8'", - "version": "==0.11.3" - }, - "six": { - "hashes": [ - "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", - "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.17.0" - }, "sqlparse": { "hashes": [ "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", @@ -1251,54 +1016,13 @@ "markers": "python_version >= '3.9'", "version": "==4.15.0" }, - "tzdata": { - "hashes": [ - "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9", - "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98" - ], - "markers": "python_version >= '2'", - "version": "==2026.1" - }, "urllib3": { "hashes": [ - "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", - "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" - ], - "markers": "python_version >= '3.9'", - "version": "==2.6.3" - }, - "uvicorn": { - "hashes": [ - "sha256:6c942071b68f07e178264b9152f1f16dfac5da85880c4ce06366a96d70d4f31e", - "sha256:ce937c99a2cc70279556967274414c087888e8cec9f9c94644dfca11bd3ced89" + "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", + "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897" ], "markers": "python_version >= '3.10'", - "version": "==0.44.0" - }, - "uvicorn-worker": { - "hashes": [ - "sha256:65dcef25ab80a62e0919640f9582216ee05b3bb1dc2f0e58b354ca0511c398fb", - "sha256:f6894544391796be6eeed37d48cae9d7739e5a105f7e37061eccef2eac5a0295" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==0.2.0" - }, - "vine": { - "hashes": [ - "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", - "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0" - ], - "markers": "python_version >= '3.6'", - "version": "==5.1.0" - }, - "wcwidth": { - "hashes": [ - "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", - "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159" - ], - "markers": "python_version >= '3.8'", - "version": "==0.6.0" + "version": "==2.7.0" } }, "develop": { @@ -1347,41 +1071,13 @@ "markers": "python_version >= '3.8'", "version": "==24.8.0" }, - "boto3-stubs": { - "extras": [ - "essential" - ], - "hashes": [ - "sha256:6cf3965964a9a22e895d75b4b1d8f21582de4bd114b9865c23da9d5b1cc7853c", - "sha256:a6066470f97da5810afeaa6d30028c4b198c9364c94180dc764597224321ae10" - ], - "markers": "python_version >= '3.8'", - "version": "==1.38.39" - }, - "botocore-stubs": { - "hashes": [ - "sha256:9423110fb0e391834bd2ed44ae5f879d8cb370a444703d966d30842ce2bcb5f0", - "sha256:dbeac2f744df6b814ce83ec3f3777b299a015cbea57a2efc41c33b8c38265825" - ], - "markers": "python_version >= '3.9'", - "version": "==1.42.41" - }, - "celery-types": { - "hashes": [ - "sha256:0cc495b8d7729891b7e070d0ec8d4906d2373209656a6e8b8276fe1ed306af9a", - "sha256:402ed0555aea3cd5e1e6248f4632e4f18eec8edb2435173f9e6dc08449fa101e" - ], - "index": "pypi", - "markers": "python_version >= '3.9' and python_version < '4.0'", - "version": "==0.23.0" - }, "certifi": { "hashes": [ - "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", - "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" + "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", + "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580" ], "markers": "python_version >= '3.7'", - "version": "==2026.2.25" + "version": "==2026.4.22" }, "charset-normalizer": { "hashes": [ @@ -1520,11 +1216,11 @@ }, "click": { "hashes": [ - "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", - "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d" + "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", + "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613" ], "markers": "python_version >= '3.10'", - "version": "==8.3.2" + "version": "==8.3.3" }, "coverage": { "extras": [ @@ -1646,17 +1342,17 @@ "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa" ], - "markers": "python_version >= '3.11'", + "markers": "python_version >= '3.9'", "version": "==0.4.1" }, "django": { "hashes": [ - "sha256:5788fce61da23788a8ce6f02583765ab060d396720924789f97fa42119d37f7a", - "sha256:a31589db5188d074c63f0945c3888fad104627dfcc236fb2b97f71f89da33bc4" + "sha256:58a63ba841662e5c686b57ba1fec52ddd68c0b93bd96ac3029d55728f00bf8a2", + "sha256:6f712143bd3064310d1f50fac859c3e9a274bdcfc9595339853be7779297fc76" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==5.2.13" + "version": "==5.2.14" }, "django-extensions": { "hashes": [ @@ -1680,11 +1376,11 @@ }, "django-stubs-ext": { "hashes": [ - "sha256:70b7b7ae837e7a6036e2facb64435550bf7cf8143c1a6e802864d4824ce6058c", - "sha256:b35bdec1995bf49765cc39fa89aa7c23f120a23d0cb0152ab7fb4e48ff7d667b" + "sha256:3307d42132bc295d5744de6276bc5fdf6896efc70f891e21c0ae8bdf529d2762", + "sha256:9e4105955419ae310d7da9cfd808e039d4dae3092c628f021057bb4f2c237f8f" ], "markers": "python_version >= '3.10'", - "version": "==6.0.2" + "version": "==6.0.3" }, "django-test-migrations": { "hashes": [ @@ -1716,11 +1412,11 @@ }, "idna": { "hashes": [ - "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", - "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902" + "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242", + "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3" ], "markers": "python_version >= '3.8'", - "version": "==3.11" + "version": "==3.13" }, "iniconfig": { "hashes": [ @@ -1786,62 +1482,6 @@ "markers": "python_version >= '3.9'", "version": "==1.15.0" }, - "mypy-boto3-cloudformation": { - "hashes": [ - "sha256:1016508783c1263aba9bb24dd29afbea6f0c8c7cee79e9d073c4ed5524ce53f5", - "sha256:f4185231faab97bfb50b25dc1323333c630a092ffa8c15356f21116fc92a7f42" - ], - "markers": "python_version >= '3.8'", - "version": "==1.38.31" - }, - "mypy-boto3-dynamodb": { - "hashes": [ - "sha256:5cf3787631e312b3d75f89a6cbbbd4ad786a76f5d565af023febf03fbf23c0b5", - "sha256:6b29d89c649eeb1e894118bee002cb8b1304c78da735b1503aa08e46b0abfdec" - ], - "markers": "python_version >= '3.8'", - "version": "==1.38.4" - }, - "mypy-boto3-ec2": { - "hashes": [ - "sha256:0d15dd2f36507febc216ff1f312533ff7a8d74387f19e699ffbfaed41f345e2e", - "sha256:ea7bb4fcb74e60d8bc3e42ca107f83b6e2221f334d29e56620cd0549b9471362" - ], - "markers": "python_version >= '3.8'", - "version": "==1.38.45" - }, - "mypy-boto3-lambda": { - "hashes": [ - "sha256:04bd5f4ad032f86cd0d5b8f573c0384a388dc8549ea6bb648dcef5b2c6664064", - "sha256:145c9f7de2da29fb651b21515c9052697f7579c821ee3ab7bcbf7ef4993dcd25" - ], - "markers": "python_version >= '3.8'", - "version": "==1.38.40" - }, - "mypy-boto3-rds": { - "hashes": [ - "sha256:27a92d463333a56f36c89376e18d817c34aa6dddc93fbdfd583f02ac3ddbee9d", - "sha256:c2194335e3e11cea2f982381ecf376fd66f3cefae0434cef4166ec70491c00b9" - ], - "markers": "python_version >= '3.8'", - "version": "==1.38.46" - }, - "mypy-boto3-s3": { - "hashes": [ - "sha256:9a9d305af1eebf246b6d6195bf88902ff553fe5aa0a78e5f517817875339f5e4", - "sha256:eacf15c9164e56ecb86a76152adb414cf8f7f22b46cd80a843f6b7ef41a1cd8f" - ], - "markers": "python_version >= '3.8'", - "version": "==1.38.44" - }, - "mypy-boto3-sqs": { - "hashes": [ - "sha256:39aebc121a2fe20f962fd83b617fd916003605d6f6851fdf195337a0aa428fe1", - "sha256:8e881c8492f6f51dcbe1cce9d9f05334f4b256b5843e227fa925e0f6e702b31d" - ], - "markers": "python_version >= '3.8'", - "version": "==1.38.0" - }, "mypy-extensions": { "hashes": [ "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", @@ -1852,19 +1492,19 @@ }, "packaging": { "hashes": [ - "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", - "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" + "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", + "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661" ], "markers": "python_version >= '3.8'", - "version": "==26.0" + "version": "==26.2" }, "pathspec": { "hashes": [ - "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", - "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723" + "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", + "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189" ], "markers": "python_version >= '3.9'", - "version": "==1.0.4" + "version": "==1.1.1" }, "platformdirs": { "hashes": [ @@ -2014,14 +1654,6 @@ "markers": "python_version >= '3.9'", "version": "==0.14.0" }, - "types-awscrt": { - "hashes": [ - "sha256:09d3eaf00231e0f47e101bd9867e430873bc57040050e2a3bd8305cb4fc30865", - "sha256:e5ce65a00a2ab4f35eacc1e3d700d792338d56e4823ee7b4dbe017f94cfc4458" - ], - "markers": "python_version >= '3.8'", - "version": "==0.31.3" - }, "types-cachetools": { "hashes": [ "sha256:0d8ae2dd5ba0b4cfe6a55c34396dd0415f1be07d0033d84781cdc4ed9c2ebc6b", @@ -2042,11 +1674,11 @@ }, "types-pyyaml": { "hashes": [ - "sha256:92a73f2b8d7f39ef392a38131f76b970f8c66e4c42b3125ae872b7c93b556307", - "sha256:fbc42037d12159d9c801ebfcc79ebd28335a7c13b08a4cfbc6916df78fee9384" + "sha256:5ae42149c3ebf7aaaf6c65ee49af590c80f0ba52e9e3f75a75c5564b33556fa6", + "sha256:edc094ed3a918b0c6232f71a5b67fdf38e76e17517b7d87bfbb9fc27d442fb51" ], "markers": "python_version >= '3.10'", - "version": "==6.0.12.20260408" + "version": "==6.0.12.20260508" }, "types-regex": { "hashes": [ @@ -2059,19 +1691,11 @@ }, "types-requests": { "hashes": [ - "sha256:81f31d5ea4acb39f03be7bc8bed569ba6d5a9c5d97e89f45ac43d819b68ca50f", - "sha256:95b9a86376807a216b2fb412b47617b202091c3ea7c078f47cc358d5528ccb7b" + "sha256:81b2ae5f0d20967714a6aa5ef9284c05570d7cb06b7de8f2a77b918b63ddd411", + "sha256:fa01459cca184229713df03709db46a905325906d27e042cd4fd7ea3d15d3400" ], "markers": "python_version >= '3.10'", - "version": "==2.33.0.20260408" - }, - "types-s3transfer": { - "hashes": [ - "sha256:1c0cd111ecf6e21437cb410f5cddb631bfb2263b77ad973e79b9c6d0cb24e0ef", - "sha256:b4636472024c5e2b62278c5b759661efeb52a81851cde5f092f24100b1ecb443" - ], - "markers": "python_version >= '3.9'", - "version": "==0.16.0" + "version": "==2.33.0.20260508" }, "typing-extensions": { "hashes": [ @@ -2083,11 +1707,11 @@ }, "urllib3": { "hashes": [ - "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", - "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" + "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", + "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897" ], - "markers": "python_version >= '3.9'", - "version": "==2.6.3" + "markers": "python_version >= '3.10'", + "version": "==2.7.0" } } } diff --git a/codeforlife/__init__.py b/codeforlife/__init__.py index d611c98d..003941da 100644 --- a/codeforlife/__init__.py +++ b/codeforlife/__init__.py @@ -5,12 +5,7 @@ import os import sys -import typing as t -from io import StringIO from pathlib import Path -from types import SimpleNamespace - -from .types import Env # Do NOT set manually! # This is auto-updated by python-semantic-release in the pipeline. @@ -22,52 +17,24 @@ USER_DIR = BASE_DIR.joinpath("user") -if t.TYPE_CHECKING: - from mypy_boto3_s3.client import S3Client - - -# pylint: disable-next=too-few-public-methods -class Secrets(SimpleNamespace): - """The secrets for this service. - - If a key does not exist, the value None will be returned. - """ - - def __getattribute__(self, name: str) -> t.Optional[str]: - try: - return super().__getattribute__(name) - except AttributeError: - return None - - def set_up_settings(service_base_dir: Path, service_name: str): """Set up the settings for the service. *This needs to be called before importing the CFL settings!* - To expose a secret to your Django project, you'll need to create a setting - for it following Django's conventions. - Examples: ``` from codeforlife import set_up_settings # Must set up settings before importing them! - secrets = set_up_settings(BASE_DIR, service_name="my-service") + set_up_settings(BASE_DIR, service_name="my-service") from codeforlife.settings import * - - # Expose secret to django project. - SECRET_KEY = secrets.SECRET_KEY ``` Args: service_base_dir: The base directory of the service. service_name: The name of the service. - - Returns: - The secrets. These are not loaded as environment variables so that 3rd - party packages cannot read them. """ # Validate CFL settings have not been imported yet. @@ -76,57 +43,6 @@ def set_up_settings(service_base_dir: Path, service_name: str): "You must set up the CFL settings before importing them." ) - # pylint: disable-next=import-outside-toplevel - from dotenv import dotenv_values, load_dotenv - # Set required environment variables. os.environ["SERVICE_BASE_DIR"] = str(service_base_dir) os.environ["SERVICE_NAME"] = service_name - - # Get environment name. - os.environ.setdefault("ENV", "local") - env = t.cast(Env, os.environ["ENV"]) - - # Load environment variables. - load_dotenv(service_base_dir / f"env/.env.{env}", override=False) - load_dotenv(service_base_dir / "env/.env", override=False) - - # Get secrets. - if env == "local": - secrets_path = service_base_dir / "env/.env.local.secrets" - # TODO: move this to the dev container setup script. - if not os.path.exists(secrets_path): - # pylint: disable=line-too-long - secrets_file_comment = ( - "# 📝 Local Secret Variables 📝\n" - "# These secret variables are only loaded in your local environment (on your PC).\n" - "#\n" - "# This file is git-ignored intentionally to keep these variables a secret.\n" - "#\n" - "# 🚫 DO NOT PUSH SECRETS TO THE CODE REPO 🚫\n" - "\n" - ) - # pylint: enable=line-too-long - - with open(secrets_path, "w+", encoding="utf-8") as secrets_file: - secrets_file.write(secrets_file_comment) - - secrets = dotenv_values(secrets_path) - else: - # pylint: disable-next=import-outside-toplevel - import boto3 - - s3: "S3Client" = boto3.client("s3") - secrets_object = s3.get_object( - Bucket=os.environ["aws_s3_app_bucket"], - Key=( - os.environ["aws_s3_app_folder"] - + f"/secure/.env.secrets.{service_name}" - ), - ) - - secrets = dotenv_values( - stream=StringIO(secrets_object["Body"].read().decode("utf-8")) - ) - - return Secrets(**secrets) diff --git a/codeforlife/auth.py b/codeforlife/auth.py deleted file mode 100644 index 0be7f677..00000000 --- a/codeforlife/auth.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -© Ocado Group -Created on 04/12/2025 at 18:58:44(+00:00). - -Authentication credentials. -""" - -import typing as t - -from boto3 import Session as AwsSession -from django.conf import settings -from google.auth.aws import ( - AwsSecurityCredentials, - AwsSecurityCredentialsSupplier, -) -from google.auth.aws import Credentials as AwsCredentials -from google.auth.credentials import Credentials -from google.oauth2.service_account import ( - Credentials as GcpServiceAccountCredentials, -) - - -class AwsSessionSecurityCredentialsSupplier(AwsSecurityCredentialsSupplier): - """Supplies AWS security credentials from the current boto3 session.""" - - def get_aws_region(self, _, __): - return settings.AWS_REGION - - def get_aws_security_credentials(self, _, __) -> AwsSecurityCredentials: - aws_credentials = AwsSession().get_credentials() - assert aws_credentials - - aws_read_only_credentials = aws_credentials.get_frozen_credentials() - assert aws_read_only_credentials.access_key - assert aws_read_only_credentials.secret_key - assert aws_read_only_credentials.token - - return AwsSecurityCredentials( - access_key_id=aws_read_only_credentials.access_key, - secret_access_key=aws_read_only_credentials.secret_key, - session_token=aws_read_only_credentials.token, - ) - - -# pylint: disable-next=abstract-method,too-many-ancestors -class GcpWifCredentials(AwsCredentials): - """Workload Identity Federation credentials for GCP using AWS IAM roles.""" - - def __init__(self, token_lifetime_seconds: int = 600): - super().__init__( - subject_token_type="urn:ietf:params:aws:token-type:aws4_request", - audience=settings.GCP_WIF_AUDIENCE, - universe_domain="googleapis.com", - token_url="https://sts.googleapis.com/v1/token", - service_account_impersonation_url=( - "https://iamcredentials.googleapis.com/v1/projects/-/" - f"serviceAccounts/{settings.GCP_WIF_SERVICE_ACCOUNT}" - ":generateAccessToken" - ), - service_account_impersonation_options={ - "token_lifetime_seconds": token_lifetime_seconds - }, - aws_security_credentials_supplier=( - AwsSessionSecurityCredentialsSupplier() - ), - ) - - -def get_gcp_service_account_credentials( - token_lifetime_seconds: int = 600, - service_account_json: t.Optional[str] = None, -) -> Credentials: - """Get GCP service account credentials. - - Args: - token_lifetime_seconds: The lifetime of the token in seconds. - service_account_json: The path to the service account JSON file. - - Returns: - The GCP service account credentials. - """ - if settings.ENV != "local": - return GcpWifCredentials(token_lifetime_seconds=token_lifetime_seconds) - assert ( - service_account_json - ), "Service account JSON file path must be provided in local environment." - - return GcpServiceAccountCredentials.from_service_account_file( - service_account_json - ) diff --git a/codeforlife/encryption.py b/codeforlife/encryption.py index b0a2ea4c..6a322432 100644 --- a/codeforlife/encryption.py +++ b/codeforlife/encryption.py @@ -10,10 +10,11 @@ primitive. A simple check for the environment (e.g., `settings.ENV == "local"`) determines whether to use the real `GcpKmsClient` or a `FakeGcpKmsClient`. -The fake client mimics the behavior of the real one. Instead of performing real -encryption, it simulates it by encoding the plaintext in base64 and adding a -prefix. This allows the application to run without needing cloud credentials -while still being able to distinguish between "encrypted" and plaintext data. +The fake client mimics the behavior of the real one while still using a real +AEAD algorithm locally. `FakeAead` uses `AESGCM` with random nonces and +associated data support, then prefixes the resulting blob so local ciphertext +can be identified as fake-KMS output. This keeps local behavior close to cloud +execution while avoiding cloud credentials. While the `FakeAead` and `FakeGcpKmsClient` are sufficient for running a local development server, they are not `unittest.mock.MagicMock` instances by default. @@ -27,13 +28,13 @@ """ import typing as t -from base64 import b64decode, b64encode -from dataclasses import dataclass +from functools import cache from io import BytesIO +from os import urandom from unittest.mock import MagicMock, create_autospec +from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings -from django.utils.crypto import get_random_string from tink import ( # type: ignore[import-untyped] BinaryKeysetReader, BinaryKeysetWriter, @@ -48,27 +49,39 @@ _GcpKmsClient = gcpkms.GcpKmsClient -@dataclass class FakeAead: """A fake AEAD primitive for local testing.""" - @staticmethod - # pylint: disable-next=unused-argument - def encrypt(plaintext: bytes, associated_data: bytes = b""): - """Simulate ciphertext by wrapping in base64 and adding a prefix.""" - return b"fake_enc:" + b64encode(plaintext) + ciphertext_prefix = b"fake_enc:" + nonce_size = 12 - @staticmethod - # pylint: disable-next=unused-argument - def decrypt(ciphertext: bytes, associated_data: bytes = b""): - """Simulate decryption by removing prefix and base64 decoding.""" - if not ciphertext.startswith(b"fake_enc:"): + def __init__(self, key: bytes): + self._aesgcm = AESGCM(key) + + def encrypt(self, plaintext: bytes, associated_data: bytes): + """Encrypt data using AES-GCM and return a prefixed blob.""" + nonce = urandom(self.nonce_size) + ciphertext = self._aesgcm.encrypt( + nonce=nonce, + data=plaintext, + associated_data=associated_data, + ) + + return self.ciphertext_prefix + nonce + ciphertext + + def decrypt(self, ciphertext: bytes, associated_data: bytes): + """Decrypt a prefixed AES-GCM blob.""" + if not ciphertext.startswith(self.ciphertext_prefix): raise ValueError("Invalid ciphertext for fake mock") + ciphertext = ciphertext.removeprefix(self.ciphertext_prefix) - return b64decode(ciphertext.replace(b"fake_enc:", b"")) + return self._aesgcm.decrypt( + nonce=ciphertext[: self.nonce_size], + data=ciphertext[self.nonce_size :], + associated_data=associated_data, + ) - @classmethod - def as_mock(cls): + def as_mock(self): """ Returns the class as a functional MagicMock for testing. The mock tracks calls while still performing the fake encryption and decryption by using @@ -76,17 +89,21 @@ def as_mock(cls): the mock behaves as an instance of the class, not the class itself. """ mock: MagicMock = create_autospec(Aead, instance=True) - mock.encrypt.side_effect = cls.encrypt - mock.decrypt.side_effect = cls.decrypt + mock.encrypt.side_effect = self.encrypt + mock.decrypt.side_effect = self.decrypt return mock -@dataclass class FakeGcpKmsClient: """A fake GcpKmsClient for local testing.""" - key_uri: str + # Fake key encryption key (KEK) for local testing. + # pylint: disable-next=line-too-long + kek = b"fake_kek:\xba\xcc\x8c;\xa9\x85k\n\x93\xb7\x1b2\xab\x86\x9d\xea\xb1+\x88\xc0\xc4y3" + + def __init__(self, key_uri: str): + self.key_uri = key_uri @staticmethod def register_client( @@ -97,7 +114,7 @@ def register_client( # pylint: disable-next=unused-argument def get_aead(self, key_uri: str) -> Aead: """Return the fake AEAD primitive.""" - return FakeAead() + return FakeAead(self.kek) @classmethod def as_mock(cls): @@ -108,15 +125,16 @@ def as_mock(cls): itself. """ mock: MagicMock = create_autospec(_GcpKmsClient, instance=True) - mock.get_aead.return_value = FakeAead.as_mock() + mock.get_aead.return_value = FakeAead(cls.kek).as_mock() return mock +@cache def _get_kek_aead(): """Get the AEAD primitive for the key encryption key (KEK).""" - return GcpKmsClient(key_uri=settings.GCP_KMS_KEY_URI).get_aead( - key_uri=settings.GCP_KMS_KEY_URI + return GcpKmsClient(key_uri=settings.GCP_KMS_KEY_URI()).get_aead( + key_uri=settings.GCP_KMS_KEY_URI() ) @@ -127,7 +145,7 @@ def create_dek(): """ # In local environment, return a fake encrypted DEK. if settings.ENV == "local": - return FakeAead.encrypt(get_random_string(32).encode()) + return _get_kek_aead().encrypt(urandom(32), b"") stream = BytesIO() new_keyset_handle(key_template=aead_key_templates.AES256_GCM).write( @@ -148,7 +166,7 @@ def get_dek_aead(dek: bytes) -> Aead: # In local environment, return the fake AEAD primitive. if settings.ENV == "local": - return FakeAead() + return FakeAead(_get_kek_aead().decrypt(dek, b"")) return read_keyset_handle( keyset_reader=BinaryKeysetReader(dek), @@ -164,5 +182,5 @@ def get_dek_aead(dek: bytes) -> Aead: # Register the GCP KMS client with Tink. GcpKmsClient.register_client( - key_uri=settings.GCP_KMS_KEY_URI, credentials_path=None + key_uri=settings.GCP_KMS_KEY_URI(), credentials_path=None ) diff --git a/codeforlife/filters.py b/codeforlife/filters.py index 831ca89d..c4df621a 100644 --- a/codeforlife/filters.py +++ b/codeforlife/filters.py @@ -15,11 +15,12 @@ class FilterSet(_FilterSet): """Base filter set all other filter sets must inherit.""" @staticmethod - def make_exclude_field_list_method(field: str): + def make_exclude_field_list_method(field: str, lookup="in"): """Make a class-method that excludes a list of values for a field. Args: field: The field to exclude a list of values for. + lookup: The lookup type to use for exclusion. Returns: A class-method. @@ -27,7 +28,7 @@ def make_exclude_field_list_method(field: str): def method(self: FilterSet, queryset: QuerySet, name: str, *args): return queryset.exclude( - **{f"{field}__in": self.request.GET.getlist(name)} + **{f"{field}__{lookup}": self.request.GET.getlist(name)} ) return method diff --git a/codeforlife/mail.py b/codeforlife/mail.py index 306f54f3..b1e3324a 100644 --- a/codeforlife/mail.py +++ b/codeforlife/mail.py @@ -74,7 +74,7 @@ def add_contact( # pylint: enable=line-too-long if auth is None: - auth = settings.MAIL_AUTH + auth = settings.MAIL_AUTH() contact: JsonDict = {"email": email.lower()} if opt_in_type is not None: @@ -186,7 +186,7 @@ def remove_contact( return True if auth is None: - auth = settings.MAIL_AUTH + auth = settings.MAIL_AUTH() response = requests.delete( # pylint: disable-next=line-too-long @@ -256,7 +256,7 @@ def send_mail( # pylint: enable=line-too-long if auth is None: - auth = settings.MAIL_AUTH + auth = settings.MAIL_AUTH() body = { "campaignId": campaign_id, diff --git a/codeforlife/models/abstract_base_user.py b/codeforlife/models/abstract_base_user.py index fadb2a92..958c669c 100644 --- a/codeforlife/models/abstract_base_user.py +++ b/codeforlife/models/abstract_base_user.py @@ -11,6 +11,8 @@ from django.contrib.auth.models import AbstractBaseUser as _AbstractBaseUser from django.utils.translation import gettext_lazy as _ +from .base import Model + if t.TYPE_CHECKING: from django_stubs_ext.db.models import TypedModelMeta @@ -19,7 +21,7 @@ TypedModelMeta = object -class AbstractBaseUser(_AbstractBaseUser): +class AbstractBaseUser(Model, _AbstractBaseUser): """ Base user class to be inherited by all user classes. https://docs.djangoproject.com/en/5.1/topics/auth/customizing/#using-a-custom-user-model-when-starting-a-project diff --git a/codeforlife/models/base.py b/codeforlife/models/base.py index 8da23226..dc5ccc92 100644 --- a/codeforlife/models/base.py +++ b/codeforlife/models/base.py @@ -3,6 +3,11 @@ Created on 19/01/2024 at 15:18:48(+00:00). Base model for all Django models. + +Supports optional `field_aliases` expansion during `save(update_fields=...)`. +When an alias is present in `update_fields`, it is replaced by its mapped real +field names so property-driven transformations can persist all dependent +columns in a single save. """ import typing as t @@ -10,6 +15,7 @@ from django.db import models if t.TYPE_CHECKING: + from django.db.models.base import ModelBase from django_stubs_ext.db.models import TypedModelMeta else: TypedModelMeta = object @@ -18,10 +24,37 @@ class Model(models.Model): """Base for all models.""" + field_aliases: t.Dict[str, t.Set[str]] objects: models.Manager[t.Self] class Meta(TypedModelMeta): abstract = True + def save( + self, + *args, + force_insert: t.Union[bool, t.Tuple["ModelBase", ...]] = False, + force_update: bool = False, + using: t.Optional[str] = None, + update_fields: t.Optional[t.Iterable[str]] = None + ): + """Save the model, expanding any field aliases in `update_fields`.""" + if update_fields: + new_update_fields = set(update_fields) + # Replace any fields in update_fields with their aliased fields. + for field in update_fields: + if field in self.field_aliases: + new_update_fields.remove(field) + new_update_fields.update(self.field_aliases[field]) + update_fields = new_update_fields + + super().save( + *args, + force_insert=force_insert, + force_update=force_update, + using=using, + update_fields=update_fields, + ) + AnyModel = t.TypeVar("AnyModel", bound=Model) diff --git a/codeforlife/models/base_data_encryption_key.py b/codeforlife/models/base_data_encryption_key.py index 600d5a2f..b1fd48d5 100644 --- a/codeforlife/models/base_data_encryption_key.py +++ b/codeforlife/models/base_data_encryption_key.py @@ -10,18 +10,26 @@ import typing as t from cachetools import TTLCache -from django.core.exceptions import ValidationError +from django.core import checks +from django.core.exceptions import FieldDoesNotExist, ValidationError from ..encryption import create_dek, get_dek_aead +from ..types import KwArgs from .encrypted import EncryptedModel +from .utils import is_real_model_class if t.TYPE_CHECKING: from django_stubs_ext.db.models import TypedModelMeta + from tink.aead import Aead # type: ignore[import] - from .fields import DataEncryptionKeyField + from .fields.data_encryption_key import Dek else: TypedModelMeta = object +AnyBaseDataEncryptionKeyModel = t.TypeVar( + "AnyBaseDataEncryptionKeyModel", bound="BaseDataEncryptionKeyModel" +) + class BaseDataEncryptionKeyModel(EncryptedModel): """Model to store and manage a data encryption key.""" @@ -32,6 +40,9 @@ class BaseDataEncryptionKeyModel(EncryptedModel): # In-memory cache for the decrypted DEK AEAD primitive. DEK_AEAD_CACHE: TTLCache + # A class-level reference to the DataEncryptionKeyField instance. + # This is set by the `contribute_to_class` method of the field. + DEK_FIELD: str def __init_subclass__(cls): super().__init_subclass__() @@ -39,38 +50,121 @@ def __init_subclass__(cls): maxsize=cls.dek_aead_cache_maxsize, ttl=cls.dek_aead_cache_ttl ) - # A class-level reference to the DataEncryptionKeyField instance. - # This is set by the `contribute_to_class` method of the field. - DEK_FIELD: t.Optional["DataEncryptionKeyField"] = None - class Meta(TypedModelMeta): abstract = True + # pylint: disable-next=too-few-public-methods + class Manager( + EncryptedModel.Manager[AnyBaseDataEncryptionKeyModel], + t.Generic[AnyBaseDataEncryptionKeyModel], + ): + """Base manager for models with a data encryption key.""" + + def _inject_dek_kwarg(self, kwargs: KwArgs): + """Inject a DEK into the kwargs.""" + kwargs[self.model.DEK_FIELD] = create_dek() + + def create(self, **kwargs): + """Ensure a DEK is created for new instances.""" + self._inject_dek_kwarg(kwargs) + return super().create(**kwargs) + + base_manager_class = Manager + + @classmethod + def _check_dek_field(cls): + """ + Check that the DEK_FIELD is defined correctly and exists on the model. + """ + errors: t.List[checks.Error] = [] + + # Skip non-real models. + if not is_real_model_class(cls): + return errors + + if not hasattr(cls, "DEK_FIELD"): + errors.append( + checks.Error( + f"'{cls.__module__}.{cls.__name__}' must have DEK_FIELD " + "defined.", + hint="Set `dek = DataEncryptionKeyField()` on the model.", + obj=cls, + id="base_data_encryption_key.E001", + ) + ) + elif not isinstance(cls.DEK_FIELD, str): + errors.append( + checks.Error( + f"'{cls.__module__}.{cls.__name__}' DEK_FIELD must be a " + "string.", + hint="Set `dek = DataEncryptionKeyField()` on the model.", + obj=cls, + id="base_data_encryption_key.E002", + ) + ) + elif not cls.DEK_FIELD: + errors.append( + checks.Error( + f"'{cls.__module__}.{cls.__name__}' DEK_FIELD cannot be " + "empty.", + hint="Set `dek = DataEncryptionKeyField()` on the model.", + obj=cls, + id="base_data_encryption_key.E003", + ) + ) + else: + try: + cls._meta.get_field(cls.DEK_FIELD) + except FieldDoesNotExist: + errors.append( + checks.Error( + f"'{cls.__module__}.{cls.__name__}' DEK_FIELD " + f"'{cls.DEK_FIELD}' does not exist.", + hint="Set `dek = DataEncryptionKeyField()` on the " + "model.", + obj=cls, + id="base_data_encryption_key.E004", + ) + ) + + return errors + + @classmethod + def check(cls, **kwargs): + """Run model checks, including custom checks for encrypted models.""" + errors = super().check(**kwargs) + errors.extend(cls._check_dek_field()) + + return errors + @property def dek_aead(self): """ Provides the AEAD primitive for the DEK, caching it for performance. """ - # Ensure the instance is saved before accessing the DEK AEAD. - if self.pk is None: + # Get the DEK and return None if it's not set. + dek: t.Optional["Dek"] = getattr(self, self.DEK_FIELD) + if dek is None: raise ValidationError( - "Instance must be saved before accessing dek_aead.", - code="unsaved_instance", + "Cannot retrieve the AEAD primitive for the data encryption " + "key (DEK) because the DEK is None.", + code="dek_is_none", ) - # Return None if there is no DEK. - if self.DEK_FIELD is None: - return None - # Check the cache for the DEK AEAD. - if self.pk in self.DEK_AEAD_CACHE: - return self.DEK_AEAD_CACHE[self.pk] + if self.pk is not None and self.pk in self.DEK_AEAD_CACHE: + dek_aead = t.cast( + t.Optional["Aead"], self.DEK_AEAD_CACHE.get(self.pk, None) + ) + if dek_aead is not None: + return dek_aead # Get the AEAD primitive for the data encryption key. - dek_aead = get_dek_aead(self.DEK_FIELD) + dek_aead = get_dek_aead(bytes(dek)) # Cache the DEK AEAD for future access. - self.DEK_AEAD_CACHE[self.pk] = dek_aead + if self.pk is not None: + self.DEK_AEAD_CACHE[self.pk] = dek_aead return dek_aead @@ -83,9 +177,10 @@ def save( update_fields=None, ): # Lazily create a new DEK for new instances. - if self.pk is None and self.__class__.DEK_FIELD is not None: - self.__dict__[self.__class__.DEK_FIELD.field.attname] = create_dek() + if self.pk is None and getattr(self, self.DEK_FIELD) is None: + self.__dict__[self.DEK_FIELD] = create_dek() + # pylint: disable=duplicate-code return super().save( # type: ignore[misc] *args, force_insert=force_insert, @@ -93,3 +188,4 @@ def save( using=using, update_fields=update_fields, ) + # pylint: enable=duplicate-code diff --git a/codeforlife/models/base_data_encryption_key_test.py b/codeforlife/models/base_data_encryption_key_test.py index 565a1e3b..72f39907 100644 --- a/codeforlife/models/base_data_encryption_key_test.py +++ b/codeforlife/models/base_data_encryption_key_test.py @@ -6,7 +6,7 @@ import typing as t from unittest.mock import MagicMock, patch -from ..encryption import FakeAead, create_dek +from ..encryption import FakeAead, create_dek, get_dek_aead from ..tests import ModelTestCase from .base_data_encryption_key import BaseDataEncryptionKeyModel from .fields import DataEncryptionKeyField @@ -42,53 +42,52 @@ def set_dek_for_test(self): def get_model_instance(self, *args, **kwargs): return self.get_model_class()(*args, **kwargs) - def test_dek_aead__none(self): - """Returns None when dek is None on a saved instance.""" + def test_dek_aead__dek_is_none(self): + """Raise ValidationError when dek is None on a saved instance.""" instance = self.get_model_instance(pk=1, dek=None) - assert instance.dek_aead is None - - def test_dek_aead__unsaved_instance(self): - """Cannot get dek before saving the instance.""" - instance = self.get_model_instance() - with self.assert_raises_validation_error(code="unsaved_instance"): + with self.assert_raises_validation_error(code="dek_is_none"): _ = instance.dek_aead - @patch("codeforlife.models.base_data_encryption_key.get_dek_aead") - def test_dek_aead__not_cached(self, get_dek_aead_mock: MagicMock): + def test_dek_aead__not_cached(self): """Returns dek_aead and caches it when not cached.""" # Create an instance with a primary key to mimic a saved instance. instance = self.get_model_instance(pk=1) instance.set_dek_for_test() # Setup the mock to return a FakeAead instance. - dek_aead_mock = FakeAead.as_mock() - get_dek_aead_mock.return_value = dek_aead_mock - - # Initially, the cache should not have the dek_aead. After accessing - # dek_aead, it should be cached. - assert instance.pk not in instance.DEK_AEAD_CACHE - assert instance.dek_aead is dek_aead_mock - assert instance.pk in instance.DEK_AEAD_CACHE - - # Ensure the get_dek_aead function was called with the correct dek. - get_dek_aead_mock.assert_called_once_with(instance.dek) - - @patch("codeforlife.models.base_data_encryption_key.get_dek_aead") - def test_dek_aead__cached(self, get_dek_aead_mock: MagicMock): + dek_aead_mock = t.cast(FakeAead, get_dek_aead(instance.dek)).as_mock() + with patch( + "codeforlife.models.base_data_encryption_key.get_dek_aead", + return_value=dek_aead_mock, + ) as get_dek_aead_mock: + # Initially, the cache should not have the dek_aead. After accessing + # dek_aead, it should be cached. + assert instance.pk not in instance.DEK_AEAD_CACHE + assert instance.dek_aead is dek_aead_mock + assert instance.pk in instance.DEK_AEAD_CACHE + assert instance.DEK_AEAD_CACHE[instance.pk] is dek_aead_mock + + # Ensure the get_dek_aead function was called with the correct dek. + get_dek_aead_mock.assert_called_once_with(instance.dek) + + def test_dek_aead__cached(self): """Returns the cached dek_aead.""" # Create an instance with a primary key to mimic a saved instance. instance = self.get_model_instance(pk=1) instance.set_dek_for_test() # Pre-populate the cache with a FakeAead instance. - dek_aead_mock = FakeAead.as_mock() - instance.DEK_AEAD_CACHE[instance.pk] = dek_aead_mock + dek_aead_mock = t.cast(FakeAead, get_dek_aead(instance.dek)).as_mock() + with patch( + "codeforlife.models.base_data_encryption_key.get_dek_aead" + ) as get_dek_aead_mock: + instance.DEK_AEAD_CACHE[instance.pk] = dek_aead_mock - # Accessing dek_aead should return the cached value. - assert instance.dek_aead is dek_aead_mock + # Accessing dek_aead should return the cached value. + assert instance.dek_aead is dek_aead_mock - # Ensure the get_dek_aead function was not called as its cached. - get_dek_aead_mock.assert_not_called() + # Ensure the get_dek_aead function was not called as its cached. + get_dek_aead_mock.assert_not_called() @patch("django.db.models.base.Model.save", autospec=True) @patch( @@ -114,3 +113,53 @@ def test_save__creates_dek( ) assert instance.dek is not None + + def test_check__e001(self): + """Raises an error if the DEK field is missing.""" + + class BaseDekE001(BaseDataEncryptionKeyModel): + class Meta(TypedModelMeta): + app_label = "codeforlife.user" + + self.assert_check( + error_id="base_data_encryption_key.E001", model_class=BaseDekE001 + ) + + def test_check__e002(self): + """Raises an error if the DEK field is not a string.""" + + class BaseDekE002(BaseDataEncryptionKeyModel): + DEK_FIELD = 123 # type: ignore[assignment] + + class Meta(TypedModelMeta): + app_label = "codeforlife.user" + + self.assert_check( + error_id="base_data_encryption_key.E002", model_class=BaseDekE002 + ) + + def test_check__e003(self): + """Raises an error if the DEK field name is empty.""" + + class BaseDekE003(BaseDataEncryptionKeyModel): + DEK_FIELD = "" + + class Meta(TypedModelMeta): + app_label = "codeforlife.user" + + self.assert_check( + error_id="base_data_encryption_key.E003", model_class=BaseDekE003 + ) + + def test_check__e004(self): + """Raises an error if the DEK field does not exist on the model.""" + + class BaseDekE004(BaseDataEncryptionKeyModel): + DEK_FIELD = "non_existent_field" + + class Meta(TypedModelMeta): + app_label = "codeforlife.user" + + self.assert_check( + error_id="base_data_encryption_key.E004", model_class=BaseDekE004 + ) diff --git a/codeforlife/models/encrypted.py b/codeforlife/models/encrypted.py index 69c9f0b4..3ad8d571 100644 --- a/codeforlife/models/encrypted.py +++ b/codeforlife/models/encrypted.py @@ -34,6 +34,7 @@ from django.db import models from .base import Model +from .utils import is_real_model_class if t.TYPE_CHECKING: from django_stubs_ext.db.models import TypedModelMeta @@ -55,7 +56,9 @@ class EncryptedModel(Model): ENCRYPTED_FIELDS: t.List["BaseEncryptedField"] def __init__(self, *args, **kwargs): - # Each instance gets its own dict of decrypted values. + # Each instance gets its own dict of pending-encryption and decrypted + # values. + self.__pending_encryption_values__: t.Dict[str, t.Any] = {} self.__decrypted_values__: t.Dict[str, t.Any] = {} super().__init__(*args, **kwargs) @@ -89,7 +92,6 @@ def update(self, **kwargs): return super().update(**kwargs) # Disable bulk operations that would bypass field-level encryption. - aupdate: t.Never = None # type: ignore[assignment] bulk_update: t.Never = None # type: ignore[assignment] abulk_update: t.Never = None # type: ignore[assignment] bulk_create: t.Never = None # type: ignore[assignment] @@ -97,20 +99,22 @@ def update(self, **kwargs): in_bulk: t.Never = None # type: ignore[assignment] ain_bulk: t.Never = None # type: ignore[assignment] + base_manager_class: t.Type[Manager] = Manager objects: Manager["EncryptedModel"] = Manager() # type: ignore[assignment] class Meta(TypedModelMeta): abstract = True @classmethod - def _check_associated_data(cls, **kwargs): + def _check_associated_data(cls): """ Check 'associated_data' values are unique across all EncryptedModel subclasses. """ errors: t.List[checks.Error] = [] - if cls._meta.abstract: + # Skip non-real models. + if not is_real_model_class(cls): return errors # Ensure associated_data is defined. @@ -149,7 +153,7 @@ def _check_associated_data(cls, **kwargs): if ( # pylint: disable-next=too-many-boolean-expressions not model is cls - and not model._meta.abstract + and is_real_model_class(model) and issubclass(model, EncryptedModel) and hasattr(model, "associated_data") and isinstance(model.associated_data, str) @@ -175,16 +179,16 @@ def _check_associated_data(cls, **kwargs): def check(cls, **kwargs): """Run model checks, including custom checks for encrypted models.""" errors = super().check(**kwargs) - errors.extend(cls._check_associated_data(**kwargs)) + errors.extend(cls._check_associated_data()) - if not issubclass(cls.objects.__class__, EncryptedModel.Manager): + if not issubclass(cls.objects.__class__, cls.base_manager_class): errors.append( checks.Error( - "EncryptedModel subclasses must use the" - " EncryptedModel.Manager.", + f"{cls.__name__} must have a manager that is a subclass of" + f" {cls.base_manager_class.__name__}.", hint=( - f"Set 'objects = EncryptedModel.Manager()' on" - f" {cls.__module__}.{cls.__name__}." + f"Set 'objects = {cls.base_manager_class.__name__}()' " + f"on {cls.__module__}.{cls.__name__}." ), obj=cls, id="encrypted.E005", diff --git a/codeforlife/models/encrypted_test.py b/codeforlife/models/encrypted_test.py index 35e346f6..1986d962 100644 --- a/codeforlife/models/encrypted_test.py +++ b/codeforlife/models/encrypted_test.py @@ -37,10 +37,6 @@ def test_objects___update__cannot_update(self): with self.assert_raises_validation_error(code="cannot_update"): Person.objects.update(name="Alice") - def test_objects___aupdate(self): - """Cannot aupdate encrypted field via objects.aupdate().""" - assert Person.objects.aupdate is None - def test_objects___bulk_update(self): """Cannot bulk update encrypted field via objects.bulk_update().""" assert Person.objects.bulk_update is None @@ -75,53 +71,53 @@ def test_check__e001(self): """Check for missing associated_data.""" # pylint: disable-next=abstract-method - class E001(EncryptedModel): + class EncryptedE001(EncryptedModel): class Meta(TypedModelMeta): app_label = "codeforlife.user" - self.assert_check(error_id="encrypted.E001", model_class=E001) + self.assert_check(error_id="encrypted.E001", model_class=EncryptedE001) def test_check__e002(self): """Check for string associated_data.""" # pylint: disable-next=abstract-method - class E002(EncryptedModel): + class EncryptedE002(EncryptedModel): associated_data = 123 # type: ignore[assignment] class Meta(TypedModelMeta): app_label = "codeforlife.user" - self.assert_check(error_id="encrypted.E002", model_class=E002) + self.assert_check(error_id="encrypted.E002", model_class=EncryptedE002) def test_check__e003(self): """Check for non-empty associated_data.""" # pylint: disable-next=abstract-method - class E003(EncryptedModel): + class EncryptedE003(EncryptedModel): associated_data = "" class Meta(TypedModelMeta): app_label = "codeforlife.user" - self.assert_check(error_id="encrypted.E003", model_class=E003) + self.assert_check(error_id="encrypted.E003", model_class=EncryptedE003) def test_check__e004(self): """Check for unique associated_data.""" # pylint: disable-next=abstract-method - class E004(EncryptedModel): + class EncryptedE004(EncryptedModel): associated_data = OtpBypassToken.associated_data class Meta(TypedModelMeta): app_label = "codeforlife.user" - self.assert_check(error_id="encrypted.E004", model_class=E004) + self.assert_check(error_id="encrypted.E004", model_class=EncryptedE004) def test_check__e005(self): """Check manager subclasses EncryptedModel.Manager.""" # pylint: disable-next=abstract-method - class E005(EncryptedModel): + class EncryptedE005(EncryptedModel): associated_data = "example" objects = models.Manager() # type: ignore[assignment] @@ -129,4 +125,4 @@ class E005(EncryptedModel): class Meta(TypedModelMeta): app_label = "codeforlife.user" - self.assert_check(error_id="encrypted.E005", model_class=E005) + self.assert_check(error_id="encrypted.E005", model_class=EncryptedE005) diff --git a/codeforlife/models/fields/__init__.py b/codeforlife/models/fields/__init__.py index 19cfac94..6da52f1b 100644 --- a/codeforlife/models/fields/__init__.py +++ b/codeforlife/models/fields/__init__.py @@ -7,3 +7,4 @@ from .data_encryption_key import DataEncryptionKeyField from .deferred_attribute import DeferredAttribute from .encrypted_text import EncryptedTextField +from .sha256 import Sha256Field diff --git a/codeforlife/models/fields/base_encrypted.py b/codeforlife/models/fields/base_encrypted.py index 621ddf36..673a3fc3 100644 --- a/codeforlife/models/fields/base_encrypted.py +++ b/codeforlife/models/fields/base_encrypted.py @@ -2,188 +2,63 @@ © Ocado Group Created on 19/01/2026 at 09:57:04(+00:00). -This is where the core logic of transparent encryption and decryption happens. -`BaseEncryptedField` is a generic field that stores encrypted data as bytes. The -magic is in its descriptor, `EncryptedAttribute`. - -The descriptor intercepts get and set operations: - -- On `set`: The value is not immediately encrypted. It's wrapped in a - `_PendingEncryption` object and stored in the model instance's `__dict__`. - This is a performance optimization to avoid encrypting a value multiple - times if it's changed repeatedly before saving. Any previously cached - decrypted value is cleared. -- On `get`: The descriptor first checks if a decrypted value is already cached - on the model instance. If so, it returns it immediately. Otherwise, it checks - the value in `__dict__`. If it's ciphertext (bytes), it's decrypted - on-the-fly, cached on the instance, and then returned. If it's a pending - plaintext value, it's returned directly. -- On `save`: The field's `pre_save` method is called. It checks for a - `_PendingEncryption` object. If found, it encrypts the plaintext value using - the `dek_aead` and replaces it with the resulting ciphertext bytes, which are - then written to the database. - -A key challenge is differentiating between a value that has just been loaded -from the database (and is therefore encrypted ciphertext) and a new plaintext -value that a developer is setting. - -- When a developer sets `user.email = "new@example.com"`, this is **new - plaintext** that needs to be encrypted on save. -- When Django loads a `User` from the database, the `email` field contains - **existing ciphertext** (raw bytes). - -We solve this with two wrapper classes: - -1. `_PendingEncryption(value)`: When a developer sets a value on the field, the - `EncryptedAttribute` descriptor wraps it in this class. This marks the data - as "dirty" or "pending encryption." The `pre_save` method looks for this - wrapper to know what needs to be encrypted. -2. `_TrustedCiphertext(value)`: When Django loads data from the database, the - `from_db_value` method on the field is called. We wrap the raw bytes from the - database in this class. The `EncryptedAttribute` descriptor's `__set__` - method sees this wrapper and knows the value is already-encrypted ciphertext, - preventing it from being re-wrapped as `_PendingEncryption`. This avoids - unnecessary re-encryption of data that hasn't changed. - -This distinction allows the field to correctly handle both new data and existing -data without ambiguity. +This module contains the core field-level logic for transparent encryption and +decryption. + +`BaseEncryptedField` stores ciphertext in the model field (bytes/memoryview) +and exposes static convenience methods, `get()` and `set()`, to work with +typed plaintext values. + +- `set(instance, value, field_name)` stores plaintext in + `instance.__pending_encryption_values__`. +- `pre_save()` encrypts pending plaintext and writes ciphertext to the DB. +- `get(instance, field_name)` returns plaintext by reading pending data, + returning a cached decrypted value, or decrypting stored ciphertext. + +The descriptor class remains intentionally minimal: it only clears cached +decrypted values when the underlying binary value changes. It does not perform +type conversion or wrapper-based state transitions. """ import typing as t -from dataclasses import dataclass from django.core.exceptions import ValidationError from django.db.models import BinaryField from ...types import Args, KwArgs from ..encrypted import EncryptedModel +from ..utils import is_real_model_class from .deferred_attribute import DeferredAttribute T = t.TypeVar("T") - - -@dataclass(frozen=True) -class _PendingEncryption(t.Generic[T]): - """A wrapper for plaintext that is pending encryption.""" - - value: T - - -@dataclass(frozen=True) -class _TrustedCiphertext: - """A wrapper for ciphertext that comes directly from the database.""" - - ciphertext: bytes - - -Value: t.TypeAlias = t.Union[_TrustedCiphertext, _PendingEncryption[T]] - +Ciphertext: t.TypeAlias = t.Union[bytes, memoryview] AnyBaseEncryptedField = t.TypeVar( "AnyBaseEncryptedField", bound="BaseEncryptedField" ) class EncryptedAttribute( - DeferredAttribute[AnyBaseEncryptedField, EncryptedModel, Value[T]], - t.Generic[AnyBaseEncryptedField, T], + DeferredAttribute[AnyBaseEncryptedField, EncryptedModel, Ciphertext], + t.Generic[AnyBaseEncryptedField], ): - """ - Descriptor that handles the get/set mechanics for encrypted fields. - """ - - def __get__(self, instance, cls=None): - # Get the internal value from the instance. - internal_value = super().__get__(instance, cls) - - # Return the descriptor itself when accessed on the class. - if internal_value is self: - return self - - # No data to decrypt. - if internal_value is None: - return None - - # The user just set this value, return it directly. - if isinstance(internal_value, _PendingEncryption): - return internal_value.value - - if isinstance(internal_value, _TrustedCiphertext): - # If we have a cached decrypted value, return it. - if self.field.attname in instance.__decrypted_values__: - return t.cast( - T, instance.__decrypted_values__[self.field.attname] - ) - - # Decrypt the value before returning it. - decrypted_value = t.cast( - T, self.field.decrypt_value(instance, internal_value.ciphertext) - ) - - # Cache the decrypted value on the instance. - instance.__decrypted_values__[self.field.attname] = decrypted_value - - return decrypted_value - - raise ValidationError( - "Unexpected internal value type for encrypted field.", - code="invalid_internal_value_type", - ) + """Descriptor that clears cached decrypted values on assignment.""" - def __set__( - self, - instance, - value: t.Optional[ # type: ignore[override] - t.Union[memoryview, _TrustedCiphertext, T] - ], - ): + def __set__(self, instance, value): # Clear any cached decrypted value. instance.__decrypted_values__.pop(self.field.attname, None) - # Determine the internal value to set. - internal_value: t.Optional[Value[T]] - if value is None: - internal_value = None - # When Django loads data from a fixture (e.g., a JSON file), it - # provides binary data as a `memoryview` object. Our descriptor - # handles this by extracting the raw bytes from the `memoryview`. - elif isinstance(value, memoryview): - if not isinstance(value.obj, bytes): - raise ValidationError( - "Expected bytes in memoryview for encrypted field.", - code="invalid_memoryview_type", - ) - internal_value = _TrustedCiphertext(value.obj) - elif isinstance(value, _TrustedCiphertext): # From DB. - internal_value = value - else: # From user input. - internal_value = _PendingEncryption(value) - # Set the internal value on the instance. - super().__set__(instance, internal_value) + super().__set__(instance, value) class BaseEncryptedField(BinaryField, t.Generic[T]): - """Encrypted field base class.""" + """Binary field base class for storing encrypted typed values.""" model: t.Type[EncryptedModel] descriptor_class = EncryptedAttribute - # -------------------------------------------------------------------------- - # Construction & Deconstruction - # -------------------------------------------------------------------------- - - def set_init_kwargs(self, kwargs: KwArgs): - """Sets common init kwargs.""" - kwargs.setdefault("db_column", self.associated_data) - - def __init__( - self, - associated_data: str, - # Set type for default to match T. - default: t.Optional[t.Union[T, t.Callable[[], T]]] = None, - **kwargs, - ): + def __init__(self, associated_data: str, **kwargs): if not associated_data: raise ValidationError( "Associated data cannot be empty.", @@ -191,23 +66,17 @@ def __init__( ) self.associated_data = associated_data - self.set_init_kwargs(kwargs) - super().__init__(**kwargs, default=default) + super().__init__(**kwargs) def deconstruct(self): name, path, args, kwargs = t.cast( t.Tuple[str, str, Args, KwArgs], super().deconstruct() ) - self.set_init_kwargs(kwargs) kwargs["associated_data"] = self.associated_data return name, path, args, kwargs - # -------------------------------------------------------------------------- - # Django Model Field Integration - # -------------------------------------------------------------------------- - def contribute_to_class(self, cls, name, private_only=False): """ Called by Django when the field is added to a model. This method @@ -215,8 +84,8 @@ def contribute_to_class(self, cls, name, private_only=False): """ super().contribute_to_class(cls, name, private_only) - # Skip fake models used for migrations. - if cls.__module__ == "__fake__": + # Skip non-real models. + if not is_real_model_class(cls): return # Ensure the model subclasses EncryptedModel. @@ -246,70 +115,51 @@ def contribute_to_class(self, cls, name, private_only=False): # Register this field as an encrypted field on the model. cls.ENCRYPTED_FIELDS.append(self) - # -------------------------------------------------------------------------- - # Descriptor Methods - # -------------------------------------------------------------------------- - @t.overload # type: ignore[override] - def __get__( # Get the descriptor with the correct types. + def __get__( # Get the descriptor. self, instance: None, owner: t.Any - ) -> EncryptedAttribute[t.Self, T]: ... + ) -> EncryptedAttribute[t.Self]: ... @t.overload - def __get__( # Get the internal value when accessed on an instance. + def __get__( # Get the value. self, instance: EncryptedModel, owner: t.Any - ) -> t.Optional[T]: ... + ) -> t.Optional[Ciphertext]: ... # Actual implementation of __get__. def __get__(self, instance: t.Optional[EncryptedModel], owner: t.Any): return t.cast( - t.Union[EncryptedAttribute[t.Self, T], t.Optional[T]], + t.Union[EncryptedAttribute[t.Self], t.Optional[Ciphertext]], # pylint: disable-next=no-member super().__get__(instance, owner), ) - # Set the internal value when assigned on an instance. - def __set__(self, instance: EncryptedModel, value: t.Optional[T]): ... - - # pylint: disable-next=unused-argument - def from_db_value(self, value: t.Optional[bytes], expression, connection): - """ - Converts a value as returned by the database to a Python object. - We wrap the raw bytes in _TrustedCiphertext to signal that this is - existing ciphertext from the database, not new plaintext. - """ - if value is None: - return None + def __set__( # Set the value. + self, instance: EncryptedModel, value: t.Optional[Ciphertext] + ): ... - # Wrap it so __set__ knows this is NOT new user input. - return _TrustedCiphertext(value) + def value_from_object(self, obj): + return t.cast(t.Optional[Ciphertext], super().value_from_object(obj)) def pre_save( self, model_instance: EncryptedModel, add # type: ignore[override] ): - """Before saving, encrypt any pending values.""" - value: t.Optional[Value[T]] = model_instance.__dict__.get(self.attname) - - # No data to encrypt. - if value is None: - return None + """Encrypt pending plaintext values before writing to the database.""" # Data needs encrypting. - if isinstance(value, _PendingEncryption): - return self.encrypt_value(model_instance, value.value) + if self.attname in model_instance.__pending_encryption_values__: + value = t.cast( + T, + model_instance.__pending_encryption_values__.pop(self.attname), + ) + encrypted_value = self._encrypt(model_instance, value) - # Already encrypted data from DB, store as-is. - if isinstance(value, _TrustedCiphertext): - return value.ciphertext + model_instance.__decrypted_values__[self.attname] = value + model_instance.__dict__[self.attname] = encrypted_value - raise ValidationError( - f"Unexpected value type '{type(value)}' for encryption.", - code="invalid_value_type", - ) + return encrypted_value - # -------------------------------------------------------------------------- - # Crypto Logic - # -------------------------------------------------------------------------- + # If data is already encrypted or None, return it as-is. + return super().pre_save(model_instance, add) def bytes_to_value(self, data: bytes) -> T: """ @@ -330,13 +180,8 @@ def full_associated_data(self): """Returns the fully qualified associated data for this field.""" return f"{self.model.associated_data}:{self.associated_data}".encode() - def decrypt_value( - self, instance: EncryptedModel, ciphertext: t.Optional[bytes] - ): + def _decrypt(self, instance: EncryptedModel, ciphertext: bytes): """Decrypts a single value using the DEK and associated data.""" - if ciphertext is None: - return None - data = instance.dek_aead.decrypt( ciphertext=ciphertext, associated_data=self.full_associated_data, @@ -344,13 +189,77 @@ def decrypt_value( return self.bytes_to_value(data) - def encrypt_value(self, instance: EncryptedModel, plaintext: t.Optional[T]): + def _encrypt(self, instance: EncryptedModel, plaintext: T): """Encrypts a single value using the DEK and associated data.""" - return ( - None - if plaintext is None - else instance.dek_aead.encrypt( - plaintext=self.value_to_bytes(plaintext), - associated_data=self.full_associated_data, + return instance.dek_aead.encrypt( + plaintext=self.value_to_bytes(plaintext), + associated_data=self.full_associated_data, + ) + + @staticmethod + def get(instance: EncryptedModel, field_name: str): + """Get a typed plaintext value for an encrypted field. + + Args: + instance: The model instance from which to decrypt the value. + field_name: The name of the encrypted field to decrypt. + + Returns: + The plaintext value, or None if the field is empty. + + Notes: + Internal model storage remains ciphertext bytes. This helper applies + the conversion/decryption path and cache handling. + """ + field = t.cast( + BaseEncryptedField[T], instance._meta.get_field(field_name) + ) + + # If we have a cached pending encryption value, return it. + if field.attname in instance.__pending_encryption_values__: + return t.cast( + T, instance.__pending_encryption_values__[field.attname] ) + + # If we have a cached decrypted value, return it. + if field.attname in instance.__decrypted_values__: + return t.cast(T, instance.__decrypted_values__[field.attname]) + + # Get the internal value from the instance's __dict__. + value = field.value_from_object(instance) + if value is None: + return None + + # Decrypt the value before returning it. + # pylint: disable-next=protected-access + decrypted_value = field._decrypt(instance, bytes(value)) + + # Cache the decrypted value on the instance. + instance.__decrypted_values__[field.attname] = decrypted_value + + return decrypted_value + + @staticmethod + def set(instance: EncryptedModel, value: t.Optional[T], field_name: str): + """Set a typed plaintext value for an encrypted field. + + The plaintext is staged in pending-encryption storage and encrypted at + save time by `pre_save`. + + Args: + instance: The model instance on which to set the value. + value: The plaintext value to set. If None, the field is cleared. + field_name: The name of the encrypted field to set. + """ + field = t.cast( + BaseEncryptedField[T], instance._meta.get_field(field_name) ) + + # Set the pending encryption value. + if value is None: + instance.__pending_encryption_values__.pop(field.attname, None) + else: + instance.__pending_encryption_values__[field.attname] = value + + # In all cases we need to clear the internal and cached-decrypted value. + setattr(instance, field_name, None) diff --git a/codeforlife/models/fields/base_encrypted_test.py b/codeforlife/models/fields/base_encrypted_test.py index 4c206d1e..ac81e02c 100644 --- a/codeforlife/models/fields/base_encrypted_test.py +++ b/codeforlife/models/fields/base_encrypted_test.py @@ -9,14 +9,10 @@ from django.db import models -from ...encryption import FakeAead +from ...encryption import FakeAead, create_dek, get_dek_aead from ...tests import InterruptPipelineError, TestCase from ..encrypted import EncryptedModel -from .base_encrypted import ( - BaseEncryptedField, - _PendingEncryption, - _TrustedCiphertext, -) +from .base_encrypted import BaseEncryptedField if t.TYPE_CHECKING: from django_stubs_ext.db.models import TypedModelMeta @@ -26,6 +22,7 @@ # pylint: disable=missing-class-docstring # pylint: disable=too-few-public-methods # pylint: disable=too-many-instance-attributes +# pylint: disable=protected-access class FakeEncryptedModel(EncryptedModel): @@ -38,27 +35,53 @@ class Meta(TypedModelMeta): @cached_property def dek_aead(self): - return FakeAead.as_mock() + return t.cast(FakeAead, get_dek_aead(create_dek())).as_mock() - def get_stored_value(self, field: BaseEncryptedField): - """Gets the stored value for the given field.""" + def get_pending_encryption_value(self, field: BaseEncryptedField) -> str: + """Gets the pending encryption value for the given field.""" assert field in self.ENCRYPTED_FIELDS - return self.__dict__[field.attname] + return self.__pending_encryption_values__[field.attname] - def set_stored_value(self, field: BaseEncryptedField, value): - """Sets the stored value for the given field.""" + def set_pending_encryption_value( + self, field: BaseEncryptedField, value: str + ): + """Sets the pending encryption value for the given field.""" assert field in self.ENCRYPTED_FIELDS - self.__dict__[field.attname] = value + self.__pending_encryption_values__[field.attname] = value - def assert_value_is_pending_encryption( + def assert_pending_encryption_value_is_cached( self, field: BaseEncryptedField, value: str ): - """ - Asserts the value for the given field is pending encryption. - """ - pending_encryption = self.get_stored_value(field) - assert isinstance(pending_encryption, _PendingEncryption) - assert pending_encryption.value == value + """Asserts the value for the given field is cached.""" + assert field.attname in self.__pending_encryption_values__ + assert self.__pending_encryption_values__[field.attname] == value + + def assert_pending_encryption_value_is_not_cached( + self, field: BaseEncryptedField + ): + """Asserts the value for the given field is not cached.""" + assert field.attname not in self.__pending_encryption_values__ + + def get_decrypted_value(self, field: BaseEncryptedField) -> str: + """Gets the decrypted value for the given field.""" + assert field in self.ENCRYPTED_FIELDS + return self.__decrypted_values__[field.attname] + + def set_decrypted_value(self, field: BaseEncryptedField, value: str): + """Sets the decrypted value for the given field.""" + assert field in self.ENCRYPTED_FIELDS + self.__decrypted_values__[field.attname] = value + + def assert_decrypted_value_is_cached( + self, field: BaseEncryptedField, value: str + ): + """Asserts the value for the given field is cached.""" + assert field.attname in self.__decrypted_values__ + assert self.__decrypted_values__[field.attname] == value + + def assert_decrypted_value_is_not_cached(self, field: BaseEncryptedField): + """Asserts the value for the given field is not cached.""" + assert field.attname not in self.__decrypted_values__ class FakeModelMeta(TypedModelMeta): @@ -73,8 +96,8 @@ class FakeEncryptedField(BaseEncryptedField[str]): value_to_bytes: MagicMock bytes_to_value: MagicMock - encrypt_value: MagicMock - decrypt_value: MagicMock + _encrypt: MagicMock + _decrypt: MagicMock @staticmethod def _value_to_bytes(value: str): @@ -84,13 +107,13 @@ def _value_to_bytes(value: str): def _bytes_to_value(data: bytes): return data.decode() - def __init__(self, associated_data, default=None, **kwargs): - super().__init__(associated_data, default, **kwargs) + def __init__(self, associated_data, **kwargs): + super().__init__(associated_data, **kwargs) self.value_to_bytes = MagicMock(side_effect=self._value_to_bytes) self.bytes_to_value = MagicMock(side_effect=self._bytes_to_value) - self.encrypt_value = MagicMock(side_effect=super().encrypt_value) - self.decrypt_value = MagicMock(side_effect=super().decrypt_value) + self._encrypt = MagicMock(side_effect=super()._encrypt) + self._decrypt = MagicMock(side_effect=super()._decrypt) # pylint: disable-next=too-many-public-methods @@ -122,18 +145,14 @@ def _get_model_instance(self, **kwargs): def setUp(self): # Set up the first field with a non-callable default for testing. self.field_associated_data = "field" - self.field_default = "default" self.field = FakeEncryptedField( - associated_data=self.field_associated_data, - default=self.field_default, + associated_data=self.field_associated_data ) # Set up a second field with a callable default for testing. self.field2_associated_data = "field2" - self.field2_default = "default2" self.field2 = FakeEncryptedField( - associated_data=self.field2_associated_data, - default=lambda: self.field2_default, + associated_data=self.field2_associated_data ) # -------------------------------------------------------------------------- @@ -148,14 +167,12 @@ def test_init__no_associated_data(self): def test_init(self): """BaseEncryptedField is constructed correctly.""" assert self.field.associated_data == self.field_associated_data - assert self.field.db_column == self.field_associated_data def test_deconstruct(self): """BaseEncryptedField is deconstructed correctly.""" _, _, _, kwargs = self.field.deconstruct() assert kwargs["associated_data"] == self.field_associated_data - assert kwargs["db_column"] == self.field_associated_data # -------------------------------------------------------------------------- # Django Model Field Integration Tests @@ -238,15 +255,11 @@ def test_decrypt_value(self): decrypt_mock: MagicMock = instance.dek_aead.decrypt bytes_to_value_mock: MagicMock = self.field.bytes_to_value - # When ciphertext is None, no decryption occurs. - decrypted_value = self.field.decrypt_value(instance, ciphertext=None) - assert decrypted_value is None - decrypt_mock.assert_not_called() - bytes_to_value_mock.assert_not_called() - # When ciphertext is provided, decryption occurs. - ciphertext = FakeAead.encrypt(b"value") - decrypted_value = self.field.decrypt_value(instance, ciphertext) + ciphertext = instance.dek_aead.encrypt( + b"value", associated_data=self.field.full_associated_data + ) + decrypted_value = self.field._decrypt(instance, ciphertext) decrypt_kwargs = { "ciphertext": ciphertext, "associated_data": self.field.full_associated_data, @@ -258,22 +271,16 @@ def test_decrypt_value(self): decrypted_bytes ) - def test_encrypt_value(self): - """encrypt_value encrypts the given plaintext.""" + def test__encrypt(self): + """_encrypt encrypts the given plaintext.""" # Create instance and mock shorthands. instance = self._get_model_instance() encrypt_mock: MagicMock = instance.dek_aead.encrypt value_to_bytes_mock: MagicMock = self.field.value_to_bytes - # When plaintext is None, no encryption occurs. - encrypted_bytes = self.field.encrypt_value(instance, plaintext=None) - assert encrypted_bytes is None - value_to_bytes_mock.assert_not_called() - encrypt_mock.assert_not_called() - # When plaintext is provided, encryption occurs. - plaintext = self.field_default - encrypted_bytes = self.field.encrypt_value(instance, plaintext) + plaintext = "some value" + encrypted_bytes = self.field._encrypt(instance, plaintext) value_to_bytes_mock.assert_called_once_with(plaintext) decrypted_bytes = value_to_bytes_mock.side_effect(plaintext) encrypt_kwargs = { @@ -281,149 +288,88 @@ def test_encrypt_value(self): "associated_data": self.field.full_associated_data, } encrypt_mock.assert_called_once_with(**encrypt_kwargs) - assert encrypted_bytes == encrypt_mock.side_effect(**encrypt_kwargs) + assert self.field._decrypt(instance, encrypted_bytes) == plaintext - # -------------------------------------------------------------------------- - # Descriptor Methods Tests - # -------------------------------------------------------------------------- - - def test_set__default(self): - """Setting field to default value stores pending encryption.""" - # Field must have a non-callable default for this test. - assert self.field.default is not None and not callable( - self.field.default - ) - # Field2 must have a callable default for this test. - assert self.field2.default is not None and callable(self.field2.default) - - instance = self._get_model_instance() - instance.assert_value_is_pending_encryption( - self.field, self.field_default - ) - instance.assert_value_is_pending_encryption( - self.field2, self.field2_default - ) - - def test_set__init(self): - """Setting field to initial value stores pending encryption.""" - value = "initial_value" - instance = self._get_model_instance(field=value) - instance.assert_value_is_pending_encryption(self.field, value) + def test_get__descriptor(self): + """Getting field from class returns the descriptor.""" + Model = self._get_model_class() + assert isinstance(Model.field, BaseEncryptedField.descriptor_class) + assert Model.field.field == self.field def test_set__none(self): """Setting field to None stores None.""" - assert self.field.default is not None - instance = self._get_model_instance(field=None) - assert instance.get_stored_value(self.field) is None - - def test_set__trusted_ciphertext(self): - """Setting field to _TrustedCiphertext stores ciphertext directly.""" - trusted_ciphertext = _TrustedCiphertext(b"encrypted_value") - instance = self._get_model_instance(field=trusted_ciphertext) - assert instance.get_stored_value(self.field) is trusted_ciphertext - - def test_set__memoryview(self): - """Setting field to memoryview stores bytes directly.""" - memoryview_value = memoryview(b"byte_value") - instance = self._get_model_instance(field=memoryview_value) - trusted_ciphertext = instance.get_stored_value(self.field) - assert isinstance(trusted_ciphertext, _TrustedCiphertext) - assert trusted_ciphertext.ciphertext == memoryview_value.obj - - def test_set__memoryview__invalid_memoryview_type(self): - """Setting field to invalid memoryview type raises ValidationError.""" - value = bytearray(b"Hello") - memoryview_value = memoryview(value) - with self.assert_raises_validation_error( - code="invalid_memoryview_type" - ): - self._get_model_instance(field=memoryview_value) - - def test_set__new_value(self): - """ - Setting field to new value stores pending encryption and clears cache. - """ - assert self.field.default is not None + instance = self._get_model_instance() + assert instance.field == b"" + instance.set_decrypted_value(self.field, "decrypted value") - value = "new_value" - assert self.field.default != value + FakeEncryptedField.set(instance, None, "field") + assert instance.field is None + instance.assert_decrypted_value_is_not_cached(self.field) + instance.assert_pending_encryption_value_is_not_cached(self.field) + def test_set__value(self): + """Setting field to a valid value stores it as pending encryption.""" instance = self._get_model_instance() + assert instance.field == b"" + instance.set_decrypted_value(self.field, "decrypted value") - # Cache the value on the instance. - instance.__decrypted_values__[self.field.attname] = value - - # Clear cache by setting to new value. - instance.field = value - instance.assert_value_is_pending_encryption(self.field, value) + value = "value" + FakeEncryptedField.set(instance, value, "field") + assert instance.field is None + instance.assert_decrypted_value_is_not_cached(self.field) + instance.assert_pending_encryption_value_is_cached(self.field, value) - # Ensure cached value is cleared. - assert self.field.attname not in instance.__decrypted_values__ + self.field._decrypt.assert_not_called() - def test_get__invalid_internal_value_type(self): + def test_get__pending_encryption(self): """ - Getting field with invalid internal value type raises ValidationError. + Getting a field that's pending encryption returns the pending value. """ + value = "pending value" instance = self._get_model_instance() - instance.set_stored_value(self.field, b"data") # Invalid type. + instance.set_pending_encryption_value(self.field, value) - with self.assert_raises_validation_error( - code="invalid_internal_value_type" - ): - _ = instance.field + assert FakeEncryptedField.get(instance, "field") == value - def test_get__descriptor(self): - """Getting field from class returns the descriptor.""" - Model = self._get_model_class() - assert isinstance(Model.field, BaseEncryptedField.descriptor_class) - assert Model.field.field == self.field - - def test_get__cached(self): - """Getting field when cached returns cached value.""" + def test_get__decrypted(self): + """ + Getting a field that's already decrypted returns the decrypted value. + """ + value = "decrypted value" instance = self._get_model_instance() - instance.set_stored_value(self.field, _TrustedCiphertext(b"irrelevant")) + instance.set_decrypted_value(self.field, value) - value = "decrypted_value" - instance.__decrypted_values__[self.field.attname] = value - assert instance.field == value + assert FakeEncryptedField.get(instance, "field") == value + + self.field._decrypt.assert_not_called() def test_get__none(self): - """Getting field when stored value is None returns None.""" - instance = self._get_model_instance() - instance.set_stored_value(self.field, None) + """Getting a field when stored value is None returns None.""" + instance = self._get_model_instance(field=None) + assert FakeEncryptedField.get(instance, "field") is None - assert instance.field is None - self.field.decrypt_value.assert_not_called() + self.field._decrypt.assert_not_called() - def test_get__pending_encryption(self): + def test_get__encrypted(self): """ - Getting field when stored value is pending encryption returns value. + Getting a field with stored ciphertext decrypts, caches and returns + the value. """ - instance = self._get_model_instance() - value = "decrypted_value" - pending_encryption = _PendingEncryption(value) - instance.set_stored_value(self.field, pending_encryption) - - assert instance.field == value - self.field.decrypt_value.assert_not_called() - - def test_get__decrypted_value(self): - """Getting field when stored value is ciphertext returns decrypted.""" plaintext = "decrypted_value" - ciphertext = FakeAead.encrypt(plaintext.encode()) # Create instance with stored ciphertext. instance = self._get_model_instance() - instance.set_stored_value(self.field, _TrustedCiphertext(ciphertext)) - # Ensure cache is not set initially. - assert self.field.attname not in instance.__decrypted_values__ + ciphertext = instance.dek_aead.encrypt( + plaintext.encode(), self.field.full_associated_data + ) + instance.field = ciphertext # Get the field value, which should decrypt the ciphertext. - assert instance.field == plaintext - self.field.decrypt_value.assert_called_once_with(instance, ciphertext) + instance.assert_decrypted_value_is_not_cached(self.field) + assert FakeEncryptedField.get(instance, "field") == plaintext + instance.assert_decrypted_value_is_cached(self.field, plaintext) - # Ensure decrypted value is cached on the instance. - assert instance.__decrypted_values__[self.field.attname] == plaintext + self.field._decrypt.assert_called_once_with(instance, ciphertext) # -------------------------------------------------------------------------- # pre_save Tests @@ -432,18 +378,15 @@ def test_get__decrypted_value(self): def test_pre_save__pending_encryption(self): """pre_save encrypts pending encryption before saving.""" # Create instance with pending encryption. + value = "pending value" instance = self._get_model_instance() - pending_encryption = instance.get_stored_value(self.field) - assert isinstance(pending_encryption, _PendingEncryption) + instance.set_pending_encryption_value(self.field, value) # Assert the value is encrypted in pre_save. def assert_pre_save(result): - self.field.encrypt_value.assert_called_once_with( - instance, pending_encryption.value - ) - assert result == self.field.encrypt_value.side_effect( - instance, pending_encryption.value - ) + self.field._encrypt.assert_called_once_with(instance, value) + assert result != value + assert self.field._decrypt.side_effect(instance, result) == value # Run the save pipeline, interrupting at pre_save. InterruptPipelineError.run( @@ -457,13 +400,12 @@ def assert_pre_save(result): def test_pre_save__none(self): """pre_save with no value does nothing.""" # Create instance with no stored value. - instance = self._get_model_instance() - instance.set_stored_value(self.field, None) + instance = self._get_model_instance(field=None) # Assert pre_save does nothing. def assert_pre_save(result): assert result is None - self.field.encrypt_value.assert_not_called() + self.field._encrypt.assert_not_called() # Run the save pipeline, interrupting at pre_save. InterruptPipelineError.run( @@ -474,17 +416,16 @@ def assert_pre_save(result): pipeline=instance.save, ) - def test_pre_save__trusted_ciphertext(self): - """pre_save with trusted ciphertext does nothing.""" - # Create instance with trusted ciphertext. + def test_pre_save__ciphertext(self): + """pre_save with ciphertext does nothing.""" + # Create instance with ciphertext. ciphertext = b"encrypted_value" - trusted_ciphertext = _TrustedCiphertext(ciphertext) - instance = self._get_model_instance(field=trusted_ciphertext) + instance = self._get_model_instance(field=ciphertext) # Assert pre_save returns the ciphertext directly. def assert_pre_save(result): assert result == ciphertext - self.field.encrypt_value.assert_not_called() + self.field._encrypt.assert_not_called() # Run the save pipeline, interrupting at pre_save. InterruptPipelineError.run( @@ -494,13 +435,3 @@ def assert_pre_save(result): assert_step=assert_pre_save, pipeline=instance.save, ) - - def test_pre_save__invalid_value_type(self): - """pre_save with invalid value type raises ValidationError.""" - # Create instance with invalid stored value. - instance = self._get_model_instance() - instance.set_stored_value(self.field, b"data") # Invalid type. - - # Run the save pipeline, interrupting at pre_save. - with self.assert_raises_validation_error(code="invalid_value_type"): - instance.save() diff --git a/codeforlife/models/fields/data_encryption_key.py b/codeforlife/models/fields/data_encryption_key.py index f4e71610..fd3087ab 100644 --- a/codeforlife/models/fields/data_encryption_key.py +++ b/codeforlife/models/fields/data_encryption_key.py @@ -19,31 +19,28 @@ """ import typing as t -from dataclasses import dataclass from django.core.exceptions import ValidationError from django.db.models import BinaryField from django.utils.translation import gettext_lazy as _ -from ...types import KwArgs from ..base_data_encryption_key import BaseDataEncryptionKeyModel +from ..utils import is_real_model_class from .deferred_attribute import DeferredAttribute +if t.TYPE_CHECKING: # pragma: no cover + from django_stubs_ext import StrOrPromise + + AnyDataEncryptionKeyField = t.TypeVar( "AnyDataEncryptionKeyField", bound="DataEncryptionKeyField" ) - - -@dataclass(frozen=True) -class _TrustedDek: - """A wrapper for a DEK that comes directly from the database.""" - - dek: bytes +Dek: t.TypeAlias = t.Union[bytes, memoryview] class DataEncryptionKeyAttribute( DeferredAttribute[ - AnyDataEncryptionKeyField, BaseDataEncryptionKeyModel, bytes + AnyDataEncryptionKeyField, BaseDataEncryptionKeyModel, Dek ], t.Generic[AnyDataEncryptionKeyField], ): @@ -51,26 +48,13 @@ class DataEncryptionKeyAttribute( Descriptor for DataEncryptionKeyField that handles data shredding. """ - def __set__( - self, - instance, - value: t.Optional[_TrustedDek], # type: ignore[override] - ): + def __set__(self, instance, value): # Clear any cached DEK AEAD. if instance.pk is not None and instance.pk in instance.DEK_AEAD_CACHE: instance.DEK_AEAD_CACHE.pop(instance.pk, None) - if isinstance(value, _TrustedDek): # From DB. - internal_value = value.dek - elif value is None: # Data is being shredded. - internal_value = None - else: - raise ValidationError( - "DataEncryptionKeyField can only be set to None.", - code="cannot_set_value", - ) - - super().__set__(instance, internal_value) + # Set the internal value on the instance. + super().__set__(instance, value) class DataEncryptionKeyField(BinaryField): @@ -91,15 +75,17 @@ class DataEncryptionKeyField(BinaryField): "The encrypted data encryption key (DEK) for this model." ) - def set_init_kwargs(self, kwargs: KwArgs): - """Sets common init kwargs.""" - kwargs["editable"] = False # DEK should not be editable in admin forms - kwargs["null"] = True # Allow null for data shredding - kwargs.setdefault("verbose_name", _(self.default_verbose_name)) - kwargs.setdefault("help_text", _(self.default_help_text)) - - def __init__(self, **kwargs): - if kwargs.get("editable", False): + def __init__( + self, + # DEK should not be editable in admin forms. + editable: t.Literal[False] = False, + # Allow null for data shredding. + null: t.Literal[True] = True, + verbose_name: t.Optional["StrOrPromise"] = _(default_verbose_name), + help_text: "StrOrPromise" = _(default_help_text), + **kwargs, + ): + if editable: raise ValidationError( "DataEncryptionKeyField cannot be editable.", code="editable_not_allowed", @@ -109,20 +95,20 @@ def __init__(self, **kwargs): "DataEncryptionKeyField cannot have a default value.", code="default_not_allowed", ) - if not kwargs.get("null", True): + if not null: raise ValidationError( "DataEncryptionKeyField must allow null to support data" " shredding.", code="null_not_allowed", ) - self.set_init_kwargs(kwargs) - super().__init__(**kwargs) - - def deconstruct(self): - name, path, args, kwargs = super().deconstruct() - self.set_init_kwargs(kwargs) - return name, path, args, kwargs + super().__init__( + **kwargs, + editable=editable, + null=null, + verbose_name=verbose_name, + help_text=help_text, + ) # -------------------------------------------------------------------------- # Django Model Field Integration @@ -131,8 +117,8 @@ def deconstruct(self): def contribute_to_class(self, cls, name, private_only=False): super().contribute_to_class(cls, name, private_only) - # Skip fake models used for migrations. - if cls.__module__ == "__fake__": + # Skip non-real models. + if not is_real_model_class(cls): return # Ensure the model subclasses BaseDataEncryptionKeyModel. @@ -145,7 +131,7 @@ def contribute_to_class(self, cls, name, private_only=False): ) # Ensure only one DEK field per model. - if cls.DEK_FIELD is not None: + if hasattr(cls, "DEK_FIELD"): raise ValidationError( f"'{cls.__module__}.{cls.__name__}' already has a" " DataEncryptionKeyField defined.", @@ -153,7 +139,7 @@ def contribute_to_class(self, cls, name, private_only=False): ) # Set the class DEK field reference. - cls.DEK_FIELD = getattr(cls, self.name) + cls.DEK_FIELD = self.attname # -------------------------------------------------------------------------- # Descriptor Methods @@ -180,16 +166,3 @@ def __get__( # Actual implementation of __get__. # Can only be set to None to allow data shredding. def __set__(self, instance: BaseDataEncryptionKeyModel, value: None): ... - - # pylint: disable-next=unused-argument - def from_db_value(self, value: t.Optional[bytes], expression, connection): - """ - Converts a value as returned by the database to a Python object. - We wrap the raw bytes in _TrustedDek to signal that this is an - existing DEK from the database, not new plaintext. - """ - if value is None: - return None - - # Wrap it so __set__ knows this is NOT new user input. - return _TrustedDek(value) diff --git a/codeforlife/models/fields/data_encryption_key_test.py b/codeforlife/models/fields/data_encryption_key_test.py index 42d8c3b8..be4d1ddb 100644 --- a/codeforlife/models/fields/data_encryption_key_test.py +++ b/codeforlife/models/fields/data_encryption_key_test.py @@ -10,7 +10,7 @@ from ...encryption import create_dek from ...tests import TestCase from ..base_data_encryption_key import BaseDataEncryptionKeyModel -from .data_encryption_key import DataEncryptionKeyField, _TrustedDek +from .data_encryption_key import DataEncryptionKeyField if t.TYPE_CHECKING: from django_stubs_ext.db.models import TypedModelMeta @@ -67,7 +67,7 @@ def setUp(self): def test_init__editable_not_allowed(self): """Cannot create DataEncryptionKeyField with editable=True.""" with self.assert_raises_validation_error(code="editable_not_allowed"): - DataEncryptionKeyField(editable=True) + DataEncryptionKeyField(editable=True) # type: ignore[arg-type] def test_init__default_not_allowed(self): """Cannot create DataEncryptionKeyField with default value.""" @@ -77,7 +77,7 @@ def test_init__default_not_allowed(self): def test_init__null_allowed(self): """Cannot create DataEncryptionKeyField with null=True.""" with self.assert_raises_validation_error(code="null_not_allowed"): - DataEncryptionKeyField(null=False) + DataEncryptionKeyField(null=False) # type: ignore[arg-type] def test_init(self): """DataEncryptionKeyField is constructed correctly.""" @@ -93,7 +93,6 @@ def test_deconstruct(self): """DataEncryptionKeyField is deconstructed correctly.""" _, _, _, kwargs = self.field.deconstruct() - assert kwargs["editable"] is False assert kwargs["null"] is True assert ( kwargs["verbose_name"] @@ -136,11 +135,11 @@ def test_contribute_to_class(self): """DataEncryptionKeyField is contributed to model correctly.""" with self.subTest("Class attribute set correctly"): Model = self._get_model_class() - assert Model.DEK_FIELD == Model.dek + assert Model.DEK_FIELD == Model.dek.field.attname with self.subTest("Instance attribute set correctly"): instance = Model() - assert instance.DEK_FIELD == instance.dek + assert instance.DEK_FIELD == Model.dek.field.attname # -------------------------------------------------------------------------- # Descriptor Methods Tests @@ -152,6 +151,11 @@ def test_get__descriptor(self): assert isinstance(Model.dek, DataEncryptionKeyField.descriptor_class) assert Model.dek.field == self.field + def test_get__none(self): + """Getting field from instance returns None if DEK is not set.""" + instance = self._get_model_instance() + assert instance.dek is None + def test_get__value(self): """Getting field from instance returns the DEK bytes.""" instance = self._get_model_instance() @@ -160,13 +164,6 @@ def test_get__value(self): assert isinstance(dek_value, bytes) assert dek_value == instance.__dict__["dek"] - def test_set__default(self): - """Setting field to _TrustedDek sets to DEK bytes.""" - instance = self._get_model_instance() - trusted_dek = _TrustedDek(b"dek") - instance.dek = trusted_dek - assert trusted_dek.dek == instance.__dict__["dek"] - def test_set__none(self): """Setting field to None sets to None.""" instance = self._get_model_instance() @@ -174,8 +171,9 @@ def test_set__none(self): instance.dek = None assert instance.__dict__["dek"] is None - def test_set__cannot_set_value(self): - """Setting field to any value other than None or _TrustedDek raises.""" + def test_set__bytes(self): + """Setting field to bytes with valid prefix sets to DEK bytes.""" instance = self._get_model_instance() - with self.assert_raises_validation_error(code="cannot_set_value"): - instance.dek = b"some_value" + dek = b"some_value" + instance.dek = dek + assert instance.__dict__["dek"] == dek diff --git a/codeforlife/models/fields/deferred_attribute.py b/codeforlife/models/fields/deferred_attribute.py index f4d13286..3acf32c8 100644 --- a/codeforlife/models/fields/deferred_attribute.py +++ b/codeforlife/models/fields/deferred_attribute.py @@ -34,7 +34,7 @@ def __get__( self, instance: t.Optional[AnyModel], cls=None # type: ignore[override] ): return t.cast( - t.Optional[T], + t.Union[t.Self, t.Optional[T]], super().__get__(instance, cls), # type: ignore[misc] ) diff --git a/codeforlife/models/fields/sha256.py b/codeforlife/models/fields/sha256.py new file mode 100644 index 00000000..ddb2df7c --- /dev/null +++ b/codeforlife/models/fields/sha256.py @@ -0,0 +1,135 @@ +""" +© Ocado Group +Created on 16/03/2026 at 17:35:19(+00:00). + +Deterministic one-way hashing helpers for queryable sensitive values. + +`Sha256Field` stores an HMAC-SHA256 digest (hex string) derived from a +plaintext input and the Django `SECRET_KEY`. This makes equality matching +possible without storing plaintext values. + +Custom lookups are registered to keep querying ergonomic: + +- `__sha256`: hashes a single right-hand side plaintext value. +- `__sha256_in`: hashes each plaintext value in an iterable. + +This pattern is typically paired with encrypted fields when data must remain +encrypted at rest but still needs deterministic lookup support. +""" + +import hmac +import typing as t +from hashlib import sha256 + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db.models import CharField, Model, lookups + + +class Sha256Field(CharField): + """A CharField for deterministic, one-way hashed values.""" + + def __init__( + self, + editable: t.Literal[False] = False, + max_length: t.Literal[64] = 64, # Length of SHA-256 hash in hexadecimal + **kwargs, + ): + if editable: + raise ValidationError( + f"{self.__class__.__name__} cannot be editable.", + code="editable_not_allowed", + ) + if max_length != 64: + raise ValidationError( + f"{self.__class__.__name__} must have max_length of 64 to " + "store a SHA-256 hash in hexadecimal.", + code="max_length_not_64", + ) + + super().__init__(editable=editable, max_length=max_length, **kwargs) + + @staticmethod + def hash(value: str): + """Create a consistent, salted hash of a value. + + Args: + value: The value to hash. + + Returns: + A hash of the value salted with the Django secret key. + """ + return hmac.new( + key=settings.SECRET_KEY.encode("utf-8"), + msg=value.encode("utf-8"), + digestmod=sha256, + ).hexdigest() + + @classmethod + def set(cls, instance: Model, value: t.Optional[str], field_name: str): + """ + Hash and assign a plaintext value to a Sha256Field. + + Args: + instance: The model instance on which to set the value. + value: The plaintext value to hash and set. + field_name: The name of the Sha256Field on the model. + """ + if value is not None: + value = cls.hash(value) + + setattr(instance, field_name, value) + + +# pylint: disable-next=abstract-method +class Sha256ExactLookup(lookups.Exact): + """ + A lookup that hashes a plaintext right-hand side value before comparing. + + This allows querying a hashed field with a plain text value, e.g.: + `User.objects.filter(_email_hash__sha256="user@example.com")` + """ + + rhs: t.Optional[str] + + lookup_name = "sha256" + + def process_rhs(self, compiler, connection): + sql, params = super().process_rhs(compiler, connection) + + return sql, params if self.rhs is None else [Sha256Field.hash(self.rhs)] + + def get_rhs_op(self, connection, rhs): + """ + Get the operator for the right-hand side of the expression. + + We force it to use the '=' operator from the 'exact' lookup. + """ + return connection.operators["exact"] % rhs + + +# pylint: disable-next=abstract-method,too-many-ancestors +class Sha256InLookup(lookups.In): + """ + A lookup that hashes plaintext right-hand side values before comparing. + + This allows querying a hashed field with plain text values, e.g.: + `User.objects.filter(_email_hash__sha256_in=["user@example.com"])` + """ + + rhs: t.Optional[t.Iterable[str]] + + lookup_name = f"{Sha256ExactLookup.lookup_name}_in" + + def process_rhs(self, compiler, connection): + sql, params = super().process_rhs(compiler, connection) + + return sql, ( + params + if self.rhs is None + else [Sha256Field.hash(value) for value in self.rhs] + ) + + +Sha256Field.register_lookup(Sha256ExactLookup) +Sha256Field.register_lookup(Sha256InLookup) diff --git a/codeforlife/models/fields/sha256_test.py b/codeforlife/models/fields/sha256_test.py new file mode 100644 index 00000000..60dff338 --- /dev/null +++ b/codeforlife/models/fields/sha256_test.py @@ -0,0 +1,58 @@ +""" +© Ocado Group +Created on 16/03/2026 at 15:01:24(+00:00). +""" + +from ...tests import TestCase +from ...user.models import User +from .sha256 import Sha256Field + + +# pylint: disable-next=missing-class-docstring +class Sha256FieldTests(TestCase): + fixtures = ["school_1"] + + def test_init__editable_not_allowed(self): + """Cannot create Sha256Field with editable=True.""" + with self.assert_raises_validation_error(code="editable_not_allowed"): + Sha256Field(editable=True) # type: ignore[arg-type] + + def test_init__max_length_not_64(self): + """Cannot create Sha256Field with max_length not equal to 64.""" + with self.assert_raises_validation_error(code="max_length_not_64"): + Sha256Field(max_length=32) # type: ignore[arg-type] + + def test_set__none(self): + """Setting field to None sets to None.""" + user = User(_email_hash=None) + assert user.__dict__["_email_hash"] is None + + def test_hash(self): + """Hashing the same value produces the same hash of 64 characters.""" + value = "consistent_value" + hashed_value = Sha256Field.hash(value) + assert hashed_value == Sha256Field.hash(value) + assert hashed_value != Sha256Field.hash("different_value") + assert len(hashed_value) == 64 + + def test_lookup__sha256(self): + """ + `sha256` lookup hashes the right-hand side value before doing an exact + match. + """ + user = User.objects.filter(_email_hash__isnull=False).first() + assert user + # pylint: disable-next=protected-access + assert user.email != user._email_hash + assert User.objects.get(_email_hash__sha256=user.email) == user + + def test_lookup__sha256_in(self): + """ + `sha256_in` lookup hashes each value in the list before doing an exact + match. + """ + user = User.objects.filter(_email_hash__isnull=False).first() + assert user + # pylint: disable-next=protected-access + assert user.email != user._email_hash + assert User.objects.get(_email_hash__sha256_in=[user.email]) == user diff --git a/codeforlife/models/utils.py b/codeforlife/models/utils.py new file mode 100644 index 00000000..212a04bf --- /dev/null +++ b/codeforlife/models/utils.py @@ -0,0 +1,17 @@ +""" +© Ocado Group +Created on 16/03/2026 at 11:34:42(+00:00). +""" + +import typing as t + +from django.db.models import Model + + +def is_real_model_class(cls: t.Type[Model]): + """Determine if the class is a real model class that should be validated.""" + return ( + cls.__module__ != "__fake__" # used for migrations + and not cls._meta.abstract + and not cls._meta.proxy + ) diff --git a/codeforlife/permissions/auth_header_is_github_oidc_token.py b/codeforlife/permissions/auth_header_is_github_oidc_token.py index 9ac6de82..5751785b 100644 --- a/codeforlife/permissions/auth_header_is_github_oidc_token.py +++ b/codeforlife/permissions/auth_header_is_github_oidc_token.py @@ -11,7 +11,7 @@ import requests from django.conf import settings from django.utils import timezone -from jwt.algorithms import RSAAlgorithm +from jwt.types import JWKDict, Options from ..types import JsonDict from .base import BasePermission @@ -79,7 +79,7 @@ def _decode_token(self, token: str): header = jwt.get_unverified_header(token) kid = header.get("kid") - jwk: t.Optional[JsonDict] = None + jwk: t.Optional[JWKDict] = None for _jwk in jwks: if _jwk.get("kid") == kid: jwk = _jwk @@ -91,12 +91,11 @@ def _decode_token(self, token: str): return jwt.decode( token, - key=RSAAlgorithm.from_jwk(jwk), # type: ignore[arg-type] + key=jwt.PyJWK.from_dict(jwk), algorithms=["RS256", "RS384", "RS512"], audience=settings.SERVICE_DOMAIN, issuer=self.issuer, - # pylint: disable-next=line-too-long - options={"require_exp": True, "verify_signature": True}, # type: ignore[arg-type] + options=Options(require=["exp"], verify_signature=True), ) except jwt.exceptions.ExpiredSignatureError: diff --git a/codeforlife/pprint/__init__.py b/codeforlife/pprint/__init__.py new file mode 100644 index 00000000..ca55bfaa --- /dev/null +++ b/codeforlife/pprint/__init__.py @@ -0,0 +1,8 @@ +""" +© Ocado Group +Created on 18/03/2026 at 14:37:45(+00:00). +""" + +from .ansi import ANSI +from .pretty_printer import PrettyPrinter, pprint +from .style import Style diff --git a/codeforlife/pprint/ansi.py b/codeforlife/pprint/ansi.py new file mode 100644 index 00000000..4c8c53db --- /dev/null +++ b/codeforlife/pprint/ansi.py @@ -0,0 +1,27 @@ +""" +© Ocado Group +Created on 18/03/2026 at 14:37:45(+00:00). +""" + +from enum import Enum + + +class ANSI(Enum): + """ANSI escape codes for styling terminal output.""" + + RESET = "\033[0m" + + BLACK = "\033[30m" + WHITE = "\033[37m" + + RED = "\033[31m" + GREEN = "\033[32m" + BLUE = "\033[34m" + + YELLOW = "\033[33m" + MAGENTA = "\033[35m" + CYAN = "\033[36m" + + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + OVERLINE = "\033[53m" diff --git a/codeforlife/pprint/pretty_printer.py b/codeforlife/pprint/pretty_printer.py new file mode 100644 index 00000000..14e8aeb6 --- /dev/null +++ b/codeforlife/pprint/pretty_printer.py @@ -0,0 +1,168 @@ +""" +© Ocado Group +Created on 18/03/2026 at 14:37:45(+00:00). +""" + +import os +import typing as t +from timeit import default_timer + +from .ansi import ANSI +from .style import Style + + +# pylint: disable-next=too-many-instance-attributes +class PrettyPrinter: + """A utility class for pretty-printing styled messages to the terminal.""" + + def __init__( + self, + write: Style.Write, + name: str, + indent_level=0, + disable_styles=False, + ): + self.write = write + self.name = name + self.indent_level = indent_level + self.disable_styles = disable_styles + self.start_time: t.Optional[float] = None + self.end_time: t.Optional[float] = None + + self.style = Style.with_defaults( + write=self.__call__, enabled=not disable_styles + ) + + # Black and white. + self.black = self.style.ansi(ANSI.BLACK) + self.white = self.style.ansi(ANSI.WHITE) + + # Red, green, and blue. + self.red = self.style.ansi(ANSI.RED) + self.green = self.style.ansi(ANSI.GREEN) + self.blue = self.style.ansi(ANSI.BLUE) + + # Cyan, magenta, and yellow. + self.cyan = self.style.ansi(ANSI.CYAN) + self.magenta = self.style.ansi(ANSI.MAGENTA) + self.yellow = self.style.ansi(ANSI.YELLOW) + + # Common text styles. + self.bold = self.style.ansi(ANSI.BOLD) + self.underline = self.style.ansi(ANSI.UNDERLINE) + self.overline = self.style.ansi(ANSI.OVERLINE) + + # Status styles. + self.success = self.style.combine(self.green, self.bold) + self.error = self.style.combine(self.red, self.bold) + self.warn = self.warning = self.style.combine(self.yellow, self.bold) + self.info = self.notice = self.style.combine(self.blue, self.bold) + + # Heading styles. + self.h1 = self.style( + lambda message, **kwargs: "\n".join( + [ + self.bold.apply(self.divider("="), **kwargs), + self.bold.apply(message, **kwargs), + self.bold.apply(self.divider("="), **kwargs), + ] + ) + ) + self.h2 = self.style( + lambda message, **kwargs: "\n".join( + [ + self.bold.apply(self.divider("-"), **kwargs), + self.bold.apply(message, **kwargs), + self.bold.apply(self.divider("-"), **kwargs), + ] + ) + ) + self.h3 = self.style.combine(self.overline, self.underline, self.bold) + + def __call__(self, message: str, *args, **kwargs): + # pylint: disable=line-too-long + """Write a message to the terminal. + + Args: + message: The message to write. + *args: Additional positional arguments to pass to the write function. + **kwargs: Additional keyword arguments to pass to the write function. + """ + # pylint: enable=line-too-long + indented_message = self.indent(self.indent_level) + message + self.write(indented_message, *args, **kwargs) + + def __enter__(self): + self.bold(self.name) + self.indent_level = max(0, self.indent_level + 1) + + self.start_time = default_timer() + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.end_time = default_timer() + elapsed_time = self.end_time - (self.start_time or self.end_time) + + self.indent_level = max(0, self.indent_level - 1) + self( + self.bold.apply(f"{self.name} ") + + (self.error.apply("✘") if exc_type else self.success.apply("✔")) + + self.bold.apply(f" ({elapsed_time:.2f}s elapsed)") + ) + + def process( + self, + name: str, + indent_level: t.Optional[int] = None, + disable_styles: t.Optional[bool] = None, + ): + """ + A context manager for processing a message with automatic success or + error handling. If an exception is raised within the context, the + message is marked as an error; otherwise, it is marked as a success. + + Any messages written within the context are indented by the specified + level and written when the context is entered. + """ + return self.__class__( + write=self.write, + name=name, + indent_level=( + self.indent_level if indent_level is None else indent_level + ), + disable_styles=( + self.disable_styles + if disable_styles is None + else disable_styles + ), + ) + + def divider(self, char="-", default_columns=80): + # pylint: disable=line-too-long + """Write a divider line with the specified character. + + Args: + default_columns: The default number of columns to use if the terminal width cannot be determined. + char: The character to use for the divider. + """ + # pylint: enable=line-too-long + try: + columns = os.get_terminal_size().columns + except OSError: + columns = default_columns + + return char * columns + + def indent(self, count: int, spaces=4, char=" "): + """Write an indentation of the specified number of spaces. + + Args: + count: The number of indentation levels. + spaces: The number of spaces per indentation level. + char: The character to use for indentation. + """ + return char * count * spaces + + +pprint = PrettyPrinter(write=print, name="main") diff --git a/codeforlife/pprint/style.py b/codeforlife/pprint/style.py new file mode 100644 index 00000000..6888f7a6 --- /dev/null +++ b/codeforlife/pprint/style.py @@ -0,0 +1,107 @@ +""" +© Ocado Group +Created on 18/03/2026 at 14:37:45(+00:00). +""" + +import typing as t + +from .ansi import ANSI + +if t.TYPE_CHECKING: + from typing_extensions import Protocol + + # pylint: disable-next=too-few-public-methods + class ApplyProtocol(Protocol): + """A protocol for a callable that applies a style to a message.""" + + def __call__(self, message: str, **kwargs) -> str: ... + + +class Style: + """A callable class that applies styles to messages.""" + + Apply: t.TypeAlias = "ApplyProtocol" + Write: t.TypeAlias = t.Callable[[str], None] + + def __init__(self, apply: Apply, write: Write = print, enabled=True): + self.original_apply = apply + self.write = write + self.enabled = enabled + + def __call__(self, message: str, *args, **kwargs): + styled_message = self.apply(message) + self.write(styled_message, *args, **kwargs) + + def apply(self, message: str, **kwargs): + """Apply the style to the given message. + + Args: + message: The message to apply the style to. + **kwargs: Keyword arguments that may be used by the style. + + Returns: + The styled message. + """ + return ( + self.original_apply(message, **kwargs) if self.enabled else message + ) + + @classmethod + def ansi(cls, code: ANSI): + """Create a style that applies the given ANSI code to messages. + + Args: + code: The ANSI code to apply. + + Returns: + A style that applies the given ANSI code to messages. + """ + + def apply(message: str, **kwargs): + if kwargs.get("reset", True): + message += ANSI.RESET.value + + return code.value + message + + return cls(apply) + + @classmethod + def combine(cls, *styles: "Style"): + """Combine multiple styles into a single style. + + Args: + *styles: The styles to combine. + + Returns: + A style that applies all the given styles to messages. + """ + + def apply(message: str, **kwargs): + for style in styles: + message = style.apply(message, **kwargs) + + return message + + return cls(apply) + + @classmethod + def with_defaults(cls, write: Write, enabled=True): + """Create a style class that uses the given defaults. + + Args: + write: The function to use for writing the styled message. + enabled: Whether the style is enabled. + + Returns: + A style class that uses the given defaults. + """ + + class StyleWithDefaults(cls): # type: ignore[valid-type,misc] + """A style class that uses the given defaults.""" + + def __init__( + self, apply: Style.Apply, write=write, enabled=enabled + ): + super().__init__(apply, write, enabled) + + return StyleWithDefaults diff --git a/codeforlife/server.py b/codeforlife/server.py deleted file mode 100644 index 9a669175..00000000 --- a/codeforlife/server.py +++ /dev/null @@ -1,249 +0,0 @@ -""" -© Ocado Group -Created on 05/06/2025 at 17:33:56(+01:00). -""" - -import atexit -import logging -import multiprocessing -import os -import subprocess -import sys -import typing as t -from functools import cached_property -from importlib import import_module - -from celery import Celery -from django import setup as setup_django -from django.core.asgi import get_asgi_application as get_django_asgi_app -from django.core.management import call_command as call_django_command -from django.core.wsgi import get_wsgi_application as get_django_wsgi_app -from gunicorn.app.base import BaseApplication # type: ignore[import-untyped] - -from .tasks import get_task_name -from .types import DatabaseEngine, Env, LogLevel - - -# pylint: disable-next=abstract-method,too-many-instance-attributes -class Server(BaseApplication): - """Serves a service in different modes.""" - - Mode = t.Literal["django", "celery"] - - # The entrypoint module. - main_module = os.path.splitext(os.path.basename(sys.argv[0]))[0] - # The dot-path of the application module. - app_module: str = "application" - # The dot-path of the settings module. - settings_module: str = "settings" - # The dot-path of Django's manage module. - django_manage_module: str = "manage" - # The dot-path of the source-code module. - src_module: str = "src" - # The port the app is served on. - app_port: int = 8080 - - @cached_property - def app_server_is_running(self): - """Whether or not the app server is running.""" - return self.main_module == self.app_module - - @cached_property - def django_dev_server_is_running(self): - """Whether or not the Django development server is running.""" - return ( - self.main_module == self.django_manage_module - and sys.argv[1] == "runserver" - ) - - # pylint: disable-next=too-many-arguments,too-many-positional-arguments - def __init__( - self, - mode: Mode = t.cast(Mode, os.getenv("SERVER_MODE", "django")), - workers: int = int(os.getenv("SERVER_WORKERS", "0")), - log_level: t.Optional[LogLevel] = t.cast( - LogLevel, os.getenv("LOG_LEVEL", "INFO") - ), - db_engine: DatabaseEngine = "postgresql", - dump_request: bool = False, - ): - # pylint: disable=line-too-long - """Initialize a service's app-server. - - Examples: - ``` - from codeforlife.server import Server - - Server().run() - ``` - - Args: - mode: The mode to run in. Note, "celery" will start Django with only the health-check url. - workers: The number of workers. 0 will auto-calculate. Note, "celery" will create 1 Django worker. - log_level: The log level. None uses the default. - db_engine: The database's engine type. - dump_request: A flag designating whether to add the dump_request Celery task (useful for debugging). - """ - # pylint: enable=line-too-long - - if mode != "django" and self.django_dev_server_is_running: - mode = "django" - os.environ["SERVER_MODE"] = mode - self.mode = mode - - if log_level: - os.environ["LOG_LEVEL"] = log_level - self.log_level = log_level - - os.environ["DB_ENGINE"] = db_engine - self.db_engine = db_engine - - if mode == "django": - # https://docs.gunicorn.org/en/stable/design.html#how-many-workers - workers = workers or (multiprocessing.cpu_count() * 2) + 1 - self.workers = workers - - if self.app_server_is_running: - os.environ["SERVICE_PORT"] = str(self.app_port) - - os.environ["DJANGO_SETTINGS_MODULE"] = self.settings_module - setup_django() - - self.django_asgi_app = get_django_asgi_app() - self.django_wsgi_app = get_django_wsgi_app() - self.celery_app = Celery() - - self.options = { - "bind": f"0.0.0.0:{self.app_port}", - "workers": 1 if mode == "celery" else workers, - "worker_class": "uvicorn.workers.UvicornWorker", - "forwarded_allow_ips": "*", - } - - if mode == "celery": - # Using a string here means the worker doesn't have to serialize - # the configuration object to child processes. - # - namespace='CELERY' means all celery-related configuration keys - # should have a `CELERY_` prefix. - self.celery_app.config_from_object( - "django.conf:settings", namespace="CELERY" - ) - - # Load task modules from all registered Django apps. - self.celery_app.autodiscover_tasks([self.src_module]) - - if dump_request: - - @self.celery_app.task( - name=get_task_name("dump_request"), - bind=True, - ignore_result=True, - ) - def _dump_request(self, *args, **kwargs): - """Dumps its own request information.""" - - logging.info("Request: %s", self.request) - - super().__init__() - - # Set the apps as global variables in the app module. - app = import_module(self.app_module) - app.django_wsgi = self.django_wsgi_app # type: ignore[attr-defined] - app.celery = self.celery_app # type: ignore[attr-defined] - - def load_config(self): - config = { - key: value - for key, value in self.options.items() - if key in self.cfg.settings and value is not None - } - for key, value in config.items(): - self.cfg.set(key.lower(), value) - - def load(self): - return self.django_asgi_app - - # pylint: disable-next=dangerous-default-value - def run( - self, - migrate: bool = True, - collect_static: bool = True, - create_sites: bool = True, - load_fixtures: t.Optional[t.Set[str]] = {src_module}, - ): - """Run the server in the set mode. - - Args: - migrate: A flag designating whether to migrate the models. - collect_static: A flag designating whether to collect static files. - create_sites: A flag designating whether to create the django-sites. - load_fixtures: An array of fixtures to load. None to skip. - """ - - if self.mode == "django": - # NOTE: Imports come after django setup in server initialization. - # pylint: disable=import-outside-toplevel - from django.conf import settings - from django.contrib.sites.models import Site - - # pylint: enable=import-outside-toplevel - - if self.db_engine == "sqlite": - migrate = False - create_sites = False - load_fixtures = None - - if not self.django_dev_server_is_running: - load_fixtures = None - collect_static = False - - if migrate: - call_django_command("migrate", interactive=False) - if load_fixtures: - call_django_command("load_fixtures", *load_fixtures) - if collect_static: - call_django_command("collectstatic", "--noinput", "--clear") - if create_sites: - - def create_site(domain: str): - Site.objects.get_or_create( - domain=domain, - defaults={"name": settings.SERVICE_NAME}, - ) - - if t.cast(Env, settings.ENV) == "local": - create_site(domain=f"localhost:{settings.SERVICE_PORT}") - create_site(domain=f"127.0.0.1:{settings.SERVICE_PORT}") - else: - create_site(domain=settings.SERVICE_DOMAIN) - create_site(domain=settings.SERVICE_HOST) - - if self.app_server_is_running: - if self.mode == "celery": - self.run_celery_worker_as_subprocess() - - super().run() - - def run_celery_worker_as_subprocess(self): - """Starts a worker using the 'celery worker' command.""" - - command = ["celery", f"--app={self.app_module}", "worker"] - if self.workers: - command.append(f"--concurrency={self.workers}") - if self.log_level: - command.append(f"--loglevel={self.log_level}") - - stdout, stderr = (None, None) # Use defaults. - else: - stdout, stderr = (subprocess.DEVNULL, subprocess.DEVNULL) - - try: - # pylint: disable-next=consider-using-with - process = subprocess.Popen(command, stdout=stdout, stderr=stderr) - - atexit.register(process.terminate) - - os.environ["SERVER_CELERY_WORKER_PID"] = str(process.pid) - - except Exception as ex: # pylint: disable=broad-exception-caught - print(f"Error starting Celery worker: {ex}") diff --git a/codeforlife/settings/__init__.py b/codeforlife/settings/__init__.py index 0166da8d..37248ad1 100644 --- a/codeforlife/settings/__init__.py +++ b/codeforlife/settings/__init__.py @@ -14,8 +14,8 @@ ` """ +from ._secrets import LatestSecret, TypedLatestSecret, get_secret from .custom import * from .django import * from .google import * -from .otp import * from .third_party import * diff --git a/codeforlife/settings/_secrets.py b/codeforlife/settings/_secrets.py new file mode 100644 index 00000000..faa71b0a --- /dev/null +++ b/codeforlife/settings/_secrets.py @@ -0,0 +1,277 @@ +""" +© Ocado Group +Created on 09/04/2026 at 10:28:21(+00:00). +""" + +import os +import typing as t +from functools import lru_cache +from threading import Event, RLock + +import google_crc32c +from cachetools import Cache +from google.api_core.exceptions import NotFound +from google.cloud.secretmanager import SecretManagerServiceClient +from typing_extensions import TypeVar + +if t.TYPE_CHECKING: + from ..types import Env + +T = TypeVar("T", default=None) +# pylint: disable-next=invalid-name +T2 = TypeVar("T2", default=None) + +_LATEST_CACHE: Cache[str, t.Tuple[str, str]] = Cache(maxsize=256) +_LATEST_CACHE_LOCK = RLock() +_LATEST_CACHE_EVENTS: t.Dict[str, Event] = {} + + +@lru_cache(maxsize=1) # This is a singleton, so we only want to create it once. +def _client(): + return SecretManagerServiceClient( + client_options={ + "api_endpoint": ( + "secretmanager." + + os.environ["GCP_SECRET_MANAGER_LOCATION"] + + ".rep.googleapis.com" + ) + } + ) + + +def _get_full_secret_name(name: str, version: str): + return ( + f"projects/{os.environ['GOOGLE_CLOUD_PROJECT_ID']}" + f"/locations/{os.environ['GCP_SECRET_MANAGER_LOCATION']}" + f"/secrets/{name}" + f"/versions/{version}" + ) + + +def _get_secret_metadata(name: str, version: str): + try: + return _client().get_secret_version( + request={"name": _get_full_secret_name(name, version)} + ) + except NotFound: + return None + + +def _get_secret(name: str, version: str): + try: + response = _client().access_secret_version( + request={"name": _get_full_secret_name(name, version)} + ) + except NotFound: + return None + + # Verify payload checksum. + crc32c = google_crc32c.Checksum() + crc32c.update(response.payload.data) + if response.payload.data_crc32c != int(crc32c.hexdigest(), 16): + raise ValueError(f"Invalid checksum for secret {name}.") + + return response.payload.data.decode("utf-8") + + +def _get_secret_with_latest_version_cache(name: str): + version = "latest" + + # Single-flight: only one thread per secret does remote calls to prevent a + # 'thundering herd' of remote calls when a secret is first accessed. + with _LATEST_CACHE_LOCK: + event = _LATEST_CACHE_EVENTS.get(name) + if event is None: + event = Event() + _LATEST_CACHE_EVENTS[name] = event + leader = True + else: + leader = False + + # Wait for leader to finish, then read cache result. + if not leader: + event.wait() + with _LATEST_CACHE_LOCK: + cached = _LATEST_CACHE.get(name) + return None if cached is None else cached[1] + + try: + meta = _get_secret_metadata(name, version) + if meta is None: + with _LATEST_CACHE_LOCK: + _LATEST_CACHE.pop(name, None) + return None + + with _LATEST_CACHE_LOCK: + cached = _LATEST_CACHE.get(name) + if cached is not None: + etag, secret = cached + if meta.etag == etag: + return secret + + secret = _get_secret(name, version) + + with _LATEST_CACHE_LOCK: + if secret is None: + _LATEST_CACHE.pop(name, None) + else: + _LATEST_CACHE[name] = (meta.etag, secret) + + return secret + finally: + with _LATEST_CACHE_LOCK: + done = _LATEST_CACHE_EVENTS.pop(name, None) + if done is not None: + done.set() + + +@lru_cache(maxsize=200) # Increase as needed, but be mindful of memory usage. +def _get_secret_with_specific_version_cache(name: str, version: str): + return _get_secret(name, version) + + +@t.overload +def get_secret( + name: str, default: None = None, version="latest", cache=True +) -> t.Union[str, None]: ... + + +@t.overload +def get_secret( + name: str, default: T, version="latest", cache=True +) -> t.Union[str, T]: ... + + +def get_secret( + name: str, default: t.Optional[T] = None, version="latest", cache=True +): + """Get a secret from GCP Secret Manager. + + https://docs.cloud.google.com/secret-manager/docs/samples/secretmanager-access-regional-secret-version#secretmanager_access_regional_secret_version-python + + If running locally, this value will be read from environment variables. + + Args: + name: The name of the secret. + default: The default value to return if the secret does not exist. + version: The version of the secret to access. + cache: Whether to get & set the secret from/in the cache. + + Raises: + ValueError: If the secret exists but has an invalid checksum. + + Returns: + The value of the secret, or the default value if the secret does not + exist. + """ + env = t.cast("Env", os.getenv("ENV", "local")) + if env == "local": + return os.getenv(name, default) + + name = f"{env}_{name}".upper() + value = ( + ( + _get_secret_with_latest_version_cache(name) + if version == "latest" + else _get_secret_with_specific_version_cache(name, version) + ) + if cache + else _get_secret(name, version) + ) + + return default if value is None else value + + +# pylint: disable-next=too-few-public-methods +class LatestSecret(t.Generic[T]): + """ + A callable wrapper around `get_secret()` that allows you to define a secret + once and then call it to always get the latest value. + + This is useful for Django settings, where values are only evaluated once at + startup. + + Examples: + ``` + ### In settings.py + from codeforlife.settings import LatestSecret + + API_KEY = LatestSecret(name="API_KEY", default="a") # Define once. + + ### In your code + from django.conf import settings + + # Always returns the latest value. + # If the secret does not exist, it returns the default value "a". + value = settings.API_KEY() + + # You can also override the default value at call time. + # If the secret does not exist, it returns the default value "b". + value = settings.API_KEY(default="b") + + # You can also bypass the cache to get the latest value directly from GCP. + value = settings.API_KEY(cache=False) + ``` + """ + + def __init__(self, name: str, default: t.Optional[T] = None): + self.name = name + self.default = default + + @t.overload + def __call__(self, default: None = None, cache=True) -> t.Union[str, T]: ... + + @t.overload + def __call__(self, default: T2, cache=True) -> t.Union[str, T2]: ... + + def __call__(self, default: t.Optional[T2] = None, cache=True): + return get_secret( + name=self.name, + default=t.cast(T, self.default) if default is None else default, + version="latest", + cache=cache, + ) + + +# pylint: disable-next=too-few-public-methods +class TypedLatestSecret(LatestSecret[T], t.Generic[T, T2]): + """A `LatestSecret` that also casts the secret value to a specific type.""" + + @t.overload + def __init__( + self, + name: str, + cast: t.Callable[[str], T], + default: t.Optional[T2] = None, + ): ... + + @t.overload + def __init__(self, name: str, cast: t.Callable[[str], T], default: T): ... + + def __init__( + self, + name: str, + cast: t.Callable[[str], T], + default: t.Optional[t.Union[T, T2]] = None, + ): + super().__init__(name=name, default=t.cast(t.Optional[T], default)) + + self.cast = cast + + @t.overload # type: ignore[override] + def __call__(self, default: None = None, cache=True) -> t.Union[T, T2]: ... + + @t.overload + # pylint: disable-next=signature-differs + def __call__(self, default: T, cache=True) -> T: ... + + def __call__(self, default: t.Optional[T] = None, cache=True): + secret = super().__call__(default=default, cache=cache) + + # If the secret is None, default, or self.default, return it as-is + # without casting. + return ( + t.cast(t.Union[T, T2], secret) + if secret is None or not isinstance(secret, str) + else self.cast(secret) + ) diff --git a/codeforlife/settings/custom.py b/codeforlife/settings/custom.py index 13a66574..d3ac2455 100644 --- a/codeforlife/settings/custom.py +++ b/codeforlife/settings/custom.py @@ -5,37 +5,20 @@ This file contains all of our custom settings we define for our own purposes. """ -import json import os import re import typing as t from pathlib import Path -import boto3 - -from .otp import ( - AWS_S3_APP_BUCKET, - AWS_S3_APP_FOLDER, - AWS_S3_STATIC_FOLDER, - CACHE_DB_DATA_PATH, -) +from ._secrets import LatestSecret if t.TYPE_CHECKING: - from mypy_boto3_s3.client import S3Client - - from ..server import Server - from ..types import CookieSamesite, DatabaseEngine, Env, JsonDict + from ..types import CookieSamesite, Env # The name of the current environment. ENV = t.cast("Env", os.getenv("ENV", "local")) -# The database's engine type. -DB_ENGINE = t.cast("DatabaseEngine", os.getenv("DB_ENGINE", "postgresql")) - -# The mode the service is being served in. -SERVER_MODE = t.cast("Server.Mode", os.getenv("SERVER_MODE", "django")) - # The level of the logs. LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") @@ -68,14 +51,9 @@ # The frontend url of the current service. SERVICE_SITE_URL = os.getenv("SERVICE_SITE_URL", "http://localhost:5173") -# The location of the service's folder in the s3 buckets. -SERVICE_S3_APP_LOCATION = f"{AWS_S3_APP_FOLDER}/{SERVICE_NAME}/{SERVER_MODE}" -SERVICE_S3_STATIC_LOCATION = ( - f"{AWS_S3_STATIC_FOLDER}/{SERVICE_NAME}/{SERVER_MODE}" -) # The authorization bearer token used to authenticate with Dotdigital. -MAIL_AUTH = os.getenv("MAIL_AUTH", "REPLACE_ME") +MAIL_AUTH = LatestSecret("MAIL_AUTH", "REPLACE_ME") # A global flag to enable/disable sending emails. # If disabled, emails will be logged to the console instead. @@ -89,44 +67,6 @@ SESSION_METADATA_COOKIE_SAMESITE: "CookieSamesite" = "Strict" -def get_redis_url(): - """Get the Redis URL for the current environment. - - Raises: - ConnectionAbortedError: If the engine is not Redis. - - Returns: - The Redis URL. - """ - - if ENV == "local": - host = os.getenv("REDIS_HOST", "cache") - port = int(os.getenv("REDIS_PORT", "6379")) - path = os.getenv("REDIS_PATH", "0") - url = f"{host}:{port}/{path}" - else: - # Get the dbdata object. - s3: "S3Client" = boto3.client("s3") - db_data_object = s3.get_object( - Bucket=t.cast(str, AWS_S3_APP_BUCKET), Key=CACHE_DB_DATA_PATH - ) - - # Load the object as a JSON dict. - db_data: "JsonDict" = json.loads( - db_data_object["Body"].read().decode("utf-8") - ) - if not db_data or db_data["Engine"] != "Redis": - raise ConnectionAbortedError("Invalid database data.") - - endpoint = t.cast(dict, db_data["Endpoint"]) - url = t.cast(str, endpoint["0001"]) - - return f"redis://{url}" - - -# The URL to connect to the Redis cache. -REDIS_URL = get_redis_url() - # A flag to indicate whether the old system is the current runtime to # conditionally run code that is still needed for the old system to work but is # no longer needed in the new system. Once the old system is fully deprecated, diff --git a/codeforlife/settings/django.py b/codeforlife/settings/django.py index 231f3c3e..fc052d0a 100644 --- a/codeforlife/settings/django.py +++ b/codeforlife/settings/django.py @@ -6,54 +6,28 @@ https://docs.djangoproject.com/en/5.1/ref/settings/ """ -import json import os -import typing as t -import boto3 from django.utils.translation import gettext_lazy as _ from .. import TEMPLATES_DIR +from ._secrets import get_secret from .custom import ( - DB_ENGINE, ENV, LOG_LEVEL, - REDIS_URL, SERVICE_BASE_DIR, SERVICE_BASE_URL, SERVICE_DOMAIN, SERVICE_EXTERNAL_DOMAIN, SERVICE_NAME, - SERVICE_S3_APP_LOCATION, - SERVICE_S3_STATIC_LOCATION, SERVICE_SITE_URL, ) -from .otp import ( - AWS_REGION, - AWS_S3_APP_BUCKET, - AWS_S3_APP_DEFAULT_ACL, - AWS_S3_APP_DOMAIN, - AWS_S3_APP_QUERYSTRING_AUTH, - AWS_S3_APP_QUERYSTRING_EXPIRE, - AWS_S3_STATIC_BUCKET, - AWS_S3_STATIC_DEFAULT_ACL, - AWS_S3_STATIC_DOMAIN, - AWS_S3_STATIC_QUERYSTRING_AUTH, - AWS_S3_STATIC_QUERYSTRING_EXPIRE, - RDS_DB_DATA_PATH, -) - -if t.TYPE_CHECKING: - from mypy_boto3_s3.client import S3Client - - from ..types import JsonDict - # SECURITY WARNING: don't run with debug turned on in production! DEBUG = bool(int(os.getenv("DEBUG", "1"))) # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.getenv("SECRET_KEY", "replace-me") +SECRET_KEY = get_secret("SECRET_KEY", "replace-me") # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ @@ -63,65 +37,17 @@ # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases - -def get_databases(): - """Get the databases for the current environment. - - Raises: - ConnectionAbortedError: If the engine is not postgres. - - Returns: - The database configs. - """ - - if DB_ENGINE == "sqlite": - return { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": ":memory:", - } - } - - if ENV == "local": - name = os.getenv("DB_NAME", SERVICE_NAME) - user = os.getenv("DB_USER", "root") - password = os.getenv("DB_PASSWORD", "password") - host = os.getenv("DB_HOST", "db") - port = int(os.getenv("DB_PORT", "5432")) - else: - # Get the dbdata object. - s3: "S3Client" = boto3.client("s3") - db_data_object = s3.get_object( - Bucket=t.cast(str, AWS_S3_APP_BUCKET), Key=RDS_DB_DATA_PATH - ) - - # Load the object as a JSON dict. - db_data: "JsonDict" = json.loads( - db_data_object["Body"].read().decode("utf-8") - ) - if not db_data or db_data["DBEngine"] != "postgres": - raise ConnectionAbortedError("Invalid database data.") - - name = t.cast(str, db_data["Database"]) - user = t.cast(str, db_data["user"]) - password = t.cast(str, db_data["password"]) - host = t.cast(str, db_data["Endpoint"]) - port = t.cast(int, db_data["Port"]) - - return { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": name, - "USER": user, - "PASSWORD": password, - "HOST": host, - "PORT": port, - "ATOMIC_REQUESTS": True, - } +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": get_secret("DB_NAME", SERVICE_NAME), + "USER": get_secret("DB_USER", "root"), + "PASSWORD": get_secret("DB_PASSWORD", "password"), + "HOST": get_secret("DB_HOST", "db"), + "PORT": int(get_secret("DB_PORT", "5432")), + "ATOMIC_REQUESTS": True, } - - -DATABASES = get_databases() +} # Application definition @@ -288,29 +214,8 @@ def get_databases(): "corsheaders", "rest_framework", "django_filters", - "storages", ] -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/5.1/howto/static-files/ - -STATIC_ROOT = SERVICE_BASE_DIR / "static" -STATIC_URL = ( - f"https://{AWS_S3_STATIC_DOMAIN}/{SERVICE_S3_STATIC_LOCATION}/" - if ENV != "local" - else "/static/" -) - -# User-uploaded files -# https://docs.djangoproject.com/en/5.1/topics/files/ - -MEDIA_ROOT = SERVICE_BASE_DIR / "media" -MEDIA_URL = ( - f"https://{AWS_S3_APP_DOMAIN}/{SERVICE_S3_APP_LOCATION}/" - if ENV != "local" - else "/media/" -) - # Templates # https://docs.djangoproject.com/en/5.1/ref/templates/ @@ -332,51 +237,3 @@ def get_databases(): }, }, ] - -# Storages -# https://docs.djangoproject.com/en/5.1/ref/settings/#storages - -STORAGES: t.Dict[str, t.Any] = { - "default": ( - {"BACKEND": "django.core.files.storage.FileSystemStorage"} - if ENV == "local" - else { - "BACKEND": "storages.backends.s3.S3Storage", - # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings - "OPTIONS": { - "bucket_name": AWS_S3_APP_BUCKET, - "location": SERVICE_S3_APP_LOCATION, - "region_name": AWS_REGION, - "default_acl": AWS_S3_APP_DEFAULT_ACL, - "querystring_auth": AWS_S3_APP_QUERYSTRING_AUTH, - "querystring_expire": AWS_S3_APP_QUERYSTRING_EXPIRE, - }, - } - ), - "staticfiles": ( - {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"} - if ENV == "local" - else { - "BACKEND": "storages.backends.s3.S3Storage", - # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings - "OPTIONS": { - "bucket_name": AWS_S3_STATIC_BUCKET, - "location": SERVICE_S3_STATIC_LOCATION, - "region_name": AWS_REGION, - "default_acl": AWS_S3_STATIC_DEFAULT_ACL, - "querystring_auth": AWS_S3_STATIC_QUERYSTRING_AUTH, - "querystring_expire": AWS_S3_STATIC_QUERYSTRING_EXPIRE, - }, - } - ), -} - -# Caches -# https://docs.djangoproject.com/en/5.1/topics/cache/ - -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": REDIS_URL, - } -} diff --git a/codeforlife/settings/google.py b/codeforlife/settings/google.py index 37e160e3..14bff328 100644 --- a/codeforlife/settings/google.py +++ b/codeforlife/settings/google.py @@ -7,35 +7,39 @@ import os +from ._secrets import LatestSecret + +# pylint: disable=invalid-name + # Our Google OAuth 2.0 client credentials # https://console.cloud.google.com/auth/clients GOOGLE_CLIENT_ID = os.getenv( "GOOGLE_CLIENT_ID", "354656325390-o5n12nbaivhi4do8lalkh29q403uu9u4.apps.googleusercontent.com", ) -GOOGLE_CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET", "REPLACE_ME") +GOOGLE_CLIENT_SECRET = LatestSecret("GOOGLE_CLIENT_SECRET", "REPLACE_ME") # The ID of our GCP project. GOOGLE_CLOUD_PROJECT_ID = os.getenv( "GOOGLE_CLOUD_PROJECT_ID", "decent-digit-629" ) -# The ID of our BigQuery dataset. -GOOGLE_CLOUD_BIGQUERY_DATASET_ID = os.getenv( - "GOOGLE_CLOUD_BIGQUERY_DATASET_ID", "REPLACE_ME" -) - # Key management service (KMS) # https://docs.cloud.google.com/python/docs/reference/cloudkms/latest/summary_overview -GCP_KMS_KEY_RING_LOCATION = os.getenv("GCP_KMS_KEY_RING_LOCATION", "REPLACE_ME") -GCP_KMS_KEY_RING_NAME = os.getenv("GCP_KMS_KEY_RING_NAME", "REPLACE_ME") -GCP_KMS_KEY_NAME = os.getenv("GCP_KMS_KEY_NAME", "REPLACE_ME") -# The URI of the KMS key encryption key (KEK). -GCP_KMS_KEY_URI = ( - "gcp-kms://" - f"projects/{GOOGLE_CLOUD_PROJECT_ID}/" - f"locations/{GCP_KMS_KEY_RING_LOCATION}/" - f"keyRings/{GCP_KMS_KEY_RING_NAME}/" - f"cryptoKeys/{GCP_KMS_KEY_NAME}" +GCP_KMS_KEY_RING_LOCATION = LatestSecret( + "GCP_KMS_KEY_RING_LOCATION", "REPLACE_ME" ) +GCP_KMS_KEY_RING_NAME = LatestSecret("GCP_KMS_KEY_RING_NAME", "REPLACE_ME") +GCP_KMS_KEY_NAME = LatestSecret("GCP_KMS_KEY_NAME", "REPLACE_ME") + + +def GCP_KMS_KEY_URI() -> str: + """The URI of the KMS key encryption key (KEK).""" + return ( + "gcp-kms://" + f"projects/{GOOGLE_CLOUD_PROJECT_ID}/" + f"locations/{GCP_KMS_KEY_RING_LOCATION()}/" + f"keyRings/{GCP_KMS_KEY_RING_NAME()}/" + f"cryptoKeys/{GCP_KMS_KEY_NAME()}" + ) diff --git a/codeforlife/settings/otp.py b/codeforlife/settings/otp.py deleted file mode 100644 index 30ebc790..00000000 --- a/codeforlife/settings/otp.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -© Ocado Group -Created on 29/11/2024 at 15:59:40(+00:00). - -This file contains all the variables required and/or exposed by Ocado's -Technology Platform. - -NOTE: All variables should be retrieved like so: `os.getenv("key")`. -""" - -import os - -# App - -APP_ID = os.getenv("APP_ID") -APP_VERSION = os.getenv("APP_VERSION") - -# AWS - -AWS_REGION = os.getenv("aws_region") - -# AWS S3 - -AWS_S3_APP_BUCKET = os.getenv("aws_s3_app_bucket") -AWS_S3_APP_FOLDER = os.getenv("aws_s3_app_folder") -AWS_S3_APP_DOMAIN = f"{AWS_S3_APP_BUCKET}.s3.amazonaws.com" -AWS_S3_APP_DEFAULT_ACL = os.getenv("AWS_S3_APP_DEFAULT_ACL") -AWS_S3_APP_QUERYSTRING_AUTH = bool( - int(os.getenv("AWS_S3_APP_QUERYSTRING_AUTH", "1")) -) -AWS_S3_APP_QUERYSTRING_EXPIRE = int( - os.getenv("AWS_S3_APP_QUERYSTRING_EXPIRE", "3600") -) -AWS_S3_STATIC_BUCKET = os.getenv("AWS_S3_STATIC_BUCKET") -AWS_S3_STATIC_FOLDER = os.getenv("AWS_S3_STATIC_FOLDER") -AWS_S3_STATIC_DOMAIN = f"{AWS_S3_STATIC_BUCKET}.s3.amazonaws.com" -AWS_S3_STATIC_DEFAULT_ACL = os.getenv("AWS_S3_STATIC_DEFAULT_ACL") -AWS_S3_STATIC_QUERYSTRING_AUTH = bool( - int(os.getenv("AWS_S3_STATIC_QUERYSTRING_AUTH", "1")) -) -AWS_S3_STATIC_QUERYSTRING_EXPIRE = int( - os.getenv("AWS_S3_STATIC_QUERYSTRING_EXPIRE", "3600") -) - -# RDS - -RDS_DB_NAME = os.getenv("RDS_DB_NAME") -RDS_SCHEMA_NAME = os.getenv("RDS_SCHEMA_NAME") -RDS_INSTANCE_NAME = os.getenv("RDS_INSTANCE_NAME") -RDS_DB_DATA_PATH = ( - f"{AWS_S3_APP_FOLDER}/dbMetadata/" - + (f"{RDS_INSTANCE_NAME}/" if RDS_INSTANCE_NAME else "") - + f"{RDS_DB_NAME}/{RDS_SCHEMA_NAME}.dbdata" -) - -# ElastiCache - -CACHE_CLUSTER_ID = os.getenv("CACHE_CLUSTER_ID") -CACHE_DB_DATA_PATH = ( - f"{AWS_S3_APP_FOLDER}/elasticacheMetadata/{CACHE_CLUSTER_ID}.dbdata" -) - -# GCP - -GCP_WIF_AUDIENCE = os.getenv("GCP_WIF_AUDIENCE") -GCP_WIF_SERVICE_ACCOUNT = os.getenv("GCP_WIF_SERVICE_ACCOUNT") - -# SQS - -SQS_URL = os.getenv("SQS_URL") diff --git a/codeforlife/settings/third_party.py b/codeforlife/settings/third_party.py index 43c10f39..7689936b 100644 --- a/codeforlife/settings/third_party.py +++ b/codeforlife/settings/third_party.py @@ -5,13 +5,7 @@ This file contains custom settings defined by third party extensions. """ -import os - -from ..tasks import get_local_sqs_url as _get_local_sqs_url -from .custom import ENV, SERVICE_NAME, SERVICE_SITE_URL -from .django import TIME_ZONE -from .otp import AWS_REGION as OTP_AWS_REGION -from .otp import SQS_URL +from .custom import ENV, SERVICE_SITE_URL # CORS # https://pypi.org/project/django-cors-headers/ @@ -33,34 +27,3 @@ "DEFAULT_PAGINATION_CLASS": "codeforlife.pagination.LimitOffsetPagination", "NON_FIELD_ERRORS_KEY": "__all__", } - -# AWS CLI -# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html -# NOTE: These are set in the dev container: -# https://github.com/ocadotechnology/codeforlife-workspace/blob/main/.devcontainer/docker-compose.yml - -AWS_REGION = os.getenv("AWS_REGION", "us-east-1") -AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "test") -AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "test") -AWS_ENDPOINT_URL = os.getenv("AWS_ENDPOINT_URL", "http://aws:4566") - -# Celery -# https://docs.celeryq.dev/en/v5.4.0/userguide/configuration.html -# https://docs.celeryq.dev/en/v5.4.0/getting-started/backends-and-brokers/sqs.html - -CELERY_TIMEZONE = TIME_ZONE -CELERY_BROKER_URL = "sqs://" -CELERY_BROKER_TRANSPORT_OPTIONS = { - "region": OTP_AWS_REGION if ENV != "local" else AWS_REGION, - "predefined_queues": { - SERVICE_NAME: { - "url": ( - SQS_URL - if ENV != "local" - else _get_local_sqs_url(AWS_REGION, SERVICE_NAME) - ) - } - }, -} -CELERY_TASK_DEFAULT_QUEUE = SERVICE_NAME -CELERY_TASK_TIME_LIMIT = 60 * 30 diff --git a/codeforlife/tasks/__init__.py b/codeforlife/tasks/__init__.py deleted file mode 100644 index 265c0975..00000000 --- a/codeforlife/tasks/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -© Ocado Group -Created on 06/10/2025 at 17:14:31(+01:00). -""" - -from .bigquery import BigQueryTask -from .utils import get_local_sqs_url, get_task_name, shared_task diff --git a/codeforlife/tasks/bigquery.py b/codeforlife/tasks/bigquery.py deleted file mode 100644 index 05ad2c8f..00000000 --- a/codeforlife/tasks/bigquery.py +++ /dev/null @@ -1,410 +0,0 @@ -""" -© Ocado Group -Created on 06/10/2025 at 17:15:37(+01:00). -""" - -import csv -import io -import logging -import typing as t -from dataclasses import dataclass, field -from datetime import date, datetime, time, timezone -from tempfile import NamedTemporaryFile, _TemporaryFileWrapper - -from celery import Task -from celery import shared_task as _shared_task -from django.conf import settings as django_settings -from django.core.exceptions import ValidationError -from django.db.models.query import QuerySet -from google.cloud.bigquery import ( - Client, - CreateDisposition, - LoadJobConfig, - SourceFormat, - WriteDisposition, -) - -from ..auth import get_gcp_service_account_credentials -from ..types import KwArgs -from .utils import get_task_name - -if t.TYPE_CHECKING: - CsvFile = _TemporaryFileWrapper[bytes] - - -# pylint: disable-next=abstract-method -class BigQueryTask(Task): - """A task which loads data from a Django queryset into a BigQuery table.""" - - TABLE_NAMES: t.Set[str] = set() - - WriteDisposition: t.TypeAlias = WriteDisposition # shorthand - GetQuerySet: t.TypeAlias = t.Callable[..., QuerySet[t.Any]] - - @dataclass(frozen=True) - # pylint: disable-next=too-many-instance-attributes - class Settings: - """The settings for a BigQuery task.""" - - # The BigQuery table's write disposition. - write_disposition: str - # The number of rows to write at a time. Must be a multiple of 10. - chunk_size: int - # The [Django model] fields to include in the CSV. - fields: t.List[str] - # The name of the field used to identify each row. - id_field: str = "id" - # The maximum amount of time this task is allowed to take before it's - # hard-killed. - time_limit: int = 3600 - # The name of the BigQuery table where the data will ultimately be - # saved. If not provided, the name of the decorated function is used. - table_name: t.Optional[str] = None - # The maximum number of retries allowed. - max_retries: int = 5 - # The countdown before attempting the next retry. - retry_countdown: int = 10 - # The additional keyword arguments to pass to the Celery task decorator. - kwargs: KwArgs = field(default_factory=dict) - - def __post_init__(self): - # Set required values as defaults. - self.kwargs.setdefault("bind", True) - self.kwargs.setdefault("base", BigQueryTask) - - # Ensure the ID field is always present. - if self.id_field not in self.fields: - self.fields.append(self.id_field) - - # Validate args. - if not self.write_disposition.startswith("WRITE_") or not hasattr( - WriteDisposition, self.write_disposition - ): - raise ValidationError( - f'The write disposition "{self.write_disposition}"' - " does not exist.", - code="write_disposition_does_not_exist", - ) - if self.chunk_size <= 0: - raise ValidationError( - "The chunk size must be > 0.", - code="chunk_size_lte_0", - ) - if self.chunk_size % 10 != 0: - raise ValidationError( - "The chunk size must be a multiple of 10.", - code="chunk_size_not_multiple_of_10", - ) - if len(self.fields) <= 1: - raise ValidationError( - "Must provide at least 1 field (not including ID field).", - code="no_fields", - ) - if len(self.fields) != len(set(self.fields)): - raise ValidationError( - "Fields must be unique.", - code="duplicate_fields", - ) - if self.time_limit <= 0: - raise ValidationError( - "The time limit must be > 0.", - code="time_limit_lte_0", - ) - if self.time_limit > 3600: - raise ValidationError( - "The time limit must be <= 3600 (1 hour).", - code="time_limit_gt_3600", - ) - if self.max_retries < 0: - raise ValidationError( - "The max retries must be >= 0.", - code="max_retries_lt_0", - ) - if self.retry_countdown < 0: - raise ValidationError( - "The retry countdown must be >= 0.", - code="retry_countdown_lt_0", - ) - if self.kwargs["bind"] is not True: - raise ValidationError( - "The task must be bound.", code="task_unbound" - ) - if not issubclass(self.kwargs["base"], BigQueryTask): - raise ValidationError( - f"The base must be a subclass of " - f"'{BigQueryTask.__module__}." - f"{BigQueryTask.__qualname__}'.", - code="base_not_subclass", - ) - - settings: Settings - get_queryset: GetQuerySet - - @classmethod - def register_table_name(cls, table_name: str): - """Register a table name to ensure it is unique. - - Args: - table_name: The name of the table to register. - - Raises: - ValidationError: If the table name is already registered. - """ - - if table_name in cls.TABLE_NAMES: - raise ValidationError( - f'The table name "{table_name}" is already registered.', - code="table_name_already_registered", - ) - - cls.TABLE_NAMES.add(table_name) - - def get_ordered_queryset(self, *task_args, **task_kwargs): - """Get the ordered queryset. - - Args: - task_args: The positional arguments passed to the task. - task_kwargs: The keyword arguments passed to the task. - - Returns: - The ordered queryset. - """ - - queryset = self.get_queryset(*task_args, **task_kwargs) - if not queryset.ordered: - queryset = queryset.order_by(self.settings.id_field) - - return queryset - - @staticmethod - def format_value_for_csv(value: t.Any) -> str: - """Format a value for inclusion in a CSV file. - - Args: - value: The value to format. - - Returns: - The formatted value as a string. - """ - - if value is None: - return "" # BigQuery treats an empty string as NULL/None. - if isinstance(value, datetime): - return ( - value.astimezone(timezone.utc) - .replace(tzinfo=None) - .isoformat(sep=" ") - ) - if isinstance(value, (date, time)): - return value.isoformat() - if not isinstance(value, str): - return str(value) - - return value - - @classmethod - def write_queryset_to_csv( - cls, - fields: t.List[str], - chunk_size: int, - queryset: QuerySet[t.Any], - csv_file: "CsvFile", - ): - """Write a queryset to a CSV file. - - Args: - fields: The list of fields to include in the CSV. - chunk_size: The number of rows to write at a time. - queryset: The queryset to write. - csv_file: The CSV file to write to. - - Returns: - Whether any values were written to the CSV file. - """ - - text_wrapper = io.TextIOWrapper(csv_file, encoding="utf-8", newline="") - - csv_writer = csv.writer( - text_wrapper, lineterminator="\n", quoting=csv.QUOTE_MINIMAL - ) - csv_writer.writerow(fields) # Write the headers. - - chunk_index = 1 # 1 based index. For logging. - wrote_values = False # Track if any values were written. - - for row_index, values in enumerate( - t.cast( - t.Iterator[t.Tuple[t.Any, ...]], - # Iterate chunks to avoid OOM for large querysets. - queryset.values_list(*fields).iterator(chunk_size), - ) - ): - if row_index % chunk_size == 0: - logging.info("Writing chunk %d", chunk_index) - chunk_index += 1 - - csv_row = [cls.format_value_for_csv(value) for value in values] - csv_writer.writerow(csv_row) - wrote_values = True - - # Move back 1 byte (because lineterminator is "\n"). - text_wrapper.seek(text_wrapper.tell() - 1) - # Chop off the trailing newline. - text_wrapper.truncate() - # Detach the wrapper to flush data to the binary file. - text_wrapper.detach() - - return wrote_values - - @staticmethod - def load_csv_into_bq( - write_disposition: str, - time_limit: int, - table_name: str, - csv_file: "CsvFile", - ): - """Load a CSV file into a BigQuery table. - - Args: - write_disposition: Write disposition for the BigQuery table. - time_limit: The maximum time to wait for the load job to complete. - table_name: The table name in BigQuery. - csv_file: The CSV file to load into BigQuery. - """ - - bq_client = Client( - project=django_settings.GOOGLE_CLOUD_PROJECT_ID, - credentials=get_gcp_service_account_credentials( - token_lifetime_seconds=time_limit - ), - ) - - full_table_id = ".".join( - [ - django_settings.GOOGLE_CLOUD_PROJECT_ID, - django_settings.GOOGLE_CLOUD_BIGQUERY_DATASET_ID, - table_name, - ] - ) - - csv_file.seek(0) # Reset file pointer to the start. - - logging.info("Starting BigQuery load job.") - # Load the temporary CSV file into BigQuery. - bq_load_job = bq_client.load_table_from_file( - file_obj=csv_file, - destination=full_table_id, - job_config=LoadJobConfig( - create_disposition=CreateDisposition.CREATE_IF_NEEDED, - source_format=SourceFormat.CSV, - skip_leading_rows=1, - write_disposition=write_disposition, - time_zone="Etc/UTC", - date_format="YYYY-MM-DD", - time_format="HH24:MI:SS", - datetime_format="YYYY-MM-DD HH24:MI:SS", - ), - ) - - bq_load_job.result() - logging.info( - "Successfully loaded %d rows into to BigQuery table %s.", - bq_load_job.output_rows, - full_table_id, - ) - - @staticmethod - # pylint: disable-next=too-many-locals,bad-staticmethod-argument - def _load_data_into_bq( - self: "BigQueryTask", table_name: str, *task_args, **task_kwargs - ): - queryset = self.get_ordered_queryset(*task_args, **task_kwargs) - - with NamedTemporaryFile( - mode="w+b", suffix=".csv", delete=True - ) as csv_file: - if self.write_queryset_to_csv( - fields=self.settings.fields, - chunk_size=self.settings.chunk_size, - queryset=queryset, - csv_file=csv_file, - ): - self.load_csv_into_bq( - write_disposition=self.settings.write_disposition, - time_limit=self.settings.time_limit, - table_name=table_name, - csv_file=csv_file, - ) - - @classmethod - def shared(cls, settings: Settings): - """Create a shared BigQuery task. - - This decorator creates a Celery task that saves the queryset to a - BigQuery table. - - Each task *must* be given a distinct table name and queryset to avoid - unintended consequences. - - Examples: - ``` - @BigQueryTask.shared( - BigQueryTask.Settings( - # table_name = "example", <- explicitly set the table name - write_disposition=BigQueryTask.WriteDisposition.WRITE_TRUNCATE, - chunk_size=1000, - fields=["first_name", "joined_at", "is_active"], - ) - ) - def user(): # All users will be saved to a BQ table named "user". - return User.objects.all() - ``` - - Args: - settings: The settings for this BigQuery task. - - Returns: - A wrapper-function which expects to receive a callable that returns - a queryset and returns a Celery task to save the queryset to - BigQuery. - """ - - def wrapper(get_queryset: "BigQueryTask.GetQuerySet"): - table_name = settings.table_name or get_queryset.__name__ - cls.register_table_name(table_name) - - # Wraps the task with retry logic. - def task(self: "BigQueryTask", *task_args, **task_kwargs): - try: - cls._load_data_into_bq( - self, table_name, *task_args, **task_kwargs - ) - except Exception as exc: - raise self.retry( - args=task_args, - kwargs=task_kwargs, - exc=exc, - countdown=settings.retry_countdown, - ) - - # Namespace the task with service's name. If the name is not - # explicitly provided, it defaults to the name of the decorated - # function. - name = settings.kwargs.pop("name", None) - name = get_task_name( - name if isinstance(name, str) else get_queryset - ) - - return t.cast( - BigQueryTask, - _shared_task( # type: ignore[call-overload] - **settings.kwargs, - name=name, - time_limit=settings.time_limit, - max_retries=settings.max_retries, - settings=settings, - get_queryset=staticmethod(get_queryset), - )(task), - ) - - return wrapper diff --git a/codeforlife/tasks/bigquery_test.py b/codeforlife/tasks/bigquery_test.py deleted file mode 100644 index 41b43b72..00000000 --- a/codeforlife/tasks/bigquery_test.py +++ /dev/null @@ -1,379 +0,0 @@ -""" -© Ocado Group -Created on 02/10/2025 at 17:22:38(+01:00). -""" - -import csv -import io -import os -import typing as t -from datetime import date, datetime, time, timedelta, timezone -from tempfile import NamedTemporaryFile -from unittest.mock import MagicMock - -from celery import Celery -from django.conf import settings -from django.db.models.query import QuerySet -from google.cloud.bigquery import CreateDisposition, SourceFormat - -from ..tests import CeleryTestCase -from ..types import KwArgs -from ..user.models import User -from .bigquery import BigQueryTask - -if t.TYPE_CHECKING: - from tempfile import _TemporaryFileWrapper - -CsvFile = t.Union[io.BufferedReader, "_TemporaryFileWrapper[bytes]"] - -# pylint: disable=missing-class-docstring - - -# pylint: disable-next=too-many-instance-attributes,too-many-public-methods -class TestLoadDataIntoBigQueryTask(CeleryTestCase): - fixtures = ["school_1"] - - append_users: BigQueryTask - truncate_users: BigQueryTask - - @staticmethod - def _get_users(order_by: t.Optional[str] = None): - queryset = User.objects.all() - if order_by: - queryset = queryset.order_by(order_by) - return queryset - - @classmethod - def setUpClass(cls): - cls.app = Celery(broker="memory://") - - cls.append_users = BigQueryTask.shared( - BigQueryTask.Settings( - table_name="user__append", - write_disposition=BigQueryTask.WriteDisposition.WRITE_APPEND, - chunk_size=10, - fields=["first_name", "is_active"], - ) - )(cls._get_users) - - cls.truncate_users = BigQueryTask.shared( - BigQueryTask.Settings( - table_name="user__truncate", - write_disposition=BigQueryTask.WriteDisposition.WRITE_TRUNCATE, - chunk_size=10, - fields=["first_name", "is_active"], - ) - )(cls._get_users) - - return super().setUpClass() - - def setUp(self): - def target(relative_dot_path: str): # Shortcut for patching. - return f"{BigQueryTask.__module__}.{relative_dot_path}" - - # Mock creating a NamedTemporaryFile. - # pylint: disable-next=consider-using-with - self.csv_file = NamedTemporaryFile( - mode="w+b", suffix=".csv", delete=False - ) - self.addCleanup(os.remove, self.csv_file.name) - self.mock_named_temporary_file = self.patch( - target("NamedTemporaryFile"), return_value=self.csv_file - ) - - # Mock getting GCP service account credentials. - self.credentials = "I can haz cheezburger?" - self.mock_get_gcp_service_account_credentials = self.patch( - target("get_gcp_service_account_credentials"), - return_value=self.credentials, - ) - - # Mock BigQuery client and its methods. - self.mock_bq_client = MagicMock() - self.mock_bq_client_class = self.patch( - target("Client"), return_value=self.mock_bq_client - ) - - # Mock load_table_from_file method and its result(). - self.mock_load_table_from_file: MagicMock = ( - self.mock_bq_client.load_table_from_file - ) - self.mock_load_job = MagicMock() - self.mock_load_table_from_file.return_value = self.mock_load_job - self.mock_load_job_result: MagicMock = self.mock_load_job.result - self.job_config = MagicMock() - self.mock_load_job_config_class = self.patch( - target("LoadJobConfig"), return_value=self.job_config - ) - - return super().setUp() - - # assertions - - def _assert_queryset_written_to_csv( - self, - queryset: QuerySet[t.Any], - fields: t.List[str], - csv_file: t.Optional[CsvFile] = None, - ): - # Read the actual CSV content. - csv_file = csv_file or self.csv_file - csv_file.seek(0) - actual_content = csv_file.read().decode("utf-8") - - # Generate the expected CSV content. - csv_content = io.StringIO() - csv_writer = csv.writer( - csv_content, lineterminator="\n", quoting=csv.QUOTE_MINIMAL - ) - csv_writer.writerow(fields) # Write the headers. - for obj in queryset: - csv_writer.writerow( - [ - BigQueryTask.format_value_for_csv(getattr(obj, field)) - for field in fields - ] - ) - expected_content = csv_content.getvalue().rstrip() - - # Assert the actual CSV content matches the expected content. - assert actual_content == expected_content - - def _assert_csv_file_loaded_into_bigquery( - self, - table_name: str, - token_lifetime_seconds: int, - write_disposition: str, - csv_file: CsvFile, - ): - # Assert BigQuery client was created. - self.mock_get_gcp_service_account_credentials.assert_called_once_with( - token_lifetime_seconds=token_lifetime_seconds - ) - self.mock_bq_client_class.assert_called_once_with( - project=settings.GOOGLE_CLOUD_PROJECT_ID, - credentials=self.credentials, - ) - - # Assert load job was created and run. - self.mock_load_job_config_class.assert_called_once_with( - create_disposition=CreateDisposition.CREATE_IF_NEEDED, - source_format=SourceFormat.CSV, - skip_leading_rows=1, - write_disposition=write_disposition, - time_zone="Etc/UTC", - date_format="YYYY-MM-DD", - time_format="HH24:MI:SS", - datetime_format="YYYY-MM-DD HH24:MI:SS", - ) - self.mock_load_table_from_file.assert_called_once_with( - file_obj=csv_file, - destination=".".join( - [ - settings.GOOGLE_CLOUD_PROJECT_ID, - settings.GOOGLE_CLOUD_BIGQUERY_DATASET_ID, - table_name, - ] - ), - job_config=self.job_config, - ) - self.mock_load_job_result.assert_called_once_with() - - # settings - - # pylint: disable-next=too-many-arguments,too-many-positional-arguments - def _test_settings( - self, - code: str, - write_disposition: str = BigQueryTask.WriteDisposition.WRITE_APPEND, - chunk_size: int = 10, - fields: t.Optional[t.List[str]] = None, - kwargs: t.Optional[KwArgs] = None, - **settings_kwargs, - ): - with self.assert_raises_validation_error(code=code): - BigQueryTask.Settings( - write_disposition=write_disposition, - chunk_size=chunk_size, - fields=fields or ["some_field"], - kwargs=kwargs or {}, - **settings_kwargs, - ) - - def test_settings__write_disposition_does_not_exist(self): - """Write disposition must exist.""" - self._test_settings( - code="write_disposition_does_not_exist", - write_disposition="WRITE_INVALID", - ) - - def test_settings__chunk_size_lte_0(self): - """Chunk size must be > 0.""" - self._test_settings(code="chunk_size_lte_0", chunk_size=0) - - def test_settings__chunk_size_not_multiple_of_10(self): - """Chunk size must be a multiple of 10.""" - self._test_settings(code="chunk_size_not_multiple_of_10", chunk_size=9) - - def test_settings__no_fields(self): - """Must provide at least 1 field (not including ID field).""" - self._test_settings(code="no_fields", fields=["id"]) - - def test_settings__duplicate_fields(self): - """Fields must be unique.""" - self._test_settings(code="duplicate_fields", fields=["email", "email"]) - - def test_settings__time_limit_lte_0(self): - """Time limit must be > 0.""" - self._test_settings(code="time_limit_lte_0", time_limit=0) - - def test_settings__time_limit_gt_3600(self): - """Time limit must be <= 3600 (1 hour).""" - self._test_settings(code="time_limit_gt_3600", time_limit=3601) - - def test_settings__max_retries_lt_0(self): - """Max retries must be >= 0.""" - self._test_settings(code="max_retries_lt_0", max_retries=-1) - - def test_settings__retry_countdown_lt_0(self): - """Retry countdown must be >= 0.""" - self._test_settings(code="retry_countdown_lt_0", retry_countdown=-1) - - def test_settings__task_unbound(self): - """BigQueryTask must be bound.""" - self._test_settings(code="task_unbound", kwargs={"bind": False}) - - def test_settings__base_not_subclass(self): - """Base must be a subclass of BigQueryTask.""" - self._test_settings(code="base_not_subclass", kwargs={"base": int}) - - # register_table_name - - def test_register_table_name__registered(self): - """An already registered table name raises a ValidationError.""" - table_name = self.append_users.settings.table_name - assert table_name - assert table_name in BigQueryTask.TABLE_NAMES - with self.assert_raises_validation_error( - code="table_name_already_registered" - ): - BigQueryTask.register_table_name(table_name) - - def test_register_table_name__unregistered(self): - """An unregistered table name does not raise an error.""" - table_name = "some_unique_table_name" - assert table_name not in BigQueryTask.TABLE_NAMES - BigQueryTask.register_table_name(table_name) - assert table_name in BigQueryTask.TABLE_NAMES - - # format_value_for_csv - - def test_format_value_for_csv__none(self): - """None is converted to an empty string.""" - assert "" == BigQueryTask.format_value_for_csv(None) - - def test_format_value_for_csv__bool(self): - """Booleans are converted to 0 or 1.""" - assert "True" == BigQueryTask.format_value_for_csv(True) - assert "False" == BigQueryTask.format_value_for_csv(False) - - def test_format_value_for_csv__datetime(self): - """Datetimes are converted to ISO 8601 format with a space separator.""" - assert "2025-02-01 11:30:15" == BigQueryTask.format_value_for_csv( - datetime( - year=2025, month=2, day=1, hour=12, minute=30, second=15 - ).replace(tzinfo=timezone(timedelta(hours=1))) - ) - - def test_format_value_for_csv__date(self): - """Dates are converted to ISO 8601 format.""" - assert "2025-02-01" == BigQueryTask.format_value_for_csv( - date(year=2025, month=2, day=1) - ) - - def test_format_value_for_csv__time(self): - """Times are converted to ISO 8601 format, ignoring timezone info.""" - assert "12:30:15" == BigQueryTask.format_value_for_csv( - time(hour=12, minute=30, second=15) - ) - - # get_ordered_queryset - - def _test_get_ordered_queryset(self, order_by: t.Optional[str] = None): - task = self.append_users - queryset = task.get_ordered_queryset(order_by=order_by) - assert queryset.ordered - assert list(queryset) == list( - User.objects.order_by(order_by or task.settings.id_field) - ) - - def test_get_ordered_queryset__pre_ordered(self): - """Does not reorder an already ordered queryset.""" - self._test_get_ordered_queryset(order_by="first_name") - - def test_get_ordered_queryset__post_ordered(self): - """Orders the queryset if not already ordered. The default is by ID.""" - self._test_get_ordered_queryset() - - # write_queryset_to_csv - - def _test_write_queryset_to_csv( - self, - queryset: QuerySet[t.Any], - fields: t.List[str], - chunk_size: int = 10, - ): - assert queryset.exists() == BigQueryTask.write_queryset_to_csv( - fields=fields, - chunk_size=chunk_size, - queryset=queryset, - csv_file=self.csv_file, - ) - - self._assert_queryset_written_to_csv(queryset, fields) - - def test_write_queryset_to_csv__all(self): - """Values are written to the CSV file.""" - queryset = User.objects.all() - assert queryset.exists() - self._test_write_queryset_to_csv(queryset, fields=["first_name"]) - - def test_write_queryset_to_csv__none(self): - """No values are written to the CSV file.""" - queryset = User.objects.none() - assert not queryset.exists() - self._test_write_queryset_to_csv(queryset, fields=["first_name"]) - - # shared - - def _test_shared__write(self, task: BigQueryTask): - self.apply_task(name=task.name) - - # Assert CSV file was created. - self.mock_named_temporary_file.assert_called_once_with( - mode="w+b", suffix=".csv", delete=True - ) - - # Assert queryset was written to CSV. - assert self.csv_file.closed - with open(self.csv_file.name, "rb") as csv_file: - self._assert_queryset_written_to_csv( - queryset=task.get_ordered_queryset(), - fields=task.settings.fields, - csv_file=csv_file, - ) - - self._assert_csv_file_loaded_into_bigquery( - table_name=task.settings.table_name or task.get_queryset.__name__, - token_lifetime_seconds=task.settings.time_limit, - write_disposition=task.settings.write_disposition, - csv_file=self.csv_file, - ) - - def test_shared__write_append(self): - """The append_users task writes data to BigQuery in append mode.""" - self._test_shared__write(self.append_users) - - def test_shared__write_truncate(self): - """The append_users task writes data to BigQuery in truncate mode.""" - self._test_shared__write(self.truncate_users) diff --git a/codeforlife/tasks/utils.py b/codeforlife/tasks/utils.py deleted file mode 100644 index 18b095d8..00000000 --- a/codeforlife/tasks/utils.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -© Ocado Group -Created on 28/03/2025 at 14:37:46(+00:00). - -Custom utilities for Celery tasks. -""" - -import typing as t - -from celery import shared_task as _shared_task -from django.conf import settings - - -def get_task_name(task: t.Union[str, t.Callable]): - """Namespace a task by the service it's in. - - Args: - task: The name of the task. - - Returns: - The name of the task in the format: "{SERVICE_NAME}.{TASK_NAME}". - """ - - if callable(task): - task = f"{task.__module__}.{task.__name__}" - - if not task.startswith(settings.SERVICE_NAME): - task = f"{settings.SERVICE_NAME}.{task}" - - return task - - -def shared_task(*args, **kwargs): - """ - Wrapper around Celery's default shared_task decorator which namespaces all - tasks to a specific service. - """ - - if len(args) == 1 and callable(args[0]): - task = args[0] - return _shared_task(name=get_task_name(task))(task) - - def wrapper(task: t.Callable): - name = kwargs.pop("name", None) - name = get_task_name(name if isinstance(name, str) else task) - return _shared_task(*args, **kwargs, name=name)(task) - - return wrapper - - -def get_local_sqs_url(aws_region: str, service_name: str): - """Get the URL of an SQS queue in the local environment. - - Args: - aws_region: The AWS region. - service_name: The service this SQS queue belongs to. - - Returns: - The SQS queue's URL. - """ - return ( - f"http://sqs.{aws_region}.localhost.localstack.cloud:4566" - f"/000000000000/{service_name}" - ) diff --git a/codeforlife/tests/__init__.py b/codeforlife/tests/__init__.py index 185ecf84..da013ebe 100644 --- a/codeforlife/tests/__init__.py +++ b/codeforlife/tests/__init__.py @@ -8,7 +8,6 @@ from .api import APITestCase, BaseAPITestCase from .api_client import APIClient, BaseAPIClient from .api_request_factory import APIRequestFactory, BaseAPIRequestFactory -from .celery import CeleryTestCase from .exceptions import InterruptPipelineError from .model import ModelTestCase from .model_list_serializer import ( diff --git a/codeforlife/tests/celery.py b/codeforlife/tests/celery.py deleted file mode 100644 index 98a902c2..00000000 --- a/codeforlife/tests/celery.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -© Ocado Group -Created on 01/04/2025 at 16:57:19(+01:00). -""" - -import typing as t -from importlib import import_module - -from celery import Celery, Task -from django.db.models import QuerySet - -from ..tasks import BigQueryTask, get_task_name -from ..types import Args, KwArgs -from .test import TestCase - - -class CeleryTestCase(TestCase): - """A test case for celery tasks.""" - - # The dot-path of the module containing the Celery app. - app_module: str = "application" - # The name of the Celery app. - app_name: str = "celery" - # The Celery app instance. Auto-imported if not set. - app: Celery - - @classmethod - def setUpClass(cls): - if not hasattr(cls, "app"): - cls.app = getattr(import_module(cls.app_module), cls.app_name) - - return super().setUpClass() - - def apply_task( - self, - name: str, - args: t.Optional[Args] = None, - kwargs: t.Optional[KwArgs] = None, - **options - ): - """Apply a task. - - Args: - name: The name of the task. - args: The args to pass to the task. - kwargs: The keyword args to pass to the task. - """ - task: Task = self.app.tasks[get_task_name(name)] - task.apply(args=args, kwargs=kwargs, **options) - - def assert_bigquery_task( - self, - task: BigQueryTask, - args: t.Optional[Args] = None, - kwargs: t.Optional[KwArgs] = None, - ): - """Assert that the decorated data warehouse task returns a queryset. - - Args: - task: The data warehouse task. - args: The args to pass to the task. - kwargs: The keyword args to pass to the task. - """ - args, kwargs = args or tuple(), kwargs or {} - queryset = task.get_queryset(*args, **kwargs) - self.assertIsInstance(queryset, QuerySet) - queryset.values_list(*task.settings.fields) # assert fields in queryset diff --git a/codeforlife/tests/model_view_set.py b/codeforlife/tests/model_view_set.py index 9b2fb342..1918df87 100644 --- a/codeforlife/tests/model_view_set.py +++ b/codeforlife/tests/model_view_set.py @@ -83,9 +83,8 @@ def reverse_action( if model is not None: lookup_url_kwarg = self.model_view_set_class.lookup_url_kwarg lookup_field = self.model_view_set_class.lookup_field - reverse_kwargs[lookup_url_kwarg or lookup_field] = getattr( - model, lookup_field - ) + lookup = lookup_url_kwarg or lookup_field + reverse_kwargs[lookup] = getattr(model, lookup) name = name.replace("_", "-") return reverse( diff --git a/codeforlife/types.py b/codeforlife/types.py index 6396e9b1..2f660587 100644 --- a/codeforlife/types.py +++ b/codeforlife/types.py @@ -11,8 +11,6 @@ Env = t.Literal["local", "development", "staging", "production"] -DatabaseEngine = t.Literal["postgresql", "sqlite"] - Args = t.Tuple[t.Any, ...] KwArgs = t.Dict[str, t.Any] diff --git a/codeforlife/urls/patterns.py b/codeforlife/urls/patterns.py index 2c649e4f..d5566f9f 100644 --- a/codeforlife/urls/patterns.py +++ b/codeforlife/urls/patterns.py @@ -5,30 +5,22 @@ import typing as t -from django.conf import settings from django.contrib import admin from django.urls import URLPattern, URLResolver, include, path -from ..views import ( - CsrfCookieView, - HealthCheckView, - LogoutView, - session_expired_view, -) +from ..views import CsrfCookieView, LogoutView, session_expired_view UrlPatterns = t.List[t.Union[URLResolver, URLPattern]] def get_urlpatterns( api_url_patterns: UrlPatterns, - health_check_view: t.Type[HealthCheckView] = HealthCheckView, include_user_urls: bool = True, ) -> UrlPatterns: """Generate standard url patterns for each service. Args: api_urls_path: The path to the api's urls. - health_check_view: The health check view to use. include_user_urls: Whether or not to include the CFL's user urls. Returns: @@ -36,17 +28,6 @@ def get_urlpatterns( """ urlpatterns: UrlPatterns = [ - path( - "health-check/", - health_check_view.as_view(), - name="health-check", - ), - ] - - if settings.SERVER_MODE == "celery": - return urlpatterns - - urlpatterns += [ # https://www.django-rest-framework.org/topics/browsable-api/#authentication path( "/", diff --git a/codeforlife/user/auth/backends/email.py b/codeforlife/user/auth/backends/email.py index 58f215e7..14601f89 100644 --- a/codeforlife/user/auth/backends/email.py +++ b/codeforlife/user/auth/backends/email.py @@ -22,13 +22,16 @@ def authenticate( # type: ignore[override] if email is None or password is None: return None + email = self.user_class.objects.normalize_email(email) + # pylint: disable=duplicate-code try: - user = self.user_class.objects.get(email__iexact=email) - if user.check_password(password): - return user + user = self.user_class.objects.get(_email_hash__sha256=email) except self.user_class.DoesNotExist: return None # pylint: enable=duplicate-code + if user.check_password(password): + return user + return None diff --git a/codeforlife/user/auth/backends/google.py b/codeforlife/user/auth/backends/google.py index d40108a6..12fa8170 100644 --- a/codeforlife/user/auth/backends/google.py +++ b/codeforlife/user/auth/backends/google.py @@ -42,7 +42,7 @@ def authenticate( # type: ignore[override] "code_verifier": code_verifier, "redirect_uri": redirect_uri, "client_id": settings.GOOGLE_CLIENT_ID, - "client_secret": settings.GOOGLE_CLIENT_SECRET, + "client_secret": settings.GOOGLE_CLIENT_SECRET(), "grant_type": "authorization_code", }, timeout=5, diff --git a/codeforlife/user/auth/backends/student.py b/codeforlife/user/auth/backends/student.py index aed1cb0b..aab78dac 100644 --- a/codeforlife/user/auth/backends/student.py +++ b/codeforlife/user/auth/backends/student.py @@ -29,13 +29,14 @@ def authenticate( # type: ignore[override] # pylint: disable=duplicate-code try: user = self.user_class.objects.get( - first_name=first_name, - new_student__class_field__access_code=class_id, + _first_name_hash__sha256=first_name, + new_student__class_field___access_code_hash__sha256=class_id, ) - if user.check_password(password): - return user except self.user_class.DoesNotExist: return None # pylint: enable=duplicate-code + if user.check_password(password): + return user + return None diff --git a/codeforlife/user/caches/google_oauth2_token.py b/codeforlife/user/caches/google_oauth2_token.py index 969ca2a6..ddaed9fd 100644 --- a/codeforlife/user/caches/google_oauth2_token.py +++ b/codeforlife/user/caches/google_oauth2_token.py @@ -55,7 +55,7 @@ def get(cls, key, default=None, version=None): "grant_type": "refresh_token", "refresh_token": user.userprofile.google_refresh_token, "client_id": settings.GOOGLE_CLIENT_ID, - "client_secret": settings.GOOGLE_CLIENT_SECRET, + "client_secret": settings.GOOGLE_CLIENT_SECRET(), }, timeout=10, ) diff --git a/codeforlife/user/filters/klass.py b/codeforlife/user/filters/klass.py index 10bfcc91..39e8f08b 100644 --- a/codeforlife/user/filters/klass.py +++ b/codeforlife/user/filters/klass.py @@ -3,7 +3,6 @@ Created on 24/07/2024 at 13:19:57(+01:00). """ -from django.db.models import Q # isort: skip from django.db.models.query import QuerySet # isort: skip from django_filters import ( # type: ignore[import-untyped] # isort: skip rest_framework as filters, @@ -16,15 +15,24 @@ # pylint: disable-next=missing-class-docstring class ClassFilterSet(FilterSet): _id = filters.CharFilter(method="_id__method") - _id__method = FilterSet.make_exclude_field_list_method("access_code") + _id__method = FilterSet.make_exclude_field_list_method( + "_access_code_hash", "sha256_in" + ) id_or_name = filters.CharFilter(method="id_or_name__method") def id_or_name__method(self, queryset: QuerySet[Class], _: str, value: str): """Get classes where the id or the name contain a substring.""" - return queryset.filter( - Q(access_code__icontains=value) | Q(name__icontains=value) - ) + value = value.lower() + pks = [ + klass.pk + for klass in queryset.select_related("teacher__school").only( + "_access_code_enc", "_name_enc", "teacher__school__dek" + ) + if value in klass.name.lower() or value in klass.access_code.lower() + ] + + return queryset.filter(pk__in=pks) class Meta: model = Class diff --git a/codeforlife/user/filters/user.py b/codeforlife/user/filters/user.py index 9a62dc6e..013466f8 100644 --- a/codeforlife/user/filters/user.py +++ b/codeforlife/user/filters/user.py @@ -5,7 +5,6 @@ import typing as t -from django.db.models import Q # isort: skip from django.db.models.query import QuerySet # isort: skip from django_filters import ( # type: ignore[import-untyped] # isort: skip rest_framework as filters, @@ -23,8 +22,8 @@ # pylint: disable-next=missing-class-docstring class UserFilterSet(FilterSet): students_in_class = filters.CharFilter( - "new_student__class_field__access_code", - "exact", + "new_student__class_field___access_code_hash", + "sha256", ) _id = filters.NumberFilter(method="_id__method") @@ -50,13 +49,18 @@ def name__method( first_name, last_name = ( names if len(names) == 2 else (names[0], names[0]) ) + first_name, last_name = first_name.lower(), last_name.lower() - # TODO: use PostgreSQL specific lookup - # https://docs.djangoproject.com/en/5.0/ref/contrib/postgres/lookups/#std-fieldlookup-trigram_similar - return queryset.filter( - Q(first_name__icontains=first_name) - | Q(last_name__icontains=last_name) - ) + pks = [ + user.pk + for user in queryset.only( + "dek", "_first_name_enc", "_last_name_enc" + ) + if first_name in user.first_name.lower() + or last_name in user.last_name.lower() + ] + + return queryset.filter(pk__in=pks) def type__method( self: FilterSet, diff --git a/codeforlife/user/fixtures/google_users.json b/codeforlife/user/fixtures/google_users.json index 13cd2a2a..c9d02b8b 100644 --- a/codeforlife/user/fixtures/google_users.json +++ b/codeforlife/user/fixtures/google_users.json @@ -3,10 +3,18 @@ "model": "user.user", "pk": 34, "fields": { - "first_name": "Google", - "last_name": "Teacher", - "username": "google.teacher@noschool.com", - "email": "google.teacher@noschool.com", + "_email_enc": "ZmFrZV9lbmM6yMZoeJAUbdxMaVb2rDtkRymEfNCh6Z57unJt0wfyDhpR2SuwDh6iqEWHcwKOiTLFSd2PBjTXnQ==", + "_email_hash": "ee95f43c0012fa1a9d5771313a7034cf94af568b0588b34ca20c34c25701af78", + "_email_plain": "google.teacher@noschool.com", + "_first_name_enc": "ZmFrZV9lbmM6KXn5yJbvaIAq1O8qATyemFxIK7GJRkWh1tdv/QaBc/4PQw==", + "_first_name_hash": "5463f3f79e73077256f212d9e43a2752447cb7b238cda23004926bc0b9c5076f", + "_first_name_plain": "Google", + "_last_name_enc": "ZmFrZV9lbmM6DYRgnaINtnv2v6s09apDac1iGUCekmo7k4MuZ5TVKwhWdUE=", + "_last_name_plain": "Teacher", + "_username_enc": "ZmFrZV9lbmM6neI4BdO9uaVQxpJirgXjv1zaBdn5G850jNO4G+yLhIFggC2qe6BUBfvGOIfPHcRpOLNjeC/eVA==", + "_username_hash": "ee95f43c0012fa1a9d5771313a7034cf94af568b0588b34ca20c34c25701af78", + "_username_plain": "google.teacher@noschool.com", + "dek": "ZmFrZV9lbmM658n6erabwmKyOMZuC5pc34LlWgk2C+OounRdomC5y4HgPHs82e9t36Ht5XDh3oMcduegpgH6KtzPYofw", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -14,16 +22,16 @@ "model": "user.userprofile", "pk": 34, "fields": { - "user": 34, - "is_verified": true + "is_verified": true, + "user": 34 } }, { "model": "user.teacher", "pk": 13, "fields": { - "user": 34, - "new_user": 34 + "new_user": 34, + "user": 34 } } ] diff --git a/codeforlife/user/fixtures/independent.json b/codeforlife/user/fixtures/independent.json index 71d2f958..67feca8b 100644 --- a/codeforlife/user/fixtures/independent.json +++ b/codeforlife/user/fixtures/independent.json @@ -3,10 +3,18 @@ "model": "user.user", "pk": 28, "fields": { - "first_name": "Indy", - "last_name": "Requester", - "username": "indy.requester@email.com", - "email": "indy.requester@email.com", + "_email_enc": "ZmFrZV9lbmM6BbOMTZgfEvfGWqqJMXewiatccqnrYAwXP0TL2Yg0XcRCkDjU88u6fN7m92841onGLZd38g==", + "_email_hash": "8efc7683cec3a31792d0c22f49a1cc374ba2ae2199b3cb8d228a7cbabe8370b2", + "_email_plain": "indy.requester@email.com", + "_first_name_enc": "ZmFrZV9lbmM6pCLIN/sWZNWKMxn/nDrhIxPqZJOvgjlMCMMbSP6LavQ=", + "_first_name_hash": "c90d2a9004eccdad8bec752f90ffe656955c44a81f984d3d32b827f17b473d7c", + "_first_name_plain": "Indy", + "_last_name_enc": "ZmFrZV9lbmM6/3o0FjosbmbS1oVTg10ezTJdBjsJBeuKxIHRFNBmZPASWT+9dA==", + "_last_name_plain": "Requester", + "_username_enc": "ZmFrZV9lbmM6JGWj0OAH3zI4LTiClJre31xNzlULtOKIzFOS6JgTsFfMkeDtW6VSWme6PwPwn294DfkYbw==", + "_username_hash": "8efc7683cec3a31792d0c22f49a1cc374ba2ae2199b3cb8d228a7cbabe8370b2", + "_username_plain": "indy.requester@email.com", + "dek": "ZmFrZV9lbmM6L2LWK8EitcxU86uC69XSb5nbOso2Hsy7FjBH14+a3er944CPFMfyBfV0x3Hs3fxjYHMqP/hUpoPxRx6g", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -14,27 +22,35 @@ "model": "user.userprofile", "pk": 28, "fields": { - "user": 28, - "is_verified": true + "is_verified": true, + "user": 28 } }, { "model": "user.student", "pk": 18, "fields": { - "user": 28, "new_user": 28, - "pending_class_request": 6 + "pending_class_request": 6, + "user": 28 } }, { "model": "user.user", "pk": 30, "fields": { - "first_name": "Indy", - "last_name": "NoRequest", - "username": "indy@email.com", - "email": "indy@email.com", + "_email_enc": "ZmFrZV9lbmM6cGgLFQ0NIcL8dvu9xyEsz5Stwyvl7Qp9OYPPnCQAv/+ZO6yF2vjR6iI1", + "_email_hash": "63e90930d7617f596f1006362382d81aecb8308c5a9893007f78c50e954e6d1a", + "_email_plain": "indy@email.com", + "_first_name_enc": "ZmFrZV9lbmM6P0Fsz+sx2cuXNgTPn0AoikMvFz67Uy2F6X+I2kCPTOg=", + "_first_name_hash": "c90d2a9004eccdad8bec752f90ffe656955c44a81f984d3d32b827f17b473d7c", + "_first_name_plain": "Indy", + "_last_name_enc": "ZmFrZV9lbmM6aOhhzg9mD3C0ROh1lCuDC1XKskZ6DYh6ajmv7jRUFFx4GyBY4w==", + "_last_name_plain": "NoRequest", + "_username_enc": "ZmFrZV9lbmM6Mex03gOlRJsMhPARyGhxY10G8lrOTR0l7LSV3AeH2IgllqHHMkd8fkvB", + "_username_hash": "63e90930d7617f596f1006362382d81aecb8308c5a9893007f78c50e954e6d1a", + "_username_plain": "indy@email.com", + "dek": "ZmFrZV9lbmM6kNpc5tLn74rFhZeIxmToumntwifCY5oqPfrL3VTp7Xa962lNlx3jEdSyUbn2WjHaAHLrKWINQQX9f6dp", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -42,16 +58,16 @@ "model": "user.userprofile", "pk": 30, "fields": { - "user": 30, - "is_verified": true + "is_verified": true, + "user": 30 } }, { "model": "user.student", "pk": 20, "fields": { - "user": 30, - "new_user": 30 + "new_user": 30, + "user": 30 } } ] diff --git a/codeforlife/user/fixtures/legacy.json b/codeforlife/user/fixtures/legacy.json index ff6bb9dc..3a1f2d37 100644 --- a/codeforlife/user/fixtures/legacy.json +++ b/codeforlife/user/fixtures/legacy.json @@ -3,241 +3,243 @@ "model": "user.userprofile", "pk": 1, "fields": { - "user": 2, - "otp_secret": null, - "last_otp_for_time": null, "developer": true, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 2 } }, { "model": "user.userprofile", "pk": 2, "fields": { - "user": 3, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 3 } }, { "model": "user.userprofile", "pk": 3, "fields": { - "user": 4, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 4 } }, { "model": "user.userprofile", "pk": 4, "fields": { - "user": 5, - "otp_secret": null, - "last_otp_for_time": null, "developer": true, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 5 } }, { "model": "user.userprofile", "pk": 5, "fields": { - "user": 6, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 6 } }, { "model": "user.userprofile", "pk": 6, "fields": { - "user": 7, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 7 } }, { "model": "user.userprofile", "pk": 7, "fields": { - "user": 8, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 8 } }, { "model": "user.userprofile", "pk": 8, "fields": { - "user": 9, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 9 } }, { "model": "user.userprofile", "pk": 9, "fields": { - "user": 10, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 10 } }, { "model": "user.userprofile", "pk": 10, "fields": { - "user": 11, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 11 } }, { "model": "user.userprofile", "pk": 11, "fields": { - "user": 12, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 12 } }, { "model": "user.userprofile", "pk": 12, "fields": { - "user": 13, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 13 } }, { "model": "user.userprofile", "pk": 13, "fields": { - "user": 14, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 14 } }, { "model": "user.userprofile", "pk": 14, "fields": { - "user": 15, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 15 } }, { "model": "user.userprofile", "pk": 15, "fields": { - "user": 16, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 16 } }, { "model": "user.userprofile", "pk": 16, "fields": { - "user": 17, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 17 } }, { "model": "user.userprofile", "pk": 17, "fields": { - "user": 18, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 18 } }, { "model": "user.userprofile", "pk": 18, "fields": { - "user": 19, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 19 } }, { "model": "user.userprofile", "pk": 19, "fields": { - "user": 20, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 20 } }, { "model": "user.userprofile", "pk": 20, "fields": { - "user": 1, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": true + "is_verified": true, + "last_otp_for_time": null, + "otp_secret": null, + "user": 1 } }, { "model": "user.userprofile", "pk": 21, "fields": { - "user": 21, - "otp_secret": null, - "last_otp_for_time": null, "developer": false, - "is_verified": false + "is_verified": false, + "last_otp_for_time": null, + "otp_secret": null, + "user": 21 } }, { "model": "user.school", "pk": 1, "fields": { - "name": "Swiss Federal Polytechnic", + "_name_enc": "ZmFrZV9lbmM6OZ5fpw/7lnlu6VCmZ1D9j/ypiKMWMpPS4xgpUVojgcT7kmoAI8VUeao08V7yLsHnE+SyTew=", + "_name_plain": "Swiss Federal Polytechnic", "country": "GB", "county": "nan", "creation_time": null, + "dek": "ZmFrZV9lbmM6+bOK8/jOfj3VDE4HWSqQTMr9+j/QCITdy63oBgTo7tLAfLBvdQBtdP9NkAe1hRpvA8yHhxE9Wd+nBwvh", "is_active": true } }, @@ -245,343 +247,366 @@ "model": "user.teacher", "pk": 1, "fields": { - "user": 1, + "blocked_time": null, + "invited_by": null, + "is_admin": true, "new_user": 2, "school": 1, - "is_admin": true, - "blocked_time": null, - "invited_by": null + "user": 1 } }, { "model": "user.teacher", "pk": 2, "fields": { - "user": 2, + "blocked_time": null, + "invited_by": null, + "is_admin": false, "new_user": 3, "school": 1, - "is_admin": false, - "blocked_time": null, - "invited_by": null + "user": 2 } }, { "model": "user.teacher", "pk": 3, "fields": { - "user": 3, + "blocked_time": null, + "invited_by": null, + "is_admin": true, "new_user": 4, "school": 1, - "is_admin": true, - "blocked_time": null, - "invited_by": null + "user": 3 } }, { "model": "user.teacher", "pk": 4, "fields": { - "user": 20, + "blocked_time": null, + "invited_by": null, + "is_admin": true, "new_user": 1, "school": 1, - "is_admin": true, - "blocked_time": null, - "invited_by": null + "user": 20 } }, { "model": "user.class", "pk": 1, "fields": { - "name": "Class 101", - "teacher": 1, - "access_code": "AB123", - "classmates_data_viewable": true, - "always_accept_requests": true, + "_access_code_enc": "ZmFrZV9lbmM6yepC98pMREdly4JPlDeSOnh4SE5txcFKptSmTM+RP03F", + "_access_code_hash": "7dcbfe80fd523896acdad8310fe14d804dad5ae7e4dc4975815f3d5ff76aa963", + "_access_code_plain": "AB123", + "_name_enc": "ZmFrZV9lbmM6rfVJBo3PCRlQYodcqpqSFxZp7dYSoex1a8m6nuPs+pP1xkSkBg==", + "_name_plain": "Class 101", "accept_requests_until": null, + "always_accept_requests": true, + "classmates_data_viewable": true, + "created_by": null, "creation_time": null, "is_active": true, - "created_by": null + "teacher": 1 } }, { "model": "user.class", "pk": 2, "fields": { - "name": "Class 102", - "teacher": 2, - "access_code": "AB124", - "classmates_data_viewable": true, - "always_accept_requests": true, + "_access_code_enc": "ZmFrZV9lbmM633mwB2JzuTaoAcD+HEj6ZQUYY9ZM2Ch0gwrBtbawUJce", + "_access_code_hash": "ee16d5af3f31578ab0327da3257b212fe2beefcadef4f6b8ad6c992d153b90aa", + "_access_code_plain": "AB124", + "_name_enc": "ZmFrZV9lbmM6q8B+3LwD7ohrKPUED05T82/7CtGc2VslUmlVOG7o9tba5SG+Ag==", + "_name_plain": "Class 102", "accept_requests_until": null, + "always_accept_requests": true, + "classmates_data_viewable": true, + "created_by": null, "creation_time": null, "is_active": true, - "created_by": null + "teacher": 2 } }, { "model": "user.class", "pk": 3, "fields": { - "name": "Class 103", - "teacher": 2, - "access_code": "AB125", - "classmates_data_viewable": true, - "always_accept_requests": true, + "_access_code_enc": "ZmFrZV9lbmM6ZZ3FDZRvlw+E1lK4dxpki1idTSRAOt9hCFPUfcdeEmfm", + "_access_code_hash": "a895a482b72ec418311b8a9677bcf2c690425993f563f58d4a15b9586753b88b", + "_access_code_plain": "AB125", + "_name_enc": "ZmFrZV9lbmM60/Rt59QRAWw4AcoT0nKDqrFm25OgmMhKElgPi2A/e2oTsy9ZZg==", + "_name_plain": "Class 103", "accept_requests_until": null, + "always_accept_requests": true, + "classmates_data_viewable": true, + "created_by": null, "creation_time": null, "is_active": true, - "created_by": null + "teacher": 2 } }, { "model": "user.class", "pk": 4, "fields": { - "name": "Young Coders 101", - "teacher": 3, - "access_code": "RL123", - "classmates_data_viewable": true, - "always_accept_requests": true, + "_access_code_enc": "ZmFrZV9lbmM6lX8pCVUFOyvveBTehUr60jztsv49yrwUJTsZALhc2w89", + "_access_code_hash": "38045227bf3c575c643dc1b34a1e70669f35b90225495bbeba0a53ae5ba82c81", + "_access_code_plain": "RL123", + "_name_enc": "ZmFrZV9lbmM61dGjSEe2fv5xma2w6WNAaxOJ4yJ28pd7TJkdh1eh9LqgDchS/3hDrY+j77I=", + "_name_plain": "Young Coders 101", "accept_requests_until": null, + "always_accept_requests": true, + "classmates_data_viewable": true, + "created_by": null, "creation_time": null, "is_active": true, - "created_by": null + "teacher": 3 } }, { "model": "user.class", "pk": 5, "fields": { - "name": "Portaladmin's class", - "teacher": 4, - "access_code": "PO123", - "classmates_data_viewable": true, - "always_accept_requests": true, + "_access_code_enc": "ZmFrZV9lbmM6tmPl174+K04tllPCnHhyfux6Kw7XPt/dWsDEfr62yjHP", + "_access_code_hash": "732f2cd0915bdea50771ce7c92a4e1b8ea83f6d3323b95dd07619e5ea7f0fd06", + "_access_code_plain": "PO123", + "_name_enc": "ZmFrZV9lbmM6SwqcaZ2noGygc4dHOKUsHTL7lWU79/WrqGgK9XqIgAUQjQmUpJZKLGE9nTHRxQ0=", + "_name_plain": "Portaladmin's class", "accept_requests_until": null, + "always_accept_requests": true, + "classmates_data_viewable": true, + "created_by": null, "creation_time": null, "is_active": true, - "created_by": null + "teacher": 4 } }, { "model": "user.student", "pk": 1, "fields": { + "blocked_time": null, "class_field": 1, "login_id": null, - "user": 4, "new_user": 5, "pending_class_request": null, - "blocked_time": null + "user": 4 } }, { "model": "user.student", "pk": 2, "fields": { + "blocked_time": null, "class_field": 1, "login_id": null, - "user": 5, "new_user": 6, "pending_class_request": null, - "blocked_time": null + "user": 5 } }, { "model": "user.student", "pk": 3, "fields": { + "blocked_time": null, "class_field": null, "login_id": null, - "user": 6, "new_user": 7, "pending_class_request": null, - "blocked_time": null + "user": 6 } }, { "model": "user.student", "pk": 4, "fields": { + "blocked_time": null, "class_field": 2, "login_id": null, - "user": 7, "new_user": 8, "pending_class_request": null, - "blocked_time": null + "user": 7 } }, { "model": "user.student", "pk": 5, "fields": { + "blocked_time": null, "class_field": 2, "login_id": null, - "user": 8, "new_user": 9, "pending_class_request": null, - "blocked_time": null + "user": 8 } }, { "model": "user.student", "pk": 6, "fields": { + "blocked_time": null, "class_field": 3, "login_id": null, - "user": 9, "new_user": 10, "pending_class_request": null, - "blocked_time": null + "user": 9 } }, { "model": "user.student", "pk": 7, "fields": { + "blocked_time": null, "class_field": null, "login_id": null, - "user": 10, "new_user": 11, "pending_class_request": null, - "blocked_time": null + "user": 10 } }, { "model": "user.student", "pk": 8, "fields": { + "blocked_time": null, "class_field": 4, "login_id": null, - "user": 11, "new_user": 12, "pending_class_request": null, - "blocked_time": null + "user": 11 } }, { "model": "user.student", "pk": 9, "fields": { + "blocked_time": null, "class_field": 4, "login_id": null, - "user": 12, "new_user": 13, "pending_class_request": null, - "blocked_time": null + "user": 12 } }, { "model": "user.student", "pk": 10, "fields": { + "blocked_time": null, "class_field": 4, "login_id": null, - "user": 13, "new_user": 14, "pending_class_request": null, - "blocked_time": null + "user": 13 } }, { "model": "user.student", "pk": 11, "fields": { + "blocked_time": null, "class_field": 4, "login_id": null, - "user": 14, "new_user": 15, "pending_class_request": null, - "blocked_time": null + "user": 14 } }, { "model": "user.student", "pk": 12, "fields": { + "blocked_time": null, "class_field": 4, "login_id": null, - "user": 15, "new_user": 16, "pending_class_request": null, - "blocked_time": null + "user": 15 } }, { "model": "user.student", "pk": 13, "fields": { + "blocked_time": null, "class_field": 4, "login_id": null, - "user": 16, "new_user": 17, "pending_class_request": null, - "blocked_time": null + "user": 16 } }, { "model": "user.student", "pk": 14, "fields": { + "blocked_time": null, "class_field": 4, "login_id": null, - "user": 17, "new_user": 18, "pending_class_request": null, - "blocked_time": null + "user": 17 } }, { "model": "user.student", "pk": 15, "fields": { + "blocked_time": null, "class_field": 4, "login_id": null, - "user": 18, "new_user": 19, "pending_class_request": null, - "blocked_time": null + "user": 18 } }, { "model": "user.student", "pk": 16, "fields": { + "blocked_time": null, "class_field": 5, "login_id": null, - "user": 21, "new_user": 21, "pending_class_request": null, - "blocked_time": null + "user": 21 } }, { "model": "user.totalactivity", "pk": 1, "fields": { - "teacher_registrations": 4, - "student_registrations": 14, - "independent_registrations": 2, + "anonymised_unverified_independents": 0, "anonymised_unverified_teachers": 0, - "anonymised_unverified_independents": 0 + "independent_registrations": 2, + "student_registrations": 14, + "teacher_registrations": 4 } }, { "model": "user.user", "pk": 1, "fields": { - "password": "pbkdf2_sha256$1000000$y2lUeK8u5GnocWVuonlViT$TnNTL7sG9pcvz6MotXSSptrHdQzAFxmLnqNbj7syLS4=", - "last_login": null, - "is_superuser": true, - "username": "codeforlife-portal@ocado.com", - "first_name": "Portal", - "last_name": "Admin", - "email": "codeforlife-portal@ocado.com", - "is_staff": true, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6dl/MQRUjrYLzJ8SGT14bzsvvpDfI00N7KlFT01peYBfPDY0mfBC3iT5Dj7hDmvX/yaUeY56NT20=", + "_email_hash": "6229d653107a83cfe416912e62307e9b3bfa5185b06deb74db27576e291ba301", + "_email_plain": "codeforlife-portal@ocado.com", + "_first_name_enc": "ZmFrZV9lbmM6ZPlIinlRmmI7yo7cj16Hxm4N1WVBWgTkcKwRvxvB3Jfjmg==", + "_first_name_hash": "dc0c24809d18768f01857048c816eb3885cb627a759bd2dd5314970adafdaf3e", + "_first_name_plain": "Portal", + "_last_name_enc": "ZmFrZV9lbmM6aRN4l2rGBRhy1m+Qh2RWRvNmADOqa+g4Sn/eUOwvUULe", + "_last_name_plain": "Admin", + "_username_enc": "ZmFrZV9lbmM6W68OaTj6IrJ6dAduESf+9QJrchey291ox5c1UWYQxdiU2zCxNL54aG0oFKCTBZxBJM77J8bHIpk=", + "_username_hash": "6229d653107a83cfe416912e62307e9b3bfa5185b06deb74db27576e291ba301", + "_username_plain": "codeforlife-portal@ocado.com", "date_joined": "2026-02-04T16:02:33.631Z", + "dek": "ZmFrZV9lbmM6nS1csorNRFGQ3FknHozzLOObhQ/iXXf2hH92eeOBy8roL2HNlrK14npw6TYEV7VJpPVyXo17WZp/C6J+", "groups": [], + "is_active": true, + "is_staff": true, + "is_superuser": true, + "last_login": null, + "password": "pbkdf2_sha256$1000000$y2lUeK8u5GnocWVuonlViT$TnNTL7sG9pcvz6MotXSSptrHdQzAFxmLnqNbj7syLS4=", "user_permissions": [] } }, @@ -589,17 +614,25 @@ "model": "user.user", "pk": 2, "fields": { - "password": "pbkdf2_sha256$1000000$IOkFLrqDnlDiEoASMl5OSY$C7gzJeOhwbfq/nJp47LaErHOxgA/3DIkaUI93ZUqauc=", - "last_login": null, - "is_superuser": false, - "username": "alberteinstein@codeforlife.com", - "first_name": "Albert", - "last_name": "Einstein", - "email": "alberteinstein@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM67sZxLliYMKm3uz98nOwt8214HZeWxLDLAiwm2MgLgBzkU4wxAvou4swZvmSm6kM8pnR5XxyuHXR5TA==", + "_email_hash": "9cb6a6152b4d3dfbad65576fb4f6688b56d73cbc65b36b2a7fbb313a1352d3c4", + "_email_plain": "alberteinstein@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6dygqD/GL7Zx+cDPU0dlncOJFQhUZrEJkEWlTHALit+s5Xg==", + "_first_name_hash": "920e5149d016b9ef005052b8f52ec833b0974ecf7fb25dc8efd42d8cb1912bf2", + "_first_name_plain": "Albert", + "_last_name_enc": "ZmFrZV9lbmM6AnE2DA2bdWH5SpZDB5F5PlypOzoZ07TeA2vLqfILyZbm9Y5c", + "_last_name_plain": "Einstein", + "_username_enc": "ZmFrZV9lbmM6kEa5T9wBkXTu33fwbGtFe9rZz2whtqtE+NmBnapbN8aFySphoMJNKjgGIxm9kl1OlmbiYVDjcIQcyQ==", + "_username_hash": "9cb6a6152b4d3dfbad65576fb4f6688b56d73cbc65b36b2a7fbb313a1352d3c4", + "_username_plain": "alberteinstein@codeforlife.com", "date_joined": "2026-02-04T16:02:34.051Z", + "dek": "ZmFrZV9lbmM6BLODLjFqwOGQ2F1gfhBxUDu0x+ysKL/THNOI4YaJWkERqeJ176yQerprvm1m8/kUquwMqV56MMYOcJg+", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$IOkFLrqDnlDiEoASMl5OSY$C7gzJeOhwbfq/nJp47LaErHOxgA/3DIkaUI93ZUqauc=", "user_permissions": [] } }, @@ -607,17 +640,25 @@ "model": "user.user", "pk": 3, "fields": { - "password": "pbkdf2_sha256$1000000$QHwukC6Cv822F2nSjQjLu0$09yKE5c8mFVwIHS5BWccM+q33Rs6XSCcc5rik3RD7eY=", - "last_login": null, - "is_superuser": false, - "username": "maxplanck@codeforlife.com", - "first_name": "Max", - "last_name": "Planck", - "email": "maxplanck@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6kzvUIAUyszIHT6fIJ1fyXPwH+XJScZHIf7FU+g5WkdvRj07vlPDJFJKRzoEWiDQ3JTT0izE=", + "_email_hash": "586984c6c88ed9d06cbb6d64e746ca1c7b71dc8782f68a075aece08fd2b5557a", + "_email_plain": "maxplanck@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6Ago4UQZ3oxNQyigyT60Lb+fyVC87pZtkjk1MpUf41Q==", + "_first_name_hash": "337aa91f0cb585a80fee292721d7e78119da4aa52be054785bf1069302979e5e", + "_first_name_plain": "Max", + "_last_name_enc": "ZmFrZV9lbmM6ZdaW5MUAIckllddasnE72Xm6ZSxl9Z7lR3Hjyc8TDeeNhg==", + "_last_name_plain": "Planck", + "_username_enc": "ZmFrZV9lbmM6NdA7tbr7z1HLyLHeAH4ndHuTQjFHvhixbXqR2vU6ImdteCCd7aceSCjXXCn3U/1JPMnOWKo=", + "_username_hash": "586984c6c88ed9d06cbb6d64e746ca1c7b71dc8782f68a075aece08fd2b5557a", + "_username_plain": "maxplanck@codeforlife.com", "date_joined": "2026-02-04T16:02:34.252Z", + "dek": "ZmFrZV9lbmM6V20JnUxKv0GdaX13KdF574HPbkmjD7RJj7/rFr4wQVtQfCIjK2MHcFryVDsMq2eIJ72uaxXc/1FFSkDp", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$QHwukC6Cv822F2nSjQjLu0$09yKE5c8mFVwIHS5BWccM+q33Rs6XSCcc5rik3RD7eY=", "user_permissions": [] } }, @@ -625,17 +666,25 @@ "model": "user.user", "pk": 4, "fields": { - "password": "pbkdf2_sha256$1000000$OcLHQ75ALkslmiUxFwe1Ft$7hpkII5kIi1lytO4PQ0p/6HdtT6//mPcZfZNjROXqrg=", - "last_login": null, - "is_superuser": false, - "username": "ramleith@codeforlife.com", - "first_name": "Ram", - "last_name": "Leith", - "email": "ramleith@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6LwlZEk/dUtTEdlbhapucNrJocYIk9xPu5Y1NMg9hnmIndIj2+gvjuruwfVXOgGZbA6vQUw==", + "_email_hash": "6df8a9b30d61359c2f434a45e881fd4461899af9f5df0e05fd15ba57e5885878", + "_email_plain": "ramleith@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6MP5pw/kzO6fiWqUFK0ZxqwsWrY66EuDvC1jVIAtulw==", + "_first_name_hash": "62ea6bdedfb21184d0fb2342a6109dfd8ecbe0ed5b80ec8db8b2b414ac01423c", + "_first_name_plain": "Ram", + "_last_name_enc": "ZmFrZV9lbmM6Jn1K31v4LzeQ/qZffGAGMZG0/J6biO5+Y1DEnEOI4CeK", + "_last_name_plain": "Leith", + "_username_enc": "ZmFrZV9lbmM69F34/PbRct5acqudNvNPo4mrJ5YZqvhfjDtKBrdAT/JR7XRfzfPE3dklvAmdu4uEOnIBZQ==", + "_username_hash": "6df8a9b30d61359c2f434a45e881fd4461899af9f5df0e05fd15ba57e5885878", + "_username_plain": "ramleith@codeforlife.com", "date_joined": "2026-02-04T16:02:34.448Z", + "dek": "ZmFrZV9lbmM6TcuSFYFFWC8rXPukd/0YiijU8IaKU4lHNAHMjD2eDz8ncLZsgebHW5bRnJAx+WvX8vZoCtlDPcHrEAuh", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$OcLHQ75ALkslmiUxFwe1Ft$7hpkII5kIi1lytO4PQ0p/6HdtT6//mPcZfZNjROXqrg=", "user_permissions": [] } }, @@ -643,17 +692,25 @@ "model": "user.user", "pk": 5, "fields": { - "password": "pbkdf2_sha256$1000000$6QiofvSo5LPJ0DlGshTh0h$XdFmFi/Al5vuhxw02nlZE/rUG8g2uHxFnHXs46XgT/c=", - "last_login": null, - "is_superuser": false, - "username": "leonardodavinci@codeforlife.com", - "first_name": "Leonardo", - "last_name": "DaVinci", - "email": "leonardodavinci@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6btpo9a5rZlczOWXCdwHL5xOSQxCZc/emn8X0E7IcKyFF7C0VS6LmOVBYa/MqEmol91LPCDXB225KEmo=", + "_email_hash": "55a63981ea426d7470560ecfaf3b53c5ecd90e036edd8eca2b1a33c84178908e", + "_email_plain": "leonardodavinci@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6ZT6hCR6NYDU208WG5Zt2Q18itxlQtSKfLZ/3c5K+bxRw43K6", + "_first_name_hash": "9e84472777869ca6205d06421ed265c1c3cfcdfc35a7de158201227c3c541198", + "_first_name_plain": "Leonardo", + "_last_name_enc": "ZmFrZV9lbmM6Eiq20D36p1cMlSdqcy5Mtqq3hm22MTDChEcibQg985YsRXE=", + "_last_name_plain": "DaVinci", + "_username_enc": "ZmFrZV9lbmM6KUios32zumvdsLvZifvFVDEdGgpgw5dYlyh5EokO0X7851hQXMj1HPI8WoPePOtAQF+ZYLIv31AumHU=", + "_username_hash": "55a63981ea426d7470560ecfaf3b53c5ecd90e036edd8eca2b1a33c84178908e", + "_username_plain": "leonardodavinci@codeforlife.com", "date_joined": "2026-02-04T16:02:34.641Z", + "dek": "ZmFrZV9lbmM6Muw/vyJnAHIelig6sSUZZCY8bk1SS5TCasFP5Sk7U8o5llwR60mq7jAJoLkVme26sZJ0ULm27NyLEvKm", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$6QiofvSo5LPJ0DlGshTh0h$XdFmFi/Al5vuhxw02nlZE/rUG8g2uHxFnHXs46XgT/c=", "user_permissions": [] } }, @@ -661,17 +718,25 @@ "model": "user.user", "pk": 6, "fields": { - "password": "pbkdf2_sha256$1000000$HF1HBQVpMLdAuYb0vt55r1$gY3+SetJqidFfylntSqmmrtremtNk8QuGeAdlHNqD9A=", - "last_login": null, - "is_superuser": false, - "username": "galileogalilei@codeforlife.com", - "first_name": "Galileo", - "last_name": "Galilei", - "email": "galileogalilei@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM65Pk8pnQv+HKcPHLwoXfdCg5l++CiG/neKwfiJtMFNQhqRVU7N4SrdhFq08gcZm19S/L4zYbSbXvLeg==", + "_email_hash": "1c5f6e85fe62be2048d5a3867eff53db48f9bcd6da889b92e68668a4a464afd2", + "_email_plain": "galileogalilei@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6FGiykuz+Sx8x0lgoaRGOOBtiL2fRSKJDSIMLZoxttrQl4fo=", + "_first_name_hash": "cae481802ba39e14537dda2d0125c791bab6e0b4e323e9833591a5de6c50207e", + "_first_name_plain": "Galileo", + "_last_name_enc": "ZmFrZV9lbmM6zQGTj4h6SuBQ+GHF0VPEIZO+bu/PsWXhzhyuHeHIefBYmHE=", + "_last_name_plain": "Galilei", + "_username_enc": "ZmFrZV9lbmM6Y4pOAxErjYVL1xULqX20AJveq6DGfsJqA6tQNtZdoOaeRFAwKUNvdCM/A/vm8LBq6uTpNLGUpudfOA==", + "_username_hash": "1c5f6e85fe62be2048d5a3867eff53db48f9bcd6da889b92e68668a4a464afd2", + "_username_plain": "galileogalilei@codeforlife.com", "date_joined": "2026-02-04T16:02:34.839Z", + "dek": "ZmFrZV9lbmM60JSmixBDvrfWaRyP8idNvMR78k1YyavGZ+lBOARPt05D1PqVrxAP+HK3qajPuD2iizikhEgSh6dF5iyb", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$HF1HBQVpMLdAuYb0vt55r1$gY3+SetJqidFfylntSqmmrtremtNk8QuGeAdlHNqD9A=", "user_permissions": [] } }, @@ -679,17 +744,25 @@ "model": "user.user", "pk": 7, "fields": { - "password": "pbkdf2_sha256$1000000$BUSt8V8F4DwWZPlCZeS1Sc$WMQElSAhvIE7LRHx6lwTde9YG5P74dc7Az+639AHdYc=", - "last_login": null, - "is_superuser": false, - "username": "isaacnewton@codeforlife.com", - "first_name": "Isaac", - "last_name": "Newton", - "email": "isaacnewton@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6nE5NOakc9jQBJf7sLqY2uiRil9e9sUEShSNLiSoHg5FqK/CY2Do/sbV+1A6oxxB85d3HZuL+4Q==", + "_email_hash": "4fc8d589140865b27bc27623cb547e2a24f9b985ab2e1dd0b5b5c035d367c4b6", + "_email_plain": "isaacnewton@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6eCpZHFZkWzQ/hv8G9psJxOAqvezmqYDq0RSpDJlEptGK", + "_first_name_hash": "c49fb23f0176885dece511c58087b7f23019c343b9861e2d70926600577a4611", + "_first_name_plain": "Isaac", + "_last_name_enc": "ZmFrZV9lbmM6xN/iK8bQby0voDF9a35nJ83At3HgLkGbAptBWLtTNtnZzg==", + "_last_name_plain": "Newton", + "_username_enc": "ZmFrZV9lbmM62SGXHdjBqA5Y+dD5VvE3AykKRYr7Rr6b+QvCaibuKwggnxtlx0AGgi1fNYQswsuoa3sskN5e3Q==", + "_username_hash": "4fc8d589140865b27bc27623cb547e2a24f9b985ab2e1dd0b5b5c035d367c4b6", + "_username_plain": "isaacnewton@codeforlife.com", "date_joined": "2026-02-04T16:02:35.036Z", + "dek": "ZmFrZV9lbmM6AgiWjZ7GQyBK5l4VurLVygmXkszQHL3kZhN33iYzyC1HMfWWMVI/ggxa+oNbq2olRmVHguF8uNzBvTo3", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$BUSt8V8F4DwWZPlCZeS1Sc$WMQElSAhvIE7LRHx6lwTde9YG5P74dc7Az+639AHdYc=", "user_permissions": [] } }, @@ -697,17 +770,25 @@ "model": "user.user", "pk": 8, "fields": { - "password": "pbkdf2_sha256$1000000$TaYcwU3iqPA4c4co14ZYHx$FAwypkSE7UucZod6BY/YY+A4zV4JbySNxDwd4ypOZT8=", - "last_login": null, - "is_superuser": false, - "username": "richardfeynman@codeforlife.com", - "first_name": "Richard", - "last_name": "Feynman", - "email": "richardfeynman@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6QTQVq47caqRoq8EK0uUN1ygFV+XvmYPEXq5onCbdj13XdI6DfrCAlx7W7gFAABbpkBCJhQYZ0cETjw==", + "_email_hash": "6bd23daa66a426a27655016f084d3fa65a49e2e5d264701b6d84e550e415638d", + "_email_plain": "richardfeynman@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6KD7poCabt24+OPDhCz9fiZcUqF+PrVEHoIT9gwFbShzh3JU=", + "_first_name_hash": "0a8a5deb9f8f9a8f0e96af7630e59b6fdc0a2528095a5a382c3d417da9c97f7e", + "_first_name_plain": "Richard", + "_last_name_enc": "ZmFrZV9lbmM6JgmcA+18EuPfUeWJSuXju16Z0h+HDj9m/Xn51LaSe+Ef2TA=", + "_last_name_plain": "Feynman", + "_username_enc": "ZmFrZV9lbmM6ahl5fcghPgCSSN8QVcwIyupa5kZvvO9vOiUoJdZRSUbB/aLgN2tPBaCA8QiR6/lkO2182l5Y/mr3Aw==", + "_username_hash": "6bd23daa66a426a27655016f084d3fa65a49e2e5d264701b6d84e550e415638d", + "_username_plain": "richardfeynman@codeforlife.com", "date_joined": "2026-02-04T16:02:35.230Z", + "dek": "ZmFrZV9lbmM67Iqyt69l+yrc2BMF1r5L+UE0BIUZ0HfFN3baguyKwLTeNtTFj1HOcVJAuFtoV+hvIT+1oY7x0Fqul03j", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$TaYcwU3iqPA4c4co14ZYHx$FAwypkSE7UucZod6BY/YY+A4zV4JbySNxDwd4ypOZT8=", "user_permissions": [] } }, @@ -715,17 +796,25 @@ "model": "user.user", "pk": 9, "fields": { - "password": "pbkdf2_sha256$1000000$mCuu3Ptx9EGmKQ7FSNIqzw$dqwQK/ovljow4iGxfLIQTm8PPR8Iu5AITzQ9tVQxv9w=", - "last_login": null, - "is_superuser": false, - "username": "alexanderflemming@codeforlife.com", - "first_name": "Alexander", - "last_name": "Flemming", - "email": "alexanderflemming@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6kQeK+uVf0/b+O1SZqONvKrsfffa8C3pSKCQa6uyF1lPK6FwDVl2vQNCPL5ak/4+a7ja7A00S4i5lpJZr8g==", + "_email_hash": "1ab9d2f1ae8ef6a5e99cee8948c326e3404aceb89b8d72215e297a1cfc8393a6", + "_email_plain": "alexanderflemming@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6ZVz+1VG3scf9MoEfVA/ShO5cJNBE9riNqQ/2I20l/OsroQMZrA==", + "_first_name_hash": "e78f1c3e180b8af2c8dfc79ec04d65ac565361c887584100e67890cdb70ab52b", + "_first_name_plain": "Alexander", + "_last_name_enc": "ZmFrZV9lbmM6jxxUTKIDlEj/scNJ2jRVZujwmG8nYCfRqepcQi7lnBz/vYSI", + "_last_name_plain": "Flemming", + "_username_enc": "ZmFrZV9lbmM6g1cd/om7Tco3iOqnDa5z3mU8wFau/Mq7xTEH6EJWEGe9d68cGq0SmwovpAWW6HmDxF3STUJ6PV8kECAZiA==", + "_username_hash": "1ab9d2f1ae8ef6a5e99cee8948c326e3404aceb89b8d72215e297a1cfc8393a6", + "_username_plain": "alexanderflemming@codeforlife.com", "date_joined": "2026-02-04T16:02:35.422Z", + "dek": "ZmFrZV9lbmM6hGzSBoLIbLU4BDj32rxVFgg6uArfWek5BfmFo5ZoBjV2MhQN5m0dBvuVjzb0t6fTY5d2nMzlUcdIofrF", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$mCuu3Ptx9EGmKQ7FSNIqzw$dqwQK/ovljow4iGxfLIQTm8PPR8Iu5AITzQ9tVQxv9w=", "user_permissions": [] } }, @@ -733,17 +822,25 @@ "model": "user.user", "pk": 10, "fields": { - "password": "pbkdf2_sha256$1000000$isUqjEe0ZtdHC0tDFilALA$k20Gy//4APEqsGNYGa8aQPUTUBqMPKe6hI5GMIWiMnE=", - "last_login": null, - "is_superuser": false, - "username": "danielbernoulli@codeforlife.com", - "first_name": "Daniel", - "last_name": "Bernoulli", - "email": "danielbernoulli@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM64e5Vzz8oKJYjbYI7+2ZF7XuH8wPevFgtQC2X5bc0pWJ5H9bETgDrFyA4sMZaU9Tg9kv+qws4haGfGTg=", + "_email_hash": "0c92dd15cc629b84ea9436c965f944543cfedd41ced6c8f2594a26ee50cf3ea5", + "_email_plain": "danielbernoulli@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM65SWBLZni/laQtNAl64mXA79twEZho+651VCt311rAguFzA==", + "_first_name_hash": "6bc6d4fc17ad3131d3417aebb876bdec85a697de1a69734b9d113c800e72aaf3", + "_first_name_plain": "Daniel", + "_last_name_enc": "ZmFrZV9lbmM60eUQSPe5OAsCeRt1gzrwLa7EVSar6AnRbZG3gHztcvdth8IrvA==", + "_last_name_plain": "Bernoulli", + "_username_enc": "ZmFrZV9lbmM6hCfOHVID4fPRgzyP+oOM6gHabOlxDQwjLsbLuw0ngMJ1SapLlj2nC9fk48FecXELl/JXwRS1/PI5sEY=", + "_username_hash": "0c92dd15cc629b84ea9436c965f944543cfedd41ced6c8f2594a26ee50cf3ea5", + "_username_plain": "danielbernoulli@codeforlife.com", "date_joined": "2026-02-04T16:02:35.611Z", + "dek": "ZmFrZV9lbmM6J4zSpKtOmn7y3Y3C6x4OVGcb+e0E+AW85tNIEZb2pSDinQqzUkft/oBWqEYTqk8llh1JDVFmia1hKBjl", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$isUqjEe0ZtdHC0tDFilALA$k20Gy//4APEqsGNYGa8aQPUTUBqMPKe6hI5GMIWiMnE=", "user_permissions": [] } }, @@ -751,17 +848,25 @@ "model": "user.user", "pk": 11, "fields": { - "password": "pbkdf2_sha256$1000000$mjbmbYr7G3WRyxxHlDcH6d$w1wf3D0lDP6pF3nSoTUD3vSRioCzPEloW3r8sXnLSJw=", - "last_login": null, - "is_superuser": false, - "username": "indianajones@codeforlife.com", - "first_name": "Indiana", - "last_name": "Jones", - "email": "indianajones@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6Bkpg2XLc10D1UmbpVkCI0lg4MJD0zeGOfT07lzxvvkgA52wMafu63U9ylmvgukYRBJr3bWG1XLI=", + "_email_hash": "1a93bc8026b39e9d71124c74cef2f072d4e41625ea3b12f4009f104dce962e2f", + "_email_plain": "indianajones@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM62ghGxpAhcBGTgECH3XG5lhv3/nGPelk1SYi2on3HmiQnnEs=", + "_first_name_hash": "9859180db8b64c329244146e3cfda4e6c42fd38f8ef9fc0f272af81298c6c02a", + "_first_name_plain": "Indiana", + "_last_name_enc": "ZmFrZV9lbmM6excgvA44+7Tzd9kxwsurRR0/7JddJ4ONlEhYSqS+e1XH", + "_last_name_plain": "Jones", + "_username_enc": "ZmFrZV9lbmM6u79RxJjXjQOn3DUiaBeGPzBlfNd5dfa4/yAAH5j3Wwnc57AflACzCpfF713nnD00CTFftFYTf9Y=", + "_username_hash": "1a93bc8026b39e9d71124c74cef2f072d4e41625ea3b12f4009f104dce962e2f", + "_username_plain": "indianajones@codeforlife.com", "date_joined": "2026-02-04T16:02:35.803Z", + "dek": "ZmFrZV9lbmM6RJvZthmhY//D7zh7NTsTeIBZxSsBClnGVaP30RcuabLz1xYLUNNnumUOdygscg+BDZwxu8cwTD2anSyk", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$mjbmbYr7G3WRyxxHlDcH6d$w1wf3D0lDP6pF3nSoTUD3vSRioCzPEloW3r8sXnLSJw=", "user_permissions": [] } }, @@ -769,17 +874,22 @@ "model": "user.user", "pk": 12, "fields": { - "password": "pbkdf2_sha256$1000000$qm1oQx3HNeIx6dpY1aRzXb$xMKwoqXoW6MuHqoABww+Yq+3A1uwkLbkpaWWytNu6l0=", - "last_login": null, - "is_superuser": false, - "username": "media noah", - "first_name": "Noah", - "last_name": "Monaghan", - "email": "", - "is_staff": false, - "is_active": true, + "_first_name_enc": "ZmFrZV9lbmM6kjfxs345zoTZfqKXkDHtfcIY8poSjk8VFfhMq1knz/8=", + "_first_name_hash": "91886b94f7298f93affde2a4c4bb1e1ec836b5da66507834c964dc8db418fa14", + "_first_name_plain": "Noah", + "_last_name_enc": "ZmFrZV9lbmM6eHb/bRCUVrG5ESV5qHfbsiBvRelPu7wOHL8ci370OvxasRpm", + "_last_name_plain": "Monaghan", + "_username_enc": "ZmFrZV9lbmM6AtzaoZyVd75YQrEOaNqUxr14E2Z+9i3KJuJ0cgFi2gzAI7zw85E=", + "_username_hash": "a1e49779f7ae17103934e74a1eea820997bea85334002ec14ffbdeaa2d626eee", + "_username_plain": "media noah", "date_joined": "2026-02-04T16:02:35.999Z", + "dek": "ZmFrZV9lbmM6ODiZ6w5NfOc9/9oP8JH+F0AKguKkTdwnguKY7IUw4yMv7bkTGS0ZDr6lRMeUSB2bUDAAeG/s4o4Rvbsa", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$qm1oQx3HNeIx6dpY1aRzXb$xMKwoqXoW6MuHqoABww+Yq+3A1uwkLbkpaWWytNu6l0=", "user_permissions": [] } }, @@ -787,17 +897,22 @@ "model": "user.user", "pk": 13, "fields": { - "password": "pbkdf2_sha256$1000000$vQZKG1iO3Np3cdOHSrG85F$QnuZLe1KvMt64uWXjz3RKOEnr6owhloxfgs3u8xB6GQ=", - "last_login": null, - "is_superuser": false, - "username": "media elliot", - "first_name": "Elliot", - "last_name": "Sharp", - "email": "", - "is_staff": false, - "is_active": true, + "_first_name_enc": "ZmFrZV9lbmM6BWoSB7BYxH4FxpQhkd4TvieVMj0N3sPdLhCNohLeetFyrg==", + "_first_name_hash": "050fa7990069876e75ceb3fe387a3f10b5beec63612c5b4e500cfcf246daa312", + "_first_name_plain": "Elliot", + "_last_name_enc": "ZmFrZV9lbmM69YjNMSfcXFg3pGr58K77NGxqGG3nV21qsO+sSnFPRvbd", + "_last_name_plain": "Sharp", + "_username_enc": "ZmFrZV9lbmM6P8vG8taJXnEQWOdAwlh7FzalVSbQzMwdx2aku3PZK9wt99K3A9K9/g==", + "_username_hash": "efa5e3d6015a02b88470d47080508a671bfb539b41d18e1893db820f95a228cb", + "_username_plain": "media elliot", "date_joined": "2026-02-04T16:02:36.195Z", + "dek": "ZmFrZV9lbmM6E2af4DEGdnVy8AdysOxQWAr/XZjM1paohrFtzwcU8ErHa4Mds1bm5zK1WoPEwxqoEPD99/XZfBuw7/oa", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$vQZKG1iO3Np3cdOHSrG85F$QnuZLe1KvMt64uWXjz3RKOEnr6owhloxfgs3u8xB6GQ=", "user_permissions": [] } }, @@ -805,17 +920,22 @@ "model": "user.user", "pk": 14, "fields": { - "password": "pbkdf2_sha256$1000000$iCO3IsvmN5C8rpX5J0d1ob$TFK2zYume/i4StDly6EUiqFej/sOnuzV/wYE9rgr/cg=", - "last_login": null, - "is_superuser": false, - "username": "media tajmae", - "first_name": "Tajmae", - "last_name": "Joseph", - "email": "", - "is_staff": false, - "is_active": true, + "_first_name_enc": "ZmFrZV9lbmM663tdyyomEQ2QzpHpj3ZX8IldQVNNvwZoKn7SuywzsKUDvA==", + "_first_name_hash": "969bd503b94e99bb283caa34c5e28fcb8cebfb278111bff1f76ed1f8d08c2720", + "_first_name_plain": "Tajmae", + "_last_name_enc": "ZmFrZV9lbmM6Hb+2sb0IFHydvEnx3koqRuZDYKraKhtD1gLI7XSs5hrM6g==", + "_last_name_plain": "Joseph", + "_username_enc": "ZmFrZV9lbmM631Qr0w4wBZN1Phdldl9WY+1n3NrBZ6QUraZY4p0ZmC8v+9NnUvTvgA==", + "_username_hash": "b154013ded73a161c8ceae084060ed319504d9a3690b569cc54d39b3e985a0bc", + "_username_plain": "media tajmae", "date_joined": "2026-02-04T16:02:36.394Z", + "dek": "ZmFrZV9lbmM6lHwaHwEAEkTsDE+w+LnUw/c/ra+TdL5/dU9xPFIW92I98HzZg50AxHu4n2lKGe/FTzNrR4uCFzFQ/+IL", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$iCO3IsvmN5C8rpX5J0d1ob$TFK2zYume/i4StDly6EUiqFej/sOnuzV/wYE9rgr/cg=", "user_permissions": [] } }, @@ -823,17 +943,22 @@ "model": "user.user", "pk": 15, "fields": { - "password": "pbkdf2_sha256$1000000$6Nhm1sHtcMbFdBitdz7nxq$s5i8H7MwdH6SRQ3+DZDDfsPtqxXVDPAC5t67G4FluM4=", - "last_login": null, - "is_superuser": false, - "username": "media carlton", - "first_name": "Carlton", - "last_name": "Joseph", - "email": "", - "is_staff": false, - "is_active": true, + "_first_name_enc": "ZmFrZV9lbmM6xiOckTwack87H3A7PBretwem+4On+TLQK7vQ40/0FpPNCX8=", + "_first_name_hash": "8e450f405264e612fb62f915830166cd43c03612b82c19c0c52aaee93183b0bb", + "_first_name_plain": "Carlton", + "_last_name_enc": "ZmFrZV9lbmM69RSRhxmcba/7l7H8s49DQTlBtOLzTQUsgdggRgHLYzUELA==", + "_last_name_plain": "Joseph", + "_username_enc": "ZmFrZV9lbmM6DPlTHgfxXTVL0L2uxDpQ+IpoVZymQQMhhSOnTnW8fCtJSV25jdQ+pRw=", + "_username_hash": "e18c768a720dfab2514b66d44c2b0cbf54d5e28f9e68dbc0df5dfde76df02f5e", + "_username_plain": "media carlton", "date_joined": "2026-02-04T16:02:36.589Z", + "dek": "ZmFrZV9lbmM6An/cveM3ZXcZM7QxIgYd4WK7/M3yJiWAe8Vo+pc7PL0TNogWsWByiNucIQqlmfIZARMQcJC/p3pJx1QQ", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$6Nhm1sHtcMbFdBitdz7nxq$s5i8H7MwdH6SRQ3+DZDDfsPtqxXVDPAC5t67G4FluM4=", "user_permissions": [] } }, @@ -841,17 +966,22 @@ "model": "user.user", "pk": 16, "fields": { - "password": "pbkdf2_sha256$1000000$FRA5lx8xBTO5Rn8J9wwZZW$KiQzLItBM40bzbwRdXpKnrI1uWXv0xsSzTMkFwAewwo=", - "last_login": null, - "is_superuser": false, - "username": "media nadal", - "first_name": "Nadal", - "last_name": "Spencer-Jennings", - "email": "", - "is_staff": false, - "is_active": true, + "_first_name_enc": "ZmFrZV9lbmM6OtOk2SsoMWAc38TD0Q41fmaYTkUR0+0nt2Rw/JuDkgb5", + "_first_name_hash": "8424a2115bf1ee8fa253b59cd46cd6487c1b53b51201e593c08a12c390d70ad4", + "_first_name_plain": "Nadal", + "_last_name_enc": "ZmFrZV9lbmM6f8mE9Wc19ubz9l+FnWRJU4dXhOwbl3Vwc2ISVdXXF4GmrbeK+AMRwdZVwDI=", + "_last_name_plain": "Spencer-Jennings", + "_username_enc": "ZmFrZV9lbmM6DDox9dauGKr9FLYT8aNae6nM84ZP0SO66osdvk/A+JR5toUdN+N/", + "_username_hash": "9d75c6e6c5f4793504f8c8be317bb007ff69ffbc9caa0e80acfef15ee15f661b", + "_username_plain": "media nadal", "date_joined": "2026-02-04T16:02:36.792Z", + "dek": "ZmFrZV9lbmM6VtLPjgaz4N8xsAHY6fKs7B4Q6dUNJLcu0e8i02/cbPVf+Xcx/OYE//dzmldTRcj0BEhRE5HG6EV9yJGB", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$FRA5lx8xBTO5Rn8J9wwZZW$KiQzLItBM40bzbwRdXpKnrI1uWXv0xsSzTMkFwAewwo=", "user_permissions": [] } }, @@ -859,17 +989,22 @@ "model": "user.user", "pk": 17, "fields": { - "password": "pbkdf2_sha256$1000000$LAQQycnLxi5rY9QCD15nHw$Ad5r4JEo5CCQrwuD2F12ajXnV/bDSL+FGQ99vM2l0/k=", - "last_login": null, - "is_superuser": false, - "username": "media freddie", - "first_name": "Freddie", - "last_name": "Goff", - "email": "", - "is_staff": false, - "is_active": true, + "_first_name_enc": "ZmFrZV9lbmM6j0YLTeDH49wQDBGAGGslQq+XvbkmjBOjZz/81chsQ6CgduE=", + "_first_name_hash": "8d81abd4d9f4dd116d4473915b068c9790f2229ab72fbcd974daff57ebf62717", + "_first_name_plain": "Freddie", + "_last_name_enc": "ZmFrZV9lbmM6qqw2DvU2GxSQXO2EqczZa2U8FVS4mRcyIZoDpbhoEeI=", + "_last_name_plain": "Goff", + "_username_enc": "ZmFrZV9lbmM6eWftc/iNk2tjFElPm7fCJpsZ1iMQKF5L6M866R+GpcCflBTD9p+YzRI=", + "_username_hash": "aab046dad21372920e2322ed46ad43a8574963ffd7033e1bd5a6285ad07c3b60", + "_username_plain": "media freddie", "date_joined": "2026-02-04T16:02:37.009Z", + "dek": "ZmFrZV9lbmM6e0QkyGSRPJxtPLH/c5zTXK5jkOET732kHyRwCTpiQhiSqj3Ar80GkMDhWaXGvqx4qpDLrzZDLZftBe/8", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$LAQQycnLxi5rY9QCD15nHw$Ad5r4JEo5CCQrwuD2F12ajXnV/bDSL+FGQ99vM2l0/k=", "user_permissions": [] } }, @@ -877,17 +1012,22 @@ "model": "user.user", "pk": 18, "fields": { - "password": "pbkdf2_sha256$1000000$XHhBEg6yrF9a6TR487gNrU$DOR6TONr+9q0rLXJwB43UuezRcQkQAmEmoN0nYmm4do=", - "last_login": null, - "is_superuser": false, - "username": "media leon", - "first_name": "Leon", - "last_name": "Scott", - "email": "", - "is_staff": false, - "is_active": true, + "_first_name_enc": "ZmFrZV9lbmM6UerbiIlXuCiufRtvMUVZ+JauFHZ3y0Fp/GZl7ISGn/o=", + "_first_name_hash": "5081dca0f5b49f2fdfda5543c6bbeef8e733838de7b8e642064f9b8edc0e24b5", + "_first_name_plain": "Leon", + "_last_name_enc": "ZmFrZV9lbmM6bxT3gxf8cicQPv5nGIKLHApI/oxYN3k15TGYe4vwZec0", + "_last_name_plain": "Scott", + "_username_enc": "ZmFrZV9lbmM6iW7kADgeYsJU5l1soljcMXYut3ZM9ax1Xop0oYhAw87tLOln0wM=", + "_username_hash": "f34fecd51a3f21f82a61f6577abf7b69aac0e43bb23c6ad6003ec5df6a7ed54e", + "_username_plain": "media leon", "date_joined": "2026-02-04T16:02:37.216Z", + "dek": "ZmFrZV9lbmM6ivzHA5Vfn50gnAP3KwRspc9mnTuE4qCr43qbCfPHpFP5D5YwNqW8OQnDdDwe/o5a9DdNJ3n85j7MpPU2", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$XHhBEg6yrF9a6TR487gNrU$DOR6TONr+9q0rLXJwB43UuezRcQkQAmEmoN0nYmm4do=", "user_permissions": [] } }, @@ -895,17 +1035,22 @@ "model": "user.user", "pk": 19, "fields": { - "password": "pbkdf2_sha256$1000000$ua0WUUCFIijIjxpLdkc25v$mvl+UejCNGop0krujly8TPlMbgWPFVZ42tH+NPDltS4=", - "last_login": null, - "is_superuser": false, - "username": "media betty", - "first_name": "Betty", - "last_name": "Kessell", - "email": "", - "is_staff": false, - "is_active": true, + "_first_name_enc": "ZmFrZV9lbmM609cs4YE+Kj0atwhR0SOrJyuO9vSA74sqShdgLD7KRQ+C", + "_first_name_hash": "c25e694e58d404373d63ef5dd2b2a413dab9c64aab028d735401c166813a87fa", + "_first_name_plain": "Betty", + "_last_name_enc": "ZmFrZV9lbmM604sjV50gIcumJ2c/iDCvy3Siq/kVoTKllzvkyghd6A2hMoE=", + "_last_name_plain": "Kessell", + "_username_enc": "ZmFrZV9lbmM6yFnuAvCj5RFXC8Iv+rljmAU/zzrUYIerAMmuXI/c4TFVi7+gSaGC", + "_username_hash": "6120be5842a74a935c8d31a810ce2901fece2ab29d35d2d3fa6083b908a44434", + "_username_plain": "media betty", "date_joined": "2026-02-04T16:02:37.413Z", + "dek": "ZmFrZV9lbmM648Mk4GDSMJHDA77reE0u/16wjHi8ujc2YxH6XPgOWc4S85G389Y5//r/RJKhonqH7X1pTIhiUZFbHz5F", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$ua0WUUCFIijIjxpLdkc25v$mvl+UejCNGop0krujly8TPlMbgWPFVZ42tH+NPDltS4=", "user_permissions": [] } }, @@ -913,17 +1058,22 @@ "model": "user.user", "pk": 20, "fields": { - "password": "pbkdf2_sha256$1000000$o2AfZh0JqqIFRTZGHWEQKV$c1es5kdOakAvCGkeMVgDOX5q+rFhdQgOkMZiKAtqMrk=", - "last_login": null, - "is_superuser": false, - "username": "4271ee7b7ce94e34a58d1f4e82025280", - "first_name": "Deleted", - "last_name": "User", - "email": "", - "is_staff": false, - "is_active": false, + "_first_name_enc": "ZmFrZV9lbmM6kEW6JpB05mjWAPE/xrxMrbFNoyD6ZBNVl+YCZI/q3+LRROg=", + "_first_name_hash": "959e43f80903c037d87f933a2225789fbdbd71e0dea09fc1854a51e8d2f9b17e", + "_first_name_plain": "Deleted", + "_last_name_enc": "ZmFrZV9lbmM6gwxaGy9/aDQWDEVUwUverJ6wmk3ohaTkvF5GYq6koNg=", + "_last_name_plain": "User", + "_username_enc": "ZmFrZV9lbmM6kUOy1ae7Y1ElSiAiDXnKIZxPkOs1saBSjmH+UCEThmkOR7lKe9Nlmza/K+mW1Slr0UutzlVaqMLU3yWT", + "_username_hash": "daa37b0878d74480285044dccc67750ad597b99719dc7323462cbe16e0b0271d", + "_username_plain": "4271ee7b7ce94e34a58d1f4e82025280", "date_joined": "2026-02-04T16:02:37.614Z", + "dek": "ZmFrZV9lbmM6Ct68dRt74lesvjRcTVtlbUlPC8xt85anig+MaarMMZAUvcFmkJmno57KkQUR8S5/o5ElV9Bpn3RDpxeS", "groups": [], + "is_active": false, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$o2AfZh0JqqIFRTZGHWEQKV$c1es5kdOakAvCGkeMVgDOX5q+rFhdQgOkMZiKAtqMrk=", "user_permissions": [] } }, @@ -931,17 +1081,25 @@ "model": "user.user", "pk": 21, "fields": { - "password": "pbkdf2_sha256$1000000$cv54m0PSNCQrH0MBLU56pk$nhGzkqLXL01EEtLAsoAbaWcTxuj1SOrZ7s6rBdA7eZk=", - "last_login": null, - "is_superuser": false, - "username": "adminstudent@codeforlife.com", - "first_name": "Portaladmin", - "last_name": "Student", - "email": "adminstudent@codeforlife.com", - "is_staff": false, - "is_active": true, + "_email_enc": "ZmFrZV9lbmM6v3CMqzgcHkXFUuH/E5WTE6Y1uXORrpJHu9VhYRdzAlYByeUliJUAkavcHMXo38zMc1HN92jTgwg=", + "_email_hash": "d10b776ab1e76347e63692528b0e4e97a5e3288b5f3c918476577a96b9255463", + "_email_plain": "adminstudent@codeforlife.com", + "_first_name_enc": "ZmFrZV9lbmM6WI6wHI7iEIUv0n0GxIsHsV75X85C0vRUlSEmAMnBr8aQOaDLa8RN", + "_first_name_hash": "689afa6b012cd1c17a5b32dbf80482ccb8049f87af317883a23e7144b84242b2", + "_first_name_plain": "Portaladmin", + "_last_name_enc": "ZmFrZV9lbmM6++cVFuCbPR0sIbZk2qM4m46PJKlZleoblSe6PWeixQl61Qk=", + "_last_name_plain": "Student", + "_username_enc": "ZmFrZV9lbmM6jzxttMaup8QyTqNObb0h8Xwo1Un3ptf+BjbsOKFtTNwQ8dyEdNJLHvUa5D0ulpl85uxQzgyh898=", + "_username_hash": "d10b776ab1e76347e63692528b0e4e97a5e3288b5f3c918476577a96b9255463", + "_username_plain": "adminstudent@codeforlife.com", "date_joined": "2026-02-04T16:02:40.242Z", + "dek": "ZmFrZV9lbmM6BoYrbuFtVkzXFPjmHBZLlD971sOH30A01sG1em/a2PlmhBmN/i1FPqXZEBXkx9GU/QG5J1EQzGQqM0A3", "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": null, + "password": "pbkdf2_sha256$1000000$cv54m0PSNCQrH0MBLU56pk$nhGzkqLXL01EEtLAsoAbaWcTxuj1SOrZ7s6rBdA7eZk=", "user_permissions": [] } } diff --git a/codeforlife/user/fixtures/non_school_teacher.json b/codeforlife/user/fixtures/non_school_teacher.json index 1bcc2b7b..2615f6d2 100644 --- a/codeforlife/user/fixtures/non_school_teacher.json +++ b/codeforlife/user/fixtures/non_school_teacher.json @@ -3,10 +3,18 @@ "model": "user.user", "pk": 22, "fields": { - "first_name": "John", - "last_name": "Doe", - "username": "teacher@noschool.com", - "email": "teacher@noschool.com", + "_email_enc": "ZmFrZV9lbmM66eERLU4ywXZv8HpJEjyu3AJj13quStWLJbiKre21Mggqgh81l+viMglF/QBQ4fFM", + "_email_hash": "7cc99dd2f335a068ac421d2cec8f04687fa34d37d313718f56ebc6b6c3c09d2c", + "_email_plain": "teacher@noschool.com", + "_first_name_enc": "ZmFrZV9lbmM6naDBVRP/vWnfgWPmHiirA6COL71ARyfg0pnyeQ3DYlU=", + "_first_name_hash": "7554c6b4930e0ac2c4d7fb5e7e266797b0444953067a0d7a17e835b518229e84", + "_first_name_plain": "John", + "_last_name_enc": "ZmFrZV9lbmM6BVE1wAneS3Mcvb4o3lxNbaV8U2yFpq7pBiP5ai8FWA==", + "_last_name_plain": "Doe", + "_username_enc": "ZmFrZV9lbmM6MHbn0VZ2WdBaxfuGEuU3zzyBsvI2NKWmKsqaipMhi1ie0aNmcBfnZqPRYM0bqM/q", + "_username_hash": "7cc99dd2f335a068ac421d2cec8f04687fa34d37d313718f56ebc6b6c3c09d2c", + "_username_plain": "teacher@noschool.com", + "dek": "ZmFrZV9lbmM6t5dLPgCaHfVJ+ZUMvIpOeZEo7pODNwUpd8exyX9ilp3COjdmNReUbCIeYVVyppQpim4aOw3EaOo6AInr", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -14,26 +22,34 @@ "model": "user.userprofile", "pk": 22, "fields": { - "user": 22, - "is_verified": true + "is_verified": true, + "user": 22 } }, { "model": "user.teacher", "pk": 5, "fields": { - "user": 22, - "new_user": 22 + "new_user": 22, + "user": 22 } }, { "model": "user.user", "pk": 33, "fields": { - "first_name": "Unverified", - "last_name": "Teacher", - "username": "unverified.teacher@noschool.com", - "email": "unverified.teacher@noschool.com", + "_email_enc": "ZmFrZV9lbmM6ymTYrJvQBbledTYMf1QxnBLLJKtZOAY2eSrLKWn1Ku9Bt8UfjyAv9Kdcq22Nlw4KEUXUF9B+5uWFrII=", + "_email_hash": "7f34bbe92504e0ca5c2235f2f64bf2686a7d040690a45948074247cad2eb5c71", + "_email_plain": "unverified.teacher@noschool.com", + "_first_name_enc": "ZmFrZV9lbmM6TYaKDI0fEu4AoVsf35i4onELsjpPePS3+0aTi7bJ/cz6Uu6Cr3I=", + "_first_name_hash": "47c086ea8af2fcd653408088d2d3809d3d46fd2698755a4a208d5fb78ebc205b", + "_first_name_plain": "Unverified", + "_last_name_enc": "ZmFrZV9lbmM6045ESPZ+3oNUHhw8P27QOxgaRxBEbAkT1nI9LGBZeTcrWqU=", + "_last_name_plain": "Teacher", + "_username_enc": "ZmFrZV9lbmM6kP4+WygnDIbz2v7bQVTJ9A5dJ/ngwqjhV1FG727D9fVpjRBqpZR7ZzDCpJLKGL+O/ErnM/v9cQjyRRA=", + "_username_hash": "7f34bbe92504e0ca5c2235f2f64bf2686a7d040690a45948074247cad2eb5c71", + "_username_plain": "unverified.teacher@noschool.com", + "dek": "ZmFrZV9lbmM6CxNXI5JH7pjDVV3SMaAS/lVuDNePBDKJPAiV4Gi/4iVcgxG6hvQ20IJX34nFw/U4mHaHCpTwSgWfSydl", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -41,16 +57,16 @@ "model": "user.userprofile", "pk": 33, "fields": { - "user": 33, - "is_verified": false + "is_verified": false, + "user": 33 } }, { "model": "user.teacher", "pk": 12, "fields": { - "user": 33, - "new_user": 33 + "new_user": 33, + "user": 33 } } ] diff --git a/codeforlife/user/fixtures/school_1.json b/codeforlife/user/fixtures/school_1.json index 444fa023..95e026b3 100644 --- a/codeforlife/user/fixtures/school_1.json +++ b/codeforlife/user/fixtures/school_1.json @@ -3,19 +3,29 @@ "model": "user.school", "pk": 2, "fields": { - "name": "School 1", + "_name_enc": "ZmFrZV9lbmM67d4VVL/VNF+xT49FGteqYAb5nww953xETgWD4Dbq/Wx+Jtoe", + "_name_plain": "School 1", "country": "GB", - "county": "Hertfordshire" + "county": "Hertfordshire", + "dek": "ZmFrZV9lbmM66Fc74ze1ks+0BusoBSk4Dq31Ze7K2owjJGHhyqul8ZDPLqqAdJfTXCTK7l9f0EHNZBrAHwioXhVm3iFi" } }, { "model": "user.user", "pk": 23, "fields": { - "first_name": "John", - "last_name": "Doe", - "username": "teacher@school1.com", - "email": "teacher@school1.com", + "_email_enc": "ZmFrZV9lbmM6nDdT0za7YSuF59q1VhuJvT8qHbDLH+/ap65hrlVnwgSRkpRcj452UPQ/3IQ3H5s=", + "_email_hash": "d23e2f365e919be1cf5e29a5472801d54bd75c6f360bce7c37674506bc63a9ce", + "_email_plain": "teacher@school1.com", + "_first_name_enc": "ZmFrZV9lbmM6KHAoomsgzlenkIf3O3B1HkeoBYLCHnmBSzdbpebUfr0=", + "_first_name_hash": "7554c6b4930e0ac2c4d7fb5e7e266797b0444953067a0d7a17e835b518229e84", + "_first_name_plain": "John", + "_last_name_enc": "ZmFrZV9lbmM6UiD5C/IkRNEYEE7NgaGlAdMc+Le7fUhUCXNyEwEVYQ==", + "_last_name_plain": "Doe", + "_username_enc": "ZmFrZV9lbmM6jV4xfKt2ER8uTjc56hdndNVFL5ZUpxwF8xmVreNQRpfz9PDHF226iw7U14MmLBk=", + "_username_hash": "d23e2f365e919be1cf5e29a5472801d54bd75c6f360bce7c37674506bc63a9ce", + "_username_plain": "teacher@school1.com", + "dek": "ZmFrZV9lbmM6jirqlzqds7vw9lM1WNsHUk7eGuT1X5EcZFvj77aL6V86oRw09V90ANLPNUUjOnNYvbb0BBh0EKT/j198", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -23,35 +33,43 @@ "model": "user.userprofile", "pk": 23, "fields": { - "user": 23, - "is_verified": true + "is_verified": true, + "user": 23 } }, { "model": "user.teacher", "pk": 6, "fields": { - "user": 23, "new_user": 23, - "school": 2 + "school": 2, + "user": 23 } }, { "model": "user.class", "pk": 6, "fields": { - "name": "Class 1 @ School 1", - "access_code": "ZZ111", - "teacher": 6, - "accept_requests_until": "9999-02-09 20:26:08.298402+00:00" + "_access_code_enc": "ZmFrZV9lbmM60BDe75Jx8mLFNVE4Fj6A47qgtKnwj40qYBgA2/PdcJqJ", + "_access_code_hash": "ca366870cf5e795a3c9ff340f119df2ff646128cf164d992b0167bbca8c247eb", + "_access_code_plain": "ZZ111", + "_name_enc": "ZmFrZV9lbmM6GE5ywQd6ChB38qHTDtALv40nO1RQu1ty7Azups4SMdlBAfNUh93EdK65vyHrMg==", + "_name_plain": "Class 1 @ School 1", + "accept_requests_until": "9999-02-09 20:26:08.298402+00:00", + "teacher": 6 } }, { "model": "user.user", "pk": 27, "fields": { - "first_name": "Student1", - "username": "111111111111111111111111111111", + "_first_name_enc": "ZmFrZV9lbmM68LydJ3q0a8MRqB/aobe1wf3XPUOZZV/ibILf8bFjWeeTLVLY", + "_first_name_hash": "341d77ca3f20f1a3ddf8d9a06d076940c6e24b252a08c5dae066b43c2b236ed5", + "_first_name_plain": "Student1", + "_username_enc": "ZmFrZV9lbmM6fgGMjMvNBd0UtOYl4ZheNodBdcSjPsVmLYrIES+t5PbSSw66LNdmCAXLWefCbQu3Nlf4m3ESwd/3cw==", + "_username_hash": "1a3a828a1d7b20c8c9bd4d3677dcd8e5df6352c950b2d99e04a77cdc291fbfbe", + "_username_plain": "111111111111111111111111111111", + "dek": "ZmFrZV9lbmM6LLCQnwBkWTs27wN70fGx1S1a32s9SB4bd7w/qwEf38c4rhaXQaoBQr3yjcSD9KpBFUcngCtFpeKvwVYE", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -59,27 +77,35 @@ "model": "user.userprofile", "pk": 27, "fields": { - "user": 27, - "is_verified": true + "is_verified": true, + "user": 27 } }, { "model": "user.student", "pk": 17, "fields": { - "user": 27, + "class_field": 6, "new_user": 27, - "class_field": 6 + "user": 27 } }, { "model": "user.user", "pk": 24, "fields": { - "first_name": "Jane", - "last_name": "Doe", - "username": "admin.teacher@school1.com", - "email": "admin.teacher@school1.com", + "_email_enc": "ZmFrZV9lbmM6oYdBxX01WxnO+2XvH4GcDYl8JVoGDYqCmB8u5hDlu33j84hMIU0c0rXh7X3EbVDqPaviQGY=", + "_email_hash": "01c1e0721b31cb0636da3f5b3dc1a107bdd44fd0185fa12dd87f6aad06c91a08", + "_email_plain": "admin.teacher@school1.com", + "_first_name_enc": "ZmFrZV9lbmM6ehSIW3xreQS4cUAXbBMnqq6OA7uS9iYEI9Y7iNOVDS4=", + "_first_name_hash": "a5945caf6491a22426f870debae98c823043af86691d336f95248daff7bb989a", + "_first_name_plain": "Jane", + "_last_name_enc": "ZmFrZV9lbmM6uf/W7NPtVAiFRyiZCwzvPOWrYa1KeBZT/HcZhoO1MQ==", + "_last_name_plain": "Doe", + "_username_enc": "ZmFrZV9lbmM6PLx6EXzPQG/RgVjD718xICsjsz++Wir/jtJhP/5yqpPPNeLj7fjx3Zb7+VwO3Q5q8xDMOcY=", + "_username_hash": "01c1e0721b31cb0636da3f5b3dc1a107bdd44fd0185fa12dd87f6aad06c91a08", + "_username_plain": "admin.teacher@school1.com", + "dek": "ZmFrZV9lbmM6r5c/ViTs1yOJQrGRIkZcCR/7qyNWgWrWpbrj8lOhNUnsDhr31RvKy50cNcKauKJvpLWlVzXI2XLbQ2BJ", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -87,26 +113,29 @@ "model": "user.userprofile", "pk": 24, "fields": { - "user": 24, - "is_verified": true + "is_verified": true, + "user": 24 } }, { "model": "user.teacher", "pk": 7, "fields": { - "user": 24, + "is_admin": true, "new_user": 24, "school": 2, - "is_admin": true + "user": 24 } }, { "model": "user.class", "pk": 7, "fields": { - "name": "Class 2 @ School 1", - "access_code": "ZZ222", + "_access_code_enc": "ZmFrZV9lbmM6UjOO6ScOIUFHcbbcDkSzaWAQY3PaW7GNeRWTQ73zcpTl", + "_access_code_hash": "e90c6683965279b92914a9e2b18716f0bc826c0dccb5976130562ff88cbd7f01", + "_access_code_plain": "ZZ222", + "_name_enc": "ZmFrZV9lbmM6ghwPnoQhhjiDpIsfE2ufqAMOaXJAuiwUZ7nwuRLu208FZznOJkbXb3FKy9iE4Q==", + "_name_plain": "Class 2 @ School 1", "teacher": 7 } }, @@ -114,8 +143,13 @@ "model": "user.user", "pk": 29, "fields": { - "first_name": "Student2", - "username": "222222222222222222222222222222", + "_first_name_enc": "ZmFrZV9lbmM6WTlhK3trdBVmqjtck1x0Jemd/5F1xq/hu5ghMYR+W+HRRsCj", + "_first_name_hash": "f7ecf0b3ebd27342b1a6d30a6ef300b6dc23518fd5c83ca25cdc2b7135927cf8", + "_first_name_plain": "Student2", + "_username_enc": "ZmFrZV9lbmM6r10EMezevJm9pQjvClWd/Rx06lfTdlIRYzsYEUcAq+V0yPYcafJ63+j+6jbHBcOR8qlXYoBw10mOHA==", + "_username_hash": "69a7b577f2249a789e2a12ee80ee669c30ff20872c94153abeb3ebc05a61be08", + "_username_plain": "222222222222222222222222222222", + "dek": "ZmFrZV9lbmM6cbR+gLNfBr+zos590Nlz8I2AbW1AS5A7Jo+iFzvK4bOn3OmQAw7QaTDGb7+n+Bquc74bWG9OwNz/eu70", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -123,27 +157,30 @@ "model": "user.userprofile", "pk": 29, "fields": { - "user": 29, - "is_verified": true + "is_verified": true, + "user": 29 } }, { "model": "user.student", "pk": 19, "fields": { - "user": 29, + "class_field": 7, "new_user": 29, - "class_field": 7 + "user": 29 } }, { "model": "user.class", "pk": 10, "fields": { - "name": "Class 3 @ School 1", - "access_code": "ZZ333", - "teacher": 7, - "accept_requests_until": "2023-02-09 20:26:08.298402+00:00" + "_access_code_enc": "ZmFrZV9lbmM64FGWxjHz//y120LjO3pVfij0ZhFZixOLE8Z57vaYOPCW", + "_access_code_hash": "3e3b96d4c7775a2c8d15999f6d662b164ccf3d66b8df93329bd5e426251d8cdb", + "_access_code_plain": "ZZ333", + "_name_enc": "ZmFrZV9lbmM6d6hSPPotg2fBEgOhd6IRzrvlWYQxFaSv4v/FS7uXG7YxNB2XBq2k0Y08ZrhFtQ==", + "_name_plain": "Class 3 @ School 1", + "accept_requests_until": "2023-02-09 20:26:08.298402+00:00", + "teacher": 7 } } ] diff --git a/codeforlife/user/fixtures/school_2.json b/codeforlife/user/fixtures/school_2.json index 3c3b7a07..8fdbb509 100644 --- a/codeforlife/user/fixtures/school_2.json +++ b/codeforlife/user/fixtures/school_2.json @@ -3,19 +3,29 @@ "model": "user.school", "pk": 3, "fields": { - "name": "School 2", + "_name_enc": "ZmFrZV9lbmM6rDyP9kxRgaY6xKUohQdWebf8Gs94JU03UH5NCdROWyGOpUk5", + "_name_plain": "School 2", "country": "GB", - "county": "Hertfordshire" + "county": "Hertfordshire", + "dek": "ZmFrZV9lbmM6bC7l4CSlvQMlUQ3LGmAgdXlinpUVIUxRCKl15mBnGovwpYqrwQhpqx47OXVPRQnzsIh5oX5/it+bnEw2" } }, { "model": "user.user", "pk": 25, "fields": { - "first_name": "John", - "last_name": "Doe", - "username": "teacher@school2.com", - "email": "teacher@school2.com", + "_email_enc": "ZmFrZV9lbmM6aiPwZ1FRKtCtuUMMBINg0bJH+T0ADhxYrPmCORVX4iH+q73WJwIIXIp7I9hQ5Bs=", + "_email_hash": "8cdd687351ee31833adcb72d37e7b8f1c1c17c655c3fb182ac0127320af64284", + "_email_plain": "teacher@school2.com", + "_first_name_enc": "ZmFrZV9lbmM6SugTLn6zL+eLgD5bNHpO3WG1WS+ijY591LnSOvIxDic=", + "_first_name_hash": "7554c6b4930e0ac2c4d7fb5e7e266797b0444953067a0d7a17e835b518229e84", + "_first_name_plain": "John", + "_last_name_enc": "ZmFrZV9lbmM6JG27G2EreOMHHuSC6ydDGwMO9sE3g0VQBKNLJDomIw==", + "_last_name_plain": "Doe", + "_username_enc": "ZmFrZV9lbmM6TEP3/O1x823sRLTr68TrLNGrCdYrQF8xhHtrqkhJJKHyg4OcCq+p3IV9chaGAlQ=", + "_username_hash": "8cdd687351ee31833adcb72d37e7b8f1c1c17c655c3fb182ac0127320af64284", + "_username_plain": "teacher@school2.com", + "dek": "ZmFrZV9lbmM631EYeb8mv6nnLRT6A9OG9FBDTQY+mzzJSZ9/poVoYpJwZas0QQ+o+S+T0N1qfP8XG2j4QAtybU8BYtVb", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -23,114 +33,117 @@ "model": "user.userprofile", "pk": 25, "fields": { - "user": 25, "is_verified": true, - "otp_secret": "KI6FA34LPRQU265KQBFYS2MTDYHE2EIG" + "otp_secret": "KI6FA34LPRQU265KQBFYS2MTDYHE2EIG", + "user": 25 } }, { "model": "user.authfactor", "pk": 1, "fields": { - "user": 25, - "type": "otp" + "type": "otp", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 1, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6WVdGaFlXRmhZV0U9" + "token": "ZmFrZV9lbmM6WVdGaFlXRmhZV0U9", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 2, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6WW1KaVltSmlZbUk9" + "token": "ZmFrZV9lbmM6WW1KaVltSmlZbUk9", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 3, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6WTJOalkyTmpZMk09" + "token": "ZmFrZV9lbmM6WTJOalkyTmpZMk09", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 4, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6WkdSa1pHUmtaR1E9" + "token": "ZmFrZV9lbmM6WkdSa1pHUmtaR1E9", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 5, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6WldWbFpXVmxaV1U9" + "token": "ZmFrZV9lbmM6WldWbFpXVmxaV1U9", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 6, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6Wm1abVptWm1abVk9" + "token": "ZmFrZV9lbmM6Wm1abVptWm1abVk9", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 7, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6WjJkbloyZG5aMmM9" + "token": "ZmFrZV9lbmM6WjJkbloyZG5aMmM9", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 8, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6YUdob2FHaG9hR2c9" + "token": "ZmFrZV9lbmM6YUdob2FHaG9hR2c9", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 9, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6YVdscGFXbHBhV2s9" + "token": "ZmFrZV9lbmM6YVdscGFXbHBhV2s9", + "user": 25 } }, { "model": "user.otpbypasstoken", "pk": 10, "fields": { - "user": 25, - "token": "ZmFrZV9lbmM6YW1wcWFtcHFhbW89" + "token": "ZmFrZV9lbmM6YW1wcWFtcHFhbW89", + "user": 25 } }, { "model": "user.teacher", "pk": 8, "fields": { - "user": 25, "new_user": 25, - "school": 3 + "school": 3, + "user": 25 } }, { "model": "user.class", "pk": 8, "fields": { - "name": "Class 1 @ School 2", - "access_code": "XX111", + "_access_code_enc": "ZmFrZV9lbmM6EK8Y5SIWGbV55+NvvCUsRgaw3QB+JEUGjCgJHgFJOj7Z", + "_access_code_hash": "0157477281b2aee7660889207dee1a14756bddd428162de41bada7a02c593ff6", + "_access_code_plain": "XX111", + "_name_enc": "ZmFrZV9lbmM6QS58FDOcIpwtt0KJ08wr+sKWQCVBucwv75ScfLpy53RQGS8dRIut/FEBoNUQ2g==", + "_name_plain": "Class 1 @ School 2", "teacher": 8 } }, @@ -138,10 +151,18 @@ "model": "user.user", "pk": 26, "fields": { - "first_name": "Jane", - "last_name": "Doe", - "username": "admin.teacher@school2.com", - "email": "admin.teacher@school2.com", + "_email_enc": "ZmFrZV9lbmM6Cc97ynmcPEUq24u31O3ho+rE5m+La+iHmoc3tyz3VObKKWca658I30zgJM253BWZWax0WaY=", + "_email_hash": "4841a8d3657918adffdab6cb4a0a6b64ad771aa56a53f80fed0ee8d074841ca3", + "_email_plain": "admin.teacher@school2.com", + "_first_name_enc": "ZmFrZV9lbmM6XAAtQFeiEPJbJe9g1zErH67kShgjeyTihaxID8eHsG4=", + "_first_name_hash": "a5945caf6491a22426f870debae98c823043af86691d336f95248daff7bb989a", + "_first_name_plain": "Jane", + "_last_name_enc": "ZmFrZV9lbmM60OVDyC7hAJrEkLkpcJ7LvaWcl5ovVLPrafucEwvZsQ==", + "_last_name_plain": "Doe", + "_username_enc": "ZmFrZV9lbmM6Duu4eRtUr7kJP/pxZea76U4q0m+/ouw1xt/jEhZ7bFyVVvc1n33HOLIKzvGO4RQKdE9Ze4s=", + "_username_hash": "4841a8d3657918adffdab6cb4a0a6b64ad771aa56a53f80fed0ee8d074841ca3", + "_username_plain": "admin.teacher@school2.com", + "dek": "ZmFrZV9lbmM6acw2bmFUUi2FVvY6NaRb393PcBFtnWsUdgZokzdWNxjrCnfI3qANds0dKuEEFzufBNQe4bPsmD5RmJdO", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -149,35 +170,38 @@ "model": "user.userprofile", "pk": 26, "fields": { - "user": 26, "is_verified": true, - "otp_secret": "KI6FA34LPRQU265KQBFYS2MTDYHE2EIG" + "otp_secret": "KI6FA34LPRQU265KQBFYS2MTDYHE2EIG", + "user": 26 } }, { "model": "user.authfactor", "pk": 2, "fields": { - "user": 26, - "type": "otp" + "type": "otp", + "user": 26 } }, { "model": "user.teacher", "pk": 9, "fields": { - "user": 26, + "is_admin": true, "new_user": 26, "school": 3, - "is_admin": true + "user": 26 } }, { "model": "user.class", "pk": 9, "fields": { - "name": "Class 2 @ School 2", - "access_code": "XX222", + "_access_code_enc": "ZmFrZV9lbmM6qqcLrPU7P3AyIdfDrFRA93ClW5sJEm/zyLRdbR5FsVUd", + "_access_code_hash": "750df33e8a531e83fddf652344b680983a1555c6dcd8fda80f4af785357b444d", + "_access_code_plain": "XX222", + "_name_enc": "ZmFrZV9lbmM65DRSk4ioNDC3lTdBdMzCx8izq6MA3hx1Lv7bVmIB0klrcE/ytKj/2YAOqQF5uA==", + "_name_plain": "Class 2 @ School 2", "teacher": 9 } } diff --git a/codeforlife/user/fixtures/school_2_sessions.json b/codeforlife/user/fixtures/school_2_sessions.json index 205ca312..e26e2205 100644 --- a/codeforlife/user/fixtures/school_2_sessions.json +++ b/codeforlife/user/fixtures/school_2_sessions.json @@ -3,8 +3,8 @@ "model": "user.session", "pk": "1", "fields": { - "session_data": "", "expire_date": "9999-01-01 00:00:00.0+00:00", + "session_data": "", "user": 25 } }, @@ -12,8 +12,8 @@ "model": "user.sessionauthfactor", "pk": 1, "fields": { - "session": "1", - "auth_factor": 1 + "auth_factor": 1, + "session": "1" } } -] \ No newline at end of file +] diff --git a/codeforlife/user/fixtures/school_3.json b/codeforlife/user/fixtures/school_3.json index bbd1ec10..ebb46999 100644 --- a/codeforlife/user/fixtures/school_3.json +++ b/codeforlife/user/fixtures/school_3.json @@ -3,19 +3,29 @@ "model": "user.school", "pk": 4, "fields": { - "name": "School 3", + "_name_enc": "ZmFrZV9lbmM6LQq31+NKOlsLWDpsatuQth2lvPldBNrjK/oLXHFMGZbSkZFh", + "_name_plain": "School 3", "country": "GB", - "county": "Hertfordshire" + "county": "Hertfordshire", + "dek": "ZmFrZV9lbmM6shO69sz7AjLAiIOt+QChcplwnM2SkTEydMeG2n/a8G39H7d320r9s9Q9D18poYRBOTW1ONMu4FDE1QAE" } }, { "model": "user.user", "pk": 31, "fields": { - "first_name": "Peter", - "last_name": "Parker", - "username": "admin.teacher@school3.com", - "email": "admin.teacher@school3.com", + "_email_enc": "ZmFrZV9lbmM6IWL4ECecrQdG5hEcbqZjrua3qN4twKvoYnZ4Sxdsri4ER5WaDmBoRdlfpDxfBdt7mF4lGtw=", + "_email_hash": "22a604d4828ae8429e8303d16c7d9967145b214b827a8ff1845b19e2a10b12c3", + "_email_plain": "admin.teacher@school3.com", + "_first_name_enc": "ZmFrZV9lbmM6X/pzRYxC3wSQmFiiPbeGouhsrbGoQviFfkpXbF4nutGt", + "_first_name_hash": "083ec2a4fd4004ed6f9bd61965b170a9b5db5d5873c7217f65bede117f004a79", + "_first_name_plain": "Peter", + "_last_name_enc": "ZmFrZV9lbmM6W9e5RMeFGyGqvF5EXOFm66Sa6OeiQT/wbub//DRcXxzjUA==", + "_last_name_plain": "Parker", + "_username_enc": "ZmFrZV9lbmM6CddPXEbQUUpaRaGDpvdxqRYehBobuVa0GuIgG+mE8t/Qys7HA9KvKTx3zNWF0rwobAz7ulk=", + "_username_hash": "22a604d4828ae8429e8303d16c7d9967145b214b827a8ff1845b19e2a10b12c3", + "_username_plain": "admin.teacher@school3.com", + "dek": "ZmFrZV9lbmM6whXTmVh8Ll7uD/RveSp8tPcfOtNoJb7XPPq2qa1ZXDSymoH85slcLwDSu/wl9gCKYhotmG1eQWyTbI5B", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -23,28 +33,36 @@ "model": "user.userprofile", "pk": 31, "fields": { - "user": 31, - "is_verified": true + "is_verified": true, + "user": 31 } }, { "model": "user.teacher", "pk": 10, "fields": { - "user": 31, + "is_admin": true, "new_user": 31, "school": 4, - "is_admin": true + "user": 31 } }, { "model": "user.user", "pk": 32, "fields": { - "first_name": "Doctor", - "last_name": "Octopus", - "username": "teacher@school3.com", - "email": "teacher@school3.com", + "_email_enc": "ZmFrZV9lbmM6VhCbhP1x0NtRUqNK6RCZ4g3g6fFC8kjqD7vJDj+Jqrv3/vS6b5r6ui0BeIg2XcU=", + "_email_hash": "c66f480d5458f514966dd1a540e652d1c57850a11122a26b1760eb95d7833fa1", + "_email_plain": "teacher@school3.com", + "_first_name_enc": "ZmFrZV9lbmM6XZjkxcuqVijVuzm3HPh/C+wn9MbuGSI6FH+qbWghkHIRqg==", + "_first_name_hash": "be367a9fe2b4bf78b7f7bd98e888fac606cfafa10fc91f25745d4a60e167ba72", + "_first_name_plain": "Doctor", + "_last_name_enc": "ZmFrZV9lbmM6rZsTsYhs5sQaqfUSMArJQ7u7cb/ufIvxvUANzmjust+rIjw=", + "_last_name_plain": "Octopus", + "_username_enc": "ZmFrZV9lbmM6UyHAKfn5fsxKpVQBKNOjvFXSWeKdjx+z51Id4rtPKYOQdLPuir213o1WgrbiR6A=", + "_username_hash": "c66f480d5458f514966dd1a540e652d1c57850a11122a26b1760eb95d7833fa1", + "_username_plain": "teacher@school3.com", + "dek": "ZmFrZV9lbmM68IHXVmIH5mmTb5T4H53Dcw2J1r4tZd0/GkPNUDJPk9d0/Gzm7UfKWIlLhDiNaSAlppjgSzf2lBd1w8K+", "password": "pbkdf2_sha256$720000$Jp50WPBA6WZImUIpj3UcVm$OJWB8+UoW5lLaUkHLYo0cKgMkyRI6qnqVOWxYEsi9T0=" } }, @@ -52,17 +70,17 @@ "model": "user.userprofile", "pk": 32, "fields": { - "user": 32, - "is_verified": true + "is_verified": true, + "user": 32 } }, { "model": "user.teacher", "pk": 11, "fields": { - "user": 32, "new_user": 32, - "school": 4 + "school": 4, + "user": 32 } } ] diff --git a/codeforlife/user/management/commands/encrypt_plaintext_fields.py b/codeforlife/user/management/commands/encrypt_plaintext_fields.py new file mode 100644 index 00000000..93d4c6b5 --- /dev/null +++ b/codeforlife/user/management/commands/encrypt_plaintext_fields.py @@ -0,0 +1,450 @@ +""" +© Ocado Group +Created on 16/03/2026 at 10:58:04(+00:00). +""" + +import typing as t +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from threading import Lock + +from django.apps import apps +from django.core.exceptions import FieldDoesNotExist +from django.core.management.base import BaseCommand +from django.db import close_old_connections +from django.db.models import CharField, Field, Model, Q, QuerySet, TextField + +from ....encryption import create_dek +from ....models import BaseDataEncryptionKeyModel +from ....models.fields import EncryptedTextField, Sha256Field +from ....models.utils import is_real_model_class +from ....pprint import PrettyPrinter + +PlaintextField: t.TypeAlias = t.Union[CharField, TextField] +FieldsToUpdate: t.TypeAlias = t.List[ + t.Tuple[ + PlaintextField, + t.Optional[EncryptedTextField], + t.Optional[Sha256Field], + ] +] + + +# pylint: disable-next=missing-class-docstring +class Command(BaseCommand): + format_help = ( + "Arguments should be one or more Django app labels. " + "For each model in each app, fields ending with '_plain' will be " + "copied into matching '_enc' and '_hash' fields." + ) + help = f"Encrypts and hashes plaintext fields for app models. {format_help}" + + def add_arguments(self, parser): + parser.add_argument( + "app_labels", nargs="+", type=str, help=self.format_help + ) + parser.add_argument( + "--chunk-size", + type=int, + default=100, + help="The number of records to process in each batch.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print what would be updated without writing to the database.", + ) + parser.add_argument( + "--disable-styles", + action="store_true", + help="Disable styled output.", + ) + parser.add_argument( + "--enable-threading", + action="store_true", + help=( + "Enable threaded processing where each worker handles a " + "chunk of rows." + ), + ) + parser.add_argument( + "--max-workers", + type=int, + default=4, + help="Maximum thread workers when --enable-threading is used.", + ) + + # pylint: disable-next=too-many-locals + def _discover_model_fields( + self, + model_class: t.Type[Model], + pprint: PrettyPrinter, + ) -> FieldsToUpdate: + model_spec = ( + f"{model_class._meta.app_label}.{model_class._meta.model_name}" + ) + plain_fields_by_name = { + field.name: field + for field in model_class._meta.fields + if isinstance(field, (CharField, TextField)) + and field.name.endswith("_plain") + } + + fields_to_update: FieldsToUpdate = [] + for field_name, plain_field in plain_fields_by_name.items(): + field_name_prefix = field_name[: -len("_plain")] + enc_field_name = f"{field_name_prefix}_enc" + hash_field_name = f"{field_name_prefix}_hash" + + enc_field: t.Optional[EncryptedTextField] = None + hash_field: t.Optional[Sha256Field] = None + + try: + enc_field_obj = model_class._meta.get_field(enc_field_name) + except FieldDoesNotExist: + enc_field_obj = None + + try: + hash_field_obj = model_class._meta.get_field(hash_field_name) + except FieldDoesNotExist: + hash_field_obj = None + + if isinstance(enc_field_obj, EncryptedTextField): + enc_field = enc_field_obj + elif enc_field_obj is not None: + pprint( + "Skipping encrypted field due to type mismatch: " + + pprint.notice.apply(f"{model_spec}.{enc_field_name}") + ) + + if isinstance(hash_field_obj, Sha256Field): + hash_field = hash_field_obj + elif hash_field_obj is not None: + pprint( + "Skipping hash field due to type mismatch: " + + pprint.notice.apply(f"{model_spec}.{hash_field_name}") + ) + + if enc_field is None and hash_field is None: + if enc_field_obj is not None: + pprint( + "No valid target fields found for: " + + pprint.notice.apply(f"{model_spec}.{enc_field_name}") + ) + if hash_field_obj is not None: + pprint( + "No valid target fields found for: " + + pprint.notice.apply(f"{model_spec}.{hash_field_name}") + ) + continue + + target_fields = ", ".join( + field_name + for field_name in (enc_field_name, hash_field_name) + if (field_name == enc_field_name and enc_field is not None) + or (field_name == hash_field_name and hash_field is not None) + ) + + pprint( + "Discovered field mapping: " + + pprint.notice.apply(f"{field_name} -> {target_fields}") + ) + fields_to_update.append((plain_field, enc_field, hash_field)) + + return fields_to_update + + def _get_model_queryset( + self, + model_class: t.Type[Model], + fields_to_update: FieldsToUpdate, + pprint: PrettyPrinter, + ): + if not fields_to_update: + pprint("No fields to update.") + return model_class.objects.none() # type: ignore[attr-defined] + + models = model_class.objects.all() # type: ignore[attr-defined] + + # Pre-fetch the fields which will be read. + only_fields = [ + plain_field.name for plain_field, _, _ in fields_to_update + ] + if issubclass(model_class, BaseDataEncryptionKeyModel): + only_fields.append(model_class.DEK_FIELD) + models = models.only(*only_fields) + + # Generate a filter for a field that has a null or empty value. + def null_or_empty(field: Field, empty: t.Any): + return Q(**{f"{field.name}__isnull": True}) | Q( + **{field.name: empty} + ) + + # Filter to models where at least one field needs to be updated. + q = Q() + for plain_field, enc_field, hash_field in fields_to_update: + # Exclude models where the plaintext field is null or empty and + # include models where the encrypted or hash field is null or empty. + q |= ~null_or_empty(plain_field, "") & ( + null_or_empty(enc_field, b"") | null_or_empty(hash_field, "") + if enc_field is not None and hash_field is not None + else ( + null_or_empty(enc_field, b"") + if enc_field is not None + else null_or_empty(t.cast(Sha256Field, hash_field), "") + ) + ) + models = models.filter(q) + + # Exclude inactive records for certain models. + if model_class._meta.model_name in [ + "school", + "user", + "schoolteacherinvitation", + "class", + ]: + models = models.exclude(is_active=False) + + if model_class._meta.model_name == "class": + models = ( + models.exclude(teacher__isnull=True) + .exclude(teacher__school__isnull=True) + .exclude(teacher__school__is_active=False) + ) + elif model_class._meta.model_name == "schoolteacherinvitation": + models = models.exclude(school__isnull=True).exclude( + school__is_active=False + ) + elif model_class._meta.model_name in ["level", "workspace"]: + models = ( + models.exclude(owner__isnull=True) + .exclude(owner__user__isnull=True) + .exclude(owner__user__is_active=False) + ) + + # Exclude default levels, which are shared across users and not + # encrypted. + if model_class._meta.model_name == "level": + models = models.exclude(default=True) + + return models + + def _update_model_instance( + self, + model_class: t.Type[Model], + model: Model, + fields_to_update: FieldsToUpdate, + ): + update_fields: t.List[str] = [] + + # If the model uses DEKs and doesn't have one set, create and assign + # a DEK so that it can be used for encrypting fields. + if ( + issubclass(model_class, BaseDataEncryptionKeyModel) + and getattr(model, model_class.DEK_FIELD) is None + ): + setattr(model, model_class.DEK_FIELD, create_dek()) + update_fields.append(model_class.DEK_FIELD) + + # Iterate through the fields to update and copy the plaintext value + # into the encrypted and hash fields as appropriate. + for plain_field, enc_field, hash_field in fields_to_update: + plaintext = getattr(model, plain_field.name) + field_property_name = plain_field.name.removesuffix( + "_plain" + ).removeprefix("_") + setattr(model, field_property_name, plaintext) + + if enc_field is not None: + update_fields.append(enc_field.name) + if hash_field is not None: + update_fields.append(hash_field.name) + + # Save the model with the updated fields. + model.save(update_fields=update_fields) + + def _iter_model_batches(self, models: QuerySet[Model], chunk_size: int): + batch: t.List[Model] = [] + for model in models.iterator(chunk_size): + batch.append(model) + if len(batch) >= chunk_size: + yield batch + batch = [] + + if batch: + yield batch + + def _process_model_batch( + self, + model_class: t.Type[Model], + batch: t.List[Model], + fields_to_update: FieldsToUpdate, + ) -> int: + # Ensure this thread has a valid Django DB connection state. + close_old_connections() + + for model in batch: + self._update_model_instance(model_class, model, fields_to_update) + + close_old_connections() + return len(batch) + + # pylint: disable-next=too-many-arguments,too-many-positional-arguments + def _update_models( + self, + chunk_size: int, + model_class: t.Type[Model], + models: QuerySet[Model], + model_count: int, + fields_to_update: FieldsToUpdate, + pprint: PrettyPrinter, + ): + for model_index, model in enumerate(models.iterator(chunk_size)): + # Print progress at the start of each chunk. + if model_index % chunk_size == 0: + pprint(f"({model_index}/{model_count})") + + self._update_model_instance( + model_class=model_class, + model=model, + fields_to_update=fields_to_update, + ) + + # pylint: disable-next=too-many-arguments,too-many-locals,too-many-positional-arguments + def _update_models_threaded( + self, + chunk_size: int, + max_workers: int, + model_class: t.Type[Model], + models: QuerySet[Model], + model_count: int, + fields_to_update: FieldsToUpdate, + pprint: PrettyPrinter, + ): + progress_lock = Lock() + processed_count = 0 + submitted_batches = 0 + max_pending_futures = max_workers * 2 + + def complete_one(future: Future[int]): + nonlocal processed_count + processed_batch_size = future.result() + with progress_lock: + processed_count += processed_batch_size + pprint(f"({processed_count}/{model_count})") + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + pending_futures: t.Set[Future[int]] = set() + + for batch in self._iter_model_batches(models, chunk_size): + pending_futures.add( + executor.submit( + self._process_model_batch, + model_class, + batch, + fields_to_update, + ) + ) + submitted_batches += 1 + + if len(pending_futures) >= max_pending_futures: + done, pending_futures = wait( + pending_futures, + return_when=FIRST_COMPLETED, + ) + for future in done: + complete_one(future) + + if submitted_batches == 0: + return + + for future in pending_futures: + complete_one(future) + + # pylint: disable-next=too-many-locals + def handle(self, *args, **options): + app_labels: t.List[str] = options["app_labels"] + chunk_size: int = options["chunk_size"] + dry_run: bool = options["dry_run"] + disable_styles: bool = options["disable_styles"] + enable_threading: bool = options["enable_threading"] + max_workers: int = options["max_workers"] + + if chunk_size < 1: + raise ValueError("--chunk-size must be at least 1.") + + if max_workers < 1: + raise ValueError("--max-workers must be at least 1.") + + if max_workers > 8: + raise ValueError("--max-workers must be <= 8.") + + pprint = PrettyPrinter( + write=self.stderr.write, + name=self.__module__, + disable_styles=disable_styles, + ) + + for app_label in set(app_label.lower() for app_label in app_labels): + with pprint.process( + f"Processing app: {pprint.notice.apply(app_label)}" + ) as app_pprint: + app_config = apps.get_app_config(app_label) + + # Get real model classes. + model_classes = [ + model_class + for model_class in app_config.get_models() + if is_real_model_class(model_class) + ] + + # Sort model classes so that those saving DEKs are processed + # first, ensuring that other models with dependencies on DEKs + # can encrypt their fields. + model_classes = sorted( + model_classes, + key=lambda cls: issubclass(cls, BaseDataEncryptionKeyModel), + reverse=True, + ) + + for model_class in model_classes: + with app_pprint.process( + "Processing model: " + + app_pprint.notice.apply(model_class._meta.model_name) + ) as model_pprint: + fields_to_update = self._discover_model_fields( + model_class, model_pprint + ) + + models = self._get_model_queryset( + model_class, fields_to_update, model_pprint + ) + model_count = models.count() + + if dry_run: + model_pprint( + f"Dry run: would update {model_count} records." + ) + continue + + if model_count == 0: + model_pprint("No models to update.") + continue + + if enable_threading: + self._update_models_threaded( + chunk_size=chunk_size, + max_workers=max_workers, + model_class=model_class, + models=models, + model_count=model_count, + fields_to_update=fields_to_update, + pprint=model_pprint, + ) + else: + self._update_models( + chunk_size=chunk_size, + model_class=model_class, + models=models, + model_count=model_count, + fields_to_update=fields_to_update, + pprint=model_pprint, + ) diff --git a/codeforlife/user/migrations/0002_user_proxies_and_new_models.py b/codeforlife/user/migrations/0002_user_proxies_and_new_models.py index 0240a429..cd3748e2 100644 --- a/codeforlife/user/migrations/0002_user_proxies_and_new_models.py +++ b/codeforlife/user/migrations/0002_user_proxies_and_new_models.py @@ -8,7 +8,6 @@ import codeforlife.user.models.user.school_teacher import codeforlife.user.models.user.student import codeforlife.user.models.user.teacher -import codeforlife.user.models.user.user import django.db.models.deletion from django.conf import settings from django.db import migrations, models @@ -21,12 +20,6 @@ class Migration(migrations.Migration): ] operations = [ - # migrations.AlterModelManagers( - # name="user", - # managers=[ - # ("objects", codeforlife.user.models.user.user.UserManager()) - # ], - # ), migrations.CreateModel( name="ContactableUser", fields=[], diff --git a/codeforlife/user/migrations/0003_client_side_encryption_part_1.py b/codeforlife/user/migrations/0003_client_side_encryption_part_1.py new file mode 100644 index 00000000..0e668636 --- /dev/null +++ b/codeforlife/user/migrations/0003_client_side_encryption_part_1.py @@ -0,0 +1,290 @@ +from django.db import migrations + +from ...models.fields import ( + DataEncryptionKeyField, + EncryptedTextField, + Sha256Field, +) +from ..models.user import UserManager + +user_migrations = [ + migrations.AlterModelManagers( + name="user", + managers=[("objects", UserManager())], + ), + migrations.AddField( + model_name="user", + name="dek", + field=DataEncryptionKeyField( + editable=False, + help_text="The encrypted data encryption key (DEK) for this model.", + null=True, + verbose_name="data encryption key", + ), + ), + # Username + migrations.RenameField( + model_name="user", + old_name="username", + new_name="_username_plain", + ), + migrations.AddField( + model_name="user", + name="_username_enc", + field=EncryptedTextField( + associated_data="username", + null=True, + verbose_name="username", + db_column="username_enc", + ), + ), + migrations.AddField( + model_name="user", + name="_username_hash", + field=Sha256Field( + null=True, + unique=True, + editable=False, + max_length=64, + verbose_name="username hash", + db_column="username_hash", + ), + ), + # Email + migrations.RenameField( + model_name="user", + old_name="email", + new_name="_email_plain", + ), + migrations.AddField( + model_name="user", + name="_email_enc", + field=EncryptedTextField( + associated_data="email", + null=True, + verbose_name="email address", + db_column="email_enc", + ), + ), + migrations.AddField( + model_name="user", + name="_email_hash", + field=Sha256Field( + null=True, + editable=False, + max_length=64, + verbose_name="email hash", + db_column="email_hash", + ), + ), + # First name + migrations.RenameField( + model_name="user", + old_name="first_name", + new_name="_first_name_plain", + ), + migrations.AddField( + model_name="user", + name="_first_name_enc", + field=EncryptedTextField( + associated_data="first_name", + null=True, + verbose_name="first name", + db_column="first_name_enc", + ), + ), + migrations.AddField( + model_name="user", + name="_first_name_hash", + field=Sha256Field( + null=True, + editable=False, + max_length=64, + verbose_name="first name hash", + db_column="first_name_hash", + ), + ), + # Last name + migrations.RenameField( + model_name="user", + old_name="last_name", + new_name="_last_name_plain", + ), + migrations.AddField( + model_name="user", + name="_last_name_enc", + field=EncryptedTextField( + associated_data="last_name", + null=True, + verbose_name="last name", + db_column="last_name_enc", + ), + ), +] + +class_migrations = [ + # Access code + migrations.RenameField( + model_name="class", + old_name="access_code", + new_name="_access_code_plain", + ), + migrations.AddField( + model_name="class", + name="_access_code_enc", + field=EncryptedTextField( + associated_data="access_code", + null=True, + verbose_name="access code", + db_column="access_code_enc", + ), + ), + migrations.AddField( + model_name="class", + name="_access_code_hash", + field=Sha256Field( + null=True, + editable=False, + max_length=64, + verbose_name="access code hash", + db_column="access_code_hash", + ), + ), + # Name + migrations.RenameField( + model_name="class", + old_name="name", + new_name="_name_plain", + ), + migrations.AddField( + model_name="class", + name="_name_enc", + field=EncryptedTextField( + associated_data="name", + null=True, + verbose_name="name", + db_column="name_enc", + ), + ), +] + +school_teacher_invitation_migrations = [ + # Token + migrations.RenameField( + model_name="schoolteacherinvitation", + old_name="token", + new_name="_token_plain", + ), + migrations.AddField( + model_name="schoolteacherinvitation", + name="_token_enc", + field=EncryptedTextField( + associated_data="token", + null=True, + verbose_name="token", + db_column="token_enc", + ), + ), + migrations.AddField( + model_name="schoolteacherinvitation", + name="_token_hash", + field=Sha256Field( + null=True, + editable=False, + max_length=64, + verbose_name="token hash", + db_column="token_hash", + ), + ), + # Invited teacher first name + migrations.RenameField( + model_name="schoolteacherinvitation", + old_name="invited_teacher_first_name", + new_name="_invited_teacher_first_name_plain", + ), + migrations.AddField( + model_name="schoolteacherinvitation", + name="_invited_teacher_first_name_enc", + field=EncryptedTextField( + associated_data="invited_teacher_first_name", + null=True, + verbose_name="invited teacher first name", + db_column="invited_teacher_first_name_enc", + ), + ), + # Invited teacher last name + migrations.RenameField( + model_name="schoolteacherinvitation", + old_name="invited_teacher_last_name", + new_name="_invited_teacher_last_name_plain", + ), + migrations.AddField( + model_name="schoolteacherinvitation", + name="_invited_teacher_last_name_enc", + field=EncryptedTextField( + associated_data="invited_teacher_last_name", + null=True, + verbose_name="invited teacher last name", + db_column="invited_teacher_last_name_enc", + ), + ), + # Invited teacher email + migrations.RenameField( + model_name="schoolteacherinvitation", + old_name="invited_teacher_email", + new_name="_invited_teacher_email_plain", + ), + migrations.AddField( + model_name="schoolteacherinvitation", + name="_invited_teacher_email_enc", + field=EncryptedTextField( + associated_data="invited_teacher_email", + null=True, + verbose_name="invited teacher email", + db_column="invited_teacher_email_enc", + ), + ), +] + +school_migrations = [ + migrations.AddField( + model_name="school", + name="dek", + field=DataEncryptionKeyField( + editable=False, + help_text="The encrypted data encryption key (DEK) for this model.", + null=True, + verbose_name="data encryption key", + ), + ), + # Name + migrations.RenameField( + model_name="school", + old_name="name", + new_name="_name_plain", + ), + migrations.AddField( + model_name="school", + name="_name_enc", + field=EncryptedTextField( + associated_data="name", + null=True, + verbose_name="name", + db_column="name_enc", + ), + ), +] + + +class Migration(migrations.Migration): + + dependencies = [ + ("user", "0002_user_proxies_and_new_models"), + ] + + operations = [ + *user_migrations, + *class_migrations, + *school_teacher_invitation_migrations, + *school_migrations, + ] diff --git a/codeforlife/user/models/auth_factor.py b/codeforlife/user/models/auth_factor.py index 67c12217..44dc2a98 100644 --- a/codeforlife/user/models/auth_factor.py +++ b/codeforlife/user/models/auth_factor.py @@ -46,6 +46,8 @@ class Type(models.TextChoices): on_delete=models.CASCADE, ) + # NOTE: this is not currently used in production. when it is, it should be + # converted into an EncryptedTextField. type: str type = models.TextField(choices=Type.choices) # type: ignore[assignment] diff --git a/codeforlife/user/models/klass.py b/codeforlife/user/models/klass.py index 8d5bcced..6896a930 100644 --- a/codeforlife/user/models/klass.py +++ b/codeforlife/user/models/klass.py @@ -5,12 +5,15 @@ import typing as t from datetime import timedelta -from uuid import uuid4 from django.core.validators import MaxLengthValidator, MinLengthValidator from django.db import models +from django.db.models.query import QuerySet from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from ...models import EncryptedModel +from ...models.fields import EncryptedTextField, Sha256Field from ...types import Validators from ...validators import ( UnicodeAlphanumericCharSetValidator, @@ -22,7 +25,8 @@ from django_stubs_ext.db.models import TypedModelMeta - from .teacher import Teacher + from .student import Student + from .teacher import SchoolTeacher, Teacher else: TypedModelMeta = object @@ -41,7 +45,7 @@ ] -class ClassModelManager(models.Manager): +class ClassModelManager(EncryptedModel.Manager["Class"]): """Manager for Class model.""" def get_original_queryset(self): @@ -53,23 +57,93 @@ def get_queryset(self): return super().get_queryset().filter(is_active=True) -class Class(models.Model): +# pylint: disable-next=too-many-instance-attributes +class Class(EncryptedModel): """A class.""" - name = models.CharField(max_length=200) + students: QuerySet["Student"] + + associated_data = "class" + field_aliases = { + "name": {"_name_plain", "_name_enc"}, + "access_code": { + "_access_code_plain", + "_access_code_enc", + "_access_code_hash", + }, + } + + # -------------------------------------------------------------------------- + # Name + # -------------------------------------------------------------------------- + + _name_plain: str + _name_plain = models.CharField(max_length=200) # type: ignore[assignment] + _name_enc = EncryptedTextField( + associated_data="name", + db_column="name_enc", + null=True, + verbose_name=_("name"), + ) + + @property + def name(self): + """Get the name of the class.""" + if self._name_enc is not None: + return EncryptedTextField.get(self, "_name_enc") + return self._name_plain + + @name.setter + def name(self, value: str): + """Set the name of the class.""" + self._name_plain = value + EncryptedTextField.set(self, value, "_name_enc") - teacher: "Teacher" + # -------------------------------------------------------------------------- + + teacher: "SchoolTeacher" teacher = models.ForeignKey( # type: ignore[assignment] "user.Teacher", related_name="class_teacher", on_delete=models.CASCADE, ) - access_code: t.Optional[str] - access_code = models.CharField( # type: ignore[assignment] + # -------------------------------------------------------------------------- + # Access code + # -------------------------------------------------------------------------- + + _access_code_hash = Sha256Field( + verbose_name=_("access code hash"), + null=True, + db_column="access_code_hash", + ) + _access_code_plain: t.Optional[str] + _access_code_plain = models.CharField( # type: ignore[assignment] max_length=5, null=True, ) + _access_code_enc = EncryptedTextField( + associated_data="access_code", + null=True, + verbose_name=_("access code"), + db_column="access_code_enc", + ) + + @property + def access_code(self): + """Get the access code for the class.""" + if self._access_code_enc is not None: + return EncryptedTextField.get(self, "_access_code_enc") + return self._access_code_plain + + @access_code.setter + def access_code(self, value: t.Optional[str]): + """Set the access code for the class.""" + self._access_code_plain = value + EncryptedTextField.set(self, value, "_access_code_enc") + Sha256Field.set(self, value, "_access_code_hash") + + # -------------------------------------------------------------------------- classmates_data_viewable: bool classmates_data_viewable = models.BooleanField( # type: ignore[assignment] @@ -103,7 +177,7 @@ class Class(models.Model): on_delete=models.SET_NULL, ) - objects = ClassModelManager() + objects: ClassModelManager = ClassModelManager() # type: ignore[assignment] def __str__(self): return self.name @@ -139,8 +213,6 @@ def get_requests_message(self): def anonymise(self): """Anonymise the class.""" - self.name = uuid4().hex - self.access_code = "" self.is_active = False self.save() @@ -150,3 +222,7 @@ def anonymise(self): class Meta(TypedModelMeta): verbose_name_plural = "classes" + + @property + def dek_aead(self): + return self.teacher.school.dek_aead diff --git a/codeforlife/user/models/other.py b/codeforlife/user/models/other.py index ea000bac..ef135c26 100644 --- a/codeforlife/user/models/other.py +++ b/codeforlife/user/models/other.py @@ -12,6 +12,10 @@ from django.db import models from django.utils import timezone +from django.utils.translation import gettext_lazy as _ + +from ...models import EncryptedModel +from ...models.fields import EncryptedTextField, Sha256Field if t.TYPE_CHECKING: # pragma: no cover from datetime import datetime @@ -198,7 +202,9 @@ def __str__(self): return f"Activity on {self.date}: CSV clicks: {self.csv_click_count}, login cards clicks: {self.login_cards_click_count}, primary pack downloads: {self.primary_coding_club_downloads}, python pack downloads: {self.python_coding_club_downloads}, level control submits: {self.level_control_submits}, teacher lockout resets: {self.teacher_lockout_resets}, indy lockout resets: {self.indy_lockout_resets}, school student lockout resets: {self.school_student_lockout_resets}, unverified teachers anonymised: {self.anonymised_unverified_teachers}, unverified independents anonymised: {self.anonymised_unverified_independents}" -class SchoolTeacherInvitationModelManager(models.Manager): +class SchoolTeacherInvitationModelManager( + EncryptedModel.Manager["SchoolTeacherInvitation"] +): """ A custom model manager for the SchoolTeacherInvitation model to filter out inactive invitations by default. @@ -218,7 +224,8 @@ def get_queryset(self): return super().get_queryset().filter(is_active=True) -class SchoolTeacherInvitation(models.Model): +# pylint: disable-next=too-many-instance-attributes +class SchoolTeacherInvitation(EncryptedModel): """ A model to track invitations for teachers to join a school. This is meant to be used when a teacher invites another teacher to join their school, and the @@ -226,8 +233,56 @@ class SchoolTeacherInvitation(models.Model): the invitation, or the invitation expires. """ - token: str - token = models.CharField(max_length=88) # type: ignore[assignment] + associated_data = "school_teacher_invitation" + field_aliases = { + "token": {"_token_plain", "_token_enc", "_token_hash"}, + "invited_teacher_first_name": { + "_invited_teacher_first_name_plain", + "_invited_teacher_first_name_enc", + }, + "invited_teacher_last_name": { + "_invited_teacher_last_name_plain", + "_invited_teacher_last_name_enc", + }, + "invited_teacher_email": { + "_invited_teacher_email_plain", + "_invited_teacher_email_enc", + }, + } + + # -------------------------------------------------------------------------- + # Token + # -------------------------------------------------------------------------- + + _token_hash = Sha256Field( + verbose_name=_("token hash"), + null=True, + db_column="token_hash", + ) + _token_plain: str + _token_plain = models.CharField(max_length=88) # type: ignore[assignment] + _token_enc = EncryptedTextField( + associated_data="token", + null=True, + verbose_name=_("token"), + db_column="token_enc", + ) + + @property + def token(self): + """Get the decrypted token value.""" + if self._token_enc is not None: + return EncryptedTextField.get(self, "_token_enc") + return self._token_plain + + @token.setter + def token(self, value: str): + """Sets the token value.""" + self._token_plain = value + EncryptedTextField.set(self, value, "_token_enc") + Sha256Field.set(self, value, "_token_hash") + + # -------------------------------------------------------------------------- school: t.Optional["School"] school = models.ForeignKey( # type: ignore[assignment] @@ -245,21 +300,98 @@ class SchoolTeacherInvitation(models.Model): on_delete=models.SET_NULL, ) - invited_teacher_first_name: str - invited_teacher_first_name = models.CharField( # type: ignore[assignment] + # -------------------------------------------------------------------------- + # First name + # -------------------------------------------------------------------------- + + _invited_teacher_first_name_plain: str + # pylint: disable-next=line-too-long + _invited_teacher_first_name_plain = models.CharField( # type: ignore[assignment] max_length=150 ) # Same as User model + _invited_teacher_first_name_enc = EncryptedTextField( + associated_data="invited_teacher_first_name", + null=True, + verbose_name=_("invited teacher first name"), + db_column="invited_teacher_first_name_enc", + ) - invited_teacher_last_name: str - invited_teacher_last_name = models.CharField( # type: ignore[assignment] + @property + def invited_teacher_first_name(self): + """Get the decrypted invited teacher first name value.""" + if self._invited_teacher_first_name_enc is not None: + return EncryptedTextField.get( + self, "_invited_teacher_first_name_enc" + ) + return self._invited_teacher_first_name_plain + + @invited_teacher_first_name.setter + def invited_teacher_first_name(self, value: str): + """Sets the invited teacher first name value.""" + self._invited_teacher_first_name_plain = value + EncryptedTextField.set(self, value, "_invited_teacher_first_name_enc") + + # -------------------------------------------------------------------------- + # Last name + # -------------------------------------------------------------------------- + + _invited_teacher_last_name_plain: str + # pylint: disable-next=line-too-long + _invited_teacher_last_name_plain = models.CharField( # type: ignore[assignment] max_length=150 ) # Same as User model + _invited_teacher_last_name_enc = EncryptedTextField( + associated_data="invited_teacher_last_name", + null=True, + verbose_name=_("invited teacher last name"), + db_column="invited_teacher_last_name_enc", + ) + + @property + def invited_teacher_last_name(self): + """Get the decrypted invited teacher last name value.""" + if self._invited_teacher_last_name_enc is not None: + return EncryptedTextField.get( + self, "_invited_teacher_last_name_enc" + ) + return self._invited_teacher_last_name_plain + + @invited_teacher_last_name.setter + def invited_teacher_last_name(self, value: str): + """Sets the invited teacher last name value.""" + self._invited_teacher_last_name_plain = value + EncryptedTextField.set(self, value, "_invited_teacher_last_name_enc") + + # -------------------------------------------------------------------------- + # Email + # -------------------------------------------------------------------------- # TODO: Switch to a CharField to be able to hold hashed value - invited_teacher_email: str - invited_teacher_email = ( + _invited_teacher_email_plain: str + _invited_teacher_email_plain = ( models.EmailField() # type: ignore[assignment] ) # Same as User model + _invited_teacher_email_enc = EncryptedTextField( + associated_data="invited_teacher_email", + null=True, + verbose_name=_("invited teacher email"), + db_column="invited_teacher_email_enc", + ) + + @property + def invited_teacher_email(self): + """Get the decrypted invited teacher email value.""" + if self._invited_teacher_email_enc is not None: + return EncryptedTextField.get(self, "_invited_teacher_email_enc") + return self._invited_teacher_email_plain + + @invited_teacher_email.setter + def invited_teacher_email(self, value: str): + """Sets the invited teacher email value.""" + self._invited_teacher_email_plain = value + EncryptedTextField.set(self, value, "_invited_teacher_email_enc") + + # -------------------------------------------------------------------------- invited_teacher_is_admin: bool invited_teacher_is_admin = models.BooleanField( # type: ignore[assignment] @@ -279,7 +411,9 @@ class SchoolTeacherInvitation(models.Model): is_active = models.BooleanField(default=True) # type: ignore[assignment] # pylint: enable=duplicate-code - objects = SchoolTeacherInvitationModelManager() + objects: SchoolTeacherInvitationModelManager = ( + SchoolTeacherInvitationModelManager() # type: ignore[assignment] + ) @property def is_expired(self): @@ -300,3 +434,10 @@ def anonymise(self): self.invited_teacher_email = uuid4().hex self.is_active = False self.save() + + @property + def dek_aead(self): + if self.school: + return self.school.dek_aead + + raise KeyError("Data Encryption Key (DEK) not found.") diff --git a/codeforlife/user/models/otp_bypass_token.py b/codeforlife/user/models/otp_bypass_token.py index 23a3552f..aeb43a1d 100644 --- a/codeforlife/user/models/otp_bypass_token.py +++ b/codeforlife/user/models/otp_bypass_token.py @@ -93,7 +93,7 @@ class Meta(TypedModelMeta): @property def dek_aead(self): - return self.user.userprofile.dek_aead # type: ignore[attr-defined] + return self.user.dek_aead # type: ignore[attr-defined] def save(self, *args, **kwargs): raise IntegrityError("Cannot create or update a single instance.") diff --git a/codeforlife/user/models/school.py b/codeforlife/user/models/school.py index 40980fd0..8fca15f9 100644 --- a/codeforlife/user/models/school.py +++ b/codeforlife/user/models/school.py @@ -4,12 +4,14 @@ """ import typing as t -from uuid import uuid4 from django.db import models from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from django_countries.fields import CountryField +from ...models import DataEncryptionKeyModel +from ...models.fields import EncryptedTextField from ...types import Validators from ...validators import UnicodeAlphanumericCharSetValidator @@ -26,7 +28,7 @@ ] -class SchoolModelManager(models.Manager): +class SchoolModelManager(DataEncryptionKeyModel.Manager["School"]): """Manager for School model.""" def get_original_queryset(self): @@ -38,14 +40,46 @@ def get_queryset(self): return super().get_queryset().filter(is_active=True) -class School(models.Model): +class School(DataEncryptionKeyModel): """A school.""" - name: str - name = models.CharField( # type: ignore[assignment] + associated_data = "school" + field_aliases = { + "name": {"_name_plain", "_name_enc"}, + } + + # -------------------------------------------------------------------------- + # Name + # -------------------------------------------------------------------------- + # pylint: disable=duplicate-code + + _name_plain: str + _name_plain = models.CharField( # type: ignore[assignment] max_length=200, unique=True, ) + _name_enc = EncryptedTextField( + associated_data="name", + null=True, + verbose_name=_("name"), + db_column="name_enc", + ) + + @property + def name(self): + """Get the school's name.""" + if self._name_enc is not None: + return EncryptedTextField.get(self, "_name_enc") + return self._name_plain + + @name.setter + def name(self, value: str): + """Set the school's name.""" + self._name_plain = value + EncryptedTextField.set(self, value, "_name_enc") + + # pylint: enable=duplicate-code + # -------------------------------------------------------------------------- country: t.Optional[str] country = CountryField( # type: ignore[assignment] @@ -54,7 +88,6 @@ class School(models.Model): blank=True, ) - # TODO: Create an Address model to house address details county: t.Optional[str] county = models.CharField( # type: ignore[assignment] max_length=50, @@ -71,7 +104,9 @@ class School(models.Model): is_active: bool is_active = models.BooleanField(default=True) # type: ignore[assignment] - objects = SchoolModelManager() + objects: SchoolModelManager = ( + SchoolModelManager() # type: ignore[assignment] + ) def __str__(self): return self.name @@ -98,6 +133,6 @@ def admins(self): def anonymise(self): """Anonymize the school.""" - self.name = uuid4().hex + self.dek = None self.is_active = False - self.save() + self.save(update_fields=["dek", "is_active"]) diff --git a/codeforlife/user/models/student.py b/codeforlife/user/models/student.py index 4489fc1c..2a175a82 100644 --- a/codeforlife/user/models/student.py +++ b/codeforlife/user/models/student.py @@ -32,7 +32,9 @@ def get_random_username(self): while True: random_username = uuid4().hex[:30] # generate a random username - if not User.objects.filter(username=random_username).exists(): + if not User.objects.filter( + _username_plain=random_username + ).exists(): return random_username # pylint: disable-next=invalid-name @@ -84,7 +86,11 @@ class Student(models.Model): on_delete=models.CASCADE, ) - # hashed uuid used for the unique direct login url + # NOTE: hashed uuid used for the unique direct login url + # TODO: this is currently hashed using sha256 without a salt. this should be + # updated to use the Sha256Field which hashes the value using Django's + # secret key as a salt. This will require generating new values for existing + # students. login_id: str login_id = models.CharField( # type: ignore[assignment] max_length=64, diff --git a/codeforlife/user/models/user/admin_school_teacher.py b/codeforlife/user/models/user/admin_school_teacher.py index 54d41559..06e83d65 100644 --- a/codeforlife/user/models/user/admin_school_teacher.py +++ b/codeforlife/user/models/user/admin_school_teacher.py @@ -26,6 +26,7 @@ class AdminSchoolTeacherUserManager( SchoolTeacherUserManager["AdminSchoolTeacherUser"] ): def filter_users(self, queryset: QuerySet["User"]): + """Filter users to include only admin school teachers.""" return super().filter_users(queryset).filter(new_teacher__is_admin=True) diff --git a/codeforlife/user/models/user/contactable.py b/codeforlife/user/models/user/contactable.py index b63c2104..5b5c52f4 100644 --- a/codeforlife/user/models/user/contactable.py +++ b/codeforlife/user/models/user/contactable.py @@ -23,7 +23,9 @@ # pylint: disable-next=missing-class-docstring,too-few-public-methods class ContactableUserManager(UserManager[AnyUser], t.Generic[AnyUser]): def filter_users(self, queryset: QuerySet[User]): - return queryset.exclude(email__isnull=True).exclude(email="") + return queryset.exclude(_email_plain__isnull=True).exclude( + _email_plain="" + ) # pylint: disable-next=too-many-ancestors @@ -52,6 +54,7 @@ def email_user( # type: ignore[override] personalization_values: t.Optional[t.Dict[str, str]] = None, **kwargs, ): + """Send an email to this user using DotDigital.""" kwargs["to_addresses"] = [self.email] mail.send_mail( campaign_id=campaign_id, diff --git a/codeforlife/user/models/user/independent.py b/codeforlife/user/models/user/independent.py index c61a2854..f28a015a 100644 --- a/codeforlife/user/models/user/independent.py +++ b/codeforlife/user/models/user/independent.py @@ -40,6 +40,7 @@ def filter_users(self, queryset: QuerySet["User"]): def get_queryset(self): return super().get_queryset().prefetch_related("new_student") + # pylint: disable-next=arguments-differ def create_user( # type: ignore[override] self, first_name: str, diff --git a/codeforlife/user/models/user/non_admin_school_teacher.py b/codeforlife/user/models/user/non_admin_school_teacher.py index d5c26a40..4a09f33a 100644 --- a/codeforlife/user/models/user/non_admin_school_teacher.py +++ b/codeforlife/user/models/user/non_admin_school_teacher.py @@ -26,6 +26,7 @@ class NonAdminSchoolTeacherUserManager( SchoolTeacherUserManager["NonAdminSchoolTeacherUser"] ): def filter_users(self, queryset: QuerySet["User"]): + """Filter users to include only non-admin school teachers.""" return ( super().filter_users(queryset).filter(new_teacher__is_admin=False) ) diff --git a/codeforlife/user/models/user/non_school_teacher.py b/codeforlife/user/models/user/non_school_teacher.py index 791fac32..c00683ff 100644 --- a/codeforlife/user/models/user/non_school_teacher.py +++ b/codeforlife/user/models/user/non_school_teacher.py @@ -24,6 +24,7 @@ # pylint: disable-next=missing-class-docstring,too-few-public-methods class NonSchoolTeacherUserManager(TeacherUserManager["NonSchoolTeacherUser"]): def filter_users(self, queryset: QuerySet["User"]): + """Filter users to include only teachers not in a school.""" return ( super() .filter_users(queryset) diff --git a/codeforlife/user/models/user/school_teacher.py b/codeforlife/user/models/user/school_teacher.py index 97a55514..f1acd438 100644 --- a/codeforlife/user/models/user/school_teacher.py +++ b/codeforlife/user/models/user/school_teacher.py @@ -36,6 +36,7 @@ def create_user( # type: ignore[override] is_verified: bool = False, **extra_fields, ): + """Creates a user that is a teacher in a school.""" return super().create_user( first_name=first_name, last_name=last_name, @@ -48,6 +49,7 @@ def create_user( # type: ignore[override] ) def filter_users(self, queryset: QuerySet["User"]): + """Filter users to include only school teachers.""" return ( super() .filter_users(queryset) diff --git a/codeforlife/user/models/user/student.py b/codeforlife/user/models/user/student.py index aa94c273..fc0a8f7d 100644 --- a/codeforlife/user/models/user/student.py +++ b/codeforlife/user/models/user/student.py @@ -27,6 +27,7 @@ # pylint: disable-next=missing-class-docstring,too-few-public-methods class StudentUserManager(UserManager["StudentUser"]): + # pylint: disable-next=arguments-renamed def create_user( # type: ignore[override] self, first_name: str, klass: "Class", **extra_fields ): @@ -140,7 +141,8 @@ def get_random_username(): """Generate a random username that is unique.""" username = None while ( - username is None or User.objects.filter(username=username).exists() + username is None + or User.objects.filter(_username_hash__sha256=username).exists() ): username = get_random_string(length=30) diff --git a/codeforlife/user/models/user/teacher.py b/codeforlife/user/models/user/teacher.py index 8b6f4c40..86aacb2b 100644 --- a/codeforlife/user/models/user/teacher.py +++ b/codeforlife/user/models/user/teacher.py @@ -26,7 +26,7 @@ # pylint: disable-next=missing-class-docstring,too-few-public-methods class TeacherUserManager(ContactableUserManager[AnyUser], t.Generic[AnyUser]): - # pylint: disable-next=too-many-arguments,too-many-positional-arguments + # pylint: disable-next=too-many-arguments,too-many-positional-arguments,arguments-differ def create_user( # type: ignore[override] self, first_name: str, diff --git a/codeforlife/user/models/user/user.py b/codeforlife/user/models/user/user.py index dbb433df..343e4819 100644 --- a/codeforlife/user/models/user/user.py +++ b/codeforlife/user/models/user/user.py @@ -9,30 +9,26 @@ from datetime import datetime, timedelta from django.conf import settings - -# pylint: disable-next=imported-auth-user -from django.contrib.auth.models import AbstractUser +from django.contrib.auth.models import PermissionsMixin from django.contrib.auth.models import UserManager as _UserManager +from django.contrib.auth.validators import UnicodeUsernameValidator from django.db import models from django.db.models.query import QuerySet from django.utils import timezone from django.utils.translation import gettext_lazy as _ from pyotp import TOTP -from ....models import AbstractBaseUser +from ....models import AbstractBaseUser, DataEncryptionKeyModel +from ....models.fields import EncryptedTextField, Sha256Field from ....types import Validators from ....validators import UnicodeAlphanumericCharSetValidator if t.TYPE_CHECKING: # pragma: no cover - from django_stubs_ext.db.models import TypedModelMeta - from ..auth_factor import AuthFactor from ..otp_bypass_token import OtpBypassToken from ..session import Session from ..student import Student from ..teacher import Teacher -else: - TypedModelMeta = object # TODO: add to model validators in new schema. @@ -50,21 +46,86 @@ ] -# TODO: remove in new schema -class _AbstractBaseUser(AbstractBaseUser): - password: str = None # type: ignore[assignment] - last_login: datetime = None # type: ignore[assignment] - - class Meta(TypedModelMeta): - abstract = True +AnyUser = t.TypeVar("AnyUser", bound="User") -# pylint: disable-next=too-many-ancestors -class User( - _AbstractBaseUser, - AbstractUser, # TODO: remove this inheritance in new schema +class UserManager( + _UserManager[AnyUser], + DataEncryptionKeyModel.Manager[AnyUser], + t.Generic[AnyUser], ): - """A proxy to Django's user class.""" + """ + Manager for the User model that inherits Django's default manager and + encrypted manager to handle encrypted fields. + """ + + def _create_user_object( + self, + username: str, + email: t.Optional[str], + password: t.Optional[str], + **extra_fields, + ): + self._inject_dek_kwarg(extra_fields) + return super()._create_user_object( # type: ignore[misc] + username=username, email=email, password=password, **extra_fields + ) + + # pylint: disable=missing-function-docstring + + # @classmethod + # def normalize_email(cls, email): + # return super().normalize_email(email).lower() + + # def get_by_natural_key(self, username): + # return self.get(_username_hash__sha256=username) + + # async def aget_by_natural_key(self, username): + # return await self.aget(_username_hash__sha256=username) + + # pylint: enable=missing-function-docstring + + def filter_users(self, queryset: QuerySet["User"]): + """Filter the users to the specific type. + + Args: + queryset: The queryset of users to filter. + + Returns: + A subset of the queryset of users. + """ + return queryset + + # pylint: disable-next=missing-function-docstring + def get_queryset(self): + queryset = super().get_queryset() + return ( + queryset + if getattr(settings, "OLD_SYSTEM", True) + else self.filter_users(queryset.filter(is_active=True)) + ) + + +# pylint: disable-next=too-many-ancestors,too-many-instance-attributes +class User(AbstractBaseUser, PermissionsMixin, DataEncryptionKeyModel): + """A Code for Life user.""" + + associated_data = "user" + field_aliases = { + "username": {"_username_plain", "_username_enc", "_username_hash"}, + "first_name": { + "_first_name_plain", + "_first_name_enc", + "_first_name_hash", + }, + "last_name": {"_last_name_plain", "_last_name_enc"}, + "email": {"_email_plain", "_email_enc", "_email_hash"}, + } + + EMAIL_FIELD = "_email_plain" + USERNAME_FIELD = "_username_plain" + REQUIRED_FIELDS = ["_email_plain"] + credential_fields = frozenset(["email", "password"]) _password: t.Optional[str] @@ -75,7 +136,164 @@ class User( session: "Session" # type: ignore[assignment] userprofile: "UserProfile" - credential_fields = frozenset(["email", "password"]) + # -------------------------------------------------------------------------- + # Username + # -------------------------------------------------------------------------- + + _username_hash = Sha256Field( + verbose_name=_("username hash"), + db_column="username_hash", + unique=True, + null=True, + ) + _username_plain = models.CharField( + _("username"), + max_length=150, + unique=True, + help_text=_( + "Required. 150 characters or fewer. " + "Letters, digits and @/./+/-/_ only." + ), + validators=[UnicodeUsernameValidator()], + error_messages={ + "unique": _("A user with that username already exists."), + }, + ) + _username_enc = EncryptedTextField( + associated_data="username", + db_column="username_enc", + null=True, + verbose_name=_("username"), + ) + + @property + def username(self): + """The user's username.""" + if self._username_enc is not None: + return EncryptedTextField.get(self, "_username_enc") + return self._username_plain + + @username.setter + def username(self, value: str): + """Set the user's username.""" + self._username_plain = value + EncryptedTextField.set(self, value, "_username_enc") + Sha256Field.set(self, value, "_username_hash") + + # -------------------------------------------------------------------------- + # First name + # -------------------------------------------------------------------------- + + _first_name_hash = Sha256Field( + verbose_name=_("first name hash"), + db_column="first_name_hash", + null=True, + ) + _first_name_plain = models.CharField( + _("first name"), max_length=150, blank=True + ) + _first_name_enc = EncryptedTextField( + associated_data="first_name", + db_column="first_name_enc", + null=True, + verbose_name=_("first name"), + ) + + @property + def first_name(self): + """The user's first name.""" + if self._first_name_enc is not None: + return EncryptedTextField.get(self, "_first_name_enc") + return self._first_name_plain + + @first_name.setter + def first_name(self, value: str): + """Set the user's first name.""" + self._first_name_plain = value + EncryptedTextField.set(self, value, "_first_name_enc") + Sha256Field.set(self, value, "_first_name_hash") + + # -------------------------------------------------------------------------- + # Last name + # -------------------------------------------------------------------------- + + _last_name_plain = models.CharField( + _("last name"), max_length=150, blank=True + ) + _last_name_enc = EncryptedTextField( + associated_data="last_name", + db_column="last_name_enc", + null=True, + verbose_name=_("last name"), + ) + + @property + def last_name(self): + """The user's last name.""" + if self._last_name_enc is not None: + return EncryptedTextField.get(self, "_last_name_enc") + return self._last_name_plain + + @last_name.setter + def last_name(self, value: str): + """Set the user's last name.""" + self._last_name_plain = value + EncryptedTextField.set(self, value, "_last_name_enc") + + # -------------------------------------------------------------------------- + # Email + # -------------------------------------------------------------------------- + + _email_hash = Sha256Field( + verbose_name=_("email hash"), + db_column="email_hash", + null=True, + ) + _email_plain = models.EmailField(_("email address"), blank=True) + _email_enc = EncryptedTextField( + associated_data="email", + db_column="email_enc", + null=True, + verbose_name=_("email address"), + ) + + @property + def email(self): + """The user's email address.""" + if self._email_enc is not None: + return EncryptedTextField.get(self, "_email_enc") + return self._email_plain + + @email.setter + def email(self, value: str): + """Set the user's email address.""" + value = self.__class__.objects.normalize_email(value) + self._email_plain = value + EncryptedTextField.set(self, value, "_email_enc") + Sha256Field.set(self, value, "_email_hash") + + # -------------------------------------------------------------------------- + # Other + # -------------------------------------------------------------------------- + + is_staff = models.BooleanField( + _("staff status"), + default=False, + help_text=_( + "Designates whether the user can log into this admin site." + ), + ) + + is_active = models.BooleanField( + _("active"), + default=True, + help_text=_( + "Designates whether this user should be treated as active. " + "Unselect this instead of deleting accounts." + ), + ) + + date_joined = models.DateTimeField(_("date joined"), default=timezone.now) class Meta: verbose_name = _("user") @@ -96,6 +314,10 @@ class Meta: null=True, ) + objects: UserManager[ # type: ignore[misc] + "User" + ] = UserManager() # type: ignore[assignment] + @property def is_authenticated(self): return ( @@ -189,28 +411,21 @@ def as_type(self, user_class: t.Type["AnyUser"]): def anonymize(self): """Anonymize the user.""" - self.first_name = "" - self.last_name = "" - self.email = "" + self.dek = None self.is_active = False - self.save( - update_fields=[ - "first_name", - "last_name", - "email", - "username", - "is_active", - ] - ) + self.save(update_fields=["dek", "is_active"]) - self.userprofile.google_refresh_token = None - self.userprofile.google_sub = None - self.userprofile.save( - update_fields=[ - "google_refresh_token", - "google_sub", - ] - ) + # self.userprofile.google_refresh_token = None + # self.userprofile.google_sub = None + # self.userprofile.save( + # update_fields=[ + # "google_refresh_token", + # "google_sub", + # ] + # ) + + def __repr__(self): + return f"" if not getattr(settings, "OLD_SYSTEM", True): @@ -222,33 +437,18 @@ def is_verified(self: User): User.is_verified = property(fget=is_verified) # type: ignore[assignment] -AnyUser = t.TypeVar("AnyUser", bound=User) - - -# pylint: disable-next=missing-class-docstring -class UserManager(_UserManager[AnyUser], t.Generic[AnyUser]): - def filter_users(self, queryset: QuerySet[User]): - """Filter the users to the specific type. - - Args: - queryset: The queryset of users to filter. - - Returns: - A subset of the queryset of users. - """ - return queryset - - # pylint: disable-next=missing-function-docstring - def get_queryset(self): - return self.filter_users(super().get_queryset().filter(is_active=True)) - - +# TODO: merge the UserProfile model into the User model and delete it. class UserProfile(models.Model): """A user's profile.""" user = models.OneToOneField(User, on_delete=models.CASCADE) + # NOTE: this is not currently used in production. when it is, it should be: + # 1. moved to the User model + # 2. made non-nullable with a default generator of a random string + # 3. converted into an EncryptedTextField otp_secret = models.CharField(max_length=40, null=True, blank=True) + last_otp_for_time = models.DateTimeField(null=True, blank=True) developer = models.BooleanField(default=False) is_verified = models.BooleanField(default=False) diff --git a/codeforlife/user/serializers/user_test.py b/codeforlife/user/serializers/user_test.py index a77506ae..592b063c 100644 --- a/codeforlife/user/serializers/user_test.py +++ b/codeforlife/user/serializers/user_test.py @@ -33,7 +33,15 @@ def test_to_representation__teacher(self): "student": None, }, # TODO: remove in new schema. - non_model_fields={"requesting_to_join_class", "teacher", "student"}, + non_model_fields={ + "requesting_to_join_class", + "teacher", + "student", + # TODO: remove once plain fields are removed. + "first_name", + "last_name", + "email", + }, ) def test_to_representation__student(self): @@ -55,7 +63,15 @@ def test_to_representation__student(self): }, }, # TODO: remove in new schema. - non_model_fields={"requesting_to_join_class", "teacher", "student"}, + non_model_fields={ + "requesting_to_join_class", + "teacher", + "student", + # TODO: remove once plain fields are removed. + "first_name", + "last_name", + "email", + }, ) def test_to_representation__indy(self): @@ -73,5 +89,13 @@ def test_to_representation__indy(self): "student": None, }, # TODO: remove in new schema. - non_model_fields={"requesting_to_join_class", "teacher", "student"}, + non_model_fields={ + "requesting_to_join_class", + "teacher", + "student", + # TODO: remove once plain fields are removed. + "first_name", + "last_name", + "email", + }, ) diff --git a/codeforlife/user/views/klass.py b/codeforlife/user/views/klass.py index 7f27f45d..c55f3d14 100644 --- a/codeforlife/user/views/klass.py +++ b/codeforlife/user/views/klass.py @@ -16,7 +16,8 @@ class ClassViewSet(ModelViewSet[User, Class]): request_user_class = User model_class = Class http_method_names = ["get"] - lookup_field = "access_code" + lookup_field = "_access_code_hash__sha256" + lookup_url_kwarg = "access_code" serializer_class = ClassSerializer filterset_class = ClassFilterSet diff --git a/codeforlife/user/views/klass_test.py b/codeforlife/user/views/klass_test.py index b7383cdb..14b00d2b 100644 --- a/codeforlife/user/views/klass_test.py +++ b/codeforlife/user/views/klass_test.py @@ -20,7 +20,7 @@ class TestClassViewSet(ModelViewSetTestCase[RequestUser, Class]): def setUp(self): self.admin_school_teacher_user = AdminSchoolTeacherUser.objects.get( - email="admin.teacher@school1.com" + _email_hash__sha256="admin.teacher@school1.com" ) # test: get permissions @@ -108,18 +108,28 @@ def test_list__id_or_name(self): klass = user.teacher.classes.first() assert klass - partial_access_code = klass.access_code[:-1] - partial_name = klass.name[:-1] + partial_access_code = klass.access_code[:-1].lower() + partial_name = klass.name[:-1].lower() + + klasses = Class.objects.select_related("teacher__school") self.client.login_as(user) self.client.list( - models=user.teacher.classes.filter( - access_code__icontains=partial_access_code - ), + models=[ + klass + for klass in klasses.only( + "_access_code_enc", "teacher__school__dek" + ) + if partial_access_code in klass.access_code.lower() + ], filters={"id_or_name": partial_access_code}, ) self.client.list( - models=user.teacher.classes.filter(name__icontains=partial_name), + models=[ + klass + for klass in klasses.only("_name_enc", "teacher__school__dek") + if partial_name in klass.name.lower() + ], filters={"id_or_name": partial_name}, ) diff --git a/codeforlife/user/views/user_test.py b/codeforlife/user/views/user_test.py index c36d833e..b923cc9f 100644 --- a/codeforlife/user/views/user_test.py +++ b/codeforlife/user/views/user_test.py @@ -5,9 +5,6 @@ import typing as t -from django.db.models import Q -from django.db.models.query import QuerySet - from ...tests import ModelViewSetTestCase from ..models import ( AdminSchoolTeacherUser, @@ -33,7 +30,7 @@ class TestUserViewSet(ModelViewSetTestCase[RequestUser, User]): def setUp(self): self.admin_school_teacher_user = AdminSchoolTeacherUser.objects.get( - email="admin.teacher@school1.com" + _email_hash__sha256="admin.teacher@school1.com" ) # test: get queryset @@ -151,7 +148,7 @@ def test_list__students_in_class(self): assert user.teacher.classes.count() >= 2 klass = t.cast(Class, user.teacher.classes.first()) - students: QuerySet[Student] = klass.students.all() + students = klass.students.all() assert ( Student.objects.filter( class_field__teacher__school=user.teacher.school @@ -236,13 +233,20 @@ def test_list__name(self): school_users = user.teacher.school_users first_name, last_name = user.first_name, user.last_name[:1] + first_name, last_name = first_name.lower(), last_name.lower() + + pks = [ + user.pk + for user in school_users.only( + "dek", "_first_name_enc", "_last_name_enc" + ) + if first_name in user.first_name.lower() + or last_name in user.last_name.lower() + ] self.client.login_as(user) self.client.list( - models=school_users.filter( - Q(first_name__icontains=first_name) - | Q(last_name__icontains=last_name) - ).order_by("pk"), + models=school_users.filter(pk__in=pks).order_by("pk"), filters={"name": f"{first_name} {last_name}"}, ) diff --git a/codeforlife/views/__init__.py b/codeforlife/views/__init__.py index d069ca36..0f979ec0 100644 --- a/codeforlife/views/__init__.py +++ b/codeforlife/views/__init__.py @@ -8,6 +8,5 @@ from .base_login import BaseLoginView from .csrf import CsrfCookieView from .decorators import action -from .health_check import HealthCheckView from .model import BaseModelViewSet, ModelViewSet from .session import LogoutView, session_expired_view diff --git a/codeforlife/views/health_check.py b/codeforlife/views/health_check.py deleted file mode 100644 index 4939088b..00000000 --- a/codeforlife/views/health_check.py +++ /dev/null @@ -1,317 +0,0 @@ -""" -© Ocado Group -Created on 14/11/2024 at 16:31:56(+00:00). -""" - -import json -import logging -import os -import typing as t -from dataclasses import dataclass -from datetime import datetime -from functools import cached_property - -from django.apps import apps -from django.conf import settings -from django.contrib.sites.models import Site -from django.views.decorators.cache import cache_page -from psutil import Process -from rest_framework import status -from rest_framework.request import Request -from rest_framework.response import Response -from rest_framework.views import APIView - -from ..permissions import AllowAny - -if t.TYPE_CHECKING: - from ..server import Server - from ..types import Env, JsonDict, JsonList - -HealthStatus = t.Literal[ - "healthy", - "startingUp", - "shuttingDown", - "unhealthy", - "unknown", -] - - -@dataclass(frozen=True) -class HealthCheck: - """The health of the current service.""" - - @dataclass(frozen=True) - class Detail: - """A health detail.""" - - name: str - description: str - health: t.Optional[HealthStatus] = None - - health_status: HealthStatus - additional_info: str - details: t.Optional[t.List[Detail]] = None - - -class HealthCheckDetailList(t.List[HealthCheck.Detail]): - """Builds a list of health-check details with convenience utilities.""" - - def __init__(self, server_mode: "Server.Mode"): - super().__init__() - self.server_mode = server_mode - - @property - def health_statuses(self): - """The health statuses of all the details.""" - return t.cast( - t.FrozenSet[HealthStatus], - frozenset(detail.health for detail in self if detail.health), - ) - - def append( # type: ignore[override] - self, - name: str, - description: str, - health: t.Optional[HealthStatus] = None, - ): - return super().append( - HealthCheck.Detail( - name=f"{self.server_mode}.{name}", - description=description, - health=health, - ) - ) - - -class HealthCheckView(APIView): - """A view for load balancers to determine whether the app is healthy.""" - - http_method_names = ["get"] - permission_classes = [AllowAny] - startup_timestamp = datetime.now().isoformat() - cache_timeout: float = 30 - - def resolve_health_status( - self, *health_statuses: HealthStatus - ) -> HealthStatus: - """Given 1+ health statuses, resolve the final health status.""" - if len(health_statuses) > 0: - search_health_statuses: t.List[HealthStatus] = [ - "unhealthy", - "shuttingDown", - "startingUp", - ] - for search_health_status in search_health_statuses: - if search_health_status in health_statuses: - return search_health_status - - if all( - health_status == "healthy" for health_status in health_statuses - ): - return "healthy" - - return "unknown" - - @cached_property - def celery_worker(self): - """The celery worker started by the server.""" - return Process(int(os.environ["SERVER_CELERY_WORKER_PID"])) - - def get_celery_worker_health_check(self) -> HealthCheck: - """Check the health of the celery worker process.""" - health_check_details = HealthCheckDetailList("celery") - - _status = self.celery_worker.status() - health_check_details.append( - name="status", - description=_status, - health=( - "healthy" - if _status in ["running", "sleeping", "waking", "idle"] - else "unhealthy" - ), - ) - - health_check_details.append( - name="cpu_percent", - description=str(self.celery_worker.cpu_percent()), - ) - - health_status = self.resolve_health_status( - *health_check_details.health_statuses - ) - - return HealthCheck( - health_status=health_status, - additional_info="[celery] " - + ( - "All healthy." - if health_status == "healthy" - else "Not healthy. See details for more info." - ), - details=health_check_details, - ) - - def get_django_worker_health_check(self, request: Request) -> HealthCheck: - """Check the health of the django worker process.""" - health_check_details = HealthCheckDetailList("django") - - ready = apps.ready - health_check_details.append( - name="ready", - description=str(ready), - health="healthy" if ready else "startingUp", - ) - - apps_ready = apps.apps_ready - health_check_details.append( - name="apps_ready", - description=str(apps_ready), - health="healthy" if apps_ready else "startingUp", - ) - - models_ready = apps.models_ready - health_check_details.append( - name="models_ready", - description=str(models_ready), - health="healthy" if models_ready else "startingUp", - ) - - if settings.DB_ENGINE == "postgresql": - - def check_site_health(health_check_name: str, site_domain: str): - exists = Site.objects.filter(domain=site_domain).exists() - health_check_details.append( - name=f"site_exists.{health_check_name}", - description=str(exists), - health="healthy" if exists else "unhealthy", - ) - - if t.cast("Env", settings.ENV) == "local": - check_site_health( - health_check_name="localhost", - site_domain=f"localhost:{settings.SERVICE_PORT}", - ) - check_site_health( - health_check_name="ip_address", - site_domain=f"127.0.0.1:{settings.SERVICE_PORT}", - ) - else: - check_site_health( - health_check_name="domain", - site_domain=settings.SERVICE_DOMAIN, - ) - check_site_health( - health_check_name="host", - site_domain=settings.SERVICE_HOST, - ) - - health_status = self.resolve_health_status( - *health_check_details.health_statuses - ) - - return HealthCheck( - health_status=health_status, - additional_info="[django] " - + ( - "All healthy." - if health_status == "healthy" - else "Not healthy. See details for more info." - ), - details=health_check_details, - ) - - def get_health_check(self, request: Request) -> HealthCheck: - """Check the health of the current service.""" - details: t.List[HealthCheck.Detail] = [] - - try: - django_worker_health_check = self.get_django_worker_health_check( - request - ) - health_status = django_worker_health_check.health_status - additional_info = django_worker_health_check.additional_info - if django_worker_health_check.details: - details += django_worker_health_check.details - - if t.cast("Server.Mode", settings.SERVER_MODE) == "celery": - celery_worker_health_check = ( - self.get_celery_worker_health_check() - ) - health_status = self.resolve_health_status( - health_status, - celery_worker_health_check.health_status, - ) - additional_info += celery_worker_health_check.additional_info - if celery_worker_health_check.details: - details += celery_worker_health_check.details - - return HealthCheck( - health_status=health_status, - additional_info=additional_info, - details=details, - ) - # pylint: disable-next=broad-exception-caught - except Exception as ex: - return HealthCheck( - health_status="unknown", - additional_info=str(ex), - details=details, - ) - - def get(self, request: Request): - """Return a health check for the current service.""" - health_check = self.get_health_check(request) - - data: JsonDict = { - "appId": settings.APP_ID, - "healthStatus": health_check.health_status, - "lastCheckedTimestamp": datetime.now().isoformat(), - "additionalInformation": health_check.additional_info, - "startupTimestamp": self.startup_timestamp, - "appVersion": settings.APP_VERSION, - } - - if health_check.details: - details: JsonList = [] - for _detail in health_check.details: - detail: JsonDict = { - "name": _detail.name, - "description": _detail.description, - } - if _detail.health: - detail["health"] = _detail.health - - details.append(detail) - - data["details"] = details - - if health_check.health_status != "healthy": - logging.warning("health check: %s", json.dumps(data)) - - return Response( - data, - status={ - # The app is running normally. - "healthy": status.HTTP_200_OK, - # The app is performing app-specific initialisation which must - # complete before it will serve normal application requests - # (perhaps the app is warming a cache or something similar). You - # only need to use this status if your app will be in a start-up - # mode for a prolonged period of time. - "startingUp": status.HTTP_503_SERVICE_UNAVAILABLE, - # The app is shutting down. As with startingUp, you only need to - # use this status if your app takes a prolonged amount of time - # to shutdown, perhaps because it waits for a long-running - # process to complete before shutting down. - "shuttingDown": status.HTTP_503_SERVICE_UNAVAILABLE, - # The app is not running normally. - "unhealthy": status.HTTP_503_SERVICE_UNAVAILABLE, - # The app is not able to report its own state. - "unknown": status.HTTP_503_SERVICE_UNAVAILABLE, - }[health_check.health_status], - ) - - @classmethod - def as_view(cls, **initkwargs): - return cache_page(cls.cache_timeout)(super().as_view(**initkwargs)) diff --git a/docs/client-side-encryption.md b/docs/client-side-encryption.md index 144f2e8b..6a1031ab 100644 --- a/docs/client-side-encryption.md +++ b/docs/client-side-encryption.md @@ -1,14 +1,14 @@ # Client-Side Encryption -Client-Side Encryption with Per-User Keys and Django ORM Integration +Client-Side Encryption with Per-Entity Keys and Django ORM Integration --- ## 1. Executive Summary -This architecture implements **Application-Layer Encryption** (often called **Client-Side Encryption** relative to the database) using a **Per-User Key** strategy. +This architecture implements **Application-Layer Encryption** (often called **Client-Side Encryption** relative to the database) using a **Per-Entity Key** strategy. -Instead of relying on a single global key (which is a single point of failure), every user in the system is assigned a unique **Data Encryption Key (DEK)**. This key wraps their specific data. These DEKs are themselves encrypted by a master **Key Encryption Key (KEK)** managed by **Google Cloud KMS**. +Instead of relying on a single global key (which is a single point of failure), each key-owning entity in the system (for example, a user or a school) is assigned a unique **Data Encryption Key (DEK)**. These DEKs are themselves encrypted by a master **Key Encryption Key (KEK)** managed by **Google Cloud KMS**. **Security Guarantee:** The database (PostgreSQL) never sees plaintext data or the plaintext keys required to decrypt it. A database leak results in zero data compromise without also compromising the running application server and Google Cloud credentials. @@ -26,14 +26,14 @@ We utilize a hierarchy of keys to balance security and performance. * **Location:** Google Cloud KMS (Hardware Security Module). * **Role:** The "Master Lock." It never leaves Google. It is used only to encrypt/decrypt the User Keys (DEKs). 2. **DEK (Data Encryption Key):** - * **Location:** Encrypted in the database (e.g., in the `users` table); Decrypted only in Application Memory (RAM). - * **Role:** The "Worker Bee." Unique to every user. Used to encrypt/decrypt the actual database fields (e.g., SSNs, names). + * **Location:** Encrypted in the database (e.g., in key-owning model rows); Decrypted only in Application Memory (RAM). + * **Role:** The "Worker Bee." Unique per key-owning entity (for example, per-user or per-school). Used to encrypt/decrypt actual database fields (e.g., user email). --- ## 3. Encryption/Decryption Utilities -At the core of this system are a few utility functions that interact with Google Cloud KMS and the `tink` cryptography library. In addition, to avoid a dependency on Google Cloud KMS during local development and in CI/CD pipelines, we use fake (mock) implementations of the KMS client and its AEAD primitive. +At the core of this system are a few utility functions that interact with Google Cloud KMS and the `tink` cryptography library. In addition, to avoid a dependency on Google Cloud KMS during local development and in CI/CD pipelines, we use fake implementations of the KMS client and its AEAD primitive. The local fake path still uses a real AEAD algorithm (`AESGCM`) so behavior remains close to cloud execution. **[codeforlife/encryption.py](../codeforlife/encryption.py)** @@ -41,7 +41,7 @@ At the core of this system are a few utility functions that interact with Google ## 4. Django ORM Integration -To make working with encrypted data seamless, we've integrated the encryption logic directly into Django's ORM. This is achieved through a combination of a base model class, custom model fields, and descriptors. +To make working with encrypted data seamless, we've integrated the encryption logic directly into Django's ORM. This is achieved through a combination of a base model class, custom model fields, and explicit property accessors. ### Associated Data for Integrity @@ -69,6 +69,37 @@ The implementation details can be found in the docstring of these files. It's re 1. **[codeforlife/models/data_encryption_key.py](../codeforlife/models/data_encryption_key.py):** This model inherits from `BaseDataEncryptionKeyModel` and conveniently includes the `dek` field by default. 1. **[codeforlife/models/fields/data_encryption_key.py](../codeforlife/models/fields/data_encryption_key.py):** This field is responsible for managing the lifecycle of a DEK for a model instance. +### Field Access and Transformation Pattern + +The current `BaseEncryptedField` flow is based on explicit helpers rather than descriptor-based type mutation. + +1. **Internal storage type:** encrypted model columns store ciphertext bytes. +2. **Set path:** call `BaseEncryptedField.set(instance, plaintext, field_name)` to stage plaintext in pending-encryption storage. +3. **Save path:** `pre_save()` encrypts staged plaintext and writes ciphertext bytes to the DB. +4. **Get path:** call `BaseEncryptedField.get(instance, field_name)` to return plaintext by reading pending values, decrypted cache, or decrypted ciphertext. + +When domain logic requires additional transforms, expose a property and perform all transformation logic in that property getter/setter. + +Examples: + +* **On set:** normalize the plaintext value, then encrypt and hash it. +* **On get:** decrypt the encrypted value. + +### Querying by Sensitive Values + +Encrypted ciphertext is non-deterministic by design, so it cannot be queried with equality semantics. For lookup use-cases (e.g., username/email), store a deterministic one-way hash in a parallel `Sha256Field`. + +* Use `Sha256Field.set(instance, plaintext, "field_hash")` in property setters. +* Query exact matches with `User.objects.filter(_email_hash__sha256="user@example.com")`. +* Query multi-value matches with `User.objects.filter(_email_hash__sha256_in=["a@b.com", "c@d.com"])`. + +### Field Aliases and Partial Saves + +When saving with `update_fields`, aliases are expanded to their real fields by `Model.save()`. + +* Define alias mappings with `field_aliases`. +* Save using alias names (e.g., `update_fields={"email"}`) and all dependent columns (e.g., encrypted, hash) are persisted. + --- ## 5. Usage Patterns @@ -87,30 +118,44 @@ class User(DataEncryptionKeyModel): its own encryption key. """ associated_data = "user" # Required for EncryptedModel + field_aliases = { # Keys/Aliases are replaced with values on save(). + "email": {"_email_enc", "_email_hash"}, + } - username = models.CharField(max_length=150, unique=True) - email = EncryptedTextField(associated_data="email") + _email_enc = EncryptedTextField(associated_data="email") + _email_hash = Sha256Field() + + @property + def email(self): + return EncryptedTextField.get(self, "_email_enc") + + @email.setter + def email(self, value: str): + value = self.__class__.objects.normalize_email(value) + EncryptedTextField.set(self, value, "_email_enc") + Sha256Field.set(self, value, "_email_hash") class Meta: app_label = "auth" # --- Usage --- -# Create a new user. A new DEK is automatically generated and stored in the -# 'dek' field. The 'email' field is encrypted using this key. -user = User.objects.create( - username="johndoe", +# Create a new user. +# A new DEK is automatically generated and stored in the 'dek' field. +# The property setter handles encrypted + hash values. +user = User.objects.create_user( email="john.doe@example.com" ) -# The 'dek' and 'email' fields are stored as encrypted bytes in the database. -# But when we access the 'email' attribute, it's decrypted automatically. +# The encrypted value is stored as bytes in '_email_enc'. +# The property getter returns plaintext. print(f"User's email: {user.email}") # >>> User's email: john.doe@example.com # You can update the email as you would with a normal field. user.email = "john.doe.new@example.com" -user.save() +# Alias expansion saves all backing columns. +user.save(update_fields={"email"}) ``` ### Pattern 2: Delegated Encryption Key @@ -128,7 +173,16 @@ class Secret(EncryptedModel): associated_data = "secret" # Required for EncryptedModel user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="secrets") - secret_value = EncryptedTextField(associated_data="secret-value") + + _secret_value = EncryptedTextField(associated_data="secret-value") + + @property + def secret_value(self): + return EncryptedTextField.get(self, "_secret_value") + + @secret_value.setter + def secret_value(self, value: str): + EncryptedTextField.set(self, value, "_secret_value") class Meta: app_label = "app" @@ -163,7 +217,7 @@ This section contains diagrams that explain what the Django ORM is doing. ### 1. DEK Generation and Initial Save -This diagram shows the process that occurs when a new `EncryptedModel` instance (e.g., a `User`) is created and saved for the first time. The `EncryptedModel.save()` method is overridden to lazily manage the creation of the user-specific DEK. +This diagram shows the process that occurs when a new DEK-enabled model instance (e.g., a `User`) is created and saved for the first time. `BaseDataEncryptionKeyModel.save()` lazily manages creation of the user-specific DEK. ```mermaid sequenceDiagram @@ -193,70 +247,76 @@ sequenceDiagram ### 2. Data Encryption -This diagram illustrates what happens when a developer sets a value on an encrypted field. The `EncryptedAttribute` descriptor intercepts the assignment, wrapping the plaintext value in a `_PendingEncryption` object. The actual encryption happens later, just before the model is saved to the database. +This diagram illustrates what happens when a developer sets a value via a property that uses `BaseEncryptedField.set()`. The value is staged for encryption, and encryption happens later during `save()`. ```mermaid sequenceDiagram actor Developer participant User as User (Model Instance) - participant EncryptedAttribute as EncryptedAttribute (Descriptor) - participant _PendingEncryption as _PendingEncryption (Wrapper) - participant BaseEncryptedField as BaseEncryptedField + participant Property as User.email setter + participant SetHelper as BaseEncryptedField.set() + participant Pending as __pending_encryption_values__ + participant SaveField as BaseEncryptedField.pre_save() participant GcpKmsClient as Tink/KMS Client participant PostgreSQL - Developer->>User: user.ssn = "123-456-7890" - User->>EncryptedAttribute: __set__(user, "123-456-7890") - activate EncryptedAttribute - EncryptedAttribute->>_PendingEncryption: Create(value="123-456-7890") - _PendingEncryption-->>EncryptedAttribute: Returns _PendingEncryption instance - Note left of EncryptedAttribute: Caches are cleared - EncryptedAttribute->>User: instance.__dict__["ssn"] = _PendingEncryption(...) - deactivate EncryptedAttribute + Developer->>User: user.email = "user@example.com" + User->>Property: set transformed values + Property->>SetHelper: set(user, "user@example.com", "_email_enc") + activate SetHelper + SetHelper->>Pending: store plaintext by field attname + SetHelper->>User: clear internal binary value + deactivate SetHelper Developer->>User: user.save() - User->>BaseEncryptedField: pre_save(user, True) - activate BaseEncryptedField - Note right of BaseEncryptedField: Checks for _PendingEncryption - BaseEncryptedField->>GcpKmsClient: Decrypt user's DEK - GcpKmsClient-->>BaseEncryptedField: Returns plaintext DEK - BaseEncryptedField->>GcpKmsClient: encrypt(value, associated_data) - GcpKmsClient-->>BaseEncryptedField: Returns ciphertext - BaseEncryptedField-->>User: Returns ciphertext - deactivate BaseEncryptedField - User->>PostgreSQL: UPDATE users SET ssn=ciphertext + User->>SaveField: pre_save(user, True) + activate SaveField + Note right of SaveField: Reads pending plaintext for field + SaveField->>GcpKmsClient: Decrypt user's DEK + GcpKmsClient-->>SaveField: Returns plaintext DEK + SaveField->>GcpKmsClient: encrypt(value, associated_data) + GcpKmsClient-->>SaveField: Returns ciphertext + SaveField-->>User: Returns ciphertext + deactivate SaveField + User->>PostgreSQL: UPDATE users SET email_enc=ciphertext PostgreSQL-->>User: Returns ``` ### 3. Data Decryption -This diagram shows the process of reading an encrypted value from a model instance. The `EncryptedAttribute` descriptor checks an in-memory cache for the decrypted value first. If it's not cached, it decrypts the ciphertext from the database and populates the cache. +This diagram shows the process of reading a value via a property that uses `BaseEncryptedField.get()`. It checks pending plaintext first, then decrypted cache, then decrypts ciphertext from storage. ```mermaid sequenceDiagram actor Developer participant User as User (Model Instance) - participant EncryptedAttribute as EncryptedAttribute (Descriptor) + participant Property as User.email getter + participant BaseEncryptedField as BaseEncryptedField.get() + participant Pending as __pending_encryption_values__ + participant Cache as __decrypted_values__ participant GcpKmsClient as Tink/KMS Client - Developer->>User: print(user.ssn) - User->>EncryptedAttribute: __get__(user) - activate EncryptedAttribute + Developer->>User: print(user.email) + User->>Property: read email + Property->>BaseEncryptedField: get(user, "_email_enc") + activate BaseEncryptedField - Note over User, EncryptedAttribute: Check instance.__decrypted_values__ cache - alt Value is cached - EncryptedAttribute-->>Developer: return instance.__decrypted_values__[field_name] + alt Pending plaintext exists + BaseEncryptedField->>Pending: read pending plaintext + BaseEncryptedField-->>Property: return plaintext + else Decrypted cache hit + BaseEncryptedField->>Cache: read cached plaintext + BaseEncryptedField-->>Property: return plaintext else Value is not cached - EncryptedAttribute->>User: Get ciphertext from instance.__dict__ - User-->>EncryptedAttribute: Returns ciphertext - EncryptedAttribute->>GcpKmsClient: Decrypt user's DEK - GcpKmsClient-->>EncryptedAttribute: Returns plaintext DEK - EncryptedAttribute->>GcpKmsClient: decrypt(ciphertext, associated_data) - GcpKmsClient-->>EncryptedAttribute: Returns plaintext value - EncryptedAttribute->>User: instance.__decrypted_values__[field_name] = plaintext_value - EncryptedAttribute-->>Developer: return plaintext_value + BaseEncryptedField->>User: get internal ciphertext bytes + User-->>BaseEncryptedField: returns ciphertext + BaseEncryptedField->>GcpKmsClient: decrypt(ciphertext, associated_data) + GcpKmsClient-->>BaseEncryptedField: returns plaintext + BaseEncryptedField->>Cache: cache plaintext by field attname + BaseEncryptedField-->>Property: return plaintext end - deactivate EncryptedAttribute + Property-->>Developer: return plaintext value + deactivate BaseEncryptedField ``` ### 4. Data Shredding @@ -274,7 +334,7 @@ sequenceDiagram User->>PostgreSQL: UPDATE users SET encrypted_dek=NULL WHERE id=... PostgreSQL-->>User: Returns - Note over Developer, PostgreSQL: The user's data (e.g., SSN) is now permanently unrecoverable. + Note over Developer, PostgreSQL: The user's data (e.g., email) is now permanently unrecoverable. ``` ### 5. Encrypted Model and Field Initialization @@ -306,7 +366,7 @@ sequenceDiagram end deactivate EncryptedModel - Django->>BaseEncryptedField: contribute_to_class(EncryptedModel, "ssn") + Django->>BaseEncryptedField: contribute_to_class(EncryptedModel, "email") activate BaseEncryptedField Note over BaseEncryptedField: 4. Validate Model Subclass @@ -315,12 +375,12 @@ sequenceDiagram end Note over BaseEncryptedField: 5. Check for Duplicate Fields - alt "ssn" is already in Model.ENCRYPTED_FIELDS + alt "email" is already in Model.ENCRYPTED_FIELDS BaseEncryptedField-->>Django: raise ValidationError end Note over BaseEncryptedField: 6. Check for Duplicate Associated Data - alt "model:ssn" is already used by another field + alt "model:email" is already used by another field BaseEncryptedField-->>Django: raise ValidationError end @@ -332,7 +392,7 @@ sequenceDiagram ### 6. DEK Model and Field Initialization -This diagram details the validation that occurs when a `DataEncryptionKeyField` is added to a model. The field's `contribute_to_class` method ensures that the model is a valid `BaseDataEncryptionKeyModel` and that it contains only one DEK field. +This diagram details model-level DEK validation. `DataEncryptionKeyField.contribute_to_class()` sets the model's `DEK_FIELD`, and `BaseDataEncryptionKeyModel.check()` validates that `DEK_FIELD` is correctly defined and resolves to a real model field. ```mermaid sequenceDiagram @@ -343,20 +403,44 @@ sequenceDiagram Django->>DataEncryptionKeyField: contribute_to_class(Model, "dek") activate DataEncryptionKeyField - Note over DataEncryptionKeyField: 1. Validate Model Subclass - alt issubclass(Model, BaseDataEncryptionKeyModel) is False + Note over DataEncryptionKeyField: 1. Validate model subclass + alt Model does not subclass BaseDataEncryptionKeyModel DataEncryptionKeyField-->>Django: raise ValidationError end - Note over DataEncryptionKeyField: 2. Check for multiple DEK fields - alt Model.DEK_FIELD is not None + Note over DataEncryptionKeyField: 2. Ensure a single DEK field + alt Model already has DEK_FIELD set DataEncryptionKeyField-->>Django: raise ValidationError end - Note over DataEncryptionKeyField: 3. Register Field on Model + Note over DataEncryptionKeyField: 3. Register field on model DataEncryptionKeyField->>BaseDataEncryptionKeyModel: Model.DEK_FIELD = self - deactivate DataEncryptionKeyField + + Django->>BaseDataEncryptionKeyModel: check() + activate BaseDataEncryptionKeyModel + + Note over BaseDataEncryptionKeyModel: 4. Ensure DEK_FIELD exists + alt DEK_FIELD attribute is missing + BaseDataEncryptionKeyModel-->>Django: add checks.Error(E001) + end + + Note over BaseDataEncryptionKeyModel: 5. Ensure DEK_FIELD is a string + alt DEK_FIELD is not str + BaseDataEncryptionKeyModel-->>Django: add checks.Error(E002) + end + + Note over BaseDataEncryptionKeyModel: 6. Ensure DEK_FIELD is non-empty + alt DEK_FIELD == "" + BaseDataEncryptionKeyModel-->>Django: add checks.Error(E003) + end + + Note over BaseDataEncryptionKeyModel: 7. Ensure DEK_FIELD resolves to a field + alt _meta.get_field(DEK_FIELD) raises FieldDoesNotExist + BaseDataEncryptionKeyModel-->>Django: add checks.Error(E004) + end + + deactivate BaseDataEncryptionKeyModel ``` ### 7. DEK AEAD Caching @@ -372,13 +456,13 @@ This diagram shows how the `dek_aead` property on a `BaseDataEncryptionKeyModel` ```mermaid sequenceDiagram participant User - participant EncryptedField as EncryptedField (__get__) + participant GetHelper as BaseEncryptedField.get() participant DEKEnabledModel as DEK-Enabled Model participant DEK_AEAD_CACHE as TTLCache (in-memory) participant KMS as Google Cloud KMS - User->>+EncryptedField: Accesses encrypted attribute - EncryptedField->>+DEKEnabledModel: Accesses `dek_aead` property + User->>+GetHelper: Reads encrypted-backed property + GetHelper->>+DEKEnabledModel: Accesses `dek_aead` property alt Cache Miss DEKEnabledModel->>DEK_AEAD_CACHE: Check for cached AEAD primitive (not found) DEKEnabledModel->>+KMS: Decrypt DEK using KEK @@ -388,9 +472,9 @@ sequenceDiagram DEKEnabledModel->>DEK_AEAD_CACHE: Check for cached AEAD primitive (found) DEK_AEAD_CACHE-->>DEKEnabledModel: Return AEAD primitive end - DEKEnabledModel-->>-EncryptedField: Return AEAD primitive - EncryptedField->>EncryptedField: Decrypts data using AEAD primitive - EncryptedField-->>-User: Returns decrypted value + DEKEnabledModel-->>-GetHelper: Return AEAD primitive + GetHelper->>GetHelper: Decrypt ciphertext + cache plaintext + GetHelper-->>-User: Returns decrypted value ``` #### 7.2. Cache Invalidation diff --git a/setup.py b/setup.py index ba30c913..4cd4aaad 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,11 @@ import typing as t from pathlib import Path -# pylint: disable-next=import-error +# pylint: disable=import-error from setuptools import find_packages, setup # type: ignore[import-untyped] +from setuptools.command.build_py import build_py # type: ignore[import-untyped] + +# pylint: enable=import-error from codeforlife import DATA_DIR, TEMPLATES_DIR, __version__ from codeforlife.user import FIXTURES_DIR as USER_FIXTURES_DIR @@ -26,6 +29,24 @@ long_description = readme.read() +# pylint: disable-next=too-few-public-methods +class BuildPy(build_py): + """Custom build command to exclude test files.""" + + def find_package_modules(self, package: str, package_dir: str): + """Find all modules in the package, excluding test files.""" + return [ + module + for module in t.cast( + t.List[t.Tuple[str, str, str]], + super().find_package_modules(package, package_dir), + ) + if not ( + module[1].endswith("_test") or module[1].startswith("test_") + ) + ] + + # Walk through data directory and get relative file paths. def get_data_files(target_dir: Path): """Get the path of all files in a target directory relative to where they @@ -98,8 +119,9 @@ def parse_requirements(packages: t.Dict[str, t.Dict[str, t.Any]]): long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/ocadotechnology/codeforlife-package-python", - # TODO: exclude test files - packages=find_packages(exclude=["tests", "tests.*"]), + packages=find_packages(include=["codeforlife", "codeforlife.*"]), + package_dir={"codeforlife": "codeforlife"}, + cmdclass={"build_py": BuildPy}, include_package_data=True, data_files=[ get_data_files(DATA_DIR),