diff --git a/db/Cargo.toml b/db/Cargo.toml index 3b4a584..e78e253 100644 --- a/db/Cargo.toml +++ b/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "db-rs" -version = "0.3.3" +version = "0.3.5" edition = "2021" description = "fast, embedded, transactional, key value store" license = "BSD-3-Clause" @@ -12,6 +12,7 @@ clone = [] [dependencies] serde = { version = "1.0", features = ["derive"] } bincode = "1.3.3" +uuid = { version = "1.2.2", features = ["v4"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] fs2 = "0.4.3" diff --git a/db/src/lib.rs b/db/src/lib.rs index ceffc9a..772805c 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -109,6 +109,7 @@ pub use crate::logger::TxHandle; pub use crate::list::List; pub use crate::lookup::LookupTable; pub use crate::lookup_list::LookupList; +pub use crate::lookup_map::LookupMap; pub use crate::lookup_set::LookupSet; pub use crate::single::Single; @@ -120,9 +121,11 @@ pub mod list; pub mod logger; pub mod lookup; pub mod lookup_list; +pub mod lookup_map; pub mod lookup_set; pub mod single; pub mod table; +pub mod utils; pub type TableId = u8; pub type ByteCount = u32; diff --git a/db/src/logger.rs b/db/src/logger.rs index 6861549..24167c6 100644 --- a/db/src/logger.rs +++ b/db/src/logger.rs @@ -10,6 +10,7 @@ use std::sync::{Arc, Mutex}; use fs2::FileExt; pub struct LogFormat<'a> { + pub index: usize, pub table_id: TableId, pub bytes: &'a [u8], } @@ -77,7 +78,10 @@ impl Logger { let mut entries = vec![]; while index < buffer.len() { + let index_capture = index + 2; + println!("{index}, {index_capture}"); if buffer.len() < index + 4 + 1 { + eprintln!("size missing!"); self.inner.lock()?.incomplete_write = true; return Ok(entries); } @@ -93,16 +97,18 @@ impl Logger { index += 4; if buffer.len() < index + size { + eprintln!("expected {} found {}", index + size, buffer.len()); self.inner.lock()?.incomplete_write = true; return Ok(entries); } if table_id == 0 { + println!("TABLE_ID == 0"); continue; } let bytes = &buffer[index..index + size]; - entries.push(LogFormat { table_id, bytes }); + entries.push(LogFormat { index: index_capture, table_id, bytes }); index += size; } diff --git a/db/src/lookup_map.rs b/db/src/lookup_map.rs new file mode 100644 index 0000000..0f7e4e1 --- /dev/null +++ b/db/src/lookup_map.rs @@ -0,0 +1,155 @@ +use crate::table::Table; +use crate::{DbResult, Logger, TableId}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::hash::Hash; + +/// A special case of [crate::lookup::LookupTable] where the value of the [HashMap] is another `HashMap`. +#[derive(Debug)] +#[cfg_attr(feature = "clone", derive(Clone))] +pub struct LookupMap +where + K1: Hash + Eq + Serialize, + K2: Hash + Eq + Serialize, + V: Serialize + DeserializeOwned, +{ + table_id: TableId, + inner: HashMap>, + pub logger: Logger, +} + +#[derive(Serialize, Deserialize)] +pub enum LogEntry { + Insert(K1, K2, V), + Remove(K1, K2), + CreateKey(K1), + ClearKey(K1), + Clear, +} + +impl Table for LookupMap +where + K1: Hash + Eq + Serialize + DeserializeOwned, + K2: Hash + Eq + Serialize + DeserializeOwned, + V: Serialize + DeserializeOwned, +{ + fn init(table_id: TableId, logger: Logger) -> Self { + Self { table_id, inner: HashMap::default(), logger } + } + + fn handle_event(&mut self, bytes: &[u8]) -> DbResult<()> { + match bincode::deserialize::>(bytes)? { + LogEntry::Insert(k1, k2, v) => { + self.insert_inner(k1, k2, v); + } + LogEntry::Remove(k1, k2) => { + if let Some(map) = self.inner.get_mut(&k1) { + map.remove(&k2); + }; + } + LogEntry::CreateKey(k1) => { + self.inner.insert(k1, HashMap::new()); + } + LogEntry::ClearKey(k1) => { + self.inner.remove(&k1); + } + LogEntry::Clear => { + self.inner.clear(); + } + }; + + Ok(()) + } + + fn compact_repr(&self) -> DbResult> { + let mut repr = vec![]; + for (k1, values) in &self.inner { + if values.is_empty() { + let data = bincode::serialize(&LogEntry::<&K1, &K2, &V>::CreateKey(k1))?; + let mut data = Logger::log_entry(self.table_id, data); + repr.append(&mut data); + continue; + } + for (k2, v) in values { + let data = bincode::serialize(&LogEntry::Insert(k1, k2, v))?; + let mut data = Logger::log_entry(self.table_id, data); + repr.append(&mut data); + } + } + + Ok(repr) + } +} + +impl LookupMap +where + K1: Hash + Eq + Serialize + DeserializeOwned, + K2: Hash + Eq + Serialize + DeserializeOwned, + V: Serialize + DeserializeOwned, +{ + pub(crate) fn insert_inner(&mut self, k1: K1, k2: K2, v: V) -> Option { + if let Some(map) = self.inner.get_mut(&k1) { + map.insert(k2, v) + } else { + let mut map = HashMap::new(); + map.insert(k2, v); + self.inner.insert(k1, map); + None + } + } + + pub fn insert(&mut self, k1: K1, k2: K2, v: V) -> DbResult> { + let log_entry = LogEntry::Insert(&k1, &k2, &v); + let data = bincode::serialize(&log_entry)?; + let ret = self.insert_inner(k1, k2, v); + + self.logger.write(self.table_id, data)?; + Ok(ret) + } + + pub fn create_key(&mut self, key: K1) -> DbResult>> { + let log_entry = LogEntry::<&K1, K2, &V>::CreateKey(&key); + let data = bincode::serialize(&log_entry)?; + + let ret = self.inner.insert(key, HashMap::new()); + + self.logger.write(self.table_id, data)?; + Ok(ret) + } + + pub fn remove(&mut self, k1: &K1, k2: &K2) -> DbResult> { + if let Some(map) = self.inner.get_mut(k1) { + let log_entry = LogEntry::Remove::<&K1, &K2, V>(k1, k2); + let data = bincode::serialize(&log_entry)?; + self.logger.write(self.table_id, data)?; + Ok(map.remove(k2)) + } else { + Ok(None) + } + } + + pub fn clear_key(&mut self, k1: &K1) -> DbResult>> { + let log_entry = LogEntry::ClearKey::<&K1, K2, V>(k1); + let data = bincode::serialize(&log_entry)?; + + let ret = self.inner.remove(k1); + + self.logger.write(self.table_id, data)?; + + Ok(ret) + } + + pub fn get(&self) -> &HashMap> { + &self.inner + } + + pub fn clear(&mut self) -> DbResult<()> { + self.inner.clear(); + let log_entry = LogEntry::::Clear; + let data = bincode::serialize(&log_entry)?; + self.logger.write(self.table_id, data)?; + + Ok(()) + } +} diff --git a/db/src/utils.rs b/db/src/utils.rs new file mode 100644 index 0000000..06e8d20 --- /dev/null +++ b/db/src/utils.rs @@ -0,0 +1,5 @@ +use uuid::Uuid; + +pub fn random_test_dir() -> String { + format!("/tmp/{}", Uuid::new_v4()) +} diff --git a/db/tests/clone_feature_test.rs b/db/tests/clone_feature_test.rs index d040de7..92f07fa 100644 --- a/db/tests/clone_feature_test.rs +++ b/db/tests/clone_feature_test.rs @@ -3,7 +3,7 @@ mod clone_feature { use std::fs; - use db_rs::*; + use db_rs::{utils::random_test_dir, *}; use db_rs_derive::Schema; #[derive(Schema, Clone)] @@ -17,7 +17,7 @@ mod clone_feature { #[test] fn test() { - let dir = "/tmp/o/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = CloneFT::init(Config::in_folder(dir)).unwrap(); db.table1.insert(5, "test".to_string()).unwrap(); diff --git a/db/tests/list_tests.rs b/db/tests/list_tests.rs index a53bf96..120e994 100644 --- a/db/tests/list_tests.rs +++ b/db/tests/list_tests.rs @@ -1,4 +1,4 @@ -use db_rs::{Config, Db, List}; +use db_rs::{utils::random_test_dir, Config, Db, List}; use db_rs_derive::Schema; use std::fs; @@ -11,7 +11,7 @@ struct Schema { #[test] fn list_test() { - let dir = "/tmp/j/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = Schema::init(Config::in_folder(dir)).unwrap(); db.list1.push("a".to_string()).unwrap(); @@ -26,7 +26,7 @@ fn list_test() { #[test] fn list_test2() { - let dir = "/tmp/k/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = Schema::init(Config::in_folder(dir)).unwrap(); db.list1.push("a".to_string()).unwrap(); @@ -56,7 +56,7 @@ fn list_test2() { #[test] fn list_test3() { - let dir = "/tmp/kk/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = Schema::init(Config::in_folder(dir)).unwrap(); db.list1.push("a".to_string()).unwrap(); @@ -90,7 +90,7 @@ fn list_test3() { #[test] fn list_test_clear() { - let dir = "/tmp/kkk/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = Schema::init(Config::in_folder(dir)).unwrap(); db.list1.push("a".to_string()).unwrap(); diff --git a/db/tests/log_tests.rs b/db/tests/log_tests.rs index 3497648..4d6824b 100644 --- a/db/tests/log_tests.rs +++ b/db/tests/log_tests.rs @@ -1,4 +1,5 @@ use db_rs::compacter::BackgroundCompacter; +use db_rs::utils::random_test_dir; use db_rs::{CancelSig, Config, Db, LookupTable, Single}; use db_rs_derive::Schema; use std::fs::{remove_dir_all, OpenOptions}; @@ -15,7 +16,7 @@ pub struct LogTests { #[test] fn log_compaction() { - let dir = "/tmp/e"; + let dir = &random_test_dir(); drop(remove_dir_all(dir)); let mut db = LogTests::init(Config::in_folder(dir)).unwrap(); @@ -43,7 +44,7 @@ fn log_compaction() { #[test] fn inter_log() { - let dir = "/tmp/f"; + let dir = &random_test_dir(); drop(remove_dir_all(dir)); let mut db = LogTests::init(Config::in_folder(dir)).unwrap(); @@ -93,7 +94,7 @@ fn no_io_tests() { #[test] #[ignore] // ignored so tests don't get stuck here fn auto_log_compacter() { - let dir = "/tmp/fa"; + let dir = &random_test_dir(); drop(remove_dir_all(dir)); let db = Arc::new(Mutex::new(LogTests::init(Config::in_folder(dir)).unwrap())); let cancel = CancelSig::default(); diff --git a/db/tests/lookup_list_tests.rs b/db/tests/lookup_list_tests.rs index 274da6e..a91cb31 100644 --- a/db/tests/lookup_list_tests.rs +++ b/db/tests/lookup_list_tests.rs @@ -1,3 +1,4 @@ +use db_rs::utils::random_test_dir; use db_rs::Db; use db_rs::{Config, LookupList}; use db_rs_derive::Schema; @@ -14,7 +15,7 @@ pub struct LookupSchema { #[test] fn test() { - let dir = "/tmp/o/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); db.table1.push(5, "test".to_string()).unwrap(); @@ -28,7 +29,7 @@ fn test() { #[test] fn test2() { - let dir = "/tmp/p/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); @@ -77,7 +78,7 @@ fn test2() { #[test] fn test3() { - let dir = "/tmp/q/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); diff --git a/db/tests/lookup_map_tests.rs b/db/tests/lookup_map_tests.rs new file mode 100644 index 0000000..e8829b3 --- /dev/null +++ b/db/tests/lookup_map_tests.rs @@ -0,0 +1,129 @@ +use db_rs::utils::random_test_dir; +use db_rs::Db; +use db_rs::{Config, LookupMap}; +use db_rs_derive::Schema; +use std::fs; + +#[derive(Schema)] +pub struct LookupSchema { + table1: LookupMap, + table2: LookupMap, + table3: LookupMap, + table4: LookupMap, + table5: LookupMap, +} + +#[test] +fn test() { + let dir = &random_test_dir(); + drop(fs::remove_dir_all(dir)); + let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); + db.table1.insert(5, 0, "test".to_string()).unwrap(); + drop(db); + + let db = LookupSchema::init(Config::in_folder(dir)).unwrap(); + let target = ["test".to_string()]; + assert_eq!(db.table1.get().get(&5).unwrap().get(&0).unwrap(), &target[0]); + drop(fs::remove_dir_all(dir)); +} + +#[test] +fn test2() { + let dir = &random_test_dir(); + + drop(fs::remove_dir_all(dir)); + let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); + db.table1.insert(1, 0, "test1".to_string()).unwrap(); + assert!(db.table1.get().get(&1).is_some()); + db.table1.insert(1, 1, "test2".to_string()).unwrap(); + db.table1.insert(1, 2, "test3".to_string()).unwrap(); + db.table1.insert(2, 0, "test4".to_string()).unwrap(); + db.table1.insert(3, 0, "test5".to_string()).unwrap(); + drop(db); + + let db = LookupSchema::init(Config::in_folder(dir)).unwrap(); + assert!(db + .table1 + .get() + .get(&1) + .unwrap() + .get(&0) + .unwrap() + .contains(&"test1".to_string())); + assert!(db + .table1 + .get() + .get(&1) + .unwrap() + .get(&1) + .unwrap() + .contains(&"test2".to_string())); + assert!(db + .table1 + .get() + .get(&1) + .unwrap() + .get(&2) + .unwrap() + .contains(&"test3".to_string())); + assert!(db + .table1 + .get() + .get(&2) + .unwrap() + .get(&0) + .unwrap() + .contains(&"test4".to_string())); + assert!(db + .table1 + .get() + .get(&3) + .unwrap() + .get(&0) + .unwrap() + .contains(&"test5".to_string())); + + drop(fs::remove_dir_all(dir)); +} + +#[test] +fn test3() { + let dir = &random_test_dir(); + + drop(fs::remove_dir_all(dir)); + let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); + db.table1.create_key(1).unwrap(); + assert!(db.table1.get().get(&1).is_some()); + assert!(db.table1.get().get(&1).unwrap().is_empty()); + + drop(db); + let db = LookupSchema::init(Config::in_folder(dir)).unwrap(); + assert!(db.table1.get().get(&1).is_some()); + assert!(db.table1.get().get(&1).unwrap().is_empty()); + + drop(fs::remove_dir_all(dir)); +} + +#[test] +fn test4() { + let dir = &random_test_dir(); + + drop(fs::remove_dir_all(dir)); + let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); + db.table1.insert(0, 0, "test1".to_string()).unwrap(); + db.table1.insert(0, 1, "test2".to_string()).unwrap(); + db.table1.remove(&0, &1).unwrap(); + + db.table1.insert(1, 0, "test3".to_string()).unwrap(); + db.table1.insert(1, 1, "test4".to_string()).unwrap(); + + db.table1.clear_key(&1).unwrap(); + + drop(db); + let db = LookupSchema::init(Config::in_folder(dir)).unwrap(); + + assert!(db.table1.get().get(&1).is_none()); + assert!(db.table1.get().get(&0).unwrap().get(&1).is_none()); + + drop(fs::remove_dir_all(dir)); +} diff --git a/db/tests/lookup_set_tests.rs b/db/tests/lookup_set_tests.rs index 1395ee3..b03df84 100644 --- a/db/tests/lookup_set_tests.rs +++ b/db/tests/lookup_set_tests.rs @@ -1,3 +1,4 @@ +use db_rs::utils::random_test_dir; use db_rs::Db; use db_rs::{Config, LookupSet}; use db_rs_derive::Schema; @@ -15,7 +16,7 @@ pub struct LookupSchema { #[test] fn test() { - let dir = "/tmp/l/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); db.table1.insert(5, "test".to_string()).unwrap(); @@ -30,7 +31,7 @@ fn test() { #[test] fn test2() { - let dir = "/tmp/m/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); @@ -54,7 +55,7 @@ fn test2() { #[test] fn test3() { - let dir = "/tmp/n/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); diff --git a/db/tests/lookup_tests.rs b/db/tests/lookup_tests.rs index b0262dd..3bbb2b6 100644 --- a/db/tests/lookup_tests.rs +++ b/db/tests/lookup_tests.rs @@ -1,3 +1,4 @@ +use db_rs::utils::random_test_dir; use db_rs::Db; use db_rs::{Config, LookupTable}; use db_rs_derive::Schema; @@ -14,7 +15,7 @@ pub struct LookupSchema { #[test] fn test() { - let dir = "/tmp/a/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); db.table1.insert(5, "test".to_string()).unwrap(); @@ -27,7 +28,7 @@ fn test() { #[test] fn test2() { - let dir = "/tmp/b/"; + let dir = &random_test_dir(); drop(fs::remove_dir_all(dir)); let mut db = LookupSchema::init(Config::in_folder(dir)).unwrap(); diff --git a/db/tests/single_tests.rs b/db/tests/single_tests.rs index 86d2087..9433b05 100644 --- a/db/tests/single_tests.rs +++ b/db/tests/single_tests.rs @@ -1,3 +1,4 @@ +use db_rs::utils::random_test_dir; use db_rs::{Config, Db, Single}; use db_rs_derive::Schema; use std::fs::remove_dir_all; @@ -14,7 +15,7 @@ pub struct SingleSchema { #[test] fn test_simple() { - let dir = "/tmp/c"; + let dir = &random_test_dir(); drop(remove_dir_all(dir)); @@ -37,7 +38,7 @@ fn test_simple() { #[test] fn test_complex() { - let dir = "/tmp/d"; + let dir = &random_test_dir(); drop(remove_dir_all(dir)); diff --git a/db/tests/transaction_tests.rs b/db/tests/transaction_tests.rs index f9d0964..eca8bb9 100644 --- a/db/tests/transaction_tests.rs +++ b/db/tests/transaction_tests.rs @@ -1,8 +1,8 @@ +use db_rs::utils::random_test_dir; use db_rs::{Config, Db, LookupTable}; use db_rs_derive::Schema; use std::fs::{remove_dir_all, OpenOptions}; use std::io::{Read, Write}; - #[derive(Schema)] struct TxTest { table: LookupTable, @@ -10,7 +10,7 @@ struct TxTest { #[test] fn simple_tx() { - let dir = "/tmp/g"; + let dir = &random_test_dir(); drop(remove_dir_all(dir)); let mut cfg = Config::in_folder(dir); cfg.fs_locks = false; @@ -35,7 +35,7 @@ fn simple_tx() { #[test] fn tx_log_corrupt() { - let dir = "/tmp/h"; + let dir = &random_test_dir(); drop(remove_dir_all(dir)); let mut db = TxTest::init(Config::in_folder(dir)).unwrap(); @@ -78,7 +78,7 @@ fn tx_log_corrupt() { #[test] fn snapshot_inter() { - let dir = "/tmp/i"; + let dir = &random_test_dir(); drop(remove_dir_all(dir)); let mut db = TxTest::init(Config::in_folder(dir)).unwrap(); diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 7f27843..d1da8d7 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "db-rs-derive" -version = "0.3.3" +version = "0.3.5" edition = "2021" description = "macros for db-rs" license = "BSD-3-Clause"