This is an automated email from the ASF dual-hosted git repository. mssun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-teaclave.git
commit e5215555a01ef506f3b577dd36e6cd6e26ed9760 Author: sunhe05 <[email protected]> AuthorDate: Fri Oct 28 09:33:03 2022 +0000 Update rusty-leveldb version to v1.0.4 --- common/rusty_leveldb_sgx/Cargo.toml | 4 +- common/rusty_leveldb_sgx/README.md | 22 +++-------- common/rusty_leveldb_sgx/src/block_builder.rs | 2 +- common/rusty_leveldb_sgx/src/cache.rs | 54 +++++++++++++-------------- common/rusty_leveldb_sgx/src/db_impl.rs | 16 +++++--- common/rusty_leveldb_sgx/src/db_iter.rs | 14 ++++++- common/rusty_leveldb_sgx/src/env.rs | 2 +- common/rusty_leveldb_sgx/src/error.rs | 27 +++++++++++++- common/rusty_leveldb_sgx/src/key_types.rs | 22 +++++------ common/rusty_leveldb_sgx/src/lib.rs | 1 + common/rusty_leveldb_sgx/src/mem_env.rs | 8 ++-- common/rusty_leveldb_sgx/src/options.rs | 2 - common/rusty_leveldb_sgx/src/skipmap.rs | 2 +- common/rusty_leveldb_sgx/src/version.rs | 2 +- common/rusty_leveldb_sgx/src/version_edit.rs | 8 ++-- common/rusty_leveldb_sgx/src/version_set.rs | 4 +- common/rusty_leveldb_sgx/src/write_batch.rs | 12 +++--- 17 files changed, 113 insertions(+), 89 deletions(-) diff --git a/common/rusty_leveldb_sgx/Cargo.toml b/common/rusty_leveldb_sgx/Cargo.toml index 2c09ebf9..035aceea 100644 --- a/common/rusty_leveldb_sgx/Cargo.toml +++ b/common/rusty_leveldb_sgx/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "rusty-leveldb" -version = "0.3.0" +version = "1.0.4" authors = ["Lewin Bormann <[email protected]>"] description = "A compatible re-implementation of LevelDB in Rust" homepage = "https://github.com/dermesser/leveldb-rs"; -repository = "https://borgac.net/lbo/hg/"; +repository = "https://github.com/dermesser/leveldb-rs"; readme = "README.md" keywords = ["LevelDB", "key-value", "database", "SSTable", "Google"] license = "MIT" diff --git a/common/rusty_leveldb_sgx/README.md b/common/rusty_leveldb_sgx/README.md index 090264dc..49bead60 100644 --- a/common/rusty_leveldb_sgx/README.md +++ b/common/rusty_leveldb_sgx/README.md @@ -1,9 +1,8 @@ -# leveldb-rs -[](https://ci.mesalock-linux.org/mesalock-linux/rusty_leveldb_sgx) +# rusty-leveldb-sgx [](https://crates.io/crates/rusty-leveldb) A fully compatible implementation of LevelDB in Rust. (any incompatibility is a -bug!) +bug!) Be able to run inside SGX. The implementation is very close to the original; often, you can see the same algorithm translated 1:1, and class (struct) and method names are similar or @@ -14,18 +13,7 @@ I do care, however, about bug reports. ## Status -* User-facing methods exist: Read/Write/Delete; snapshots; iteration -* Compaction is supported, including manual ones. -* Fully synchronous: Efficiency gains by using non-atomic types, but writes may - occasionally block during a compaction. In --release mode, an average compaction - takes 0.2-0.5 seconds. -* Compatibility with the original: Compression is not implemented so far; this works - as long as compression is disabled in the original. -* Performance is decent; while usually not par with the original, due to multi-threading - in the original and language-inherent overhead (we are doing things the right way), - it will be enough for most use cases. -* Safe: While using many shared pointers, the implementation is generally safe. Many - places use asserts though, so you may see a crash -- in which case you should file a bug. +Working well, with a few rare bugs (see leveldb-rs issues). ## Goals @@ -33,10 +21,10 @@ Some of the goals of this implementation are * As few copies of data as possible; most of the time, slices of bytes (`&[u8]`) are used. Owned memory is represented as `Vec<u8>` (and then possibly borrowed - as slice). + as slice). Zero-copy is not always possible, though, and sometimes simplicity is favored. * Correctness -- self-checking implementation, good test coverage, etc. Just like the original implementation. * Clarity; commented code, clear structure (hopefully doing a better job than the original implementation). * Coming close-ish to the original implementation; clarifying the translation of - typical C++ constructs to Rust. + typical C++ constructs to Rust, and doing a better job at helping understand the internals. diff --git a/common/rusty_leveldb_sgx/src/block_builder.rs b/common/rusty_leveldb_sgx/src/block_builder.rs index ed20ef9b..fdaa1b88 100644 --- a/common/rusty_leveldb_sgx/src/block_builder.rs +++ b/common/rusty_leveldb_sgx/src/block_builder.rs @@ -39,7 +39,7 @@ impl BlockBuilder { self.counter } - pub fn last_key<'a>(&'a self) -> &'a [u8] { + pub fn last_key(&self) -> &[u8] { &self.last_key } diff --git a/common/rusty_leveldb_sgx/src/cache.rs b/common/rusty_leveldb_sgx/src/cache.rs index 386e3272..c29228c3 100644 --- a/common/rusty_leveldb_sgx/src/cache.rs +++ b/common/rusty_leveldb_sgx/src/cache.rs @@ -2,7 +2,7 @@ use std::prelude::v1::*; use std::collections::HashMap; -use std::mem::{replace, swap}; +use std::mem::swap; // No clone, no copy! That asserts that an LRUHandle exists only once. type LRUHandle<T> = *mut LRUNode<T>; @@ -88,7 +88,7 @@ impl<T> LRUList<T> { // Set up the node after the new one self.head.next.as_mut().unwrap().prev = Some(newp); // Replace head.next with None and set the new node's next to that - new.next = replace(&mut self.head.next, None); + new.next = self.head.next.take(); self.head.next = Some(new); newp @@ -111,21 +111,18 @@ impl<T> LRUList<T> { } fn remove_last(&mut self) -> Option<T> { - if self.head.prev.is_some() { - let mut lasto = unsafe { - replace( - &mut (*((*self.head.prev.unwrap()).prev.unwrap())).next, - None, - ) - }; - - if let Some(ref mut last) = lasto { - self.head.prev = last.prev; - self.count -= 1; - return replace(&mut (*last).data, None); - } else { - None - } + if self.count() == 0 { + return None; + } + let mut lasto = unsafe { (*((*self.head.prev.unwrap()).prev.unwrap())).next.take() }; + + assert!(lasto.is_some()); + if let Some(ref mut last) = lasto { + assert!(last.prev.is_some()); + assert!(self.head.prev.is_some()); + self.head.prev = last.prev; + self.count -= 1; + (*last).data.take() } else { None } @@ -133,21 +130,20 @@ impl<T> LRUList<T> { fn remove(&mut self, node_handle: LRUHandle<T>) -> T { unsafe { - // If has next - if let Some(ref mut nextp) = (*node_handle).next { - (**nextp).prev = (*node_handle).prev; - } - // If has prev - if let Some(ref mut prevp) = (*node_handle).prev { - // swap prev.next - // (node_handle will own itself now) - swap(&mut (**prevp).next, &mut (*node_handle).next); + let d = (*node_handle).data.take().unwrap(); + // Take ownership of node to be removed. + let mut current = (*(*node_handle).prev.unwrap()).next.take().unwrap(); + let prev = current.prev.unwrap(); + // Update previous node's successor. + if current.next.is_some() { + // Update next node's predecessor. + current.next.as_mut().unwrap().prev = current.prev.take(); } + (*prev).next = current.next.take(); self.count -= 1; - // node_handle now only has references/objects that point to itself, - // so it's safe to drop - replace(&mut (*node_handle).data, None).unwrap() + + d } } diff --git a/common/rusty_leveldb_sgx/src/db_impl.rs b/common/rusty_leveldb_sgx/src/db_impl.rs index 04cb7575..e2d9aa3f 100644 --- a/common/rusty_leveldb_sgx/src/db_impl.rs +++ b/common/rusty_leveldb_sgx/src/db_impl.rs @@ -191,13 +191,17 @@ impl DB { let mut log_files = vec![]; for file in &filenames { - if let Ok((num, typ)) = parse_file_name(&file) { - expected.remove(&num); - if typ == FileType::Log - && (num >= self.vset.borrow().log_num || num == self.vset.borrow().prev_log_num) - { - log_files.push(num); + match parse_file_name(&file) { + Ok((num, typ)) => { + expected.remove(&num); + if typ == FileType::Log + && (num >= self.vset.borrow().log_num + || num == self.vset.borrow().prev_log_num) + { + log_files.push(num); + } } + Err(e) => return Err(e.annotate(format!("While parsing {:?}", file))), } } if !expected.is_empty() { diff --git a/common/rusty_leveldb_sgx/src/db_iter.rs b/common/rusty_leveldb_sgx/src/db_iter.rs index 72f2098f..f7dc090c 100644 --- a/common/rusty_leveldb_sgx/src/db_iter.rs +++ b/common/rusty_leveldb_sgx/src/db_iter.rs @@ -62,7 +62,7 @@ impl DBIterator { /// record_read_sample records a read sample using the current contents of self.keybuf, which /// should be an InternalKey. - fn record_read_sample<'a>(&mut self, len: usize) { + fn record_read_sample(&mut self, len: usize) { self.byte_count -= len as isize; if self.byte_count < 0 { let v = self.vset.borrow().current(); @@ -284,7 +284,7 @@ impl LdbIterator for DBIterator { } fn random_period() -> isize { - rand::random::<isize>() % 2 * READ_BYTES_PERIOD + rand::random::<isize>() % (2 * READ_BYTES_PERIOD) } #[cfg(feature = "enclave_unit_test")] @@ -292,6 +292,7 @@ pub mod tests { use super::*; use crate::db_impl::testutil::*; use crate::db_impl::DB; + use crate::options; use crate::test_util::LdbIteratorIter; use crate::types::{current_key_val, Direction}; @@ -309,6 +310,7 @@ pub mod tests { db_iter_deleted_entry_not_returned, db_iter_deleted_entry_not_returned_memtable, db_iter_repeated_open_close, + db_iter_allow_empty_key, ) } @@ -494,4 +496,12 @@ pub mod tests { } } } + + fn db_iter_allow_empty_key() { + let opt = options::for_test(); + let mut db = DB::open("db", opt).unwrap(); + assert!(db.new_iter().unwrap().next().is_none()); + db.put(&[], &[]).unwrap(); + assert!(db.new_iter().unwrap().next().is_some()); + } } diff --git a/common/rusty_leveldb_sgx/src/env.rs b/common/rusty_leveldb_sgx/src/env.rs index 33ce9838..04451e16 100644 --- a/common/rusty_leveldb_sgx/src/env.rs +++ b/common/rusty_leveldb_sgx/src/env.rs @@ -57,7 +57,7 @@ impl Logger { Logger { dst: w } } - pub fn log(&mut self, message: &String) { + pub fn log(&mut self, message: &str) { let _ = self.dst.write(message.as_bytes()); let _ = self.dst.write("\n".as_bytes()); } diff --git a/common/rusty_leveldb_sgx/src/error.rs b/common/rusty_leveldb_sgx/src/error.rs index 3f55bf95..929202bd 100644 --- a/common/rusty_leveldb_sgx/src/error.rs +++ b/common/rusty_leveldb_sgx/src/error.rs @@ -70,6 +70,13 @@ impl Status { } return Status { code, err }; } + + pub fn annotate<S: AsRef<str>>(self, msg: S) -> Status { + Status { + code: self.code, + err: format!("{}: {}", msg.as_ref(), self.err), + } + } } /// LevelDB's result type @@ -90,7 +97,7 @@ impl From<io::Error> for Status { _ => StatusCode::IOError, }; - Status::new(c, e.description()) + Status::new(c, &e.to_string()) } } @@ -104,7 +111,23 @@ impl From<snap::Error> for Status { fn from(e: snap::Error) -> Status { Status { code: StatusCode::CompressionError, - err: e.description().to_string(), + err: e.to_string(), } } } + +#[cfg(feature = "enclave_unit_test")] +pub mod tests { + use super::{Status, StatusCode}; + use std::prelude::v1::*; + use teaclave_test_utils::*; + + pub fn run_tests() -> bool { + run_tests!(test_status_to_string,) + } + + fn test_status_to_string() { + let s = Status::new(StatusCode::InvalidData, "Invalid data!"); + assert_eq!("InvalidData: Invalid data!", s.to_string()); + } +} diff --git a/common/rusty_leveldb_sgx/src/key_types.rs b/common/rusty_leveldb_sgx/src/key_types.rs index 6c92dddc..0febc080 100644 --- a/common/rusty_leveldb_sgx/src/key_types.rs +++ b/common/rusty_leveldb_sgx/src/key_types.rs @@ -47,11 +47,11 @@ pub struct LookupKey { const U64_SPACE: usize = 8; impl LookupKey { - pub fn new<'a>(k: UserKey<'a>, s: SequenceNumber) -> LookupKey { + pub fn new(k: UserKey, s: SequenceNumber) -> LookupKey { LookupKey::new_full(k, s, ValueType::TypeValue) } - pub fn new_full<'a>(k: UserKey<'a>, s: SequenceNumber, t: ValueType) -> LookupKey { + pub fn new_full(k: UserKey, s: SequenceNumber, t: ValueType) -> LookupKey { let mut key = Vec::new(); let internal_keylen = k.len() + U64_SPACE; key.resize(k.len() + internal_keylen.required_space() + U64_SPACE, 0); @@ -61,7 +61,7 @@ impl LookupKey { writer .write_varint(internal_keylen) .expect("write to slice failed"); - writer.write(k).expect("write to slice failed"); + writer.write_all(k).expect("write to slice failed"); writer .write_fixedint(s << 8 | t as u64) .expect("write to slice failed"); @@ -74,17 +74,17 @@ impl LookupKey { } /// Returns the full memtable-formatted key. - pub fn memtable_key<'a>(&'a self) -> MemtableKey<'a> { + pub fn memtable_key(&self) -> MemtableKey { self.key.as_slice() } /// Returns only the user key portion. - pub fn user_key<'a>(&'a self) -> UserKey<'a> { + pub fn user_key(&self) -> UserKey { &self.key[self.key_offset..self.key.len() - 8] } /// Returns key and tag. - pub fn internal_key<'a>(&'a self) -> InternalKey<'a> { + pub fn internal_key(&self) -> InternalKey { &self.key[self.key_offset..] } } @@ -123,12 +123,12 @@ pub fn build_memtable_key(key: &[u8], value: &[u8], t: ValueType, seq: SequenceN { let mut writer = buf.as_mut_slice(); writer.write_varint(keysize).expect("write to slice failed"); - writer.write(key).expect("write to slice failed"); + writer.write_all(key).expect("write to slice failed"); writer .write_fixedint((t as u64) | (seq << 8)) .expect("write to slice failed"); writer.write_varint(valsize).expect("write to slice failed"); - writer.write(value).expect("write to slice failed"); + writer.write_all(value).expect("write to slice failed"); assert_eq!(writer.len(), 0); } buf @@ -137,7 +137,7 @@ pub fn build_memtable_key(key: &[u8], value: &[u8], t: ValueType, seq: SequenceN /// Parses a memtable key and returns (keylen, key offset, tag, vallen, val offset). /// If the key only contains (keylen, key, tag), the vallen and val offset return values will be /// meaningless. -pub fn parse_memtable_key<'a>(mkey: MemtableKey<'a>) -> (usize, usize, u64, usize, usize) { +pub fn parse_memtable_key(mkey: MemtableKey) -> (usize, usize, u64, usize, usize) { let (keylen, mut i): (usize, usize) = VarInt::decode_var(&mkey); let keyoff = i; i += keylen - 8; @@ -181,7 +181,7 @@ pub fn cmp_memtable_key<'a, 'b>( } /// Parse a key in InternalKey format. -pub fn parse_internal_key<'a>(ikey: InternalKey<'a>) -> (ValueType, SequenceNumber, UserKey<'a>) { +pub fn parse_internal_key(ikey: InternalKey) -> (ValueType, SequenceNumber, UserKey) { if ikey.is_empty() { return (ValueType::TypeDeletion, 0, &ikey[0..0]); } @@ -212,7 +212,7 @@ pub fn cmp_internal_key<'a, 'b>( /// truncate_to_userkey performs an in-place conversion from InternalKey to UserKey format. pub fn truncate_to_userkey(ikey: &mut Vec<u8>) { let len = ikey.len(); - assert!(len > 8); + assert!(len >= 8); ikey.truncate(len - 8); } diff --git a/common/rusty_leveldb_sgx/src/lib.rs b/common/rusty_leveldb_sgx/src/lib.rs index a5727b6e..96f82cbf 100644 --- a/common/rusty_leveldb_sgx/src/lib.rs +++ b/common/rusty_leveldb_sgx/src/lib.rs @@ -102,6 +102,7 @@ pub mod tests { db_impl::tests::run_tests(), db_iter::tests::run_tests(), disk_env::tests::run_tests(), + error::tests::run_tests(), filter::tests::run_tests(), filter_block::tests::run_tests(), key_types::tests::run_tests(), diff --git a/common/rusty_leveldb_sgx/src/mem_env.rs b/common/rusty_leveldb_sgx/src/mem_env.rs index 609e370b..cd6dd1f4 100644 --- a/common/rusty_leveldb_sgx/src/mem_env.rs +++ b/common/rusty_leveldb_sgx/src/mem_env.rs @@ -175,13 +175,15 @@ impl MemFS { fn children_of(&self, p: &Path) -> Result<Vec<PathBuf>> { let fs = self.store.lock()?; let mut prefix = path_to_string(p); - if !prefix.ends_with("/") { - prefix.push('/'); + let main_separator_str = std::path::MAIN_SEPARATOR.to_string(); + if !prefix.ends_with(&main_separator_str) { + prefix.push(std::path::MAIN_SEPARATOR); } + let mut children = Vec::new(); for k in fs.keys() { if k.starts_with(&prefix) { - children.push(Path::new(k.trim_start_matches(&prefix)).to_owned()); + children.push(Path::new(k.strip_prefix(&prefix).unwrap_or(&k)).to_owned()); } } Ok(children) diff --git a/common/rusty_leveldb_sgx/src/options.rs b/common/rusty_leveldb_sgx/src/options.rs index edd57d04..5c81040e 100644 --- a/common/rusty_leveldb_sgx/src/options.rs +++ b/common/rusty_leveldb_sgx/src/options.rs @@ -40,8 +40,6 @@ pub fn int_to_compressiontype(i: u32) -> Option<CompressionType> { /// Options contains general parameters for a LevelDB instance. Most of the names are /// self-explanatory; the defaults are defined in the `Default` implementation. -/// -/// Note: Compression is not yet implemented. #[derive(Clone)] pub struct Options { pub cmp: Rc<Box<dyn Cmp>>, diff --git a/common/rusty_leveldb_sgx/src/skipmap.rs b/common/rusty_leveldb_sgx/src/skipmap.rs index 5b368f04..2b73ab62 100644 --- a/common/rusty_leveldb_sgx/src/skipmap.rs +++ b/common/rusty_leveldb_sgx/src/skipmap.rs @@ -236,7 +236,7 @@ impl InnerSkipMap { } // Construct new node - let mut new_skips = Vec::with_capacity(new_height); + let mut new_skips = Vec::new(); new_skips.resize(new_height, None); let mut new = Box::new(Node { skips: new_skips, diff --git a/common/rusty_leveldb_sgx/src/version.rs b/common/rusty_leveldb_sgx/src/version.rs index db0d7c7b..7ac8041f 100644 --- a/common/rusty_leveldb_sgx/src/version.rs +++ b/common/rusty_leveldb_sgx/src/version.rs @@ -259,7 +259,7 @@ impl Version { /// overlap_in_level returns true if the specified level's files overlap the range [smallest; /// largest]. - pub fn overlap_in_level<'a, 'b>( + pub fn overlap_in_level<'a>( &self, level: usize, smallest: UserKey<'a>, diff --git a/common/rusty_leveldb_sgx/src/version_edit.rs b/common/rusty_leveldb_sgx/src/version_edit.rs index 81eab837..9def74db 100644 --- a/common/rusty_leveldb_sgx/src/version_edit.rs +++ b/common/rusty_leveldb_sgx/src/version_edit.rs @@ -135,7 +135,7 @@ impl VersionEdit { buf.write_varint(EditTag::Comparator as u32).unwrap(); // data is prefixed by a varint32 describing the length of the following chunk buf.write_varint(cmp.len()).unwrap(); - buf.write(cmp.as_bytes()).unwrap(); + buf.write_all(cmp.as_bytes()).unwrap(); } if let Some(lognum) = self.log_number { @@ -162,7 +162,7 @@ impl VersionEdit { buf.write_varint(EditTag::CompactPointer as u32).unwrap(); buf.write_varint(cptr.level).unwrap(); buf.write_varint(cptr.key.len()).unwrap(); - buf.write(cptr.key.as_ref()).unwrap(); + buf.write_all(cptr.key.as_ref()).unwrap(); } for df in self.deleted.iter() { @@ -178,9 +178,9 @@ impl VersionEdit { buf.write_varint(nf.1.size).unwrap(); buf.write_varint(nf.1.smallest.len()).unwrap(); - buf.write(nf.1.smallest.as_ref()).unwrap(); + buf.write_all(nf.1.smallest.as_ref()).unwrap(); buf.write_varint(nf.1.largest.len()).unwrap(); - buf.write(nf.1.largest.as_ref()).unwrap(); + buf.write_all(nf.1.largest.as_ref()).unwrap(); } buf diff --git a/common/rusty_leveldb_sgx/src/version_set.rs b/common/rusty_leveldb_sgx/src/version_set.rs index 37189762..08d690ee 100644 --- a/common/rusty_leveldb_sgx/src/version_set.rs +++ b/common/rusty_leveldb_sgx/src/version_set.rs @@ -891,8 +891,8 @@ pub fn set_current_file<P: AsRef<Path>>( let tempfile = temp_file_name(dbname, manifest_file_num); { let mut f = env.open_writable_file(Path::new(&tempfile))?; - f.write(manifest_base.as_os_str().as_bytes())?; - f.write("\n".as_bytes())?; + f.write_all(manifest_base.as_os_str().as_bytes())?; + f.write_all("\n".as_bytes())?; } let currentfile = current_file_name(dbname); if let Err(e) = env.rename(Path::new(&tempfile), Path::new(¤tfile)) { diff --git a/common/rusty_leveldb_sgx/src/write_batch.rs b/common/rusty_leveldb_sgx/src/write_batch.rs index 826e7131..b0c032b2 100644 --- a/common/rusty_leveldb_sgx/src/write_batch.rs +++ b/common/rusty_leveldb_sgx/src/write_batch.rs @@ -38,11 +38,13 @@ impl WriteBatch { /// Adds an entry to a WriteBatch, to be added to the database. #[allow(unused_assignments)] pub fn put(&mut self, k: &[u8], v: &[u8]) { - self.entries.write(&[ValueType::TypeValue as u8]).unwrap(); + self.entries + .write_all(&[ValueType::TypeValue as u8]) + .unwrap(); self.entries.write_varint(k.len()).unwrap(); - self.entries.write(k).unwrap(); + self.entries.write_all(k).unwrap(); self.entries.write_varint(v.len()).unwrap(); - self.entries.write(v).unwrap(); + self.entries.write_all(v).unwrap(); let c = self.count(); self.set_count(c + 1); @@ -52,7 +54,7 @@ impl WriteBatch { #[allow(unused_assignments)] pub fn delete(&mut self, k: &[u8]) { self.entries - .write(&[ValueType::TypeDeletion as u8]) + .write_all(&[ValueType::TypeDeletion as u8]) .unwrap(); self.entries.write_varint(k.len()).unwrap(); self.entries.write(k).unwrap(); @@ -87,7 +89,7 @@ impl WriteBatch { u64::decode_fixed(&self.entries[SEQNUM_OFFSET..SEQNUM_OFFSET + 8]) } - pub fn iter<'a>(&'a self) -> WriteBatchIter<'a> { + pub fn iter(&self) -> WriteBatchIter { WriteBatchIter { batch: self, ix: HEADER_SIZE, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
