From a9a8464e1c40b5f431eef04390cbf2dd7315e045 Mon Sep 17 00:00:00 2001 From: wangboo <5417808+wangboa@user.noreply.gitee.com> Date: Wed, 29 Mar 2023 19:21:50 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E8=BF=9B=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 97f5fa6..54ba069 100644 --- a/README.md +++ b/README.md @@ -71,32 +71,32 @@ RUSTFLAGS='--cfg CORE_DEBUG="false"' cargo build --release ### 1.1.0 版本, 完成基础零部件 -| 功能模块 | 完成人 | 进度 | -|----------------------------------------------------------------------------------|----------------------|------| -| util.Options(ReadOptions, WriteOptions) | kazeseiriou,wangboo | 0% | -| util.ENV(WritableFile, SequentialFile, RandomAccessFile, FileLock) | lxd5866 | 0% | -| util.Logger/Log日志库 | peach | 50% | -| table.Block, BlockBuilder, FilterBlockBuilder | colagy | 0% | -| FilterBlock, FilterBlockReader | colagy | 0% | -| table.format(Footer, BlockHandle) | 半支烟 | 20% | -| db.dbformat(InternalKeyComparator, InternalFilterPolicy, LookupKey, InternalKey) | 半支烟 | 20% | -| db.SkipList | wangboo | 100% | -| table.Iterator(DBIter, MergingIterator, TwoLevelIterator...) | kazeseiriou | 0% | -| IteratorWrapper | kazeseiriou | 0% | -| db.MemTable(MemTable, MemTableIterator) | wangboo,tzcyujunyong | 20% | -| SSTable | fengyang | 0% | -| table.Table | peach,tzcyujunyong | | -| db.leveldb_util | wangboo | 0% | -| db.log_format | wangboo | 90% | -| db.LogReader | wangboo | 90% | -| db.LogWriter | wangboo | 90% | -| db.TableCache | colagy | 10% | -| LinkedList | fengyang | 60% | -| db.VersionEdit(Tag, VersionEdit, FileMetaData) | fengyang | 20% | -| db.VersionSet(Version, LevelFileNumIterator, SaverState) | fengyang | 20% | -| WriteBatch | tzcyujunyong,wangboo | 50% | -| | 半支烟 | 40% | - +| 功能模块 | 完成人 | 进度 | +|------------------------------------------------------------------------------------|------------------------|--------| +| util.Options(ReadOptions, WriteOptions) | kazeseiriou,wangboo | 0% | +| util.ENV(WritableFile, SequentialFile, RandomAccessFile, FileLock) | lxd5866 | 0% | +| util.Logger/Log日志库 | peach | 50% | +| table.Block, BlockBuilder, FilterBlockBuilder | colagy | 0% | +| FilterBlock, FilterBlockReader | colagy | 0% | +| table.format(Footer, BlockHandle) | 半支烟 | 20% | +| db.dbformat(InternalKeyComparator, InternalFilterPolicy, LookupKey, InternalKey) | 半支烟 | 20% | +| db.SkipList | wangboo | 100% | +| table.Iterator(DBIter, MergingIterator, TwoLevelIterator...) | kazeseiriou | 0% | +| IteratorWrapper | kazeseiriou | 0% | +| db.MemTable(MemTable, MemTableIterator) | wangboo,tzcyujunyong | 20% | +| SSTable | fengyang | 0% | +| table.Table | peach,tzcyujunyong | | +| db.leveldb_util | wangboo | 0% | +| db.log_format | wangboo | 90% | +| db.LogReader | wangboo | 90% | +| db.LogWriter | wangboo | 90% | +| db.TableCache | colagy | 10% | +| LinkedList | fengyang | 60% | +| db.VersionEdit(Tag, VersionEdit, FileMetaData) | fengyang | 20% | +| db.VersionSet(Version, LevelFileNumIterator, SaverState) | fengyang | 20% | +| WriteBatch | tzcyujunyong,wangboo | 50% | +| | 半支烟 | 90% | +| ---------------------------------------------------------------------------------- | ---------------------- | ------ | -- Gitee From d23bf8af089d34fee2acbe95859d64442acdc440 Mon Sep 17 00:00:00 2001 From: wangboo <5417808+wangboa@user.noreply.gitee.com> Date: Thu, 30 Mar 2023 21:08:24 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E6=80=A7=E8=83=BD=E6=B5=8B=E8=AF=95=E6=A1=88=E4=BE=8B=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20jemalloc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 4 +-- benches/skiplist_memory_useage.rs | 55 ------------------------------- benches/u32_shift.rs | 46 ++++++++++++++++++++++++++ src/db/skip_list.rs | 41 +++++++++++++++++++---- src/db/skip_list_test.rs | 51 ++++++++++++++++++++++++++++ src/db/version_edit_test.rs | 2 +- src/lib.rs | 5 +-- src/util/comparator_test.rs | 4 +-- src/util/hash.rs | 3 +- src/util/mem_debug.rs | 12 +++++++ src/util/mod.rs | 1 + src/util/status_test.rs | 2 +- 12 files changed, 155 insertions(+), 71 deletions(-) delete mode 100644 benches/skiplist_memory_useage.rs create mode 100644 benches/u32_shift.rs create mode 100644 src/util/mem_debug.rs diff --git a/Cargo.toml b/Cargo.toml index b60d1d1..8b4c9c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ jemallocator = "0.5" jemalloc-sys = {version = "0.5", features = ["stats"]} [dev-dependencies] -criterion = "0.3.0" +criterion = {version = "0.4.0", features = ["html_reports"]} crc32fast = "1.3.2" skiplist = "0.4.0" @@ -34,5 +34,5 @@ name = "skiplist_bench" harness = false [[bench]] -name = "skiplist_memory_useage" +name = "u32_shift" harness = false \ No newline at end of file diff --git a/benches/skiplist_memory_useage.rs b/benches/skiplist_memory_useage.rs deleted file mode 100644 index deaee47..0000000 --- a/benches/skiplist_memory_useage.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::ffi::{c_char, c_void}; -use std::ptr::{null, null_mut}; -use std::sync::{Arc, Mutex}; -use skiplist::OrderedSkipList; -use level_db_rust::db::skip_list::SkipList; -use level_db_rust::util::Arena; -use level_db_rust::util::arena::ArenaRef; -use level_db_rust::util::comparator::BytewiseComparatorImpl; -use level_db_rust::util::unsafe_slice::TryIntoUnsafeSlice; - -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -extern "C" fn write_cb(_: *mut c_void, message: *const c_char) { - print!("{}", String::from_utf8_lossy(unsafe { - std::ffi::CStr::from_ptr(message as *const i8).to_bytes() - })); -} - -fn mem_print() { - unsafe { jemalloc_sys::malloc_stats_print(Some(write_cb), null_mut(), null()) } -} - -fn bench_default_skiplist(mut list: SkipList, arena: ArenaRef, record_count: usize) { - for j in 0..record_count { - let value = format!("key_{}", j); - list.insert(value.try_into_unsafe_slice(arena.clone()).unwrap()).unwrap(); - } - println!("bench_default_skiplist: "); - mem_print(); -} - -fn bench_skiplist_v_0_4_0(mut list: OrderedSkipList, record_count: usize) { - for j in 0..record_count { - let value = format!("key_{}", j); - list.insert(value.clone()); - } - println!("bench_skiplist_v_0_4_0: "); - mem_print(); -} - -fn main() { - let record_count = 100 * 1024; - // let cmp = Arc::new(BytewiseComparatorImpl::default()); - // let arena = Arc::new(Mutex::new(Arena::default())); - // let list = SkipList::create(cmp, arena.clone()); - // bench_default_skiplist(list, arena, record_count); - - let list: OrderedSkipList = unsafe { - OrderedSkipList::with_comp(|a: &String, b: &String| { - a.cmp(b) - }) - }; - bench_skiplist_v_0_4_0(list, record_count); -} \ No newline at end of file diff --git a/benches/u32_shift.rs b/benches/u32_shift.rs new file mode 100644 index 0000000..f9a2eb5 --- /dev/null +++ b/benches/u32_shift.rs @@ -0,0 +1,46 @@ +use std::{mem, slice}; +use std::alloc::{alloc, Layout}; +use std::io::Write; + +use criterion::{Criterion, criterion_group, criterion_main}; +use level_db_rust::debug; + +pub fn u32_shift_bench(c: &mut Criterion) { + let mut data = [0_u8; 4]; + let mut buf = data.as_mut_slice(); + let value = 12345678_u32; + let mut g = c.benchmark_group("u32_shift"); + + g.bench_function("to_ne_bytes", |g| { + g.iter(|| { + buf.write(&value.to_be_bytes()).unwrap(); + }); + }); + buf = data.as_mut_slice(); + buf.fill(0); // reset + debug!("is big endian: {}", cfg!(target_endian = "big")); + g.bench_function("raw_write", |g| { + g.iter(|| { + unsafe { + if cfg!(target_endian = "big") { + (buf.as_mut_ptr() as *mut u32).write(value); + } else { + (buf.as_mut_ptr() as *mut u32).write(value.swap_bytes()); + } + } + }); + }); + buf = data.as_mut_slice(); + buf.fill(0); // reset + g.bench_function("shift_bytes", |g| { + g.iter(|| { + buf[0] = ((value >> 0) & 0xff) as u8; + buf[1] = ((value >> 1) & 0xff) as u8; + buf[2] = ((value >> 2) & 0xff) as u8; + buf[3] = ((value >> 3) & 0xff) as u8; + }); + }); +} + +criterion_group!(benches, u32_shift_bench); +criterion_main!(benches); \ No newline at end of file diff --git a/src/db/skip_list.rs b/src/db/skip_list.rs index a043adc..4d13c12 100644 --- a/src/db/skip_list.rs +++ b/src/db/skip_list.rs @@ -5,15 +5,15 @@ use std::ptr::null_mut; use std::sync::{Arc, RwLock}; use rand::prelude::*; + use crate::debug; use crate::traits::comparator_trait::Comparator; use crate::traits::DataIterator; - -use crate::util::arena::ArenaRef; use crate::util::{Arena, Result}; +use crate::util::arena::ArenaRef; use crate::util::slice::Slice; -use crate::util::unsafe_slice::UnsafeSlice; use crate::util::status::{LevelError, Status}; +use crate::util::unsafe_slice::UnsafeSlice; type RawNode = *mut Node; @@ -213,6 +213,10 @@ impl SkipList { false } + unsafe fn find_eq_or_greater>(&self, key: &R) -> Option { + todo!() + } + #[inline] pub fn max_height(&self) -> usize { MAX_LEVEL @@ -346,6 +350,28 @@ impl Node { assert!(level < MAX_LEVEL); self.next_elems.offset(level as isize).write(node); } + + /// 找到最后一个数据元素 + unsafe fn seek_to_last(&self) -> Option { + if self.is_tail() { + return None; + } + let mut pre = self; + let mut cur = &*self.next_top_node(); + loop { + if cur.is_tail() { + return Some(pre as *const Node as *mut Node); + } + pre = cur; + cur = &*cur.next_top_node(); + } + } + + /// 找到最上层的下一个元素 + #[inline] + unsafe fn next_top_node(&self) -> RawNode { + self.get_node(self.level - 1) + } } fn rand_level() -> usize { @@ -410,7 +436,6 @@ impl Iterator for Iter { } impl DataIterator for Iter { - #[inline] fn valid(&self) -> bool { unsafe { @@ -420,12 +445,16 @@ impl DataIterator for Iter { #[inline] fn seek_to_first(&mut self) { - self.current = self.head + self.current = unsafe { + (&*self.head).get_node(0) + } } #[inline] fn seek_to_last(&mut self) { - self.current = self.tail + unsafe { + self.current = (&*self.current).seek_to_last().unwrap_or(self.tail) + } } fn seek(&mut self, key: &Slice) { diff --git a/src/db/skip_list_test.rs b/src/db/skip_list_test.rs index 11ef4bd..11b4573 100644 --- a/src/db/skip_list_test.rs +++ b/src/db/skip_list_test.rs @@ -1,11 +1,21 @@ mod test { use std::collections::HashSet; + use std::env::args; + use std::ffi::{c_char, c_void}; + use std::ptr::{null, null_mut}; use std::sync::{Arc, Mutex}; + + use criterion::{Criterion, criterion_group, criterion_main}; use rand::Rng; + use skiplist::OrderedSkipList; + use crate::db::DefaultSkipList; + use crate::db::skip_list::SkipList; use crate::debug; use crate::util::Arena; + use crate::util::arena::ArenaRef; use crate::util::comparator::BytewiseComparatorImpl; + use crate::util::mem_debug::mem_print; use crate::util::Result; use crate::util::slice::Slice; use crate::util::unsafe_slice::TryIntoUnsafeSlice; @@ -55,4 +65,45 @@ mod test { Ok(()) } + + + fn default_skiplist(mut list: SkipList, arena: ArenaRef, record_count: usize) { + for j in 0..record_count { + let value = format!("key_{}", j); + list.insert(value.try_into_unsafe_slice(arena.clone()).unwrap()).unwrap(); + } + println!("bench_default_skiplist: "); + mem_print(); + } + + fn bench_skiplist_v_0_4_0(mut list: OrderedSkipList, record_count: usize) { + for j in 0..record_count { + let value = format!("key_{}", j); + list.insert(value.clone()); + } + println!("bench_skiplist_v_0_4_0: "); + mem_print(); + } + + #[test] + fn bench_default_skiplist() { + let record_count = 100 * 1024; + println!("bench default skiplist"); + let cmp = Arc::new(BytewiseComparatorImpl::default()); + let arena = Arc::new(Mutex::new(Arena::default())); + let list = SkipList::create(cmp, arena.clone()); + default_skiplist(list, arena, record_count); + } + + #[test] + fn bench_crate_skiplist() { + let record_count = 100 * 1024; + println!("bench crate skiplist"); + let list: OrderedSkipList = unsafe { + OrderedSkipList::with_comp(|a: &String, b: &String| { + a.cmp(b) + }) + }; + bench_skiplist_v_0_4_0(list, record_count); + } } \ No newline at end of file diff --git a/src/db/version_edit_test.rs b/src/db/version_edit_test.rs index 2c2efec..c38442d 100644 --- a/src/db/version_edit_test.rs +++ b/src/db/version_edit_test.rs @@ -5,7 +5,7 @@ mod test { use crate::util::slice::Slice; #[test] - fn test_Tag() { + fn test_tag() { let tag = Tag::kCompactPointer; assert_eq!(tag.get_value(), 5); diff --git a/src/lib.rs b/src/lib.rs index c8b17d3..3b67f8e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,4 @@ #![feature(box_syntax)] -#![feature(let_else)] -#![feature(generic_associated_types)] extern crate core; @@ -9,6 +7,9 @@ mod table; pub mod util; mod traits; +#[global_allocator] +static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + mod test { #[test] diff --git a/src/util/comparator_test.rs b/src/util/comparator_test.rs index d6d311b..b018bf6 100644 --- a/src/util/comparator_test.rs +++ b/src/util/comparator_test.rs @@ -93,7 +93,7 @@ mod test { // u8max 结尾 let mut u8_vec: Vec = vec![]; - u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()); + u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()).unwrap(); u8_vec.push(u8::MAX); let u8_array_str = String::from(Slice::from_buf(u8_vec.as_slice())); @@ -106,7 +106,7 @@ mod test { // u8max 开头 let mut u8_vec: Vec = vec![]; u8_vec.push(u8::MAX); - u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()); + u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()).unwrap(); let u8_max_str = String::from(Slice::from_buf(u8_vec.as_slice())); let comp = BytewiseComparatorImpl::default(); diff --git a/src/util/hash.rs b/src/util/hash.rs index 8aa51e2..15a1a03 100644 --- a/src/util/hash.rs +++ b/src/util/hash.rs @@ -11,10 +11,9 @@ use crate::util::slice::Slice; /// 一种可以计算 hash 的特质 pub trait ToHash { - #[inline] + fn to_hash(&self) -> u32; - #[inline] fn to_hash_with_seed(&self, seed: u32) -> u32; } diff --git a/src/util/mem_debug.rs b/src/util/mem_debug.rs new file mode 100644 index 0000000..bef8a83 --- /dev/null +++ b/src/util/mem_debug.rs @@ -0,0 +1,12 @@ +use std::ffi::{c_char, c_void}; +use std::ptr::{null, null_mut}; + +extern "C" fn write_cb(_: *mut c_void, message: *const c_char) { + print!("{}", String::from_utf8_lossy(unsafe { + std::ffi::CStr::from_ptr(message as *const i8).to_bytes() + })); +} + +pub fn mem_print() { + unsafe { jemalloc_sys::malloc_stats_print(Some(write_cb), null_mut(), null()) } +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index ade5ddc..d777f71 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -41,6 +41,7 @@ pub mod debug; pub mod linked_list; mod linked_list_test; pub mod unsafe_slice; +pub mod mem_debug; /// 定义别名 pub type Result = result::Result; diff --git a/src/util/status_test.rs b/src/util/status_test.rs index a7dce28..c186cee 100644 --- a/src/util/status_test.rs +++ b/src/util/status_test.rs @@ -100,7 +100,7 @@ mod test { } #[test] - fn test_level_error_toString() { + fn test_level_error_to_string() { // ok let status: Status = LevelError::ok(); assert_eq!("OK", status.to_string()); -- Gitee