scc-3.4.8/.cargo_vcs_info.json0000644000000001360000000000100116200ustar { "git": { "sha1": "fbb2ef0065da8fe32d7a52ef09c5e4a08d3a410a" }, "path_in_vcs": "" }scc-3.4.8/.github/dependabot.yml000064400000000000000000000006601046102023000146020ustar 00000000000000# To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: "cargo" directory: "/" schedule: interval: "weekly" scc-3.4.8/.github/workflows/scc.yml000064400000000000000000000065271046102023000153120ustar 00000000000000name: SCC on: push: branches: [main] pull_request: branches: [main] env: CARGO_TERM_COLOR: always jobs: check-basic-x86_64: runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v5 - name: Debug run: RUST_BACKTRACE=1 cargo test -- --nocapture; RUST_BACKTRACE=1 cargo test -p examples - name: Release run: RUST_BACKTRACE=1 cargo test --release -- --nocapture; RUST_BACKTRACE=1 cargo test --release -p examples check-basic-x86: runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v5 - name: Prepare run: sudo apt-get install gcc-multilib; rustup target add i686-unknown-linux-gnu - name: Debug run: RUST_BACKTRACE=1 cargo test --target i686-unknown-linux-gnu -- --nocapture; RUST_BACKTRACE=1 cargo test --target i686-unknown-linux-gnu -p examples - name: Release run: RUST_BACKTRACE=1 cargo test --target i686-unknown-linux-gnu --release -- --nocapture; RUST_BACKTRACE=1 cargo test --target i686-unknown-linux-gnu --release -p examples check-basic-aarch64: runs-on: macos-latest timeout-minutes: 15 steps: - uses: actions/checkout@v5 - name: Debug run: RUST_BACKTRACE=1 cargo test -- --nocapture; RUST_BACKTRACE=1 cargo test -p examples; RUST_BACKTRACE=1 cargo test -p extended_tests - name: Release run: RUST_BACKTRACE=1 cargo test --release -- --nocapture; RUST_BACKTRACE=1 cargo test --release -p examples check-extended: runs-on: ubuntu-latest timeout-minutes: 20 steps: - uses: actions/checkout@v5 - name: Prepare run: rustup toolchain add nightly; rustup component add rust-src --toolchain nightly; rustup component add miri --toolchain nightly - name: Miri run: cargo +nightly miri test check-misc: runs-on: ubuntu-latest timeout-minutes: 20 steps: - uses: actions/checkout@v5 - name: Prepare run: rustup toolchain add nightly; rustup component add rust-src --toolchain nightly; rustup component add clippy --toolchain nightly - uses: taiki-e/install-action@v2 with: tool: cargo-msrv - name: Clippy run: cargo clippy --all; cargo clippy -p examples --all; cargo clippy -p extended_tests --all - name: MSRV run: cargo msrv verify - name: Format run: cargo fmt --check - name: Doc run: cargo doc --document-private-items - name: Examples run: cargo fmt -p examples --check - name: OOM run: cargo fmt -p extended_tests --check - name: Loom run: RUST_BACKTRACE=1 cargo test --features loom --release --lib -- --nocapture - name: Equivalent run: cargo test --features equivalent --release --lib -- --nocapture - name: Serde run: cargo test serde --features serde; cargo test --release serde --features serde - name: Nightly run: cargo +nightly test --release benchmark; cargo +nightly test -p examples --release benchmark: runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v5 - name: Benchmark run: cargo bench scc-3.4.8/.gitignore000064400000000000000000000006631046102023000124050ustar 00000000000000# Generated by Cargo # will have compiled files and executables /target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk # IntelliJ .idea # macOS **/.DS_Store # VSCode .vscode # Emacs **/#*# **/*~ # Proptest /proptest-regressions/ scc-3.4.8/CHANGELOG.md000064400000000000000000000373371046102023000122360ustar 00000000000000# Changelog ## Version 3 3.4.8 * Minor updates for `codeberg` migration. 3.4.7 * Minor `Future` size improvement for `Hash*` containers. 3.4.6 * Improve `HashIndex` entry slot recycling strategies. 3.4.5 * Minor `Hash*` performance improvements. 3.4.4 * Minor `Future` size improvement. * Simplify `HashIndex` entry removal policies. * Optimize `HashIndex` for `!needs_drop::<(K, V)>()`. 3.4.3 * Maximum capacity limit is adjusted to `2^(usize::BITS - 2)`. * Minor `Future` size improvement. 3.4.2 * Minor optimization. 3.4.1 * Migrate `Bag`, `LinkedList`, `Queue`, and `Stack` to `sdd`. 3.3.7 - 3.3.8 * Improve sampling accuracy in `Hash*` containers. 3.3.6 * Simplify `Hash*` capacity management. * Fix the `loom` feature to pass the proper loom option to `saa`. 3.3.5 * Enhance `Hash*` capacity management. 3.3.4 * Minor asynchronous code optimization. 3.3.3 * API update: add `{HashMap, HashCache}::{replace_async, replace_sync}`. 3.3.2 * Remove the `'static` bound from `HashIndex`: [#200](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/200). * Ensure that `HashIndex` entries are deallocated when the `HashIndex` is dropped: [#198](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/198). 3.3.1 * More conservative grow/shrink policies in `Hash*` containers to avoid wrong load factor estimation. * Deallocate `HashIndex` entries as early as possible: [#198](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/198). 3.3.0 * API update: `HashIndex::{peek, iter}` no longer allow entry references to outlive the `HashIndex`: [#198](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/198). * More memory usage optimization: [#194](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/194). 3.2.0 * API update: `hash_index::OccupiedEntry::update` does not consume itself. * API update: add `HashSet::replace_{async|sync}`. * Minor memory usage optimization: [#194](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/194). * Fix a data race between `TreeIndex::clear` and `TreeIndex::insert*`. 3.1.2 * Minor memory usage optimization. 3.1.1 * Remove the 128B alignment requirement for `Hash*` asynchronous methods. 3.1.0 * Optimize stack memory usage of asynchronous `Hash*` methods: [#194](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/194). 3.0.7 * Fix a rare data race in `Hash*` iterator methods. 3.0.5 - 3.0.6 * Fix a potential duplicate key issue with `Hash*` containers if `HashMap::insert_*` fails to allocate memory. 3.0.3 - 3.0.4 * Fix potential data races in asynchronous operations of `Hash*` containers. 3.0.1 - 3.0.2 * Dependencies updated to the latest versions. 3.0.0 * New API. ### API update * See [`#186`](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/186). ## Version 2 2.4.0 * Add `Hash*::try_entry`. 2.3.4 * Limit the maximum initial capacity of serialized containers to `1 << 24`. 2.3.3 * Minor performance optimization for `HashIndex`. 2.3.1 * **Yanked `[2.0.0, 2.3.0]` due to a use-after-free problem: [#176](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/176).** * Fix a problem with `HashCache::read` and `HashMap::read` where the read lock is dropped too early: [#176](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/176). * Fix `HashCache` documentation: [#175](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/175). * Implement `FromIterator` for all collection types: [#173](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/173). 2.3.0 * Fix incorrect Sync trait bounds of `Bag` and `Hash*`. 2.2.6 * Fix linting errors. 2.2.5 * `Hash*::with_hasher` is now a const function: by [Daniel-Aaron-Bloom](https://github.com/Daniel-Aaron-Bloom). * Fix the `HashMap::upsert` document: by [dfaust](https://github.com/dfaust). 2.2.4 * Minor `Hash*` performance optimization. 2.2.3 * Minor `Hash*` memory optimization. 2.2.2 * Fix a data race between `TreeIndex::clear` and `TreeIndex::insert`. * Update internal doc. 2.2.1 * Fix a minor Miri specific assertion failure. 2.2.0 * Add `Comparable` and `Equivalent` traits: [#162](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/162) by [qthree](https://github.com/qthree). 2.1.18 * Add `TreeIndex::peek_entry`: [#157](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/157) by [qthree](https://github.com/qthree). * `tree_index::Range` accepts borrowed key ranges: [#158](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/158) by [qthree](https://github.com/qthree). 2.1.17 * Optimize `TreeIndex::{clear, drop}`: [#156](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/156). 2.1.16 * Fix potential data races in `HashMap` and `TreeIndex`. 2.1.15 * Add `upsert` to `HashMap`. 2.1.14 * Fix theoretical data race issues in `TreeIndex`: replace dependent loads with load-acquires. 2.1.13 * Fix a data race in `TreeIndex::insert*`. 2.1.12 * Bump `SDD` to `3.0`. * Update doc to clearly state that `Hash*::get*` owns the entry for modification, and recommend `Hash*::read*` for read-only access. 2.1.11 * Activate Miri tests: [#88](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/88). 2.1.10 * Add `loom` support. * Fix data races in `TreeIndex`: [#153](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/153). 2.1.9 * Fix a correctness issue with `Stack`, `Queue`, and `Bag::pop`: [#153](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/153). 2.1.8 * Fix a correctness issue with `TreeIndex::remove_range`: [#153](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/153). * Add `TreeIndex::remove_range_async`: [#123](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/123). 2.1.7 * Fix a correctness issue with `HashMap` on a 32-bit CPU: [#153](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/153). 2.1.6 * Minor optimization: bump `SDD` to `2.1.0`. 2.1.5 * Optimize `LinkedList` deleted node reclamation. * Optimize `Bag` by not allowing pushed instances to be dropped without being used. 2.1.4 * Implement more aggressive entry removal in `HashIndex` ([#150](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/150)). 2.1.3 * Update [`sdd`](https://crates.io/crates/sdd). 2.1.2 * Implement `any_entry` and `any_entry_async` to `HashMap` and `HashIndex` ([#148](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/148)). 2.1.1 * Implement `Deref` and `DerefMut` for `OccupiedEntry`: [#140](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/140) by [gituser-rs](https://github.com/gituser-rs) 2.1.0 * Use [`sdd`](https://crates.io/crates/sdd) as the memory reclaimer. 2.0.20 * Relax trait bounds of `TreeIndex`. 2.0.19 * Remove unnecessary trait bounds from type definitions of `HashCache`, `HashIndex`, `HashMap`, and `HashSet`. * Fix [#135](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/135). 2.0.18 * API update: add `Entry::take_inner`. * Much faster `Queue` and `Stack` entry garbage collection. 2.0.17 * Faster `Queue` and `Stack` entry garbage collection. 2.0.16 * Fix an issue with `HashCache` where an evicted entry is dropped without notifying it when `HashCache` shrinks. 2.0.15 * Fix [#122](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/122). 2.0.14 * Fix [#120](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/120). * More aggressive `TreeIndex` root node cleanup. 2.0.13 * Add `iter` to `Queue` and `Stack`. * Add `len` to `Bag`, `Queue`, and `Stack`. 2.0.12 * Fix an issue with `tree_index::Range` where it intermittently fails to find the minimum key in a `TreeIndex` if the `TreeIndex` is being updated by other threads. 2.0.10 - 2.0.11 * Fix [#121](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/121). 2.0.9 * Add `TreeIndex::contains` and `TreeIndex::remove_range`; `TreeIndex::remove_range` is experimental and will be stabilized later ([#120](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/120)). 2.0.8 * Add support for old Rust versions >= 1.65.0. 2.0.7 * Add `bucket_index` to `HashIndex`, `HashMap`, and `HashSet`. 2.0.6 * Fix [#118](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/118). 2.0.5 * Add support for 32-bit binaries. * Fix [#116](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/116). 2.0.4 * Fix [#115](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/115). 2.0.3 * Add `IntoIter` to `Bag`. 2.0.2 * Enhance the wait queue implementation. 2.0.1 * Minor code cleanup. 2.0.0 * New API. ### API update * See [`#108`](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/108). ## Version 1 1.9.1 * API update: add the `hash_index::Entry` API. 1.9.0 * API update: add `ebr::{AtomicOwned, Owned}` for non-reference-counted instances. 1.8.3 * API update: add `ebr::AtomicArc::compare_exchange_weak`. 1.8.2 * API update: [#107](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/107) add `HashMap::{prune, prune_async}`. 1.8.1 * API update: add `HashCache::{contains, contains_async}`. 1.8.0 * API update: overhaul `hash_cache::Entry` API; values can be evicted through `hash_cache::Entry` API. 1.7.3 * Add `Bag::pop_all` and `Stack::pop_all`. 1.7.2 * Add `HashCache::{any, any_async, for_each, for_each_async, read, read_async}`. 1.7.1 * Add `Serde` support to `HashCache`. 1.7.0 * Optimize `Hash*::update*` and `HashIndex::modify*`. * API update 1: [#94](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/94) a _WORK IN PROGRESS_ `HashCache` minimal implementation. * API update 2: add `HashMap::{get, get_async}` returning an `OccupiedEntry`. * API update 3: add `Hash*::capacity_range`. 1.6.3 * API update 1: [#96](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/96) - add `HashIndex::{modify, modify_async}`, special thanks to [novacrazy](https://github.com/novacrazy). * API update 2: `Hash*::default()` for any `H: BuildHasher + Default`, by [novacrazy](https://github.com/novacrazy). 1.6.2 * API update: add `HashIndex::{retain, retain_async}`. 1.6.1 * API update: add a mutable `Bag` iterator. * Replace `compare_exchange` with `compare_exchange_weak` where spurious failures do not cost much. * Fix an issue with `hash_map::Reserve::fmt` which printed superfluous information. 1.6.0 * API update 1: remove `ebr::Barrier::defer_incremental_execute` in favor of unwind-safety. * API update 2: all the data structures except for `hash_map::OccupiedEntry` and `hash_map::VacantEntry` are now `UnwindSafe`. * API update 3: export `linked_list::Entry` as `LinkedEntry`. 1.5.0 * API update: `HashMap::remove_if*` passes `&mut V` to the supplied predicate. 1.4.4 * Major `Hash*` performance boost: vectorize `Hash*` bucket lookup operations. 1.4.3 * Add `const ARRAY_LEN: usize` type parameter to `Bag`. * Minor optimization. 1.4.2 * Optimize `TreeIndex::is_empty`. * Update documentation. 1.4.1 * Add `hash_index::Reserve` and `HashIndex::reserve`. * Add missing `H = RandomState` to several types. * Add `const` to several trivial functions. 1.4.0 * **Fix a correctness issue with LLVM 16 (Rust 1.70.0)**. * API update: `{Stack, Queue}::{peek*}` receive `FnOnce(Option<&Entry>) -> R`. * `RandomState` is now the default type parameter for `hash_*` structures. * Remove explicit `Sync` requirements. * Remove `'static` lifetime constraints from `Bag`, `LinkedList`, `Queue`, and `Stack`. * Minor `Hash*` optimization. 1.3.0 * Add `HashMap::first_occupied_entry*` for more flexible mutable iteration over entries. * Add `ebr::Arc::get_ref_with`. * Implement `Send` for `hash_map::Entry` if `(K, V): Send`. * `Hash*::remove*` methods may deallocate the entire hash table when they find the container empty. 1.2.0 * API update 1: `AtomicArc::update_tag_if` now receives `fetch_order`, and the closure can access the pointer value. * API update 2: rename `hash_map::Ticket` `hash_map::Reserve`. * `Hash*` do not allocate bucket arrays until the first write access. * `Hash*::remove*` more aggressively shrinks the bucket array. 1.1.4 - 1.1.5 * Optimize `Hash*::is_empty`. * Remove unnecessary lifetime constraints on `BuildHasher`. 1.1.3 * Fix [#86](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/86) completely. * EBR garbage instances and the garbage collector instance of a thread is now deallocated immediately when the thread exits if certain conditions are met. 1.1.2 * Fix [#86](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/86). 1.1.1 * Fix a rare problem with `HashMap` and `HashSet` violating lifetime contracts on drop. 1.1.0 * Remove `'static` bounds from `HashMap`, `HashSet`, and `ebr::{Arc, AtomicArc}`. 1.0.9 * Add `HashMap::{entry, entry_async}`. 1.0.8 * More robust panic handling. * Doc update. 1.0.7 * Minor performance optimization. * Identified a piece of blocking code in `HashIndex::read`, and make it non-blocking. 1.0.6 * Optimize `TreeIndex` for low-entropy input patterns. 1.0.5 * Add `{HashMap, HashSet}::{any, any_async}` to emulate `Iterator::any`. * Implement `PartialEq` for `{HashMap, HashSet, HashIndex, TreeIndex}`. * Add `serde` support to `{HashMap, HashSet, HashIndex, TreeIndex}`. * Remove the unnecessary `Send` bound from `TreeIndex`. 1.0.4 * Minor `Hash*` optimization. 1.0.3 * Major `TreeIndex` performance improvement. * Add `From for u8`. 1.0.2 * Optimize `TreeIndex`. 1.0.1 * Add `Stack`. * API update 1: remove `Bag::clone`. * API update 2: replace `Queue::Entry` with ``. * Optimize `Bag`. * Fix memory ordering in `Bag::drop`. 1.0.0 * Implement `Bag`. ## Version 0 0.12.4 * Remove `scopeguard`. 0.12.3 * Minor `ebr` optimization. 0.12.2 * `Hash*::remove*` accept `FnOnce`. 0.12.1 * `HashMap::read`, `HashIndex::read`, and `HashIndex::read_with` accept `FnOnce`. * Proper optimization for `T: Copy` and `!needs_drop::()`. 0.12.0 * More aggressive EBR garbage collection. 0.11.5 * Fix [#84](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/84) completely. * Micro-optimization. 0.11.4 * Optimize performance for `T: Copy`. 0.11.3 * Fix [#84](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/84). * 0.11.2 and any older versions have a serious correctness problem with Rust 1.65.0 and newer. 0.11.2 * `HashIndex` and `HashMap` cleanup entries immediately when the instance is dropped. 0.11.1 * Adjust `HashIndex` parameters to suppress latency spikes. 0.11.0 * Replace `ebr::Barrier::reclaim` with `ebr::Arc::release`. * Rename `ebr::Arc::drop_in_place` `ebr::Arc::release_drop_in_place`. * Implement `ebr::Barrier::defer`. * Make `ebr::Collectible` public. 0.10.2 * Fix [#82](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/82). * Implement `ebr::Barrier::defer_incremental_execute`. 0.10.1 * Significant `HashMap`, `HashSet`, and `HashIndex` insert performance improvement by segregating zero and non-zero memory regions. 0.9.1 * `HashMap`, `HashSet`, and `HashIndex` performance improvement. 0.9.0 * API update: `HashMap::new`, `HashIndex::new`, and `HashSet::new`. * Add `unsafe HashIndex::update` for linearizability. 0.8.4 * Implement `ebr::Barrier::defer_execute` for deferred closure execution. * Fix [#78](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/78). 0.8.3 * Fix `ebr::AtomicArc::{clone, get_arc}` to never return a null pointer if the `ebr::AtomicArc` is always non-null. 0.8.2 * Fix [#77](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/77). 0.8.1 * Implement `Debug` for container types. 0.8.0 * Add `ebr::suspend` which enables garbage instances in a dormant thread to be reclaimed by other threads. * Minor `Queue` API update. * Reduce `HashMap` memory usage. scc-3.4.8/Cargo.lock0000644000001075600000000000100076040ustar # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 4 [[package]] name = "aho-corasick" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] [[package]] name = "alloca" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" dependencies = [ "cc", ] [[package]] name = "anes" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "bit-set" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "cast" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" version = "1.2.49" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" dependencies = [ "find-msvc-tools", "shlex", ] [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "ciborium" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", "serde", ] [[package]] name = "ciborium-io" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", "half", ] [[package]] name = "clap" version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstyle", "clap_lex", ] [[package]] name = "clap_lex" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "criterion" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d883447757bb0ee46f233e9dc22eb84d93a9508c9b868687b274fc431d886bf" dependencies = [ "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", "futures", "itertools", "num-traits", "oorandom", "page_size", "plotters", "rayon", "regex", "serde", "serde_json", "tinytemplate", "walkdir", ] [[package]] name = "criterion-plot" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed943f81ea2faa8dcecbbfa50164acf95d555afec96a27871663b300e387b2e4" dependencies = [ "cast", "itertools", ] [[package]] name = "crossbeam-deque" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", "windows-sys 0.61.2", ] [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "find-msvc-tools" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", "futures-executor", "futures-io", "futures-sink", "futures-task", "futures-util", ] [[package]] name = "futures-channel" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", ] [[package]] name = "futures-core" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", "futures-util", ] [[package]] name = "futures-io" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "futures-sink" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", "futures-io", "futures-macro", "futures-sink", "futures-task", "memchr", "pin-project-lite", "pin-utils", "slab", ] [[package]] name = "generator" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" dependencies = [ "cc", "cfg-if", "libc", "log", "rustversion", "windows", ] [[package]] name = "getrandom" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", "r-efi", "wasip2", ] [[package]] name = "half" version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", "zerocopy", ] [[package]] name = "itertools" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] [[package]] name = "itoa" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", ] [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "linux-raw-sys" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "lock_api" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ "scopeguard", ] [[package]] name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "loom" version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" dependencies = [ "cfg-if", "generator", "scoped-tls", "serde", "serde_json", "tracing", "tracing-subscriber", ] [[package]] name = "matchers" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ "regex-automata", ] [[package]] name = "memchr" version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "mio" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "wasi", "windows-sys 0.61.2", ] [[package]] name = "nu-ansi-term" version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ "windows-sys 0.61.2", ] [[package]] name = "num-traits" version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "oorandom" version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "page_size" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ "libc", "winapi", ] [[package]] name = "parking_lot" version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", "windows-link 0.2.1", ] [[package]] name = "pin-project-lite" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "plotters" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", "plotters-svg", "wasm-bindgen", "web-sys", ] [[package]] name = "plotters-backend" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] [[package]] name = "ppv-lite86" version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ "zerocopy", ] [[package]] name = "proc-macro2" version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] [[package]] name = "proptest" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" dependencies = [ "bit-set", "bit-vec", "bitflags", "num-traits", "rand", "rand_chacha", "rand_xorshift", "regex-syntax", "rusty-fork", "tempfile", "unarray", ] [[package]] name = "quick-error" version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] [[package]] name = "r-efi" version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha", "rand_core", ] [[package]] name = "rand_chacha" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", "rand_core", ] [[package]] name = "rand_core" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ "getrandom", ] [[package]] name = "rand_xorshift" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" dependencies = [ "rand_core", ] [[package]] name = "rayon" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", ] [[package]] name = "rayon-core" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", ] [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ "bitflags", ] [[package]] name = "regex" version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustix" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", "windows-sys 0.61.2", ] [[package]] name = "rustversion" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rusty-fork" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" dependencies = [ "fnv", "quick-error", "tempfile", "wait-timeout", ] [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "saa" version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a0333ba2ea22f7721fc41ebdc02596d9eefae8229046bf21aae78613a929d0f" dependencies = [ "loom", ] [[package]] name = "same-file" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ "winapi-util", ] [[package]] name = "scc" version = "3.4.8" dependencies = [ "criterion", "equivalent", "fnv", "futures", "loom", "proptest", "saa", "sdd", "serde", "serde_test", "static_assertions", "tokio", ] [[package]] name = "scoped-tls" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sdd" version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7168ecf885fdd3920ade15d50189593b076e1d060b60406a745766380195d65a" dependencies = [ "loom", ] [[package]] name = "serde" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", ] [[package]] name = "serde_core" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "serde_json" version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", "serde_core", ] [[package]] name = "serde_test" version = "1.0.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" dependencies = [ "serde", ] [[package]] name = "sharded-slab" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] [[package]] name = "slab" version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" dependencies = [ "libc", "windows-sys 0.60.2", ] [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "tempfile" version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", "getrandom", "once_cell", "rustix", "windows-sys 0.61.2", ] [[package]] name = "thread_local" version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", ] [[package]] name = "tinytemplate" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" dependencies = [ "serde", "serde_json", ] [[package]] name = "tokio" version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ "bytes", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "tracing" version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "pin-project-lite", "tracing-core", ] [[package]] name = "tracing-core" version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", "valuable", ] [[package]] name = "tracing-log" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ "log", "once_cell", "tracing-core", ] [[package]] name = "tracing-subscriber" version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ "matchers", "nu-ansi-term", "once_cell", "regex-automata", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", ] [[package]] name = "unarray" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "valuable" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "wait-timeout" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" dependencies = [ "libc", ] [[package]] name = "walkdir" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", ] [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ "bumpalo", "proc-macro2", "quote", "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", ] [[package]] name = "winapi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", ] [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ "windows-sys 0.61.2", ] [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", "windows-core", "windows-future", "windows-link 0.1.3", "windows-numerics", ] [[package]] name = "windows-collections" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ "windows-core", ] [[package]] name = "windows-core" version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", "windows-link 0.1.3", "windows-result", "windows-strings", ] [[package]] name = "windows-future" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core", "windows-link 0.1.3", "windows-threading", ] [[package]] name = "windows-implement" version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "windows-interface" version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "windows-link" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-numerics" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ "windows-core", "windows-link 0.1.3", ] [[package]] name = "windows-result" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link 0.1.3", ] [[package]] name = "windows-strings" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ "windows-link 0.1.3", ] [[package]] name = "windows-sys" version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ "windows-targets", ] [[package]] name = "windows-sys" version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link 0.2.1", ] [[package]] name = "windows-targets" version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ "windows-link 0.2.1", "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", "windows_x86_64_msvc", ] [[package]] name = "windows-threading" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" dependencies = [ "windows-link 0.1.3", ] [[package]] name = "windows_aarch64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] name = "windows_i686_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "wit-bindgen" version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "zerocopy" version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", "syn", ] scc-3.4.8/Cargo.toml0000644000000042270000000000100076230ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2024" rust-version = "1.85.0" name = "scc" version = "3.4.8" authors = ["wvwwvwwv "] build = false autolib = false autobins = false autoexamples = false autotests = false autobenches = false description = "A collection of high-performance containers providing both asynchronous and synchronous interfaces" documentation = "https://docs.rs/scc" readme = "README.md" keywords = [ "async", "cache", "concurrent", "hashmap", "tree", ] categories = [ "asynchronous", "caching", "concurrency", "data-structures", ] license = "Apache-2.0" repository = "https://github.com/wvwwvwwv/scalable-concurrent-containers/" [features] loom = [ "dep:loom", "saa/loom", "sdd/loom", ] [lib] name = "scc" path = "src/lib.rs" [[bench]] name = "hash_cache" path = "benches/hash_cache.rs" harness = false [[bench]] name = "hash_index" path = "benches/hash_index.rs" harness = false [[bench]] name = "hash_map" path = "benches/hash_map.rs" harness = false [[bench]] name = "tree_index" path = "benches/tree_index.rs" harness = false [dependencies.equivalent] version = "1.0" optional = true [dependencies.loom] version = "0.7" features = ["checkpoint"] optional = true [dependencies.saa] version = "5.4" [dependencies.sdd] version = "4.5" [dependencies.serde] version = "1.0" optional = true [dev-dependencies.criterion] version = "0.8" features = ["async_futures"] [dev-dependencies.fnv] version = "1.0" [dev-dependencies.futures] version = "0.3" [dev-dependencies.proptest] version = "1.9" [dev-dependencies.serde_test] version = "1.0" [dev-dependencies.static_assertions] version = "1.1" [dev-dependencies.tokio] version = "1.48" features = ["full"] scc-3.4.8/Cargo.toml.orig000064400000000000000000000023561046102023000133050ustar 00000000000000[package] name = "scc" description = "A collection of high-performance containers providing both asynchronous and synchronous interfaces" documentation = "https://docs.rs/scc" version = "3.4.8" authors = ["wvwwvwwv "] edition = "2024" rust-version = "1.85.0" readme = "README.md" repository = "https://github.com/wvwwvwwv/scalable-concurrent-containers/" license = "Apache-2.0" categories = ["asynchronous", "caching", "concurrency", "data-structures"] keywords = ["async", "cache", "concurrent", "hashmap", "tree"] [workspace] members = [".", "examples", "extended_tests"] [dependencies] equivalent = { version = "1.0", optional = true } loom = { version = "0.7", optional = true, features = ["checkpoint"] } saa = "5.4" sdd = "4.5" serde = { version = "1.0", optional = true } [features] loom = ["dep:loom", "saa/loom", "sdd/loom"] [dev-dependencies] criterion = { version = "0.8", features = ["async_futures"] } fnv = "1.0" futures = "0.3" proptest = "1.9" serde_test = "1.0" static_assertions = "1.1" tokio = { version = "1.48", features = ["full"] } [[bench]] name = "hash_map" harness = false [[bench]] name = "hash_index" harness = false [[bench]] name = "hash_cache" harness = false [[bench]] name = "tree_index" harness = false scc-3.4.8/LICENSE000064400000000000000000000250141046102023000114170ustar 00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2020-2025 Changgyoo Park Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. scc-3.4.8/README.md000064400000000000000000000315601046102023000116740ustar 00000000000000# Scalable Concurrent Containers [![Cargo](https://img.shields.io/crates/v/scc)](https://crates.io/crates/scc) ![Crates.io](https://img.shields.io/crates/l/scc) ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/wvwwvwwv/scalable-concurrent-containers/scc.yml?branch=main) A collection of high-performance containers providing both asynchronous and synchronous interfaces. #### Features - Provides both asynchronous and synchronous interfaces. - SIMD lookup to scan multiple entries in parallel: requires `RUSTFLAGS='-C target_feature=+avx2'` on `x86_64`. - [`Equivalent`](https://github.com/indexmap-rs/equivalent), [`Loom`](https://github.com/tokio-rs/loom) and [`Serde`](https://github.com/serde-rs/serde) support: `features = ["equivalent", "loom", "serde"]`. #### Concurrent Containers - [`HashMap`](#hashmap) is a concurrent hash map. - [`HashSet`](#hashset) is a concurrent hash set. - [`HashIndex`](#hashindex) is a read-optimized concurrent hash map. - [`HashCache`](#hashcache) is a 32-way associative concurrent cache backed by [`HashMap`](#hashmap). - [`TreeIndex`](#treeindex) is a read-optimized concurrent B-plus tree. ## `HashMap` [`HashMap`](#hashmap) is a concurrent hash map optimized for highly parallel write-heavy workloads. [`HashMap`](#hashmap) is structured as a lock-free stack of entry bucket arrays. The entry bucket array is managed by [`sdd`](https://crates.io/crates/sdd), thus enabling lock-free access to it and non-blocking container resizing. Each bucket is a fixed-size array of entries, protected by a read-write lock that simultaneously provides blocking and asynchronous methods. ### Locking behavior #### Entry access: fine-grained locking Read/write access to an entry is serialized by the read-write lock in the bucket containing the entry. There are no container-level locks; therefore, the larger the container gets, the lower the chance of the bucket-level lock being contended. #### Resize: lock-free Resizing of a [`HashMap`](#hashmap) is entirely non-blocking and lock-free; resizing does not block any other read/write access to the container or resizing attempts. _Resizing is analogous to pushing a new bucket array into a lock-free stack_. Each entry in the old bucket array will be incrementally relocated to the new bucket array upon future access to the container, and the old bucket array will eventually be dropped after it becomes empty. ### Examples Inserted entries can be updated, read, and removed synchronously or asynchronously. ```rust use scc::HashMap; let hashmap: HashMap = HashMap::default(); assert!(hashmap.insert_sync(1, 0).is_ok()); assert!(hashmap.insert_sync(1, 1).is_err()); assert_eq!(hashmap.upsert_sync(1, 1).unwrap(), 0); assert_eq!(hashmap.update_sync(&1, |_, v| { *v = 3; *v }).unwrap(), 3); assert_eq!(hashmap.read_sync(&1, |_, v| *v).unwrap(), 3); assert_eq!(hashmap.remove_sync(&1).unwrap(), (1, 3)); hashmap.entry_sync(7).or_insert(17); assert_eq!(hashmap.read_sync(&7, |_, v| *v).unwrap(), 17); let future_insert = hashmap.insert_async(2, 1); let future_remove = hashmap.remove_async(&1); ``` The `Entry` API of [`HashMap`](#hashmap) is helpful if the workflow is complicated. ```rust use scc::HashMap; let hashmap: HashMap = HashMap::default(); hashmap.entry_sync(3).or_insert(7); assert_eq!(hashmap.read_sync(&3, |_, v| *v), Some(7)); hashmap.entry_sync(4).and_modify(|v| { *v += 1 }).or_insert(5); assert_eq!(hashmap.read_sync(&4, |_, v| *v), Some(5)); ``` [`HashMap`](#hashmap) does not provide an [`Iterator`](https://doc.rust-lang.org/std/iter/trait.Iterator.html) since it is impossible to confine the lifetime of [`Iterator::Item`](https://doc.rust-lang.org/std/iter/trait.Iterator.html#associatedtype.Item) to the [Iterator](https://doc.rust-lang.org/std/iter/trait.Iterator.html). The limitation can be circumvented by relying on interior mutability, e.g., letting the returned reference hold a lock. However, it may lead to a deadlock if not correctly used, and frequent acquisition of locks may impact performance. Therefore, [`Iterator`](https://doc.rust-lang.org/std/iter/trait.Iterator.html) is not implemented; instead, [`HashMap`](#hashmap) provides several methods to iterate over entries synchronously or asynchronously: `iter_{async|sync}`, `iter_mut_{async|sync}`, `retain_{async|sync}`, `begin_{async|sync}`, `OccupiedEntry::next_{async|sync}`, and `OccupiedEntry::remove_and_{async|sync}` ```rust use scc::HashMap; let hashmap: HashMap = HashMap::default(); assert!(hashmap.insert_sync(1, 0).is_ok()); assert!(hashmap.insert_sync(2, 1).is_ok()); // Entries can be modified or removed via `retain_sync`. let mut acc = 0; hashmap.retain_sync(|k, v_mut| { acc += *k; *v_mut = 2; true }); assert_eq!(acc, 3); assert_eq!(hashmap.read_sync(&1, |_, v| *v).unwrap(), 2); assert_eq!(hashmap.read_sync(&2, |_, v| *v).unwrap(), 2); // `iter_sync` returns `true` when all the entries satisfy the predicate. assert!(hashmap.insert_sync(3, 2).is_ok()); assert!(!hashmap.iter_sync(|k, _| *k == 3)); // Multiple entries can be removed through `retain_sync`. hashmap.retain_sync(|k, v| *k == 1 && *v == 2); // `hash_map::OccupiedEntry` also can return the next closest occupied entry. let first_entry = hashmap.begin_sync(); assert_eq!(*first_entry.as_ref().unwrap().key(), 1); let second_entry = first_entry.and_then(|e| e.next_sync()); assert!(second_entry.is_none()); fn is_send(f: &T) -> bool { true } // Asynchronous iteration over entries using `iter_async`. let future_scan = hashmap.iter_async(|k, v| { println!("{k} {v}"); true }); assert!(is_send(&future_scan)); // Asynchronous iteration over entries using the `Entry` API. let future_iter = async { let mut iter = hashmap.begin_async().await; while let Some(entry) = iter { // `OccupiedEntry` can be sent across awaits and threads. assert!(is_send(&entry)); assert_eq!(*entry.key(), 1); iter = entry.next_async().await; } }; assert!(is_send(&future_iter)); ``` ## `HashSet` [`HashSet`](#hashset) is a special version of [`HashMap`](#hashmap) where the value type is `()`. ### Examples Most [`HashSet`](#hashset) methods are identical to those of [`HashMap`](#hashmap) except that they do not receive a value argument, and some [`HashMap`](#hashmap) methods for value modification are not implemented for [`HashSet`](#hashset). ```rust use scc::HashSet; let hashset: HashSet = HashSet::default(); assert!(hashset.read_sync(&1, |_| true).is_none()); assert!(hashset.insert_sync(1).is_ok()); assert!(hashset.read_sync(&1, |_| true).unwrap()); let future_insert = hashset.insert_async(2); let future_remove = hashset.remove_async(&1); ``` ## `HashIndex` [`HashIndex`](#hashindex) is a read-optimized version of [`HashMap`](#hashmap). In a [`HashIndex`](#hashindex), not only is the memory of the bucket array managed by [`sdd`](https://crates.io/crates/sdd), but also that of entry buckets is protected by [`sdd`](https://crates.io/crates/sdd), enabling lock-free read access to individual entries. ### Entry lifetime The `HashIndex` does not drop removed entries immediately; instead, they are dropped only when the bucket is accessed again after the [`sdd`](https://crates.io/crates/sdd) mechanism ensures there are no potential readers for those entries. This implies that a removed entry may persist as long as there are potential readers or the bucket remains unaccessed. As a result, `HashIndex` is not an optimal choice for workloads that are write-heavy and involve large entry sizes. ### Examples The `peek` and `peek_with` methods are completely lock-free. ```rust use scc::HashIndex; let hashindex: HashIndex = HashIndex::default(); assert!(hashindex.insert_sync(1, 0).is_ok()); // `peek` and `peek_with` are lock-free. assert_eq!(hashindex.peek_with(&1, |_, v| *v).unwrap(), 0); let future_insert = hashindex.insert_async(2, 1); let future_remove = hashindex.remove_if_async(&1, |_| true); ``` The `Entry` API of [`HashIndex`](#hashindex) can update an existing entry. ```rust use scc::HashIndex; let hashindex: HashIndex = HashIndex::default(); assert!(hashindex.insert_sync(1, 1).is_ok()); if let Some(mut o) = hashindex.get_sync(&1) { // Create a new version of the entry. o.update(2); }; if let Some(mut o) = hashindex.get_sync(&1) { // Update the entry in place. unsafe { *o.get_mut() = 3; } }; ``` An [`Iterator`](https://doc.rust-lang.org/std/iter/trait.Iterator.html) is implemented for [`HashIndex`](#hashindex). ```rust use scc::HashIndex; use sdd::Guard; let hashindex: HashIndex = HashIndex::default(); assert!(hashindex.insert_sync(1, 0).is_ok()); // Existing values can be replaced with a new one. hashindex.get_sync(&1).unwrap().update(1); let guard = Guard::new(); // An `Guard` has to be supplied to `iter`. let mut iter = hashindex.iter(&guard); let entry_ref = iter.next().unwrap(); assert_eq!(iter.next(), None); ``` ## `HashCache` [`HashCache`](#hashcache) is a 32-way associative concurrent cache based on the [`HashMap`](#hashmap) implementation. [`HashCache`](#hashcache) does not keep track of the least recently used entry in the entire cache. Instead, each bucket maintains a doubly linked list of occupied entries, which is updated on entry access. ### Examples The LRU entry in a bucket is evicted when a new entry is inserted, and the bucket is full. ```rust use scc::HashCache; let hashcache: HashCache = HashCache::with_capacity(100, 2000); /// The capacity cannot exceed the maximum capacity. assert_eq!(hashcache.capacity_range(), 128..=2048); /// If the bucket corresponding to `1` or `2` is full, the LRU entry will be evicted. assert!(hashcache.put_sync(1, 0).is_ok()); assert!(hashcache.put_sync(2, 0).is_ok()); /// `1` becomes the most recently accessed entry in the bucket. assert!(hashcache.get_sync(&1).is_some()); /// An entry can be normally removed. assert_eq!(hashcache.remove_sync(&2).unwrap(), (2, 0)); ``` ## `TreeIndex` [`TreeIndex`](#treeindex) is a B-plus tree variant optimized for read operations. [`sdd`](https://crates.io/crates/sdd) protects the memory used by individual entries, thus enabling lock-free read access to them. ### Locking behavior Read access is always lock-free and non-blocking. Write access to an entry is lock-free and non-blocking as long as no structural changes are required. However, when nodes are split or merged by a write operation, other write operations on keys in the affected range are blocked. ### Entry lifetime `TreeIndex` does not drop removed entries immediately. Instead, they are dropped when the leaf node is cleared or split, making `TreeIndex` a suboptimal choice if the workload is write-heavy. ### Examples Locks are acquired or awaited when internal nodes are split or merged, however blocking operations do not affect read operations. ```rust use scc::TreeIndex; let treeindex: TreeIndex = TreeIndex::new(); assert!(treeindex.insert_sync(1, 2).is_ok()); // `peek` and `peek_with` are lock-free. assert_eq!(treeindex.peek_with(&1, |_, v| *v).unwrap(), 2); assert!(treeindex.remove_sync(&1)); let future_insert = treeindex.insert_async(2, 3); let future_remove = treeindex.remove_if_async(&1, |v| *v == 2); ``` Entries can be scanned without acquiring any locks. ```rust use scc::TreeIndex; use sdd::Guard; let treeindex: TreeIndex = TreeIndex::new(); assert!(treeindex.insert_sync(1, 10).is_ok()); assert!(treeindex.insert_sync(2, 11).is_ok()); assert!(treeindex.insert_sync(3, 13).is_ok()); let guard = Guard::new(); // `iter` iterates over entries without acquiring a lock. let mut iter = treeindex.iter(&guard); assert_eq!(iter.next().unwrap(), (&1, &10)); assert_eq!(iter.next().unwrap(), (&2, &11)); assert_eq!(iter.next().unwrap(), (&3, &13)); assert!(iter.next().is_none()); ``` A specific range of keys can be scanned. ```rust use scc::TreeIndex; use sdd::Guard; let treeindex: TreeIndex = TreeIndex::new(); for i in 0..10 { assert!(treeindex.insert_sync(i, 10).is_ok()); } let guard = Guard::new(); assert_eq!(treeindex.range(1..1, &guard).count(), 0); assert_eq!(treeindex.range(4..8, &guard).count(), 4); assert_eq!(treeindex.range(4..=8, &guard).count(), 5); ``` ## Performance ### SIMD support [`HashMap`](#hashmap) is optimized for 256-bit SIMD instructions. Therefore, it is recommended to compile with `avx2` or equivalent options on `x86-64` targets, or with respective features on other platforms. * Note that Apple M-series CPUs do not support the 256-bit SIMD instructions required for optimal performance. ### [`HashMap`](#hashmap) Tail Latency The expected tail latency of a distribution of latencies of 1048576 insertion operations (`K = u64, V = u64`) is less than 50 microseconds on Apple M4 Pro. ### [`HashMap`](#hashmap) and [`HashIndex`](#hashindex) Throughput - [Results on Intel Xeon (48 cores, avx2)](https://codeberg.org/wvwwvwwv/conc-map-bench). ## [Changelog](https://github.com/wvwwvwwv/scalable-concurrent-containers/blob/main/CHANGELOG.md) scc-3.4.8/benches/hash_cache.rs000064400000000000000000000023531046102023000144360ustar 00000000000000use std::time::Instant; use criterion::{Criterion, criterion_group, criterion_main}; use scc::HashCache; fn get(c: &mut Criterion) { c.bench_function("HashCache: get", |b| { b.iter_custom(|iters| { let hashcache: HashCache = HashCache::with_capacity(iters as usize * 2, iters as usize * 2); for i in 0..iters { assert!(hashcache.put_sync(i, i).is_ok()); } let start = Instant::now(); for i in 0..iters { drop(hashcache.get_sync(&i)); } start.elapsed() }) }); } fn put_saturated(c: &mut Criterion) { let hashcache: HashCache = HashCache::with_capacity(64, 64); for k in 0..256 { assert!(hashcache.put_sync(k, k).is_ok()); } let mut max_key = 256; c.bench_function("HashCache: put, saturated", |b| { b.iter_custom(|iters| { let start = Instant::now(); for i in max_key..(max_key + iters) { assert!(hashcache.put_sync(i, i).is_ok()); } max_key += iters; start.elapsed() }) }); } criterion_group!(hash_cache, get, put_saturated); criterion_main!(hash_cache); scc-3.4.8/benches/hash_index.rs000064400000000000000000000035111046102023000144770ustar 00000000000000use std::time::Instant; use criterion::{Criterion, criterion_group, criterion_main}; use scc::{Guard, HashIndex}; fn iter_with(c: &mut Criterion) { c.bench_function("HashIndex: iter_with", |b| { b.iter_custom(|iters| { let hashindex: HashIndex = HashIndex::with_capacity(iters as usize * 2); for i in 0..iters { assert!(hashindex.insert_sync(i, i).is_ok()); } let start = Instant::now(); let guard = Guard::new(); let iter = hashindex.iter(&guard); for e in iter { assert_eq!(e.0, e.1); } start.elapsed() }) }); } fn peek(c: &mut Criterion) { c.bench_function("HashIndex: peek", |b| { b.iter_custom(|iters| { let hashindex: HashIndex = HashIndex::with_capacity(iters as usize * 2); for i in 0..iters { assert!(hashindex.insert_sync(i, i).is_ok()); } let start = Instant::now(); let guard = Guard::new(); for i in 0..iters { assert_eq!(hashindex.peek(&i, &guard), Some(&i)); } start.elapsed() }) }); } fn peek_with(c: &mut Criterion) { c.bench_function("HashIndex: peek_with", |b| { b.iter_custom(|iters| { let hashindex: HashIndex = HashIndex::with_capacity(iters as usize * 2); for i in 0..iters { assert!(hashindex.insert_sync(i, i).is_ok()); } let start = Instant::now(); for i in 0..iters { assert_eq!(hashindex.peek_with(&i, |_, v| *v == i), Some(true)); } start.elapsed() }) }); } criterion_group!(hash_index, iter_with, peek, peek_with); criterion_main!(hash_index); scc-3.4.8/benches/hash_map.rs000064400000000000000000000134231046102023000141500ustar 00000000000000use std::time::{Duration, Instant}; use criterion::async_executor::FuturesExecutor; use criterion::{Criterion, criterion_group, criterion_main}; use scc::HashMap; fn insert_single_async(c: &mut Criterion) { c.bench_function("HashMap: insert_single_async", |b| { let hashmap: HashMap = HashMap::default(); assert!(hashmap.insert_sync(0, 0).is_ok()); async fn test(hashmap: &HashMap) { assert!(hashmap.insert_async(0, 0).await.is_err()); } b.to_async(FuturesExecutor).iter(|| test(&hashmap)); }); } fn insert_single_sync(c: &mut Criterion) { c.bench_function("HashMap: insert_single_sync", |b| { let hashmap: HashMap = HashMap::default(); assert!(hashmap.insert_sync(0, 0).is_ok()); fn test(hashmap: &HashMap) { assert!(hashmap.insert_sync(0, 0).is_err()); } b.iter(|| test(&hashmap)); }); } fn insert_remove_single_async(c: &mut Criterion) { c.bench_function("HashMap: insert_remove_single_async", |b| { let hashmap: HashMap = HashMap::default(); assert!(hashmap.insert_sync(0, 0).is_ok()); async fn test(hashmap: &HashMap) { assert!(hashmap.insert_async(1, 1).await.is_ok()); assert!(hashmap.remove_async(&1).await.is_some()); } b.to_async(FuturesExecutor).iter(|| test(&hashmap)); }); } fn insert_remove_single_sync(c: &mut Criterion) { c.bench_function("HashMap: insert_remove_single_sync", |b| { let hashmap: HashMap = HashMap::default(); assert!(hashmap.insert_sync(0, 0).is_ok()); fn test(hashmap: &HashMap) { assert!(hashmap.insert_sync(1, 1).is_ok()); assert!(hashmap.remove_sync(&1).is_some()); } b.iter(|| test(&hashmap)); }); } fn insert_cold_async(c: &mut Criterion) { c.bench_function("HashMap: insert_cold_async", |b| { b.to_async(FuturesExecutor).iter_custom(async |iters| { let hashmap: HashMap = HashMap::default(); let start = Instant::now(); for i in 0..iters { assert!(hashmap.insert_async(i, i).await.is_ok()); } start.elapsed() }) }); } fn insert_cold_sync(c: &mut Criterion) { c.bench_function("HashMap: insert_cold_sync", |b| { b.iter_custom(|iters| { let hashmap: HashMap = HashMap::default(); let start = Instant::now(); for i in 0..iters { assert!(hashmap.insert_sync(i, i).is_ok()); } start.elapsed() }) }); } fn insert_warmed_up_async(c: &mut Criterion) { c.bench_function("HashMap: insert_warmed_up_async", |b| { b.to_async(FuturesExecutor).iter_custom(async |iters| { let hashmap: HashMap = HashMap::with_capacity(iters as usize * 2); let start = Instant::now(); for i in 0..iters { assert!(hashmap.insert_async(i, i).await.is_ok()); } start.elapsed() }) }); } fn insert_warmed_up_sync(c: &mut Criterion) { c.bench_function("HashMap: insert_warmed_up_sync", |b| { b.iter_custom(|iters| { let hashmap: HashMap = HashMap::with_capacity(iters as usize * 2); let start = Instant::now(); for i in 0..iters { assert!(hashmap.insert_sync(i, i).is_ok()); } start.elapsed() }) }); } fn read_async(c: &mut Criterion) { c.bench_function("HashMap: read_async", |b| { b.to_async(FuturesExecutor).iter_custom(async |iters| { let hashmap: HashMap = HashMap::with_capacity(iters as usize * 2); for i in 0..iters { assert!(hashmap.insert_async(i, i).await.is_ok()); } let start = Instant::now(); for i in 0..iters { assert_eq!(hashmap.read_async(&i, |_, v| *v == i).await, Some(true)); } start.elapsed() }) }); } fn read_sync(c: &mut Criterion) { c.bench_function("HashMap: read_sync", |b| { b.iter_custom(|iters| { let hashmap: HashMap = HashMap::with_capacity(iters as usize * 2); for i in 0..iters { assert!(hashmap.insert_sync(i, i).is_ok()); } let start = Instant::now(); for i in 0..iters { assert_eq!(hashmap.read_sync(&i, |_, v| *v == i), Some(true)); } start.elapsed() }) }); } fn insert_tail_latency(c: &mut Criterion) { c.bench_function("HashMap: insert, tail latency", move |b| { b.iter_custom(|iters| { let mut agg_max_latency = Duration::default(); for _ in 0..iters { let hashmap: HashMap = HashMap::default(); let mut key = 0; let mut max_latency = Duration::default(); (0..1048576).for_each(|_| { key += 1; let start = Instant::now(); assert!(hashmap.insert_sync(key, key).is_ok()); let elapsed = start.elapsed(); if elapsed > max_latency { max_latency = elapsed; } }); agg_max_latency += max_latency; } agg_max_latency }) }); } criterion_group!( hash_map, insert_single_async, insert_single_sync, insert_remove_single_async, insert_remove_single_sync, insert_cold_async, insert_cold_sync, insert_warmed_up_async, insert_warmed_up_sync, insert_tail_latency, read_async, read_sync ); criterion_main!(hash_map); scc-3.4.8/benches/tree_index.rs000064400000000000000000000050501046102023000145130ustar 00000000000000use std::time::Instant; use criterion::async_executor::FuturesExecutor; use criterion::{Criterion, criterion_group, criterion_main}; use scc::{Guard, TreeIndex}; fn insert_async(c: &mut Criterion) { c.bench_function("TreeIndex: insert_async", |b| { b.to_async(FuturesExecutor).iter_custom(async |iters| { let treeindex: TreeIndex = TreeIndex::default(); let start = Instant::now(); for i in 0..iters { assert!(treeindex.insert_async(i, i).await.is_ok()); } start.elapsed() }) }); } fn insert_sync(c: &mut Criterion) { c.bench_function("TreeIndex: insert_sync", |b| { b.iter_custom(|iters| { let treeindex: TreeIndex = TreeIndex::default(); let start = Instant::now(); for i in 0..iters { assert!(treeindex.insert_sync(i, i).is_ok()); } start.elapsed() }) }); } fn insert_rev(c: &mut Criterion) { c.bench_function("TreeIndex: insert, rev", |b| { b.iter_custom(|iters| { let treeindex: TreeIndex = TreeIndex::default(); let start = Instant::now(); for i in (0..iters).rev() { assert!(treeindex.insert_sync(i, i).is_ok()); } start.elapsed() }) }); } fn iter_with(c: &mut Criterion) { c.bench_function("TreeIndex: iter_with", |b| { b.iter_custom(|iters| { let treeindex: TreeIndex = TreeIndex::default(); for i in 0..iters { assert!(treeindex.insert_sync(i, i).is_ok()); } let start = Instant::now(); let guard = Guard::new(); let iter = treeindex.iter(&guard); for e in iter { assert_eq!(e.0, e.1); } start.elapsed() }) }); } fn peek(c: &mut Criterion) { c.bench_function("TreeIndex: peek", |b| { b.iter_custom(|iters| { let treeindex: TreeIndex = TreeIndex::default(); for i in 0..iters { assert!(treeindex.insert_sync(i, i).is_ok()); } let start = Instant::now(); let guard = Guard::new(); for i in 0..iters { assert_eq!(treeindex.peek(&i, &guard), Some(&i)); } start.elapsed() }) }); } criterion_group!( tree_index, insert_async, insert_sync, insert_rev, iter_with, peek ); criterion_main!(tree_index); scc-3.4.8/src/async_helper.rs000064400000000000000000000074231046102023000142270ustar 00000000000000use std::cell::UnsafeCell; use std::pin::{Pin, pin}; use std::ptr; use std::sync::atomic::Ordering; use saa::lock::Mode; use saa::{Lock, Pager}; use sdd::{AtomicShared, Guard}; /// [`AsyncGuard`] is used when an asynchronous task needs to be suspended without invalidating any /// references. /// /// The validity of those references must be checked and verified by the user. #[derive(Debug, Default)] pub(crate) struct AsyncGuard { /// [`Guard`] that can be dropped without invalidating any references. guard: UnsafeCell>, } #[derive(Debug)] pub(crate) struct AsyncWait { /// Allows the user to await the lock anywhere in the code. pager: Pager<'static, Lock>, } /// [`TryWait`] allows [`AsyncWait`] to be used in synchronous methods. pub(crate) trait TryWait { /// Registers the [`Pager`] in the [`Lock`], or synchronously waits for the [`Lock`] to be /// available. fn try_wait(&mut self, lock: &Lock); } impl AsyncGuard { /// Returns `true` if the [`AsyncGuard`] contains a valid [`Guard`]. #[inline] pub(crate) const fn has_guard(&self) -> bool { unsafe { (*self.guard.get()).is_some() } } /// Returns or creates a new [`Guard`]. /// /// # Safety /// /// The caller must ensure that any references derived from the returned [`Guard`] do not /// outlive the underlying instance. #[inline] pub(crate) fn guard(&self) -> &Guard { unsafe { (*self.guard.get()).get_or_insert_with(Guard::new) } } /// Resets the [`AsyncGuard`] to its initial state. #[inline] pub(crate) fn reset(&self) { unsafe { *self.guard.get() = None; } } /// Loads the content of the [`AtomicShared`] without exposing the [`Guard`] or checking tag /// bits. #[inline] pub(crate) fn load_unchecked( &self, atomic_ptr: &AtomicShared, mo: Ordering, ) -> Option<&T> { unsafe { atomic_ptr.load(mo, self.guard()).as_ref_unchecked() } } /// Checks if the reference is valid. #[inline] pub(crate) fn check_ref(&self, atomic_ptr: &AtomicShared, r: &T, mo: Ordering) -> bool { self.load_unchecked(atomic_ptr, mo) .is_some_and(|s| ptr::eq(s, r)) } } // SAFETY: this is the sole purpose of `AsyncGuard`; Send-safety should be ensured by the user, // e.g., the `AsyncGuard` should always be reset before the task is suspended. unsafe impl Send for AsyncGuard {} unsafe impl Sync for AsyncGuard {} impl AsyncWait { /// Awaits the [`Lock`] to be available. #[inline] pub async fn wait(self: &mut Pin<&mut Self>) { let this = unsafe { ptr::read(self) }; let mut pinned_pager = unsafe { Pin::new_unchecked(&mut this.get_unchecked_mut().pager) }; let _result = pinned_pager.poll_async().await; } } impl Default for AsyncWait { #[inline] fn default() -> Self { Self { pager: unsafe { std::mem::transmute::, Pager<'static, Lock>>(Pager::default()) }, } } } impl TryWait for Pin<&mut AsyncWait> { #[inline] fn try_wait(&mut self, lock: &Lock) { let this = unsafe { ptr::read(self) }; let mut pinned_pager = unsafe { let pager_ref = std::mem::transmute::<&mut Pager<'static, Lock>, &mut Pager>( &mut this.get_unchecked_mut().pager, ); Pin::new_unchecked(pager_ref) }; lock.register_pager(&mut pinned_pager, Mode::WaitExclusive, false); } } impl TryWait for () { #[inline] fn try_wait(&mut self, lock: &Lock) { let mut pinned_pager = pin!(Pager::default()); lock.register_pager(&mut pinned_pager, Mode::WaitExclusive, true); let _: Result<_, _> = pinned_pager.poll_sync(); } } scc-3.4.8/src/equivalent.rs000064400000000000000000000020201046102023000137140ustar 00000000000000//! Vendors the [`equivalent`](https://crates.io/crates/equivalent) crate to avoid conflicts. #![deny(unsafe_code)] use std::borrow::Borrow; use std::cmp::Ordering; /// Key equivalence trait. /// /// [`Hash`](std::hash::Hash) must be implemented to ensure that the same hash value /// is generated for equivalent keys. pub trait Equivalent { /// Compares `self` with `key` and returns `true` if they are equal. fn equivalent(&self, key: &K) -> bool; } impl Equivalent for Q where Q: Eq, K: Borrow, { #[inline] fn equivalent(&self, key: &K) -> bool { PartialEq::eq(self, key.borrow()) } } /// Key ordering trait. pub trait Comparable: Equivalent { /// Compares `self` with `key` and returns their ordering. fn compare(&self, key: &K) -> Ordering; } impl Comparable for Q where Q: Ord, K: Borrow, { #[inline] fn compare(&self, key: &K) -> Ordering { Ord::cmp(self, key.borrow()) } } scc-3.4.8/src/exit_guard.rs000064400000000000000000000026261046102023000137060ustar 00000000000000//! This module implements a simplified but safe version of //! [`scopeguard`](https://crates.io/crates/scopeguard). use std::mem::{ManuallyDrop, forget}; use std::ops::{Deref, DerefMut}; /// [`ExitGuard`] captures the environment and invokes a defined closure at the end of the scope. pub(crate) struct ExitGuard { drop_callback: ManuallyDrop<(T, F)>, } impl ExitGuard { /// Creates a new [`ExitGuard`] with the specified variables captured. #[inline] pub(crate) const fn new(captured: T, drop_callback: F) -> Self { Self { drop_callback: ManuallyDrop::new((captured, drop_callback)), } } /// Forgets the [`ExitGuard`] without invoking the drop callback. #[inline] pub(crate) fn forget(mut self) { unsafe { ManuallyDrop::drop(&mut self.drop_callback); } forget(self); } } impl Drop for ExitGuard { #[inline] fn drop(&mut self) { let (c, f) = unsafe { ManuallyDrop::take(&mut self.drop_callback) }; f(c); } } impl Deref for ExitGuard { type Target = T; #[inline] fn deref(&self) -> &Self::Target { &self.drop_callback.0 } } impl DerefMut for ExitGuard { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { &mut self.drop_callback.0 } } scc-3.4.8/src/hash_cache.rs000064400000000000000000001505111046102023000136160ustar 00000000000000//! [`HashCache`] is a concurrent 32-way associative cache backed by [`HashMap`](super::HashMap). use std::collections::hash_map::RandomState; use std::fmt::{self, Debug}; use std::hash::{BuildHasher, Hash}; use std::mem::replace; use std::ops::{Deref, DerefMut, RangeInclusive}; #[cfg(not(feature = "loom"))] use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; #[cfg(feature = "loom")] use loom::sync::atomic::AtomicUsize; use sdd::{AtomicShared, Guard, Shared, Tag}; use super::Equivalent; use super::hash_table::MAXIMUM_CAPACITY_LIMIT; use super::hash_table::bucket::{CACHE, DoublyLinkedList, EntryPtr}; use super::hash_table::bucket_array::BucketArray; use super::hash_table::{HashTable, LockedBucket}; /// Scalable concurrent 32-way associative cache backed by [`HashMap`](super::HashMap). /// /// [`HashCache`] is a concurrent 32-way associative cache based on the /// [`HashMap`](super::HashMap) implementation. [`HashCache`] does not keep track of the least /// recently used entry in the entire cache. Instead, each bucket maintains a doubly linked list of /// occupied entries, updated on access to entries to keep track of the least recently used entries /// within the bucket. Therefore, entries can be evicted before the cache is full. /// /// [`HashCache`] and [`HashMap`](super::HashMap) share the same runtime characteristics, except /// that each entry in a [`HashCache`] additionally uses 2-byte space for a doubly linked list and a /// [`HashCache`] starts evicting least recently used entries if the bucket is full instead of /// allocating a linked list of entries. /// /// ## Unwind safety /// /// [`HashCache`] is impervious to out-of-memory errors and panics in user-specified code under one /// condition: `H::Hasher::hash`, `K::drop` and `V::drop` must not panic. pub struct HashCache where H: BuildHasher, { bucket_array: AtomicShared>, minimum_capacity: AtomicUsize, maximum_capacity: usize, build_hasher: H, } /// The default maximum capacity of a [`HashCache`] is `256`. pub const DEFAULT_MAXIMUM_CAPACITY: usize = 256; /// [`EvictedEntry`] is a type alias for `Option<(K, V)>`. pub type EvictedEntry = Option<(K, V)>; /// [`Entry`] represents a single cache entry in a [`HashCache`]. pub enum Entry<'h, K, V, H = RandomState> where H: BuildHasher, { /// An occupied entry. Occupied(OccupiedEntry<'h, K, V, H>), /// A vacant entry. Vacant(VacantEntry<'h, K, V, H>), } /// [`OccupiedEntry`] is a view into an occupied cache entry in a [`HashCache`]. pub struct OccupiedEntry<'h, K, V, H = RandomState> where H: BuildHasher, { hashcache: &'h HashCache, locked_bucket: LockedBucket, entry_ptr: EntryPtr, } /// [`VacantEntry`] is a view into a vacant cache entry in a [`HashCache`]. pub struct VacantEntry<'h, K, V, H = RandomState> where H: BuildHasher, { hashcache: &'h HashCache, key: K, hash: u64, locked_bucket: LockedBucket, } /// [`ConsumableEntry`] is a view into an occupied entry in a [`HashCache`] when iterating over /// entries in it. pub struct ConsumableEntry<'b, K, V> { /// Holds an exclusive lock on the entry bucket. locked_bucket: &'b mut LockedBucket, /// Pointer to the entry. entry_ptr: &'b mut EntryPtr, /// Probes removal. remove_probe: &'b mut bool, } /// [`ReplaceResult`] is the result type of the [`HashCache::replace_async`] and /// [`HashCache::replace_sync`] methods. pub enum ReplaceResult<'h, K, V, H = RandomState> where H: BuildHasher, { /// The key was replaced. Replaced(OccupiedEntry<'h, K, V, H>, K), /// The key did not exist in the [`HashCache`]. /// /// An [`OccupiedEntry`] can be created from the [`VacantEntry`]. NotReplaced(VacantEntry<'h, K, V, H>), } impl HashCache where H: BuildHasher, { /// Creates an empty [`HashCache`] with the given [`BuildHasher`]. /// /// The maximum capacity is set to [`DEFAULT_MAXIMUM_CAPACITY`]. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use std::collections::hash_map::RandomState; /// /// let hashcache: HashCache = HashCache::with_hasher(RandomState::new()); /// ``` #[cfg(not(feature = "loom"))] #[inline] pub const fn with_hasher(build_hasher: H) -> Self { HashCache { bucket_array: AtomicShared::null(), minimum_capacity: AtomicUsize::new(0), maximum_capacity: DEFAULT_MAXIMUM_CAPACITY, build_hasher, } } /// Creates an empty [`HashCache`] with the given [`BuildHasher`]. #[cfg(feature = "loom")] #[inline] pub fn with_hasher(build_hasher: H) -> Self { Self { bucket_array: AtomicShared::null(), minimum_capacity: AtomicUsize::new(0), maximum_capacity: DEFAULT_MAXIMUM_CAPACITY, build_hasher, } } /// Creates an empty [`HashCache`] with the specified capacity and [`BuildHasher`]. /// /// The actual capacity is equal to or greater than `minimum_capacity` unless it is greater than /// `1 << (usize::BITS - 1)`. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use std::collections::hash_map::RandomState; /// /// let hashcache: HashCache = /// HashCache::with_capacity_and_hasher(1000, 2000, RandomState::new()); /// /// let result = hashcache.capacity(); /// assert_eq!(result, 1024); /// ``` #[inline] pub fn with_capacity_and_hasher( minimum_capacity: usize, maximum_capacity: usize, build_hasher: H, ) -> Self { let (array, minimum_capacity) = if minimum_capacity == 0 { (AtomicShared::null(), AtomicUsize::new(0)) } else { let array = unsafe { Shared::new_unchecked(BucketArray::::new( minimum_capacity, AtomicShared::null(), )) }; let minimum_capacity = array.num_slots(); ( AtomicShared::from(array), AtomicUsize::new(minimum_capacity), ) }; let maximum_capacity = maximum_capacity .max(minimum_capacity.load(Relaxed)) .max(BucketArray::::minimum_capacity()) .min(MAXIMUM_CAPACITY_LIMIT) .next_power_of_two(); HashCache { bucket_array: array, minimum_capacity, maximum_capacity, build_hasher, } } } impl HashCache where K: Eq + Hash, H: BuildHasher, { /// Gets the entry associated with the given key in the map for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// let future_entry = hashcache.entry_async('b'); /// ``` #[inline] pub async fn entry_async(&self, key: K) -> Entry<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_async(hash).await; let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Entry::Occupied(OccupiedEntry { hashcache: self, locked_bucket, entry_ptr, }) } else { let vacant_entry = VacantEntry { hashcache: self, key, hash, locked_bucket, }; Entry::Vacant(vacant_entry) } } /// Gets the entry associated with the given key in the map for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// for ch in "a short treatise on fungi".chars() { /// hashcache.entry_sync(ch).and_modify(|counter| *counter += 1).or_put(1); /// } /// /// assert_eq!(*hashcache.get_sync(&'s').unwrap().get(), 2); /// assert_eq!(*hashcache.get_sync(&'t').unwrap().get(), 3); /// assert!(hashcache.get_sync(&'y').is_none()); /// ``` #[inline] pub fn entry_sync(&self, key: K) -> Entry<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_sync(hash); let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Entry::Occupied(OccupiedEntry { hashcache: self, locked_bucket, entry_ptr, }) } else { let vacant_entry = VacantEntry { hashcache: self, key, hash, locked_bucket, }; Entry::Vacant(vacant_entry) } } /// Tries to get the entry associated with the given key in the map for in-place manipulation. /// /// Returns `None` if the entry could not be locked. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(0, 1).is_ok()); /// assert!(hashcache.try_entry(0).is_some()); /// ``` #[inline] pub fn try_entry(&self, key: K) -> Option> { let hash = self.hash(&key); let guard = Guard::new(); let locked_bucket = self.try_reserve_bucket(hash, &guard)?; let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Some(Entry::Occupied(OccupiedEntry { hashcache: self, locked_bucket, entry_ptr, })) } else { Some(Entry::Vacant(VacantEntry { hashcache: self, key, hash, locked_bucket, })) } } /// Puts a key-value pair into the [`HashCache`]. /// /// Returns `Some` if an entry was evicted for the new key-value pair. /// /// # Note /// /// If the key exists, the value is *not* updated. /// /// # Errors /// /// Returns an error containing the supplied key-value pair if the key exists. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// let future_put = hashcache.put_async(11, 17); /// ``` #[inline] pub async fn put_async(&self, key: K, val: V) -> Result, (K, V)> { let hash = self.hash(&key); let locked_bucket = self.writer_async(hash).await; if locked_bucket.search(&key, hash).is_valid() { Err((key, val)) } else { let evicted = locked_bucket.evict_lru_head(locked_bucket.data_block); let entry_ptr = locked_bucket.insert(hash, (key, val)); locked_bucket.update_lru_tail(&entry_ptr); Ok(evicted) } } /// Puts a key-value pair into the [`HashCache`]. /// /// Returns `Some` if an entry was evicted for the new key-value pair. /// /// # Note /// /// If the key exists, the value is *not* updated. /// /// # Errors /// /// Returns an error containing the supplied key-value pair if the key exists. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert_eq!(hashcache.put_sync(1, 1).unwrap_err(), (1, 1)); /// ``` #[inline] pub fn put_sync(&self, key: K, val: V) -> Result, (K, V)> { let hash = self.hash(&key); let locked_bucket = self.writer_sync(hash); let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Err((key, val)) } else { let evicted = locked_bucket .writer .evict_lru_head(locked_bucket.data_block); let entry_ptr = locked_bucket.insert(hash, (key, val)); locked_bucket.writer.update_lru_tail(&entry_ptr); Ok(evicted) } } /// Adds a key to the [`HashCache`], replacing the existing key, if any, that is equal to the /// given one. /// /// # Examples /// /// ``` /// use std::cmp::{Eq, PartialEq}; /// use std::hash::{Hash, Hasher}; /// /// use scc::HashCache; /// use scc::hash_cache::ReplaceResult; /// /// #[derive(Debug)] /// struct MaybeEqual(u64, u64); /// /// impl Eq for MaybeEqual {} /// /// impl Hash for MaybeEqual { /// fn hash(&self, state: &mut H) { /// // Do not read `self.1`. /// self.0.hash(state); /// } /// } /// /// impl PartialEq for MaybeEqual { /// fn eq(&self, other: &Self) -> bool { /// // Do not compare `self.1`. /// self.0 == other.0 /// } /// } /// /// let hashcache: HashCache = HashCache::default(); /// /// async { /// let ReplaceResult::NotReplaced(v) = hashcache.replace_async(MaybeEqual(11, 7)).await else { /// unreachable!(); /// }; /// drop(v.put_entry(17)); /// let ReplaceResult::Replaced(_, k) = hashcache.replace_async(MaybeEqual(11, 11)).await else { /// unreachable!(); /// }; /// assert_eq!(k.1, 7); /// }; /// ``` #[inline] pub async fn replace_async(&self, key: K) -> ReplaceResult<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_async(hash).await; let mut entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { let prev_key = replace( &mut entry_ptr .get_mut(locked_bucket.data_block, &locked_bucket.writer) .0, key, ); ReplaceResult::Replaced( OccupiedEntry { hashcache: self, locked_bucket, entry_ptr, }, prev_key, ) } else { ReplaceResult::NotReplaced(VacantEntry { hashcache: self, key, hash, locked_bucket, }) } } /// Adds a key to the [`HashCache`], replacing the existing key, if any, that is equal to the /// given one. /// /// # Examples /// /// ``` /// use std::cmp::{Eq, PartialEq}; /// use std::hash::{Hash, Hasher}; /// /// use scc::HashCache; /// use scc::hash_cache::ReplaceResult; /// /// #[derive(Debug)] /// struct MaybeEqual(u64, u64); /// /// impl Eq for MaybeEqual {} /// /// impl Hash for MaybeEqual { /// fn hash(&self, state: &mut H) { /// // Do not read `self.1`. /// self.0.hash(state); /// } /// } /// /// impl PartialEq for MaybeEqual { /// fn eq(&self, other: &Self) -> bool { /// // Do not compare `self.1`. /// self.0 == other.0 /// } /// } /// /// let hashcache: HashCache = HashCache::default(); /// /// let ReplaceResult::NotReplaced(v) = hashcache.replace_sync(MaybeEqual(11, 7)) else { /// unreachable!(); /// }; /// drop(v.put_entry(17)); /// let ReplaceResult::Replaced(_, k) = hashcache.replace_sync(MaybeEqual(11, 11)) else { /// unreachable!(); /// }; /// assert_eq!(k.1, 7); /// ``` #[inline] pub fn replace_sync(&self, key: K) -> ReplaceResult<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_sync(hash); let mut entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { let prev_key = replace( &mut entry_ptr .get_mut(locked_bucket.data_block, &locked_bucket.writer) .0, key, ); ReplaceResult::Replaced( OccupiedEntry { hashcache: self, locked_bucket, entry_ptr, }, prev_key, ) } else { ReplaceResult::NotReplaced(VacantEntry { hashcache: self, key, hash, locked_bucket, }) } } /// Removes a key-value pair if the key exists. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// let future_put = hashcache.put_async(11, 17); /// let future_remove = hashcache.remove_async(&11); /// ``` #[inline] pub async fn remove_async(&self, key: &Q) -> Option<(K, V)> where Q: Equivalent + Hash + ?Sized, { self.remove_if_async(key, |_| true).await } /// Removes a key-value pair if the key exists. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.remove_sync(&1).is_none()); /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert_eq!(hashcache.remove_sync(&1).unwrap(), (1, 0)); /// ``` #[inline] pub fn remove_sync(&self, key: &Q) -> Option<(K, V)> where Q: Equivalent + Hash + ?Sized, { self.remove_if_sync(key, |_| true) } /// Gets an [`OccupiedEntry`] corresponding to the key for in-place modification. /// /// [`OccupiedEntry`] exclusively owns the entry, preventing others from gaining access to it: /// use [`read_async`](Self::read_async) if read-only access is sufficient. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// let future_put = hashcache.put_async(11, 17); /// let future_get = hashcache.get_async(&11); /// ``` #[inline] pub async fn get_async(&self, key: &Q) -> Option> where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let locked_bucket = self.optional_writer_async(hash).await?; let entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() { locked_bucket.writer.update_lru_tail(&entry_ptr); return Some(OccupiedEntry { hashcache: self, locked_bucket, entry_ptr, }); } None } /// Gets an [`OccupiedEntry`] corresponding to the key for in-place modification. /// /// [`OccupiedEntry`] exclusively owns the entry, preventing others from gaining access to it: /// use [`read_sync`](Self::read_sync) if read-only access is sufficient. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.get_sync(&1).is_none()); /// assert!(hashcache.put_sync(1, 10).is_ok()); /// assert_eq!(*hashcache.get_sync(&1).unwrap().get(), 10); /// /// *hashcache.get_sync(&1).unwrap() = 11; /// assert_eq!(*hashcache.get_sync(&1).unwrap(), 11); /// ``` #[inline] pub fn get_sync(&self, key: &Q) -> Option> where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let locked_bucket = self.optional_writer_sync(hash)?; let entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() { locked_bucket.writer.update_lru_tail(&entry_ptr); return Some(OccupiedEntry { hashcache: self, locked_bucket, entry_ptr, }); } None } /// Reads a key-value pair. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// let future_put = hashcache.put_async(11, 17); /// let future_read = hashcache.read_async(&11, |_, v| *v); /// ``` #[inline] pub async fn read_async R>(&self, key: &Q, reader: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.reader_async(key, reader).await } /// Reads a key-value pair. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.read_sync(&1, |_, v| *v).is_none()); /// assert!(hashcache.put_sync(1, 10).is_ok()); /// assert_eq!(hashcache.read_sync(&1, |_, v| *v).unwrap(), 10); /// ``` #[inline] pub fn read_sync R>(&self, key: &Q, reader: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.reader_sync(key, reader) } /// Returns `true` if the [`HashCache`] contains a value for the specified key. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// let future_contains = hashcache.contains_async(&1); /// ``` #[inline] pub async fn contains_async(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.reader_async(key, |_, _| ()).await.is_some() } /// Returns `true` if the [`HashCache`] contains a value for the specified key. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(!hashcache.contains_sync(&1)); /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert!(hashcache.contains_sync(&1)); /// ``` #[inline] pub fn contains_sync(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.read_sync(key, |_, _| ()).is_some() } /// Removes a key-value pair if the key exists and the given condition is met. /// /// Returns `None` if the key does not exist or the condition was not met. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// let future_put = hashcache.put_async(11, 17); /// let future_remove = hashcache.remove_if_async(&11, |_| true); /// ``` #[inline] pub async fn remove_if_async bool>( &self, key: &Q, condition: F, ) -> Option<(K, V)> where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let mut locked_bucket = self.optional_writer_async(hash).await?; let mut entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() && condition(&mut locked_bucket.entry_mut(&mut entry_ptr).1) { Some(locked_bucket.remove(self, &mut entry_ptr)) } else { None } } /// Removes a key-value pair if the key exists and the given condition is met. /// /// Returns `None` if the key does not exist or the condition was not met. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert!(hashcache.remove_if_sync(&1, |v| { *v += 1; false }).is_none()); /// assert_eq!(hashcache.remove_if_sync(&1, |v| *v == 1).unwrap(), (1, 1)); /// ``` #[inline] pub fn remove_if_sync bool>( &self, key: &Q, condition: F, ) -> Option<(K, V)> where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let mut locked_bucket = self.optional_writer_sync(hash)?; let mut entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() && condition(&mut locked_bucket.entry_mut(&mut entry_ptr).1) { Some(locked_bucket.remove(self, &mut entry_ptr)) } else { None } } /// Iterates over entries asynchronously for reading. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// /// async { /// let result = hashcache.iter_async(|k, v| { /// false /// }).await; /// assert!(!result); /// }; /// ``` #[inline] pub async fn iter_async bool>(&self, mut f: F) -> bool { let mut result = true; self.for_each_reader_async(|reader, data_block| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; return false; } } true }) .await; result } /// Iterates over entries synchronously for reading. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert!(hashcache.put_sync(2, 1).is_ok()); /// /// let mut acc = 0_u64; /// let result = hashcache.iter_sync(|k, v| { /// acc += *k; /// acc += *v; /// true /// }); /// /// assert!(result); /// assert_eq!(acc, 4); /// ``` #[inline] pub fn iter_sync bool>(&self, mut f: F) -> bool { let mut result = true; let guard = Guard::new(); self.for_each_reader_sync(&guard, |reader, data_block| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; return false; } } true }); result } /// Iterates over entries asynchronously for modification. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert!(hashcache.put_sync(2, 1).is_ok()); /// /// async { /// let result = hashcache.iter_mut_async(|entry| { /// if entry.0 == 1 { /// entry.consume(); /// return false; /// } /// true /// }).await; /// /// assert!(!result); /// assert_eq!(hashcache.len(), 1); /// }; /// ``` #[inline] pub async fn iter_mut_async) -> bool>( &self, mut f: F, ) -> bool { let mut result = true; self.for_each_writer_async(0, 0, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let consumable_entry = ConsumableEntry { locked_bucket: &mut locked_bucket, entry_ptr: &mut entry_ptr, remove_probe: removed, }; if !f(consumable_entry) { result = false; return true; } } false }) .await; result } /// Iterates over entries synchronously for modification. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert!(hashcache.put_sync(2, 1).is_ok()); /// assert!(hashcache.put_sync(3, 2).is_ok()); /// /// let result = hashcache.iter_mut_sync(|entry| { /// if entry.0 == 1 { /// entry.consume(); /// return false; /// } /// true /// }); /// /// assert!(!result); /// assert!(!hashcache.contains_sync(&1)); /// assert_eq!(hashcache.len(), 2); /// ``` #[inline] pub fn iter_mut_sync) -> bool>(&self, mut f: F) -> bool { let mut result = true; let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let consumable_entry = ConsumableEntry { locked_bucket: &mut locked_bucket, entry_ptr: &mut entry_ptr, remove_probe: removed, }; if !f(consumable_entry) { result = false; return true; } } false }); result } /// Retains the entries specified by the predicate. /// /// This method allows the predicate closure to modify the value field. /// /// Entries that have existed since the invocation of the method are guaranteed to be visited /// if they are not removed, however the same entry can be visited more than once if the /// [`HashCache`] gets resized by another thread. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// let future_put = hashcache.put_async(1, 0); /// let future_retain = hashcache.retain_async(|k, v| *k == 1); /// ``` #[inline] pub async fn retain_async bool>(&self, mut pred: F) { self.iter_mut_async(|mut e| { let (k, v) = &mut *e; if !pred(k, v) { drop(e.consume()); } true }) .await; } /// Retains the entries specified by the predicate. /// /// This method allows the predicate closure to modify the value field. /// /// Entries that have existed since the invocation of the method are guaranteed to be visited /// if they are not removed, however the same entry can be visited more than once if the /// [`HashCache`] gets resized by another thread. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert!(hashcache.put_sync(2, 1).is_ok()); /// assert!(hashcache.put_sync(3, 2).is_ok()); /// /// hashcache.retain_sync(|k, v| *k == 1 && *v == 0); /// /// assert!(hashcache.contains_sync(&1)); /// assert!(!hashcache.contains_sync(&2)); /// assert!(!hashcache.contains_sync(&3)); /// ``` #[inline] pub fn retain_sync bool>(&self, mut pred: F) { self.iter_mut_sync(|mut e| { let (k, v) = &mut *e; if !pred(k, v) { drop(e.consume()); } true }); } /// Clears the [`HashCache`] by removing all key-value pairs. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// let future_put = hashcache.put_async(1, 0); /// let future_clear = hashcache.clear_async(); /// ``` #[inline] pub async fn clear_async(&self) { self.retain_async(|_, _| false).await; } /// Clears the [`HashCache`] by removing all key-value pairs. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// hashcache.clear_sync(); /// /// assert!(!hashcache.contains_sync(&1)); /// ``` #[inline] pub fn clear_sync(&self) { self.retain_sync(|_, _| false); } /// Returns the number of entries in the [`HashCache`]. /// /// It reads the entire metadata area of the bucket array to calculate the number of valid /// entries, making its time complexity `O(N)`. Furthermore, it may overcount entries if an old /// bucket array has yet to be dropped. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert_eq!(hashcache.len(), 1); /// ``` #[inline] pub fn len(&self) -> usize { self.num_entries(&Guard::new()) } /// Returns `true` if the [`HashCache`] is empty. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.is_empty()); /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert!(!hashcache.is_empty()); /// ``` #[inline] pub fn is_empty(&self) -> bool { !self.has_entry(&Guard::new()) } /// Returns the capacity of the [`HashCache`]. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache_default: HashCache = HashCache::default(); /// assert_eq!(hashcache_default.capacity(), 0); /// /// let hashcache: HashCache = HashCache::with_capacity(1000, 2000); /// assert_eq!(hashcache.capacity(), 1024); /// ``` #[inline] pub fn capacity(&self) -> usize { self.num_slots(&Guard::new()) } /// Returns the current capacity range of the [`HashCache`]. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert_eq!(hashcache.capacity_range(), 0..=256); /// ``` #[inline] pub fn capacity_range(&self) -> RangeInclusive { self.minimum_capacity.load(Relaxed)..=self.maximum_capacity() } } impl HashCache { /// Creates an empty default [`HashCache`]. /// /// The maximum capacity is set to [`DEFAULT_MAXIMUM_CAPACITY`]. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::new(); /// /// let result = hashcache.capacity(); /// assert_eq!(result, 0); /// ``` #[inline] #[must_use] pub fn new() -> Self { Self::default() } /// Creates an empty [`HashCache`] with the specified capacity. /// /// The actual capacity is equal to or greater than `minimum_capacity` unless it is greater than /// `1 << (usize::BITS - 1)`. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::with_capacity(1000, 2000); /// /// let result = hashcache.capacity(); /// assert_eq!(result, 1024); /// /// let hashcache: HashCache = HashCache::with_capacity(0, 0); /// let result = hashcache.capacity_range(); /// assert_eq!(result, 0..=64); /// ``` #[inline] #[must_use] pub fn with_capacity(minimum_capacity: usize, maximum_capacity: usize) -> Self { Self::with_capacity_and_hasher(minimum_capacity, maximum_capacity, RandomState::new()) } } impl Default for HashCache where H: BuildHasher + Default, { /// Creates an empty default [`HashCache`]. /// /// The maximum capacity is set to [`DEFAULT_MAXIMUM_CAPACITY`]. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// let result = hashcache.capacity(); /// assert_eq!(result, 0); /// ``` #[inline] fn default() -> Self { Self::with_hasher(H::default()) } } impl Debug for HashCache where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { /// Iterates over all the entries in the [`HashCache`] to print them. /// /// ## Locking behavior /// /// Shared locks on buckets are acquired during iteration, therefore any [`Entry`], /// [`OccupiedEntry`], or [`VacantEntry`] owned by the current thread will lead to a deadlock. #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut d = f.debug_map(); self.iter_sync(|k, v| { d.entry(k, v); true }); d.finish() } } impl Drop for HashCache where H: BuildHasher, { #[inline] fn drop(&mut self) { self.bucket_array .swap((None, Tag::None), Relaxed) .0 .map(|a| unsafe { // The entire array does not need to wait for an epoch change as no references will // remain outside the lifetime of the `HashCache`. a.drop_in_place() }); } } impl FromIterator<(K, V)> for HashCache where K: Eq + Hash, H: BuildHasher + Default, { #[inline] fn from_iter>(iter: T) -> Self { let into_iter = iter.into_iter(); let size_hint = into_iter.size_hint(); let hashcache = Self::with_capacity_and_hasher( size_hint.0, Self::capacity_from_size_hint(size_hint), H::default(), ); into_iter.for_each(|e| { let _result = hashcache.put_sync(e.0, e.1); }); hashcache } } impl HashTable for HashCache where K: Eq + Hash, H: BuildHasher, { #[inline] fn hasher(&self) -> &H { &self.build_hasher } #[inline] fn bucket_array_var(&self) -> &AtomicShared> { &self.bucket_array } #[inline] fn minimum_capacity_var(&self) -> &AtomicUsize { &self.minimum_capacity } #[inline] fn maximum_capacity(&self) -> usize { self.maximum_capacity } } impl PartialEq for HashCache where K: Eq + Hash, V: PartialEq, H: BuildHasher, { /// Compares two [`HashCache`] instances. /// /// ## Locking behavior /// /// Shared locks on buckets are acquired when comparing two instances of [`HashCache`], therefore /// this may lead to a deadlock if the instances are being modified by another thread. #[inline] fn eq(&self, other: &Self) -> bool { if self.iter_sync(|k, v| other.read_sync(k, |_, ov| v == ov) == Some(true)) { return other.iter_sync(|k, v| self.read_sync(k, |_, sv| v == sv) == Some(true)); } false } } impl<'h, K, V, H> Entry<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Ensures a value is in the entry by putting the supplied instance if empty. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(3).or_put(7); /// assert_eq!(*hashcache.get_sync(&3).unwrap().get(), 7); /// ``` #[inline] pub fn or_put(self, val: V) -> (EvictedEntry, OccupiedEntry<'h, K, V, H>) { self.or_put_with(|| val) } /// Ensures a value is in the entry by putting the result of the supplied closure if empty. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(19).or_put_with(|| 5); /// assert_eq!(*hashcache.get_sync(&19).unwrap().get(), 5); /// ``` #[inline] pub fn or_put_with V>( self, constructor: F, ) -> (EvictedEntry, OccupiedEntry<'h, K, V, H>) { self.or_put_with_key(|_| constructor()) } /// Ensures a value is in the entry by putting the result of the supplied closure if empty. /// /// The reference to the moved key is provided, therefore cloning or copying the key is /// unnecessary. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(11).or_put_with_key(|k| if *k == 11 { 7 } else { 3 }); /// assert_eq!(*hashcache.get_sync(&11).unwrap().get(), 7); /// ``` #[inline] pub fn or_put_with_key V>( self, constructor: F, ) -> (EvictedEntry, OccupiedEntry<'h, K, V, H>) { match self { Self::Occupied(o) => (None, o), Self::Vacant(v) => { let val = constructor(v.key()); v.put_entry(val) } } } /// Returns a reference to the key of this entry. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// assert_eq!(hashcache.entry_sync(31).key(), &31); /// ``` #[inline] pub fn key(&self) -> &K { match self { Self::Occupied(o) => o.key(), Self::Vacant(v) => v.key(), } } /// Provides in-place mutable access to an occupied entry. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(37).and_modify(|v| { *v += 1 }).or_put(47); /// assert_eq!(*hashcache.get_sync(&37).unwrap().get(), 47); /// /// hashcache.entry_sync(37).and_modify(|v| { *v += 1 }).or_put(3); /// assert_eq!(*hashcache.get_sync(&37).unwrap().get(), 48); /// ``` #[inline] #[must_use] pub fn and_modify(self, f: F) -> Self where F: FnOnce(&mut V), { match self { Self::Occupied(mut o) => { f(o.get_mut()); Self::Occupied(o) } Self::Vacant(_) => self, } } /// Sets the value of the entry. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// let entry = hashcache.entry_sync(11).put_entry(17).1; /// assert_eq!(entry.key(), &11); /// ``` #[inline] pub fn put_entry(self, val: V) -> (EvictedEntry, OccupiedEntry<'h, K, V, H>) { match self { Self::Occupied(mut o) => { o.put(val); (None, o) } Self::Vacant(v) => v.put_entry(val), } } } impl<'h, K, V, H> Entry<'h, K, V, H> where K: Eq + Hash, V: Default, H: BuildHasher, { /// Ensures a value is in the entry by putting the default value if empty. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// hashcache.entry_sync(11).or_default(); /// assert_eq!(*hashcache.get_sync(&11).unwrap().get(), 0); /// ``` #[inline] pub fn or_default(self) -> (EvictedEntry, OccupiedEntry<'h, K, V, H>) { match self { Self::Occupied(o) => (None, o), Self::Vacant(v) => v.put_entry(Default::default()), } } } impl Debug for Entry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Vacant(v) => f.debug_tuple("Entry").field(v).finish(), Self::Occupied(o) => f.debug_tuple("Entry").field(o).finish(), } } } impl OccupiedEntry<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Gets a reference to the key in the entry. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert_eq!(hashcache.entry_sync(29).or_default().1.key(), &29); /// ``` #[inline] #[must_use] pub fn key(&self) -> &K { &self.locked_bucket.entry(&self.entry_ptr).0 } /// Takes ownership of the key and value from the [`HashCache`]. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use scc::hash_cache::Entry; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(11).or_put(17); /// /// if let Entry::Occupied(o) = hashcache.entry_sync(11) { /// assert_eq!(o.remove_entry(), (11, 17)); /// }; /// ``` #[inline] #[must_use] pub fn remove_entry(mut self) -> (K, V) { self.locked_bucket .remove(self.hashcache, &mut self.entry_ptr) } /// Gets a reference to the value in the entry. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use scc::hash_cache::Entry; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(19).or_put(11); /// /// if let Entry::Occupied(o) = hashcache.entry_sync(19) { /// assert_eq!(o.get(), &11); /// }; /// ``` #[inline] #[must_use] pub fn get(&self) -> &V { &self.locked_bucket.entry(&self.entry_ptr).1 } /// Gets a mutable reference to the value in the entry. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use scc::hash_cache::Entry; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(37).or_put(11); /// /// if let Entry::Occupied(mut o) = hashcache.entry_sync(37) { /// *o.get_mut() += 18; /// assert_eq!(*o.get(), 29); /// } /// /// assert_eq!(*hashcache.get_sync(&37).unwrap().get(), 29); /// ``` #[inline] pub fn get_mut(&mut self) -> &mut V { &mut self.locked_bucket.entry_mut(&mut self.entry_ptr).1 } /// Sets the value of the entry, and returns the old value. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use scc::hash_cache::Entry; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(37).or_put(11); /// /// if let Entry::Occupied(mut o) = hashcache.entry_sync(37) { /// assert_eq!(o.put(17), 11); /// } /// /// assert_eq!(*hashcache.get_sync(&37).unwrap().get(), 17); /// ``` #[inline] pub fn put(&mut self, val: V) -> V { replace(self.get_mut(), val) } /// Takes the value out of the entry, and returns it. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use scc::hash_cache::Entry; /// /// let hashcache: HashCache = HashCache::default(); /// /// hashcache.entry_sync(11).or_put(17); /// /// if let Entry::Occupied(o) = hashcache.entry_sync(11) { /// assert_eq!(o.remove(), 17); /// }; /// ``` #[inline] #[must_use] pub fn remove(self) -> V { self.remove_entry().1 } } impl Debug for OccupiedEntry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("OccupiedEntry") .field("key", self.key()) .field("value", self.get()) .finish_non_exhaustive() } } impl Deref for OccupiedEntry<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { type Target = V; #[inline] fn deref(&self) -> &Self::Target { self.get() } } impl DerefMut for OccupiedEntry<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { self.get_mut() } } impl<'h, K, V, H> VacantEntry<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Gets a reference to the key. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// assert_eq!(hashcache.entry_sync(11).key(), &11); /// ``` #[inline] pub fn key(&self) -> &K { &self.key } /// Takes ownership of the key. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use scc::hash_cache::Entry; /// /// let hashcache: HashCache = HashCache::default(); /// /// if let Entry::Vacant(v) = hashcache.entry_sync(17) { /// assert_eq!(v.into_key(), 17); /// }; /// ``` #[inline] pub fn into_key(self) -> K { self.key } /// Sets the value of the entry with its key and returns an [`OccupiedEntry`]. /// /// Returns a key-value pair if an entry was evicted for the new key-value pair. /// /// # Examples /// /// ``` /// use scc::HashCache; /// use scc::hash_cache::Entry; /// /// let hashcache: HashCache = HashCache::default(); /// /// if let Entry::Vacant(o) = hashcache.entry_sync(19) { /// o.put_entry(29); /// } /// /// assert_eq!(*hashcache.get_sync(&19).unwrap().get(), 29); /// ``` #[inline] pub fn put_entry(self, val: V) -> (EvictedEntry, OccupiedEntry<'h, K, V, H>) { let evicted = self .locked_bucket .writer .evict_lru_head(self.locked_bucket.data_block); let entry_ptr = self.locked_bucket.insert(self.hash, (self.key, val)); self.locked_bucket.writer.update_lru_tail(&entry_ptr); let occupied = OccupiedEntry { hashcache: self.hashcache, locked_bucket: self.locked_bucket, entry_ptr, }; (evicted, occupied) } } impl Debug for VacantEntry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("VacantEntry").field(self.key()).finish() } } impl ConsumableEntry<'_, K, V> { /// Consumes the entry by moving out the key and value. /// /// # Examples /// /// ``` /// use scc::HashCache; /// /// let hashcache: HashCache = HashCache::default(); /// /// assert!(hashcache.put_sync(1, 0).is_ok()); /// assert!(hashcache.put_sync(2, 1).is_ok()); /// assert!(hashcache.put_sync(3, 2).is_ok()); /// /// let mut consumed = None; /// /// hashcache.iter_mut_sync(|entry| { /// if entry.0 == 1 { /// consumed.replace(entry.consume().1); /// } /// true /// }); /// /// assert!(!hashcache.contains_sync(&1)); /// assert_eq!(consumed, Some(0)); /// ``` #[inline] #[must_use] pub fn consume(self) -> (K, V) { *self.remove_probe |= true; self.locked_bucket .writer .remove(self.locked_bucket.data_block, self.entry_ptr) } } impl Deref for ConsumableEntry<'_, K, V> { type Target = (K, V); #[inline] fn deref(&self) -> &Self::Target { self.locked_bucket.entry(self.entry_ptr) } } impl DerefMut for ConsumableEntry<'_, K, V> { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { self.locked_bucket.entry_mut(self.entry_ptr) } } scc-3.4.8/src/hash_index.rs000064400000000000000000002003511046102023000136600ustar 00000000000000//! [`HashIndex`] is a read-optimized concurrent hash map. use std::collections::hash_map::RandomState; use std::fmt::{self, Debug}; use std::hash::{BuildHasher, Hash}; use std::iter::FusedIterator; use std::ops::{Deref, RangeInclusive}; use std::panic::UnwindSafe; use std::ptr; use std::sync::atomic::AtomicU8; #[cfg(not(feature = "loom"))] use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; #[cfg(feature = "loom")] use loom::sync::atomic::AtomicUsize; use sdd::{AtomicShared, Epoch, Guard, Shared, Tag}; use super::Equivalent; use super::hash_table::bucket::{Bucket, EntryPtr, INDEX}; use super::hash_table::bucket_array::BucketArray; use super::hash_table::{HashTable, LockedBucket}; /// Scalable concurrent hash index. /// /// [`HashIndex`] is a concurrent hash map data structure optimized for parallel read operations. /// The key characteristics of [`HashIndex`] are similar to that of [`HashMap`](super::HashMap) /// except its read operations are lock-free. /// /// ## The key differences between [`HashIndex`] and [`HashMap`](crate::HashMap). /// /// * Lock-free read: read and scan operations are never blocked and do not modify shared data. /// * Immutability: the data in the container is immutable until it becomes unreachable. /// * Linearizability: linearizability of read operations relies on the CPU architecture. /// /// ## The key statistics for [`HashIndex`] /// /// * The expected size of metadata for a single key-value pair: 2 bytes. /// * The expected number of atomic write operations required for an operation on a single key: 2. /// * The expected number of atomic variables accessed during a single key operation: 2. /// * The number of entries managed by a single bucket without a linked list: 32. /// * The expected maximum linked list length when resize is triggered: log(capacity) / 8. /// /// ## Unwind safety /// /// [`HashIndex`] is impervious to out-of-memory errors and panics in user-specified code under one /// condition: `H::Hasher::hash`, `K::drop` and `V::drop` must not panic. pub struct HashIndex where H: BuildHasher, { bucket_array: AtomicShared>, build_hasher: H, garbage_chain: AtomicShared>, garbage_epoch: AtomicU8, minimum_capacity: AtomicUsize, } /// [`Entry`] represents a single entry in a [`HashIndex`]. pub enum Entry<'h, K, V, H = RandomState> where H: BuildHasher, { /// An occupied entry. Occupied(OccupiedEntry<'h, K, V, H>), /// A vacant entry. Vacant(VacantEntry<'h, K, V, H>), } /// [`OccupiedEntry`] is a view into an occupied entry in a [`HashIndex`]. pub struct OccupiedEntry<'h, K, V, H = RandomState> where H: BuildHasher, { hashindex: &'h HashIndex, locked_bucket: LockedBucket, entry_ptr: EntryPtr, } /// [`VacantEntry`] is a view into a vacant entry in a [`HashIndex`]. pub struct VacantEntry<'h, K, V, H = RandomState> where H: BuildHasher, { hashindex: &'h HashIndex, key: K, hash: u64, locked_bucket: LockedBucket, } /// [`Reserve`] keeps the capacity of the associated [`HashIndex`] higher than a certain level. /// /// The [`HashIndex`] does not shrink the capacity below the reserved capacity. pub struct Reserve<'h, K, V, H = RandomState> where K: Eq + Hash, H: BuildHasher, { hashindex: &'h HashIndex, additional: usize, } /// An iterator over the entries of a [`HashIndex`]. /// /// An [`Iter`] iterates over all the entries that survive the [`Iter`]. pub struct Iter<'h, K, V, H = RandomState> where H: BuildHasher, { hashindex: &'h HashIndex, bucket_array: Option<&'h BucketArray>, index: usize, bucket: Option<&'h Bucket>, entry_ptr: EntryPtr, guard: &'h Guard, } impl HashIndex where H: BuildHasher, { /// Creates an empty [`HashIndex`] with the given [`BuildHasher`]. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use std::collections::hash_map::RandomState; /// /// let hashindex: HashIndex = /// HashIndex::with_hasher(RandomState::new()); /// ``` #[cfg(not(feature = "loom"))] #[inline] pub const fn with_hasher(build_hasher: H) -> Self { Self { bucket_array: AtomicShared::null(), build_hasher, garbage_chain: AtomicShared::null(), garbage_epoch: AtomicU8::new(u8::MAX), minimum_capacity: AtomicUsize::new(0), } } /// Creates an empty [`HashIndex`] with the given [`BuildHasher`]. #[cfg(feature = "loom")] #[inline] pub fn with_hasher(build_hasher: H) -> Self { Self { bucket_array: AtomicShared::null(), build_hasher, garbage_chain: AtomicShared::null(), garbage_epoch: AtomicU8::new(u8::MAX), minimum_capacity: AtomicUsize::new(0), } } /// Creates an empty [`HashIndex`] with the specified capacity and [`BuildHasher`]. /// /// The actual capacity is equal to or greater than `capacity` unless it is greater than /// `1 << (usize::BITS - 1)`. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use std::collections::hash_map::RandomState; /// /// let hashindex: HashIndex = /// HashIndex::with_capacity_and_hasher(1000, RandomState::new()); /// /// let result = hashindex.capacity(); /// assert_eq!(result, 1024); /// ``` #[inline] pub fn with_capacity_and_hasher(capacity: usize, build_hasher: H) -> Self { let (array, minimum_capacity) = if capacity == 0 { (AtomicShared::null(), AtomicUsize::new(0)) } else { let array = unsafe { Shared::new_unchecked(BucketArray::::new( capacity, AtomicShared::null(), )) }; let minimum_capacity = array.num_slots(); ( AtomicShared::from(array), AtomicUsize::new(minimum_capacity), ) }; Self { bucket_array: array, build_hasher, garbage_chain: AtomicShared::null(), garbage_epoch: AtomicU8::new(u8::MAX), minimum_capacity, } } } impl HashIndex where K: Eq + Hash, H: BuildHasher, { /// Temporarily increases the minimum capacity of the [`HashIndex`]. /// /// A [`Reserve`] is returned if the [`HashIndex`] could increase the minimum capacity while /// the increased capacity is not exclusively owned by the returned [`Reserve`], allowing /// others to benefit from it. The memory for the additional space may not be immediately /// allocated if the [`HashIndex`] is empty or currently being resized, however once the memory /// is reserved eventually, the capacity will not shrink below the additional capacity until /// the returned [`Reserve`] is dropped. /// /// # Errors /// /// Returns `None` if a too large number is given. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::with_capacity(1000); /// assert_eq!(hashindex.capacity(), 1024); /// /// let reserved = hashindex.reserve(10000); /// assert!(reserved.is_some()); /// assert_eq!(hashindex.capacity(), 16384); /// /// assert!(hashindex.reserve(usize::MAX).is_none()); /// assert_eq!(hashindex.capacity(), 16384); /// /// for i in 0..16 { /// assert!(hashindex.insert_sync(i, i).is_ok()); /// } /// drop(reserved); /// /// assert_eq!(hashindex.capacity(), 1024); /// ``` #[inline] pub fn reserve(&self, additional_capacity: usize) -> Option> { let additional = self.reserve_capacity(additional_capacity); if additional == 0 { None } else { Some(Reserve { hashindex: self, additional, }) } } /// Gets the entry associated with the given key in the map for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// let future_entry = hashindex.entry_async('b'); /// ``` #[inline] pub async fn entry_async(&self, key: K) -> Entry<'_, K, V, H> { self.reclaim_memory(); let hash = self.hash(&key); let locked_bucket = self.writer_async(hash).await; let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Entry::Occupied(OccupiedEntry { hashindex: self, locked_bucket, entry_ptr, }) } else { let vacant_entry = VacantEntry { hashindex: self, key, hash, locked_bucket, }; Entry::Vacant(vacant_entry) } } /// Gets the entry associated with the given key in the map for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// for ch in "a short treatise on fungi".chars() { /// unsafe { /// hashindex.entry_sync(ch).and_modify(|counter| *counter += 1).or_insert(1); /// } /// } /// /// assert_eq!(hashindex.peek_with(&'s', |_, v| *v), Some(2)); /// assert_eq!(hashindex.peek_with(&'t', |_, v| *v), Some(3)); /// assert!(hashindex.peek_with(&'y', |_, v| *v).is_none()); /// ``` #[inline] pub fn entry_sync(&self, key: K) -> Entry<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_sync(hash); let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Entry::Occupied(OccupiedEntry { hashindex: self, locked_bucket, entry_ptr, }) } else { let vacant_entry = VacantEntry { hashindex: self, key, hash, locked_bucket, }; Entry::Vacant(vacant_entry) } } /// Tries to get the entry associated with the given key in the map for in-place manipulation. /// /// Returns `None` if the entry could not be locked. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(0, 1).is_ok()); /// assert!(hashindex.try_entry(0).is_some()); /// ``` #[inline] pub fn try_entry(&self, key: K) -> Option> { self.reclaim_memory(); let hash = self.hash(&key); let guard = Guard::new(); let locked_bucket = self.try_reserve_bucket(hash, &guard)?; let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Some(Entry::Occupied(OccupiedEntry { hashindex: self, locked_bucket, entry_ptr, })) } else { Some(Entry::Vacant(VacantEntry { hashindex: self, key, hash, locked_bucket, })) } } /// Begins iterating over entries by getting the first occupied entry. /// /// The returned [`OccupiedEntry`] in combination with [`OccupiedEntry::next_async`] can act as /// a mutable iterator over entries. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// let future_entry = hashindex.begin_async(); /// ``` #[inline] pub async fn begin_async(&self) -> Option> { self.any_async(|_, _| true).await } /// Begins iterating over entries by getting the first occupied entry. /// /// The returned [`OccupiedEntry`] in combination with [`OccupiedEntry::next_sync`] can act as a /// mutable iterator over entries. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// /// let mut first_entry = hashindex.begin_sync().unwrap(); /// unsafe { /// *first_entry.get_mut() = 2; /// } /// /// assert!(first_entry.next_sync().is_none()); /// assert_eq!(hashindex.peek_with(&1, |_, v| *v), Some(2)); /// ``` #[inline] pub fn begin_sync(&self) -> Option> { self.any_sync(|_, _| true) } /// Finds any entry satisfying the supplied predicate for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// let future_entry = hashindex.any_async(|k, _| *k == 2); /// ``` #[inline] pub async fn any_async bool>( &self, mut pred: P, ) -> Option> { self.reclaim_memory(); let mut entry = None; self.for_each_writer_async(0, 0, |locked_bucket, _| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry(&entry_ptr); if pred(k, v) { entry = Some(OccupiedEntry { hashindex: self, locked_bucket, entry_ptr, }); return true; } } false }) .await; entry } /// Finds any entry satisfying the supplied predicate for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.insert_sync(2, 3).is_ok()); /// /// let mut entry = hashindex.any_sync(|k, _| *k == 2).unwrap(); /// assert_eq!(*entry.get(), 3); /// ``` #[inline] pub fn any_sync bool>( &self, mut pred: P, ) -> Option> { self.reclaim_memory(); let mut entry = None; let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |locked_bucket, _| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry(&entry_ptr); if pred(k, v) { entry = Some(OccupiedEntry { hashindex: self, locked_bucket, entry_ptr, }); return true; } } false }); entry } /// Inserts a key-value pair into the [`HashIndex`]. /// /// # Note /// /// If the key exists, the value is *not* updated. /// /// # Errors /// /// Returns an error containing the supplied key-value pair if the key exists. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// let future_insert = hashindex.insert_async(11, 17); /// ``` #[inline] pub async fn insert_async(&self, key: K, val: V) -> Result<(), (K, V)> { self.reclaim_memory(); let hash = self.hash(&key); let locked_bucket = self.writer_async(hash).await; if locked_bucket.search(&key, hash).is_valid() { Err((key, val)) } else { locked_bucket.insert(hash, (key, val)); Ok(()) } } /// Inserts a key-value pair into the [`HashIndex`]. /// /// # Note /// /// If the key exists, the value is *not* updated. /// /// # Errors /// /// Returns an error along with the supplied key-value pair if the key exists. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert_eq!(hashindex.insert_sync(1, 1).unwrap_err(), (1, 1)); /// ``` #[inline] pub fn insert_sync(&self, key: K, val: V) -> Result<(), (K, V)> { self.reclaim_memory(); let hash = self.hash(&key); let locked_bucket = self.writer_sync(hash); if locked_bucket.search(&key, hash).is_valid() { Err((key, val)) } else { locked_bucket.insert(hash, (key, val)); Ok(()) } } /// Removes a key-value pair if the key exists. /// /// Returns `false` if the key does not exist. /// /// Returns `true` if the key existed and the condition was met after marking the entry /// unreachable; the memory will be reclaimed later. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// let future_insert = hashindex.insert_async(11, 17); /// let future_remove = hashindex.remove_async(&11); /// ``` #[inline] pub async fn remove_async(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.remove_if_async(key, |_| true).await } /// Removes a key-value pair if the key exists. /// /// Returns `false` if the key does not exist. /// /// Returns `true` if the key existed and the condition was met after marking the entry /// unreachable; the memory will be reclaimed later. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(!hashindex.remove_sync(&1)); /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.remove_sync(&1)); /// ``` #[inline] pub fn remove_sync(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.remove_if_sync(key, |_| true) } /// Removes a key-value pair if the key exists and the given condition is met. /// /// Returns `false` if the key does not exist or the condition was not met. /// /// Returns `true` if the key existed and the condition was met after marking the entry /// unreachable; the memory will be reclaimed later. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// let future_insert = hashindex.insert_async(11, 17); /// let future_remove = hashindex.remove_if_async(&11, |_| true); /// ``` #[inline] pub async fn remove_if_async bool>(&self, key: &Q, condition: F) -> bool where Q: Equivalent + Hash + ?Sized, { self.reclaim_memory(); let hash = self.hash(key); let Some(mut locked_bucket) = self.optional_writer_async(hash).await else { return false; }; let mut entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() && condition(&mut locked_bucket.entry_mut(&mut entry_ptr).1) { locked_bucket.mark_removed(self, &mut entry_ptr); true } else { false } } /// Removes a key-value pair if the key exists and the given condition is met. /// /// Returns `false` if the key does not exist or the condition was not met. /// /// Returns `true` if the key existed and the condition was met after marking the entry /// unreachable; the memory will be reclaimed later. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(!hashindex.remove_if_sync(&1, |v| *v == 1)); /// assert!(hashindex.remove_if_sync(&1, |v| *v == 0)); /// ``` #[inline] pub fn remove_if_sync bool>(&self, key: &Q, condition: F) -> bool where Q: Equivalent + Hash + ?Sized, { self.reclaim_memory(); let hash = self.hash(key); let Some(mut locked_bucket) = self.optional_writer_sync(hash) else { return false; }; let mut entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() && condition(&mut locked_bucket.entry_mut(&mut entry_ptr).1) { locked_bucket.mark_removed(self, &mut entry_ptr); true } else { false } } /// Gets an [`OccupiedEntry`] corresponding to the key for in-place modification. /// /// [`OccupiedEntry`] exclusively owns the entry, preventing others from gaining access to it: /// use [`peek`](Self::peek) if read-only access is sufficient. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// let future_insert = hashindex.insert_async(11, 17); /// let future_get = hashindex.get_async(&11); /// ``` #[inline] pub async fn get_async(&self, key: &Q) -> Option> where Q: Equivalent + Hash + ?Sized, { self.reclaim_memory(); let hash = self.hash(key); let locked_bucket = self.optional_writer_async(hash).await?; let entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() { return Some(OccupiedEntry { hashindex: self, locked_bucket, entry_ptr, }); } None } /// Gets an [`OccupiedEntry`] corresponding to the key for in-place modification. /// /// [`OccupiedEntry`] exclusively owns the entry, preventing others from gaining access to it: /// use [`peek`](Self::peek) if read-only access is sufficient. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.get_sync(&1).is_none()); /// assert!(hashindex.insert_sync(1, 10).is_ok()); /// assert_eq!(*hashindex.get_sync(&1).unwrap().get(), 10); /// assert_eq!(*hashindex.get_sync(&1).unwrap(), 10); /// ``` #[inline] pub fn get_sync(&self, key: &Q) -> Option> where Q: Equivalent + Hash + ?Sized, { self.reclaim_memory(); let hash = self.hash(key); let locked_bucket = self.optional_writer_sync(hash)?; let entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() { return Some(OccupiedEntry { hashindex: self, locked_bucket, entry_ptr, }); } None } /// Returns a guarded reference to the value for the specified key without acquiring locks. /// /// Returns `None` if the key does not exist. /// /// # Note /// /// The closure may see an old snapshot of the value if the entry has recently been relocated /// due to resizing. This means that the effects of interior mutability, e.g., `Mutex` or /// `UnsafeCell`, may not be observable in the closure. /// /// # Safety /// /// This method is safe to use if the value is not modified or if `V` is a pointer type such as /// `Box` or `Arc`. However, it is unsafe if `V` contains a pointer that can be modified /// through interior mutability. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// use sdd::Guard; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 10).is_ok()); /// /// let guard = Guard::new(); /// let value_ref = hashindex.peek(&1, &guard).unwrap(); /// assert_eq!(*value_ref, 10); /// ``` #[inline] pub fn peek<'h, Q>(&'h self, key: &Q, guard: &'h Guard) -> Option<&'h V> where Q: Equivalent + Hash + ?Sized, { self.reclaim_memory(); self.peek_entry(key, guard).map(|(_, v)| v) } /// Peeks a key-value pair without acquiring locks. /// /// Returns `None` if the key does not exist. /// /// # Note /// /// The closure may see an old snapshot of the value if the entry has recently been relocated /// due to resizing. This means that the effects of interior mutability, e.g., `Mutex` or /// `UnsafeCell`, may not be observable in the closure. /// /// # Safety /// /// This method is safe to use if the value is not modified or if `V` is a pointer type such as /// `Box` or `Arc`. However, it is unsafe if `V` contains a pointer that can be modified /// through interior mutability. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.peek_with(&1, |_, v| *v).is_none()); /// assert!(hashindex.insert_sync(1, 10).is_ok()); /// assert_eq!(hashindex.peek_with(&1, |_, v| *v).unwrap(), 10); /// ``` #[inline] pub fn peek_with R>(&self, key: &Q, reader: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.reclaim_memory(); let guard = Guard::new(); self.peek_entry(key, &guard).map(|(k, v)| reader(k, v)) } /// Returns `true` if the [`HashIndex`] contains a value for the specified key. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(!hashindex.contains(&1)); /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.contains(&1)); /// ``` #[inline] pub fn contains(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.peek_with(key, |_, _| ()).is_some() } /// Iterates over entries asynchronously for reading. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// /// async { /// let result = hashindex.iter_async(|k, v| { /// false /// }).await; /// assert!(!result); /// }; /// ``` #[inline] pub async fn iter_async bool>(&self, mut f: F) -> bool { self.reclaim_memory(); let mut result = true; self.for_each_reader_async(|reader, data_block| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; return false; } } true }) .await; result } /// Iterates over entries synchronously for reading. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.insert_sync(2, 1).is_ok()); /// /// let mut acc = 0_u64; /// let result = hashindex.iter_sync(|k, v| { /// acc += *k; /// acc += *v; /// true /// }); /// /// assert!(result); /// assert_eq!(acc, 4); /// ``` #[inline] pub fn iter_sync bool>(&self, mut f: F) -> bool { self.reclaim_memory(); let mut result = true; let guard = Guard::new(); self.for_each_reader_sync(&guard, |reader, data_block| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; return false; } } true }); result } /// Retains the entries specified by the predicate. /// /// Entries that have existed since the invocation of the method are guaranteed to be visited /// if they are not removed, however the same entry can be visited more than once if the /// [`HashIndex`] gets resized by another thread. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// let future_insert = hashindex.insert_async(1, 0); /// let future_retain = hashindex.retain_async(|k, v| *k == 1); /// ``` #[inline] pub async fn retain_async bool>(&self, mut pred: F) { self.reclaim_memory(); self.for_each_writer_async(0, 0, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry_mut(&mut entry_ptr); if !pred(k, v) { locked_bucket .writer .mark_removed(&mut entry_ptr, &Guard::new()); *removed = true; } } false }) .await; } /// Retains the entries specified by the predicate. /// /// Entries that have existed since the invocation of the method are guaranteed to be visited /// if they are not removed, however the same entry can be visited more than once if the /// [`HashIndex`] gets resized by another thread. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.insert_sync(2, 1).is_ok()); /// assert!(hashindex.insert_sync(3, 2).is_ok()); /// /// hashindex.retain_sync(|k, v| *k == 1 && *v == 0); /// /// assert!(hashindex.contains(&1)); /// assert!(!hashindex.contains(&2)); /// assert!(!hashindex.contains(&3)); /// ``` #[inline] pub fn retain_sync bool>(&self, mut pred: F) { self.reclaim_memory(); let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry_mut(&mut entry_ptr); if !pred(k, v) { locked_bucket.writer.mark_removed(&mut entry_ptr, &guard); *removed = true; } } false }); } /// Clears the [`HashIndex`] by removing all key-value pairs. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// let future_insert = hashindex.insert_async(1, 0); /// let future_retain = hashindex.clear_async(); /// ``` pub async fn clear_async(&self) { self.retain_async(|_, _| false).await; } /// Clears the [`HashIndex`] by removing all key-value pairs. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// hashindex.clear_sync(); /// /// assert!(!hashindex.contains(&1)); /// ``` pub fn clear_sync(&self) { self.retain_sync(|_, _| false); } /// Returns the number of entries in the [`HashIndex`]. /// /// It reads the entire metadata area of the bucket array to calculate the number of valid /// entries, making its time complexity `O(N)`. Furthermore, it may overcount entries if an old /// bucket array has yet to be dropped. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert_eq!(hashindex.len(), 1); /// ``` #[inline] pub fn len(&self) -> usize { self.num_entries(&Guard::new()) } /// Returns `true` if the [`HashIndex`] is empty. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.is_empty()); /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(!hashindex.is_empty()); /// ``` #[inline] pub fn is_empty(&self) -> bool { !self.has_entry(&Guard::new()) } /// Returns the capacity of the [`HashIndex`]. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex_default: HashIndex = HashIndex::default(); /// assert_eq!(hashindex_default.capacity(), 0); /// /// assert!(hashindex_default.insert_sync(1, 0).is_ok()); /// assert_eq!(hashindex_default.capacity(), 64); /// /// let hashindex: HashIndex = HashIndex::with_capacity(1000); /// assert_eq!(hashindex.capacity(), 1024); /// ``` #[inline] pub fn capacity(&self) -> usize { self.num_slots(&Guard::new()) } /// Returns the current capacity range of the [`HashIndex`]. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert_eq!(hashindex.capacity_range(), 0..=(1_usize << (usize::BITS - 2))); /// /// let reserved = hashindex.reserve(1000); /// assert_eq!(hashindex.capacity_range(), 1000..=(1_usize << (usize::BITS - 2))); /// ``` #[inline] pub fn capacity_range(&self) -> RangeInclusive { self.minimum_capacity.load(Relaxed)..=self.maximum_capacity() } /// Returns the index of the bucket that may contain the key. /// /// The method returns the index of the bucket associated with the key. The number of buckets /// can be calculated by dividing the capacity by `32`. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::with_capacity(1024); /// /// let bucket_index = hashindex.bucket_index(&11); /// assert!(bucket_index < hashindex.capacity() / 32); /// ``` #[inline] pub fn bucket_index(&self, key: &Q) -> usize where Q: Equivalent + Hash + ?Sized, { self.calculate_bucket_index(key) } /// Returns an [`Iter`]. /// /// It is guaranteed to go through all the key-value pairs pertaining in the [`HashIndex`] /// at the moment, however the same key-value pair can be visited more than once if the /// [`HashIndex`] is being resized. /// /// It requires the user to supply a reference to a [`Guard`]. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// use sdd::Guard; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// /// let guard = Guard::new(); /// /// let mut iter = hashindex.iter(&guard); /// let entry_ref = iter.next().unwrap(); /// assert_eq!(iter.next(), None); /// /// for iter in hashindex.iter(&guard) { /// assert_eq!(iter, (&1, &0)); /// } /// ``` #[inline] pub fn iter<'h>(&'h self, guard: &'h Guard) -> Iter<'h, K, V, H> { Iter { hashindex: self, bucket_array: None, index: 0, bucket: None, entry_ptr: EntryPtr::null(), guard, } } /// Reclaims memory by dropping all garbage bucket arrays if they are unreachable. #[inline] fn reclaim_memory(&self) { if !self.garbage_chain.is_null(Acquire) { self.dealloc_garbage(); } } /// Deallocates garbage bucket arrays if they are unreachable. fn dealloc_garbage(&self) { let guard = Guard::new(); let head_ptr = self.garbage_chain.load(Acquire, &guard); if head_ptr.is_null() { return; } let garbage_epoch = self.garbage_epoch.load(Acquire); if Epoch::try_from(garbage_epoch).is_ok_and(|e| !e.in_same_generation(guard.epoch())) { if let Ok((mut garbage_head, _)) = self.garbage_chain.compare_exchange( head_ptr, (None, Tag::None), Acquire, Relaxed, &guard, ) { while let Some(garbage_bucket_array) = garbage_head { garbage_head = garbage_bucket_array .linked_array_var() .swap((None, Tag::None), Acquire) .0; let dropped = unsafe { garbage_bucket_array.drop_in_place() }; debug_assert!(dropped); } } } else { guard.set_has_garbage(); } } } impl Clone for HashIndex where K: Clone + Eq + Hash, V: Clone, H: BuildHasher + Clone, { #[inline] fn clone(&self) -> Self { let self_clone = Self::with_capacity_and_hasher(self.capacity(), self.hasher().clone()); for (k, v) in self.iter(&Guard::new()) { let _result = self_clone.insert_sync(k.clone(), v.clone()); } self_clone } } impl Debug for HashIndex where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.reclaim_memory(); let guard = Guard::new(); f.debug_map().entries(self.iter(&guard)).finish() } } impl HashIndex where K: Eq + Hash, { /// Creates an empty default [`HashIndex`]. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::new(); /// /// let result = hashindex.capacity(); /// assert_eq!(result, 0); /// ``` #[inline] #[must_use] pub fn new() -> Self { Self::default() } /// Creates an empty [`HashIndex`] with the specified capacity. /// /// The actual capacity is equal to or greater than `capacity` unless it is greater than /// `1 << (usize::BITS - 1)`. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::with_capacity(1000); /// /// let result = hashindex.capacity(); /// assert_eq!(result, 1024); /// ``` #[inline] #[must_use] pub fn with_capacity(capacity: usize) -> Self { Self::with_capacity_and_hasher(capacity, RandomState::new()) } } impl Default for HashIndex where H: BuildHasher + Default, { /// Creates an empty default [`HashIndex`]. /// /// The default capacity is `64`. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// let result = hashindex.capacity(); /// assert_eq!(result, 0); /// ``` #[inline] fn default() -> Self { Self::with_hasher(H::default()) } } impl Drop for HashIndex where H: BuildHasher, { #[inline] fn drop(&mut self) { self.bucket_array .swap((None, Tag::None), Relaxed) .0 .map(|a| unsafe { // The entire array does not need to wait for an epoch change as no references will // remain outside the lifetime of the `HashIndex`. a.drop_in_place() }); let mut garbage_head = self.garbage_chain.swap((None, Tag::None), Acquire).0; while let Some(garbage_bucket_array) = garbage_head { garbage_head = garbage_bucket_array .linked_array_var() .swap((None, Tag::None), Acquire) .0; let dropped = unsafe { garbage_bucket_array.drop_in_place() }; debug_assert!(dropped); } } } impl FromIterator<(K, V)> for HashIndex where K: Eq + Hash, H: BuildHasher + Default, { #[inline] fn from_iter>(iter: T) -> Self { let into_iter = iter.into_iter(); let hashindex = Self::with_capacity_and_hasher( Self::capacity_from_size_hint(into_iter.size_hint()), H::default(), ); into_iter.for_each(|e| { let _result = hashindex.insert_sync(e.0, e.1); }); hashindex } } impl HashTable for HashIndex where K: Eq + Hash, H: BuildHasher, { #[inline] fn hasher(&self) -> &H { &self.build_hasher } #[inline] fn defer_reclaim(&self, bucket_array: Shared>, guard: &Guard) { guard.accelerate(); self.reclaim_memory(); self.garbage_epoch.swap(u8::from(guard.epoch()), Release); let (Some(prev_head), _) = self .garbage_chain .swap((Some(bucket_array.clone()), Tag::None), AcqRel) else { return; }; // The bucket array will be dropped when the epoch enters the next generation. bucket_array .linked_array_var() .swap((Some(prev_head), Tag::None), Release); } #[inline] fn bucket_array_var(&self) -> &AtomicShared> { &self.bucket_array } #[inline] fn minimum_capacity_var(&self) -> &AtomicUsize { &self.minimum_capacity } } impl PartialEq for HashIndex where K: Eq + Hash, V: PartialEq, H: BuildHasher, { #[inline] fn eq(&self, other: &Self) -> bool { self.reclaim_memory(); let guard = Guard::new(); if !self .iter(&guard) .any(|(k, v)| other.peek_with(k, |_, ov| v == ov) != Some(true)) { return !other .iter(&guard) .any(|(k, v)| self.peek_with(k, |_, sv| v == sv) != Some(true)); } false } } impl<'h, K, V, H> Entry<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Ensures a value is in the entry by inserting the supplied instance if empty. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// hashindex.entry_sync(3).or_insert(7); /// assert_eq!(hashindex.peek_with(&3, |_, v| *v), Some(7)); /// ``` #[inline] pub fn or_insert(self, val: V) -> OccupiedEntry<'h, K, V, H> { self.or_insert_with(|| val) } /// Ensures a value is in the entry by inserting the result of the supplied closure if empty. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// hashindex.entry_sync(19).or_insert_with(|| 5); /// assert_eq!(hashindex.peek_with(&19, |_, v| *v), Some(5)); /// ``` #[inline] pub fn or_insert_with V>(self, constructor: F) -> OccupiedEntry<'h, K, V, H> { self.or_insert_with_key(|_| constructor()) } /// Ensures a value is in the entry by inserting the result of the supplied closure if empty. /// /// The reference to the moved key is provided, therefore cloning or copying the key is /// unnecessary. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// hashindex.entry_sync(11).or_insert_with_key(|k| if *k == 11 { 7 } else { 3 }); /// assert_eq!(hashindex.peek_with(&11, |_, v| *v), Some(7)); /// ``` #[inline] pub fn or_insert_with_key V>( self, constructor: F, ) -> OccupiedEntry<'h, K, V, H> { match self { Self::Occupied(o) => o, Self::Vacant(v) => { let val = constructor(v.key()); v.insert_entry(val) } } } /// Returns a reference to the key of this entry. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// assert_eq!(hashindex.entry_sync(31).key(), &31); /// ``` #[inline] pub fn key(&self) -> &K { match self { Self::Occupied(o) => o.key(), Self::Vacant(v) => v.key(), } } /// Provides in-place mutable access to an occupied entry. /// /// # Safety /// /// The caller has to make sure that there are no readers of the entry, e.g., a reader keeping /// a reference to the entry via [`HashIndex::iter`], [`HashIndex::peek`], or /// [`HashIndex::peek_with`], unless an instance of `V` can be safely read when there is a /// single writer, e.g., `V = [u8; 32]`. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// unsafe { /// hashindex.entry_sync(37).and_modify(|v| { *v += 1 }).or_insert(47); /// } /// assert_eq!(hashindex.peek_with(&37, |_, v| *v), Some(47)); /// /// unsafe { /// hashindex.entry_sync(37).and_modify(|v| { *v += 1 }).or_insert(3); /// } /// assert_eq!(hashindex.peek_with(&37, |_, v| *v), Some(48)); /// ``` #[inline] #[must_use] pub unsafe fn and_modify(self, f: F) -> Self where F: FnOnce(&mut V), { unsafe { match self { Self::Occupied(mut o) => { f(o.get_mut()); Self::Occupied(o) } Self::Vacant(_) => self, } } } } impl<'h, K, V, H> Entry<'h, K, V, H> where K: Eq + Hash, V: Default, H: BuildHasher, { /// Ensures a value is in the entry by inserting the default value if empty. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// hashindex.entry_sync(11).or_default(); /// assert_eq!(hashindex.peek_with(&11, |_, v| *v), Some(0)); /// ``` #[inline] pub fn or_default(self) -> OccupiedEntry<'h, K, V, H> { match self { Self::Occupied(o) => o, Self::Vacant(v) => v.insert_entry(Default::default()), } } } impl Debug for Entry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Vacant(v) => f.debug_tuple("Entry").field(v).finish(), Self::Occupied(o) => f.debug_tuple("Entry").field(o).finish(), } } } impl<'h, K, V, H> OccupiedEntry<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Gets a reference to the key in the entry. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert_eq!(hashindex.entry_sync(29).or_default().key(), &29); /// ``` #[inline] #[must_use] pub fn key(&self) -> &K { &self.locked_bucket.entry(&self.entry_ptr).0 } /// Marks that the entry is removed from the [`HashIndex`]. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// hashindex.entry_sync(11).or_insert(17); /// /// if let Entry::Occupied(o) = hashindex.entry_sync(11) { /// o.remove_entry(); /// }; /// assert_eq!(hashindex.peek_with(&11, |_, v| *v), None); /// ``` #[inline] pub fn remove_entry(mut self) { self.locked_bucket .mark_removed(self.hashindex, &mut self.entry_ptr); } /// Gets a reference to the value in the entry. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// hashindex.entry_sync(19).or_insert(11); /// /// if let Entry::Occupied(o) = hashindex.entry_sync(19) { /// assert_eq!(o.get(), &11); /// }; /// ``` #[inline] #[must_use] pub fn get(&self) -> &V { &self.locked_bucket.entry(&self.entry_ptr).1 } /// Gets a mutable reference to the value in the entry. /// /// # Safety /// /// The caller has to make sure that there are no readers of the entry, e.g., a reader keeping /// a reference to the entry via [`HashIndex::iter`], [`HashIndex::peek`], or /// [`HashIndex::peek_with`], unless an instance of `V` can be safely read when there is a /// single writer, e.g., `V = [u8; 32]`. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// hashindex.entry_sync(37).or_insert(11); /// /// if let Entry::Occupied(mut o) = hashindex.entry_sync(37) { /// // Safety: `u32` can be safely read while being modified. /// unsafe { *o.get_mut() += 18; } /// assert_eq!(*o.get(), 29); /// } /// /// assert_eq!(hashindex.peek_with(&37, |_, v| *v), Some(29)); /// ``` #[inline] pub unsafe fn get_mut(&mut self) -> &mut V { &mut self.locked_bucket.entry_mut(&mut self.entry_ptr).1 } /// Gets the next closest occupied entry after removing the entry. /// /// [`HashIndex::begin_async`] and this method together enable the [`OccupiedEntry`] to /// effectively act as a mutable iterator over entries. The method never acquires more than one /// lock even when it searches other buckets for the next closest occupied entry. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.insert_sync(2, 0).is_ok()); /// /// let second_entry_future = hashindex.begin_sync().unwrap().remove_and_async(); /// ``` #[inline] pub async fn remove_and_async(self) -> Option> { let hashindex = self.hashindex; let mut entry_ptr = self.entry_ptr.clone(); let guard = Guard::new(); self.locked_bucket .writer .mark_removed(&mut entry_ptr, &guard); self.locked_bucket.set_has_garbage(&guard); if let Some(locked_bucket) = self .locked_bucket .next_async(hashindex, &mut entry_ptr) .await { return Some(OccupiedEntry { hashindex, locked_bucket, entry_ptr, }); } None } /// Gets the next closest occupied entry after removing the entry. /// /// [`HashIndex::begin_sync`] and this method together enable the [`OccupiedEntry`] to /// effectively act as a mutable iterator over entries. The method never acquires more than one /// lock even when it searches other buckets for the next closest occupied entry. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.insert_sync(2, 0).is_ok()); /// /// let first_entry = hashindex.begin_sync().unwrap(); /// let first_key = *first_entry.key(); /// let second_entry = first_entry.remove_and_sync().unwrap(); /// assert_eq!(hashindex.len(), 1); /// /// let second_key = *second_entry.key(); /// /// assert!(second_entry.remove_and_sync().is_none()); /// assert_eq!(first_key + second_key, 3); /// assert_eq!(hashindex.len(), 0); /// ``` #[inline] #[must_use] pub fn remove_and_sync(self) -> Option { let hashindex = self.hashindex; let mut entry_ptr = self.entry_ptr.clone(); let guard = Guard::new(); self.locked_bucket .writer .mark_removed(&mut entry_ptr, &guard); self.locked_bucket.set_has_garbage(&guard); if let Some(locked_bucket) = self.locked_bucket.next_sync(hashindex, &mut entry_ptr) { return Some(OccupiedEntry { hashindex, locked_bucket, entry_ptr, }); } None } /// Gets the next closest occupied entry. /// /// [`HashIndex::begin_async`] and this method together enable the [`OccupiedEntry`] to /// effectively act as a mutable iterator over entries. The method never acquires more than one /// lock even when it searches other buckets for the next closest occupied entry. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.insert_sync(2, 0).is_ok()); /// /// let second_entry_future = hashindex.begin_sync().unwrap().next_async(); /// ``` #[inline] pub async fn next_async(self) -> Option> { let hashindex = self.hashindex; let mut entry_ptr = self.entry_ptr.clone(); if let Some(locked_bucket) = self .locked_bucket .next_async(hashindex, &mut entry_ptr) .await { return Some(OccupiedEntry { hashindex, locked_bucket, entry_ptr, }); } None } /// Gets the next closest occupied entry. /// /// [`HashIndex::begin_sync`] and this method together enable the [`OccupiedEntry`] to /// effectively act as a mutable iterator over entries. The method never acquires more than one /// lock even when it searches other buckets for the next closest occupied entry. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// assert!(hashindex.insert_sync(1, 0).is_ok()); /// assert!(hashindex.insert_sync(2, 0).is_ok()); /// /// let first_entry = hashindex.begin_sync().unwrap(); /// let first_key = *first_entry.key(); /// let second_entry = first_entry.next_sync().unwrap(); /// let second_key = *second_entry.key(); /// /// assert!(second_entry.next_sync().is_none()); /// assert_eq!(first_key + second_key, 3); /// ``` #[inline] #[must_use] pub fn next_sync(self) -> Option { let hashindex = self.hashindex; let mut entry_ptr = self.entry_ptr.clone(); if let Some(locked_bucket) = self.locked_bucket.next_sync(hashindex, &mut entry_ptr) { return Some(OccupiedEntry { hashindex, locked_bucket, entry_ptr, }); } None } } impl OccupiedEntry<'_, K, V, H> where K: Clone + Eq + Hash, H: BuildHasher, { /// Updates the entry by inserting a new entry and marking the existing entry removed. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// hashindex.entry_sync(37).or_insert(11); /// /// if let Entry::Occupied(mut o) = hashindex.entry_sync(37) { /// o.update(29); /// assert_eq!(o.get(), &29); /// } /// /// assert_eq!(hashindex.peek_with(&37, |_, v| *v), Some(29)); /// ``` #[inline] pub fn update(&mut self, val: V) { let key = self.key().clone(); let partial_hash = self.entry_ptr.partial_hash(&self.locked_bucket.writer); let entry_ptr = self .locked_bucket .insert(u64::from(partial_hash), (key, val)); let guard = Guard::new(); self.locked_bucket .writer .mark_removed(&mut self.entry_ptr, &guard); self.locked_bucket.set_has_garbage(&guard); self.entry_ptr = entry_ptr; } } impl Debug for OccupiedEntry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("OccupiedEntry") .field("key", self.key()) .field("value", self.get()) .finish_non_exhaustive() } } impl Deref for OccupiedEntry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { type Target = V; #[inline] fn deref(&self) -> &Self::Target { self.get() } } impl<'h, K, V, H> VacantEntry<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Gets a reference to the key. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// /// let hashindex: HashIndex = HashIndex::default(); /// assert_eq!(hashindex.entry_sync(11).key(), &11); /// ``` #[inline] pub fn key(&self) -> &K { &self.key } /// Takes ownership of the key. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// if let Entry::Vacant(v) = hashindex.entry_sync(17) { /// assert_eq!(v.into_key(), 17); /// }; /// ``` #[inline] pub fn into_key(self) -> K { self.key } /// Sets the value of the entry with its key, and returns an [`OccupiedEntry`]. /// /// # Examples /// /// ``` /// use scc::HashIndex; /// use scc::hash_index::Entry; /// /// let hashindex: HashIndex = HashIndex::default(); /// /// if let Entry::Vacant(o) = hashindex.entry_sync(19) { /// o.insert_entry(29); /// } /// /// assert_eq!(hashindex.peek_with(&19, |_, v| *v), Some(29)); /// ``` #[inline] pub fn insert_entry(self, val: V) -> OccupiedEntry<'h, K, V, H> { let entry_ptr = self.locked_bucket.insert(self.hash, (self.key, val)); OccupiedEntry { hashindex: self.hashindex, locked_bucket: self.locked_bucket, entry_ptr, } } } impl Debug for VacantEntry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("VacantEntry").field(self.key()).finish() } } impl Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Returns the number of reserved slots. #[inline] #[must_use] pub fn additional_capacity(&self) -> usize { self.additional } } impl AsRef> for Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn as_ref(&self) -> &HashIndex { self.hashindex } } impl Debug for Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("Reserve").field(&self.additional).finish() } } impl Deref for Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { type Target = HashIndex; #[inline] fn deref(&self) -> &Self::Target { self.hashindex } } impl Drop for Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn drop(&mut self) { let result = self .hashindex .minimum_capacity .fetch_sub(self.additional, Relaxed); debug_assert!(result >= self.additional); let guard = Guard::new(); if let Some(current_array) = self.hashindex.bucket_array(&guard) { self.try_shrink(current_array, 0, &guard); } } } impl Debug for Iter<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Iter") .field("current_index", &self.index) .field("current_entry_ptr", &self.entry_ptr) .finish() } } impl<'h, K, V, H> Iterator for Iter<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { type Item = (&'h K, &'h V); #[inline] fn next(&mut self) -> Option { let mut array = if let Some(&array) = self.bucket_array.as_ref() { array } else { // Start scanning. let current_array = self.hashindex.bucket_array(self.guard)?; let array = if let Some(old_array) = current_array.linked_array(self.guard) { old_array } else { current_array }; self.bucket_array.replace(array); self.bucket.replace(array.bucket(0)); array }; // Move to the next entry. loop { if let Some(bucket) = self.bucket.take() { // Move to the next entry in the bucket. if self.entry_ptr.move_to_next(bucket) { let (k, v) = self.entry_ptr.get(array.data_block(self.index)); self.bucket.replace(bucket); return Some((k, v)); } } self.entry_ptr = EntryPtr::null(); if self.index + 1 == array.len() { // Move to a newer bucket array. self.index = 0; let current_array = self.hashindex.bucket_array(self.guard)?; if self .bucket_array .as_ref() .is_some_and(|&a| ptr::eq(a, current_array)) { // Finished scanning. break; } array = if let Some(old_array) = current_array.linked_array(self.guard) { if self .bucket_array .as_ref() .is_some_and(|&a| ptr::eq(a, old_array)) { // Start scanning the current array. array = current_array; self.bucket_array.replace(current_array); self.bucket.replace(current_array.bucket(0)); continue; } old_array } else { current_array }; self.bucket_array.replace(array); self.bucket.replace(array.bucket(0)); } else { self.index += 1; self.bucket.replace(array.bucket(self.index)); } } None } } impl FusedIterator for Iter<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { } impl UnwindSafe for Iter<'_, K, V, H> where K: Eq + Hash + UnwindSafe, V: UnwindSafe, H: BuildHasher + UnwindSafe, { } scc-3.4.8/src/hash_map.rs000064400000000000000000002101301046102023000133220ustar 00000000000000//! [`HashMap`] is a concurrent hash map. use std::collections::hash_map::RandomState; use std::fmt::{self, Debug}; use std::hash::{BuildHasher, Hash}; use std::mem::replace; use std::ops::{Deref, DerefMut, RangeInclusive}; #[cfg(not(feature = "loom"))] use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; #[cfg(feature = "loom")] use loom::sync::atomic::AtomicUsize; use sdd::{AtomicShared, Guard, Shared, Tag}; use super::Equivalent; use super::hash_table::bucket::{EntryPtr, MAP}; use super::hash_table::bucket_array::BucketArray; use super::hash_table::{HashTable, LockedBucket}; /// Scalable concurrent hash map. /// /// [`HashMap`] is a concurrent hash map data structure optimized for highly concurrent workloads. /// [`HashMap`] has a dynamically sized array of buckets where a bucket is a fixed-size hash table /// with linear probing that can be expanded by allocating a linked list of smaller buckets when it /// is full. /// /// ## The key features of [`HashMap`] /// /// * Non-sharded: data is stored in a single array of entry buckets. /// * Non-blocking resizing: resizing does not block other threads or tasks. /// * Automatic resizing: grows or shrinks as needed. /// * Incremental resizing: entries in the old bucket array are incrementally relocated. /// * No busy waiting: no spin locks or hot loops to wait for desired resources. /// * Linearizability: [`HashMap`] manipulation methods are linearizable. /// /// ## The key statistics for [`HashMap`] /// /// * The expected size of metadata for a single entry: 2 bytes. /// * The expected number of atomic write operations required for an operation on a single key: 2. /// * The expected number of atomic variables accessed during a single key operation: 2. /// * The number of entries managed by a single bucket without a linked list: 32. /// * The expected maximum linked list length when a resize is triggered: log(capacity) / 8. /// /// ## Locking behavior /// /// ### Bucket access /// /// Bucket arrays are protected by [`sdd`], thus allowing lock-free access to them. /// /// ### Entry access /// /// Each read/write access to an entry is serialized by the read-write lock in the bucket containing /// the entry. There are no container-level locks, therefore, the larger the [`HashMap`] gets, the /// lower the chance that the bucket-level lock will be contended. /// /// ### Resize /// /// Resizing of the [`HashMap`] is non-blocking and lock-free; resizing does not block any other /// read/write access to the [`HashMap`] or resizing attempts. Resizing is analogous to pushing a /// new bucket array into a lock-free stack. Each entry in the old bucket array will be /// incrementally relocated to the new bucket array upon future access to the [`HashMap`], and the old /// bucket array is dropped when it becomes empty and unreachable. /// /// ### Synchronous methods in an asynchronous code block /// /// It is generally not recommended to use blocking methods, such as [`HashMap::insert_sync`], in an /// asynchronous code block or [`poll`](std::future::Future::poll), since it may lead to deadlocks /// or performance degradation. /// /// ## Unwind safety /// /// [`HashMap`] is impervious to out-of-memory errors and panics in user-specified code under one /// condition: `H::Hasher::hash`, `K::drop` and `V::drop` must not panic. pub struct HashMap where H: BuildHasher, { bucket_array: AtomicShared>, minimum_capacity: AtomicUsize, build_hasher: H, } /// [`Entry`] represents a single entry in a [`HashMap`]. pub enum Entry<'h, K, V, H = RandomState> where H: BuildHasher, { /// An occupied entry. Occupied(OccupiedEntry<'h, K, V, H>), /// A vacant entry. Vacant(VacantEntry<'h, K, V, H>), } /// [`OccupiedEntry`] is a view into an occupied entry in a [`HashMap`]. pub struct OccupiedEntry<'h, K, V, H = RandomState> where H: BuildHasher, { hashmap: &'h HashMap, locked_bucket: LockedBucket, entry_ptr: EntryPtr, } /// [`VacantEntry`] is a view into a vacant entry in a [`HashMap`]. pub struct VacantEntry<'h, K, V, H = RandomState> where H: BuildHasher, { hashmap: &'h HashMap, key: K, hash: u64, locked_bucket: LockedBucket, } /// [`ConsumableEntry`] is a view into an occupied entry in a [`HashMap`] when iterating over /// entries in it. pub struct ConsumableEntry<'b, K, V> { /// Holds an exclusive lock on the entry bucket. locked_bucket: &'b mut LockedBucket, /// Pointer to the entry. entry_ptr: &'b mut EntryPtr, /// Probes removal. remove_probe: &'b mut bool, } /// [`ReplaceResult`] is the result type of the [`HashMap::replace_async`] and /// [`HashMap::replace_sync`] methods. pub enum ReplaceResult<'h, K, V, H = RandomState> where H: BuildHasher, { /// The key was replaced. Replaced(OccupiedEntry<'h, K, V, H>, K), /// The key did not exist in the [`HashMap`]. /// /// An [`OccupiedEntry`] can be created from the [`VacantEntry`]. NotReplaced(VacantEntry<'h, K, V, H>), } /// [`Reserve`] keeps the capacity of the associated [`HashMap`] higher than a certain level. /// /// The [`HashMap`] does not shrink the capacity below the reserved capacity. pub struct Reserve<'h, K, V, H = RandomState> where K: Eq + Hash, H: BuildHasher, { hashmap: &'h HashMap, additional: usize, } impl HashMap where H: BuildHasher, { /// Creates an empty [`HashMap`] with the given [`BuildHasher`]. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use std::collections::hash_map::RandomState; /// /// let hashmap: HashMap = HashMap::with_hasher(RandomState::new()); /// ``` #[cfg(not(feature = "loom"))] #[inline] pub const fn with_hasher(build_hasher: H) -> Self { Self { bucket_array: AtomicShared::null(), minimum_capacity: AtomicUsize::new(0), build_hasher, } } /// Creates an empty [`HashMap`] with the given [`BuildHasher`]. #[cfg(feature = "loom")] #[inline] pub fn with_hasher(build_hasher: H) -> Self { Self { bucket_array: AtomicShared::null(), minimum_capacity: AtomicUsize::new(0), build_hasher, } } /// Creates an empty [`HashMap`] with the specified capacity and [`BuildHasher`]. /// /// The actual capacity is equal to or greater than `capacity` unless it is greater than /// `1 << (usize::BITS - 1)`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use std::collections::hash_map::RandomState; /// /// let hashmap: HashMap = /// HashMap::with_capacity_and_hasher(1000, RandomState::new()); /// /// let result = hashmap.capacity(); /// assert_eq!(result, 1024); /// ``` #[inline] pub fn with_capacity_and_hasher(capacity: usize, build_hasher: H) -> Self { let (array, minimum_capacity) = if capacity == 0 { (AtomicShared::null(), AtomicUsize::new(0)) } else { let array = unsafe { Shared::new_unchecked(BucketArray::::new( capacity, AtomicShared::null(), )) }; let minimum_capacity = array.num_slots(); ( AtomicShared::from(array), AtomicUsize::new(minimum_capacity), ) }; Self { bucket_array: array, minimum_capacity, build_hasher, } } } impl HashMap where K: Eq + Hash, H: BuildHasher, { /// Temporarily increases the minimum capacity of the [`HashMap`] to prevent shrinking. /// /// A [`Reserve`] is returned if the [`HashMap`] can increase the minimum capacity. The /// increased capacity is not exclusively owned by the returned [`Reserve`], allowing others to /// benefit from it. The memory for the additional space may not be immediately allocated if /// the [`HashMap`] is empty or currently being resized; however, once the memory is eventually reserved, /// the capacity will not shrink below the additional capacity until the returned /// [`Reserve`] is dropped. /// /// # Errors /// /// Returns `None` if a too large number is given. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::with_capacity(1000); /// assert_eq!(hashmap.capacity(), 1024); /// /// let reserved = hashmap.reserve(10000); /// assert!(reserved.is_some()); /// assert_eq!(hashmap.capacity(), 16384); /// /// assert!(hashmap.reserve(usize::MAX).is_none()); /// assert_eq!(hashmap.capacity(), 16384); /// /// for i in 0..16 { /// assert!(hashmap.insert_sync(i, i).is_ok()); /// } /// drop(reserved); /// /// assert_eq!(hashmap.capacity(), 1024); /// ``` #[inline] pub fn reserve(&self, additional_capacity: usize) -> Option> { let additional = self.reserve_capacity(additional_capacity); if additional == 0 { None } else { Some(Reserve { hashmap: self, additional, }) } } /// Gets the entry associated with the given key in the map for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// let future_entry = hashmap.entry_async('b'); /// ``` #[inline] pub async fn entry_async(&self, key: K) -> Entry<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_async(hash).await; let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Entry::Occupied(OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, }) } else { let vacant_entry = VacantEntry { hashmap: self, key, hash, locked_bucket, }; Entry::Vacant(vacant_entry) } } /// Gets the entry associated with the given key in the map for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// for ch in "a short treatise on fungi".chars() { /// hashmap.entry_sync(ch).and_modify(|counter| *counter += 1).or_insert(1); /// } /// /// assert_eq!(hashmap.read_sync(&'s', |_, v| *v), Some(2)); /// assert_eq!(hashmap.read_sync(&'t', |_, v| *v), Some(3)); /// assert!(hashmap.read_sync(&'y', |_, v| *v).is_none()); /// ``` #[inline] pub fn entry_sync(&self, key: K) -> Entry<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_sync(hash); let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Entry::Occupied(OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, }) } else { let vacant_entry = VacantEntry { hashmap: self, key, hash, locked_bucket, }; Entry::Vacant(vacant_entry) } } /// Tries to get the entry associated with the given key in the map for in-place manipulation. /// /// Returns `None` if the entry could not be locked. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(0, 1).is_ok()); /// assert!(hashmap.try_entry(0).is_some()); /// ``` #[inline] pub fn try_entry(&self, key: K) -> Option> { let hash = self.hash(&key); let guard = Guard::new(); let locked_bucket = self.try_reserve_bucket(hash, &guard)?; let entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { Some(Entry::Occupied(OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, })) } else { Some(Entry::Vacant(VacantEntry { hashmap: self, key, hash, locked_bucket, })) } } /// Begins iterating over entries by getting the first occupied entry. /// /// The returned [`OccupiedEntry`] in combination with [`OccupiedEntry::next_async`] can act as /// a mutable iterator over entries. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// let future_entry = hashmap.begin_async(); /// ``` #[inline] pub async fn begin_async(&self) -> Option> { self.any_async(|_, _| true).await } /// Begins iterating over entries by getting the first occupied entry. /// /// The returned [`OccupiedEntry`] in combination with [`OccupiedEntry::next_sync`] can act as a /// mutable iterator over entries. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// /// let mut first_entry = hashmap.begin_sync().unwrap(); /// *first_entry.get_mut() = 2; /// /// assert!(first_entry.next_sync().is_none()); /// assert_eq!(hashmap.read_sync(&1, |_, v| *v), Some(2)); /// ``` #[inline] pub fn begin_sync(&self) -> Option> { self.any_sync(|_, _| true) } /// Finds any entry satisfying the supplied predicate for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// let future_entry = hashmap.any_async(|k, _| *k == 2); /// ``` #[inline] pub async fn any_async bool>( &self, mut pred: P, ) -> Option> { let mut entry = None; self.for_each_writer_async(0, 0, |locked_bucket, _| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry(&entry_ptr); if pred(k, v) { entry = Some(OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, }); return true; } } false }) .await; entry } /// Finds any entry satisfying the supplied predicate for in-place manipulation. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 3).is_ok()); /// /// let mut entry = hashmap.any_sync(|k, _| *k == 2).unwrap(); /// assert_eq!(*entry.get(), 3); /// ``` #[inline] pub fn any_sync bool>( &self, mut pred: P, ) -> Option> { let mut entry = None; let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |locked_bucket, _| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let (k, v) = locked_bucket.entry(&entry_ptr); if pred(k, v) { entry = Some(OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, }); return true; } } false }); entry } /// Inserts a key-value pair into the [`HashMap`]. /// /// # Note /// /// If the key exists, the value is *not* updated. [`upsert_async`](Self::upsert_async) /// provides a way to update the value if the key exists. /// /// # Errors /// /// Returns an error containing the supplied key-value pair if the key exists. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// let future_insert = hashmap.insert_async(11, 17); /// ``` #[inline] pub async fn insert_async(&self, key: K, val: V) -> Result<(), (K, V)> { let hash = self.hash(&key); let locked_bucket = self.writer_async(hash).await; if locked_bucket.search(&key, hash).is_valid() { Err((key, val)) } else { locked_bucket.insert(hash, (key, val)); Ok(()) } } /// Inserts a key-value pair into the [`HashMap`]. /// /// # Note /// /// If the key exists, the value is *not* updated. [`upsert_sync`](Self::upsert_sync) /// provides a way to update the value if the key exists. /// /// # Errors /// /// Returns an error containing the supplied key-value pair if the key exists. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert_eq!(hashmap.insert_sync(1, 1).unwrap_err(), (1, 1)); /// ``` #[inline] pub fn insert_sync(&self, key: K, val: V) -> Result<(), (K, V)> { let hash = self.hash(&key); let locked_bucket = self.writer_sync(hash); if locked_bucket.search(&key, hash).is_valid() { Err((key, val)) } else { locked_bucket.insert(hash, (key, val)); Ok(()) } } /// Upserts a key-value pair into the [`HashMap`]. /// /// Returns the old value if the [`HashMap`] has this key present, or returns `None`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// let future_upsert = hashmap.upsert_async(11, 17); /// ``` #[inline] pub async fn upsert_async(&self, key: K, val: V) -> Option { match self.entry_async(key).await { Entry::Occupied(mut o) => Some(replace(o.get_mut(), val)), Entry::Vacant(v) => { v.insert_entry(val); None } } } /// Upserts a key-value pair into the [`HashMap`]. /// /// Returns the old value if the [`HashMap`] has this key present, or returns `None`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.upsert_sync(1, 0).is_none()); /// assert_eq!(hashmap.upsert_sync(1, 1).unwrap(), 0); /// assert_eq!(hashmap.read_sync(&1, |_, v| *v).unwrap(), 1); /// ``` #[inline] pub fn upsert_sync(&self, key: K, val: V) -> Option { match self.entry_sync(key) { Entry::Occupied(mut o) => Some(replace(o.get_mut(), val)), Entry::Vacant(v) => { v.insert_entry(val); None } } } /// Updates an existing key-value pair in-place. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// let future_update = hashmap.update_async(&1, |_, v| { *v = 2; *v }); /// ``` #[inline] pub async fn update_async(&self, key: &Q, updater: U) -> Option where Q: Equivalent + Hash + ?Sized, U: FnOnce(&K, &mut V) -> R, { let hash = self.hash(key); let mut locked_bucket = self.optional_writer_async(hash).await?; let mut entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() { let (k, v) = locked_bucket.entry_mut(&mut entry_ptr); Some(updater(k, v)) } else { None } } /// Updates an existing key-value pair in-place. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.update_sync(&1, |_, _| true).is_none()); /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert_eq!(hashmap.update_sync(&1, |_, v| { *v = 2; *v }).unwrap(), 2); /// assert_eq!(hashmap.read_sync(&1, |_, v| *v).unwrap(), 2); /// ``` #[inline] pub fn update_sync(&self, key: &Q, updater: U) -> Option where Q: Equivalent + Hash + ?Sized, U: FnOnce(&K, &mut V) -> R, { let hash = self.hash(key); let mut locked_bucket = self.optional_writer_sync(hash)?; let mut entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() { let (k, v) = locked_bucket.entry_mut(&mut entry_ptr); Some(updater(k, v)) } else { None } } /// Adds a key to the [`HashMap`], replacing the existing key, if any, that is equal to the /// given one. /// /// # Examples /// /// ``` /// use std::cmp::{Eq, PartialEq}; /// use std::hash::{Hash, Hasher}; /// /// use scc::HashMap; /// use scc::hash_map::ReplaceResult; /// /// #[derive(Debug)] /// struct MaybeEqual(u64, u64); /// /// impl Eq for MaybeEqual {} /// /// impl Hash for MaybeEqual { /// fn hash(&self, state: &mut H) { /// // Do not read `self.1`. /// self.0.hash(state); /// } /// } /// /// impl PartialEq for MaybeEqual { /// fn eq(&self, other: &Self) -> bool { /// // Do not compare `self.1`. /// self.0 == other.0 /// } /// } /// /// let hashmap: HashMap = HashMap::default(); /// /// async { /// let ReplaceResult::NotReplaced(v) = hashmap.replace_async(MaybeEqual(11, 7)).await else { /// unreachable!(); /// }; /// drop(v.insert_entry(17)); /// let ReplaceResult::Replaced(_, k) = hashmap.replace_async(MaybeEqual(11, 11)).await else { /// unreachable!(); /// }; /// assert_eq!(k.1, 7); /// }; /// ``` #[inline] pub async fn replace_async(&self, key: K) -> ReplaceResult<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_async(hash).await; let mut entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { let prev_key = replace( &mut entry_ptr .get_mut(locked_bucket.data_block, &locked_bucket.writer) .0, key, ); ReplaceResult::Replaced( OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, }, prev_key, ) } else { ReplaceResult::NotReplaced(VacantEntry { hashmap: self, key, hash, locked_bucket, }) } } /// Adds a key to the [`HashMap`], replacing the existing key, if any, that is equal to the /// given one. /// /// # Examples /// /// ``` /// use std::cmp::{Eq, PartialEq}; /// use std::hash::{Hash, Hasher}; /// /// use scc::HashMap; /// use scc::hash_map::ReplaceResult; /// /// #[derive(Debug)] /// struct MaybeEqual(u64, u64); /// /// impl Eq for MaybeEqual {} /// /// impl Hash for MaybeEqual { /// fn hash(&self, state: &mut H) { /// // Do not read `self.1`. /// self.0.hash(state); /// } /// } /// /// impl PartialEq for MaybeEqual { /// fn eq(&self, other: &Self) -> bool { /// // Do not compare `self.1`. /// self.0 == other.0 /// } /// } /// /// let hashmap: HashMap = HashMap::default(); /// /// let ReplaceResult::NotReplaced(v) = hashmap.replace_sync(MaybeEqual(11, 7)) else { /// unreachable!(); /// }; /// drop(v.insert_entry(17)); /// let ReplaceResult::Replaced(_, k) = hashmap.replace_sync(MaybeEqual(11, 11)) else { /// unreachable!(); /// }; /// assert_eq!(k.1, 7); /// ``` #[inline] pub fn replace_sync(&self, key: K) -> ReplaceResult<'_, K, V, H> { let hash = self.hash(&key); let locked_bucket = self.writer_sync(hash); let mut entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { let prev_key = replace( &mut entry_ptr .get_mut(locked_bucket.data_block, &locked_bucket.writer) .0, key, ); ReplaceResult::Replaced( OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, }, prev_key, ) } else { ReplaceResult::NotReplaced(VacantEntry { hashmap: self, key, hash, locked_bucket, }) } } /// Removes a key-value pair if the key exists. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// let future_insert = hashmap.insert_async(11, 17); /// let future_remove = hashmap.remove_async(&11); /// ``` #[inline] pub async fn remove_async(&self, key: &Q) -> Option<(K, V)> where Q: Equivalent + Hash + ?Sized, { self.remove_if_async(key, |_| true).await } /// Removes a key-value pair if the key exists. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.remove_sync(&1).is_none()); /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert_eq!(hashmap.remove_sync(&1).unwrap(), (1, 0)); /// ``` #[inline] pub fn remove_sync(&self, key: &Q) -> Option<(K, V)> where Q: Equivalent + Hash + ?Sized, { self.remove_if_sync(key, |_| true) } /// Removes a key-value pair if the key exists and the given condition is met. /// /// Returns `None` if the key does not exist or the condition was not met. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// let future_insert = hashmap.insert_async(11, 17); /// let future_remove = hashmap.remove_if_async(&11, |_| true); /// ``` #[inline] pub async fn remove_if_async bool>( &self, key: &Q, condition: F, ) -> Option<(K, V)> where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let mut locked_bucket = self.optional_writer_async(hash).await?; let mut entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() && condition(&mut locked_bucket.entry_mut(&mut entry_ptr).1) { Some(locked_bucket.remove(self, &mut entry_ptr)) } else { None } } /// Removes a key-value pair if the key exists and the given condition is met. /// /// Returns `None` if the key does not exist or the condition was not met. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.remove_if_sync(&1, |v| { *v += 1; false }).is_none()); /// assert_eq!(hashmap.remove_if_sync(&1, |v| *v == 1).unwrap(), (1, 1)); /// ``` #[inline] pub fn remove_if_sync bool>( &self, key: &Q, condition: F, ) -> Option<(K, V)> where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let mut locked_bucket = self.optional_writer_sync(hash)?; let mut entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() && condition(&mut locked_bucket.entry_mut(&mut entry_ptr).1) { Some(locked_bucket.remove(self, &mut entry_ptr)) } else { None } } /// Gets an [`OccupiedEntry`] corresponding to the key for in-place modification. /// /// [`OccupiedEntry`] exclusively owns the entry, preventing others from gaining access to it: /// use [`read_async`](Self::read_async) if read-only access is sufficient. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// let future_insert = hashmap.insert_async(11, 17); /// let future_get = hashmap.get_async(&11); /// ``` #[inline] pub async fn get_async(&self, key: &Q) -> Option> where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let locked_bucket = self.optional_writer_async(hash).await?; let entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() { return Some(OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, }); } None } /// Gets an [`OccupiedEntry`] corresponding to the key for in-place modification. /// /// [`OccupiedEntry`] exclusively owns the entry, preventing others from gaining access to it: /// use [`read_sync`](Self::read_sync) if read-only access is sufficient. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.get_sync(&1).is_none()); /// assert!(hashmap.insert_sync(1, 10).is_ok()); /// assert_eq!(*hashmap.get_sync(&1).unwrap().get(), 10); /// /// *hashmap.get_sync(&1).unwrap() = 11; /// assert_eq!(*hashmap.get_sync(&1).unwrap(), 11); /// ``` #[inline] pub fn get_sync(&self, key: &Q) -> Option> where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let locked_bucket = self.optional_writer_sync(hash)?; let entry_ptr = locked_bucket.search(key, hash); if entry_ptr.is_valid() { return Some(OccupiedEntry { hashmap: self, locked_bucket, entry_ptr, }); } None } /// Reads a key-value pair. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// let future_insert = hashmap.insert_async(11, 17); /// let future_read = hashmap.read_async(&11, |_, v| *v); /// ``` #[inline] pub async fn read_async R>(&self, key: &Q, reader: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.reader_async(key, reader).await } /// Reads a key-value pair. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.read_sync(&1, |_, v| *v).is_none()); /// assert!(hashmap.insert_sync(1, 10).is_ok()); /// assert_eq!(hashmap.read_sync(&1, |_, v| *v).unwrap(), 10); /// ``` #[inline] pub fn read_sync R>(&self, key: &Q, reader: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.reader_sync(key, reader) } /// Returns `true` if the [`HashMap`] contains a value for the specified key. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// let future_contains = hashmap.contains_async(&1); /// ``` #[inline] pub async fn contains_async(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.reader_async(key, |_, _| ()).await.is_some() } /// Returns `true` if the [`HashMap`] contains a value for the specified key. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(!hashmap.contains_sync(&1)); /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.contains_sync(&1)); /// ``` #[inline] pub fn contains_sync(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.read_sync(key, |_, _| ()).is_some() } /// Iterates over entries asynchronously for reading. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// /// async { /// let result = hashmap.iter_async(|k, v| { /// false /// }).await; /// assert!(!result); /// }; /// ``` #[inline] pub async fn iter_async bool>(&self, mut f: F) -> bool { let mut result = true; self.for_each_reader_async(|reader, data_block| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; return false; } } true }) .await; result } /// Iterates over entries synchronously for reading. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 1).is_ok()); /// /// let mut acc = 0_u64; /// let result = hashmap.iter_sync(|k, v| { /// acc += *k; /// acc += *v; /// true /// }); /// /// assert!(result); /// assert_eq!(acc, 4); /// ``` #[inline] pub fn iter_sync bool>(&self, mut f: F) -> bool { let mut result = true; let guard = Guard::new(); self.for_each_reader_sync(&guard, |reader, data_block| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&reader) { let (k, v) = entry_ptr.get(data_block); if !f(k, v) { result = false; return false; } } true }); result } /// Iterates over entries asynchronously for modification. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 1).is_ok()); /// /// async { /// let result = hashmap.iter_mut_async(|entry| { /// if entry.0 == 1 { /// entry.consume(); /// return false; /// } /// true /// }).await; /// /// assert!(!result); /// assert_eq!(hashmap.len(), 1); /// }; /// ``` #[inline] pub async fn iter_mut_async) -> bool>( &self, mut f: F, ) -> bool { let mut result = true; self.for_each_writer_async(0, 0, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let consumable_entry = ConsumableEntry { locked_bucket: &mut locked_bucket, entry_ptr: &mut entry_ptr, remove_probe: removed, }; if !f(consumable_entry) { result = false; return true; } } false }) .await; result } /// Iterates over entries synchronously for modification. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 1).is_ok()); /// assert!(hashmap.insert_sync(3, 2).is_ok()); /// /// let result = hashmap.iter_mut_sync(|entry| { /// if entry.0 == 1 { /// entry.consume(); /// return false; /// } /// true /// }); /// /// assert!(!result); /// assert!(!hashmap.contains_sync(&1)); /// assert_eq!(hashmap.len(), 2); /// ``` #[inline] pub fn iter_mut_sync) -> bool>(&self, mut f: F) -> bool { let mut result = true; let guard = Guard::new(); self.for_each_writer_sync(0, 0, &guard, |mut locked_bucket, removed| { let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&locked_bucket.writer) { let consumable_entry = ConsumableEntry { locked_bucket: &mut locked_bucket, entry_ptr: &mut entry_ptr, remove_probe: removed, }; if !f(consumable_entry) { result = false; return true; } } false }); result } /// Retains the entries specified by the predicate. /// /// This method allows the predicate closure to modify the value field. /// /// Entries that have existed since the invocation of the method are guaranteed to be visited /// if they are not removed, however the same entry can be visited more than once if the /// [`HashMap`] gets resized by another thread. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// let future_insert = hashmap.insert_async(1, 0); /// let future_retain = hashmap.retain_async(|k, v| *k == 1); /// ``` #[inline] pub async fn retain_async bool>(&self, mut pred: F) { self.iter_mut_async(|mut e| { let (k, v) = &mut *e; if !pred(k, v) { drop(e.consume()); } true }) .await; } /// Retains the entries specified by the predicate. /// /// This method allows the predicate closure to modify the value field. /// /// Entries that have existed since the invocation of the method are guaranteed to be visited /// if they are not removed, however the same entry can be visited more than once if the /// [`HashMap`] gets resized by another thread. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 1).is_ok()); /// assert!(hashmap.insert_sync(3, 2).is_ok()); /// /// hashmap.retain_sync(|k, v| *k == 1 && *v == 0); /// /// assert!(hashmap.contains_sync(&1)); /// assert!(!hashmap.contains_sync(&2)); /// assert!(!hashmap.contains_sync(&3)); /// ``` #[inline] pub fn retain_sync bool>(&self, mut pred: F) { self.iter_mut_sync(|mut e| { let (k, v) = &mut *e; if !pred(k, v) { drop(e.consume()); } true }); } /// Clears the [`HashMap`] by removing all key-value pairs. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// let future_insert = hashmap.insert_async(1, 0); /// let future_clear = hashmap.clear_async(); /// ``` #[inline] pub async fn clear_async(&self) { self.retain_async(|_, _| false).await; } /// Clears the [`HashMap`] by removing all key-value pairs. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// hashmap.clear_sync(); /// /// assert!(!hashmap.contains_sync(&1)); /// ``` #[inline] pub fn clear_sync(&self) { self.retain_sync(|_, _| false); } /// Returns the number of entries in the [`HashMap`]. /// /// It reads the entire metadata area of the bucket array to calculate the number of valid /// entries, making its time complexity `O(N)`. Furthermore, it may overcount entries if an old /// bucket array has not yet been dropped. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert_eq!(hashmap.len(), 1); /// ``` #[inline] pub fn len(&self) -> usize { self.num_entries(&Guard::new()) } /// Returns `true` if the [`HashMap`] is empty. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.is_empty()); /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(!hashmap.is_empty()); /// ``` #[inline] pub fn is_empty(&self) -> bool { !self.has_entry(&Guard::new()) } /// Returns the capacity of the [`HashMap`]. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap_default: HashMap = HashMap::default(); /// assert_eq!(hashmap_default.capacity(), 0); /// /// assert!(hashmap_default.insert_sync(1, 0).is_ok()); /// assert_eq!(hashmap_default.capacity(), 64); /// /// let hashmap: HashMap = HashMap::with_capacity(1000); /// assert_eq!(hashmap.capacity(), 1024); /// ``` #[inline] pub fn capacity(&self) -> usize { self.num_slots(&Guard::new()) } /// Returns the current capacity range of the [`HashMap`]. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert_eq!(hashmap.capacity_range(), 0..=(1_usize << (usize::BITS - 2))); /// /// let reserved = hashmap.reserve(1000); /// assert_eq!(hashmap.capacity_range(), 1000..=(1_usize << (usize::BITS - 2))); /// ``` #[inline] pub fn capacity_range(&self) -> RangeInclusive { self.minimum_capacity.load(Relaxed)..=self.maximum_capacity() } /// Returns the index of the bucket that may contain the key. /// /// The method returns the index of the bucket associated with the key. The number of buckets /// can be calculated by dividing the capacity by `32`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::with_capacity(1024); /// /// let bucket_index = hashmap.bucket_index(&11); /// assert!(bucket_index < hashmap.capacity() / 32); /// ``` #[inline] pub fn bucket_index(&self, key: &Q) -> usize where Q: Equivalent + Hash + ?Sized, { self.calculate_bucket_index(key) } } impl HashMap { /// Creates an empty default [`HashMap`]. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::new(); /// /// let result = hashmap.capacity(); /// assert_eq!(result, 0); /// ``` #[inline] #[must_use] pub fn new() -> Self { Self::default() } /// Creates an empty [`HashMap`] with the specified capacity. /// /// The actual capacity is equal to or greater than `capacity` unless it is greater than /// `1 << (usize::BITS - 1)`. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::with_capacity(1000); /// /// let result = hashmap.capacity(); /// assert_eq!(result, 1024); /// ``` #[inline] #[must_use] pub fn with_capacity(capacity: usize) -> Self { Self::with_capacity_and_hasher(capacity, RandomState::new()) } } impl Clone for HashMap where K: Clone + Eq + Hash, V: Clone, H: BuildHasher + Clone, { #[inline] fn clone(&self) -> Self { let self_clone = Self::with_capacity_and_hasher(self.capacity(), self.hasher().clone()); self.iter_sync(|k, v| { let _result = self_clone.insert_sync(k.clone(), v.clone()); true }); self_clone } } impl Debug for HashMap where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { /// Iterates over all the entries in the [`HashMap`] to print them. /// /// # Locking behavior /// /// Shared locks on buckets are acquired during iteration, therefore any [`Entry`], /// [`OccupiedEntry`], or [`VacantEntry`] owned by the current thread will lead to deadlocks. #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut d = f.debug_map(); self.iter_sync(|k, v| { d.entry(k, v); true }); d.finish() } } impl Default for HashMap where H: BuildHasher + Default, { /// Creates an empty default [`HashMap`]. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// let result = hashmap.capacity(); /// assert_eq!(result, 0); /// ``` #[inline] fn default() -> Self { Self::with_hasher(H::default()) } } impl Drop for HashMap where H: BuildHasher, { #[inline] fn drop(&mut self) { self.bucket_array .swap((None, Tag::None), Relaxed) .0 .map(|a| unsafe { // The entire array does not need to wait for an epoch change as no references will // remain outside the lifetime of the `HashMap`. a.drop_in_place() }); } } impl FromIterator<(K, V)> for HashMap where K: Eq + Hash, H: BuildHasher + Default, { #[inline] fn from_iter>(iter: T) -> Self { let into_iter = iter.into_iter(); let hashmap = Self::with_capacity_and_hasher( Self::capacity_from_size_hint(into_iter.size_hint()), H::default(), ); into_iter.for_each(|e| { hashmap.upsert_sync(e.0, e.1); }); hashmap } } impl HashTable for HashMap where K: Eq + Hash, H: BuildHasher, { #[inline] fn hasher(&self) -> &H { &self.build_hasher } #[inline] fn bucket_array_var(&self) -> &AtomicShared> { &self.bucket_array } #[inline] fn minimum_capacity_var(&self) -> &AtomicUsize { &self.minimum_capacity } } impl PartialEq for HashMap where K: Eq + Hash, V: PartialEq, H: BuildHasher, { /// Compares two [`HashMap`] instances. /// /// # Locking behavior /// /// Shared locks on buckets are acquired when comparing two instances of [`HashMap`], therefore /// this may lead to deadlocks if the instances are being modified by another thread. #[inline] fn eq(&self, other: &Self) -> bool { if self.iter_sync(|k, v| other.read_sync(k, |_, ov| v == ov) == Some(true)) { return other.iter_sync(|k, v| self.read_sync(k, |_, sv| v == sv) == Some(true)); } false } } impl<'h, K, V, H> Entry<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Ensures a value is in the entry by inserting the supplied instance if empty. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(3).or_insert(7); /// assert_eq!(hashmap.read_sync(&3, |_, v| *v), Some(7)); /// ``` #[inline] pub fn or_insert(self, val: V) -> OccupiedEntry<'h, K, V, H> { self.or_insert_with(|| val) } /// Ensures a value is in the entry by inserting the result of the supplied closure if empty. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(19).or_insert_with(|| 5); /// assert_eq!(hashmap.read_sync(&19, |_, v| *v), Some(5)); /// ``` #[inline] pub fn or_insert_with V>(self, constructor: F) -> OccupiedEntry<'h, K, V, H> { self.or_insert_with_key(|_| constructor()) } /// Ensures a value is in the entry by inserting the result of the supplied closure if empty. /// /// The reference to the moved key is provided, therefore cloning or copying the key is /// unnecessary. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(11).or_insert_with_key(|k| if *k == 11 { 7 } else { 3 }); /// assert_eq!(hashmap.read_sync(&11, |_, v| *v), Some(7)); /// ``` #[inline] pub fn or_insert_with_key V>( self, constructor: F, ) -> OccupiedEntry<'h, K, V, H> { match self { Self::Occupied(o) => o, Self::Vacant(v) => { let val = constructor(v.key()); v.insert_entry(val) } } } /// Returns a reference to the key of this entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// assert_eq!(hashmap.entry_sync(31).key(), &31); /// ``` #[inline] pub fn key(&self) -> &K { match self { Self::Occupied(o) => o.key(), Self::Vacant(v) => v.key(), } } /// Provides in-place mutable access to an occupied entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(37).and_modify(|v| { *v += 1 }).or_insert(47); /// assert_eq!(hashmap.read_sync(&37, |_, v| *v), Some(47)); /// /// hashmap.entry_sync(37).and_modify(|v| { *v += 1 }).or_insert(3); /// assert_eq!(hashmap.read_sync(&37, |_, v| *v), Some(48)); /// ``` #[inline] #[must_use] pub fn and_modify(self, f: F) -> Self where F: FnOnce(&mut V), { match self { Self::Occupied(mut o) => { f(o.get_mut()); Self::Occupied(o) } Self::Vacant(_) => self, } } /// Sets the value of the entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// let entry = hashmap.entry_sync(11).insert_entry(17); /// assert_eq!(entry.key(), &11); /// ``` #[inline] pub fn insert_entry(self, val: V) -> OccupiedEntry<'h, K, V, H> { match self { Self::Occupied(mut o) => { o.insert(val); o } Self::Vacant(v) => v.insert_entry(val), } } } impl<'h, K, V, H> Entry<'h, K, V, H> where K: Eq + Hash, V: Default, H: BuildHasher, { /// Ensures a value is in the entry by inserting the default value if empty. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// hashmap.entry_sync(11).or_default(); /// assert_eq!(hashmap.read_sync(&11, |_, v| *v), Some(0)); /// ``` #[inline] pub fn or_default(self) -> OccupiedEntry<'h, K, V, H> { match self { Self::Occupied(o) => o, Self::Vacant(v) => v.insert_entry(Default::default()), } } } impl Debug for Entry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Vacant(v) => f.debug_tuple("Entry").field(v).finish(), Self::Occupied(o) => f.debug_tuple("Entry").field(o).finish(), } } } impl<'h, K, V, H> OccupiedEntry<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Gets a reference to the key in the entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert_eq!(hashmap.entry_sync(29).or_default().key(), &29); /// ``` #[inline] #[must_use] pub fn key(&self) -> &K { &self.locked_bucket.entry(&self.entry_ptr).0 } /// Takes ownership of the key and value from the [`HashMap`]. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(11).or_insert(17); /// /// if let Entry::Occupied(o) = hashmap.entry_sync(11) { /// assert_eq!(o.remove_entry(), (11, 17)); /// }; /// ``` #[inline] #[must_use] pub fn remove_entry(mut self) -> (K, V) { self.locked_bucket.remove(self.hashmap, &mut self.entry_ptr) } /// Gets a reference to the value in the entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(19).or_insert(11); /// /// if let Entry::Occupied(o) = hashmap.entry_sync(19) { /// assert_eq!(o.get(), &11); /// }; /// ``` #[inline] #[must_use] pub fn get(&self) -> &V { &self.locked_bucket.entry(&self.entry_ptr).1 } /// Gets a mutable reference to the value in the entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(37).or_insert(11); /// /// if let Entry::Occupied(mut o) = hashmap.entry_sync(37) { /// *o.get_mut() += 18; /// assert_eq!(*o.get(), 29); /// } /// /// assert_eq!(hashmap.read_sync(&37, |_, v| *v), Some(29)); /// ``` #[inline] pub fn get_mut(&mut self) -> &mut V { &mut self.locked_bucket.entry_mut(&mut self.entry_ptr).1 } /// Sets the value of the entry, and returns the old value. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(37).or_insert(11); /// /// if let Entry::Occupied(mut o) = hashmap.entry_sync(37) { /// assert_eq!(o.insert(17), 11); /// } /// /// assert_eq!(hashmap.read_sync(&37, |_, v| *v), Some(17)); /// ``` #[inline] pub fn insert(&mut self, val: V) -> V { replace(self.get_mut(), val) } /// Takes the value out of the entry, and returns it. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// hashmap.entry_sync(11).or_insert(17); /// /// if let Entry::Occupied(o) = hashmap.entry_sync(11) { /// assert_eq!(o.remove(), 17); /// }; /// ``` #[inline] #[must_use] pub fn remove(self) -> V { self.remove_entry().1 } /// Removes the entry and gets the next closest occupied entry. /// /// [`HashMap::begin_async`] and this method together allow the [`OccupiedEntry`] to /// effectively act as a mutable iterator over entries. This method never acquires more than one /// lock, even when it searches other buckets for the next closest occupied entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 0).is_ok()); /// /// let second_entry_future = hashmap.begin_sync().unwrap().remove_and_async(); /// ``` #[inline] pub async fn remove_and_async(self) -> ((K, V), Option>) { let hashmap = self.hashmap; let mut entry_ptr = self.entry_ptr.clone(); let entry = self .locked_bucket .writer .remove(self.locked_bucket.data_block, &mut entry_ptr); if let Some(locked_bucket) = self.locked_bucket.next_async(hashmap, &mut entry_ptr).await { return ( entry, Some(OccupiedEntry { hashmap, locked_bucket, entry_ptr, }), ); } (entry, None) } /// Removes the entry and gets the next closest occupied entry. /// /// [`HashMap::begin_sync`] and this method together allow the [`OccupiedEntry`] to effectively /// act as a mutable iterator over entries. This method never acquires more than one lock, even /// when it searches other buckets for the next closest occupied entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 0).is_ok()); /// /// let first_entry = hashmap.begin_sync().unwrap(); /// let first_key = *first_entry.key(); /// let (removed, second_entry) = first_entry.remove_and_sync(); /// assert_eq!(removed.1, 0); /// assert_eq!(hashmap.len(), 1); /// /// let second_entry = second_entry.unwrap(); /// let second_key = *second_entry.key(); /// /// assert!(second_entry.remove_and_sync().1.is_none()); /// assert_eq!(first_key + second_key, 3); /// ``` #[inline] #[must_use] pub fn remove_and_sync(self) -> ((K, V), Option) { let hashmap = self.hashmap; let mut entry_ptr = self.entry_ptr.clone(); let entry = self .locked_bucket .writer .remove(self.locked_bucket.data_block, &mut entry_ptr); if let Some(locked_bucket) = self.locked_bucket.next_sync(hashmap, &mut entry_ptr) { return ( entry, Some(OccupiedEntry { hashmap, locked_bucket, entry_ptr, }), ); } (entry, None) } /// Gets the next closest occupied entry. /// /// [`HashMap::begin_async`] and this method together allow the [`OccupiedEntry`] to /// effectively act as a mutable iterator over entries. This method never acquires more than one /// lock, even when it searches other buckets for the next closest occupied entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 0).is_ok()); /// /// let second_entry_future = hashmap.begin_sync().unwrap().next_async(); /// ``` #[inline] pub async fn next_async(self) -> Option> { let hashmap = self.hashmap; let mut entry_ptr = self.entry_ptr.clone(); if let Some(locked_bucket) = self.locked_bucket.next_async(hashmap, &mut entry_ptr).await { return Some(OccupiedEntry { hashmap, locked_bucket, entry_ptr, }); } None } /// Gets the next closest occupied entry. /// /// [`HashMap::begin_sync`] and this method together allow the [`OccupiedEntry`] to effectively /// act as a mutable iterator over entries. This method never acquires more than one lock, even /// when it searches other buckets for the next closest occupied entry. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 0).is_ok()); /// /// let first_entry = hashmap.begin_sync().unwrap(); /// let first_key = *first_entry.key(); /// let second_entry = first_entry.next_sync().unwrap(); /// let second_key = *second_entry.key(); /// /// assert!(second_entry.next_sync().is_none()); /// assert_eq!(first_key + second_key, 3); /// ``` #[inline] #[must_use] pub fn next_sync(self) -> Option { let hashmap = self.hashmap; let mut entry_ptr = self.entry_ptr.clone(); if let Some(locked_bucket) = self.locked_bucket.next_sync(hashmap, &mut entry_ptr) { return Some(OccupiedEntry { hashmap, locked_bucket, entry_ptr, }); } None } } impl Debug for OccupiedEntry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("OccupiedEntry") .field("key", self.key()) .field("value", self.get()) .finish_non_exhaustive() } } impl Deref for OccupiedEntry<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { type Target = V; #[inline] fn deref(&self) -> &Self::Target { self.get() } } impl DerefMut for OccupiedEntry<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { self.get_mut() } } impl<'h, K, V, H> VacantEntry<'h, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Gets a reference to the key. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// assert_eq!(hashmap.entry_sync(11).key(), &11); /// ``` #[inline] pub fn key(&self) -> &K { &self.key } /// Takes ownership of the key. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// if let Entry::Vacant(v) = hashmap.entry_sync(17) { /// assert_eq!(v.into_key(), 17); /// }; /// ``` #[inline] pub fn into_key(self) -> K { self.key } /// Sets the value of the entry with its key, and returns an [`OccupiedEntry`]. /// /// # Examples /// /// ``` /// use scc::HashMap; /// use scc::hash_map::Entry; /// /// let hashmap: HashMap = HashMap::default(); /// /// if let Entry::Vacant(o) = hashmap.entry_sync(19) { /// o.insert_entry(29); /// } /// /// assert_eq!(hashmap.read_sync(&19, |_, v| *v), Some(29)); /// ``` #[inline] pub fn insert_entry(self, val: V) -> OccupiedEntry<'h, K, V, H> { let entry_ptr = self.locked_bucket.insert(self.hash, (self.key, val)); OccupiedEntry { hashmap: self.hashmap, locked_bucket: self.locked_bucket, entry_ptr, } } } impl Debug for VacantEntry<'_, K, V, H> where K: Debug + Eq + Hash, V: Debug, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("VacantEntry").field(self.key()).finish() } } impl ConsumableEntry<'_, K, V> { /// Consumes the entry by moving out the key and value. /// /// # Examples /// /// ``` /// use scc::HashMap; /// /// let hashmap: HashMap = HashMap::default(); /// /// assert!(hashmap.insert_sync(1, 0).is_ok()); /// assert!(hashmap.insert_sync(2, 1).is_ok()); /// assert!(hashmap.insert_sync(3, 2).is_ok()); /// /// let mut consumed = None; /// /// hashmap.iter_mut_sync(|entry| { /// if entry.0 == 1 { /// consumed.replace(entry.consume().1); /// } /// true /// }); /// /// assert!(!hashmap.contains_sync(&1)); /// assert_eq!(consumed, Some(0)); /// ``` #[inline] #[must_use] pub fn consume(self) -> (K, V) { *self.remove_probe |= true; self.locked_bucket .writer .remove(self.locked_bucket.data_block, self.entry_ptr) } } impl Deref for ConsumableEntry<'_, K, V> { type Target = (K, V); #[inline] fn deref(&self) -> &Self::Target { self.locked_bucket.entry(self.entry_ptr) } } impl DerefMut for ConsumableEntry<'_, K, V> { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { self.locked_bucket.entry_mut(self.entry_ptr) } } impl Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { /// Returns the number of reserved slots. #[inline] #[must_use] pub fn additional_capacity(&self) -> usize { self.additional } } impl AsRef> for Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn as_ref(&self) -> &HashMap { self.hashmap } } impl Debug for Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("Reserve").field(&self.additional).finish() } } impl Deref for Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { type Target = HashMap; #[inline] fn deref(&self) -> &Self::Target { self.hashmap } } impl Drop for Reserve<'_, K, V, H> where K: Eq + Hash, H: BuildHasher, { #[inline] fn drop(&mut self) { let result = self .hashmap .minimum_capacity .fetch_sub(self.additional, Relaxed); debug_assert!(result >= self.additional); let guard = Guard::new(); if let Some(current_array) = self.hashmap.bucket_array(&guard) { self.try_shrink(current_array, 0, &guard); } } } scc-3.4.8/src/hash_set.rs000064400000000000000000000622071046102023000133520ustar 00000000000000//! [`HashSet`] is a concurrent hash set. #![deny(unsafe_code)] use std::collections::hash_map::RandomState; use std::fmt::{self, Debug}; use std::hash::{BuildHasher, Hash}; use std::mem::swap; use std::ops::{Deref, RangeInclusive}; use super::hash_map; use super::hash_table::HashTable; use super::{Equivalent, HashMap}; /// Scalable concurrent hash set. /// /// [`HashSet`] is a concurrent hash set based on [`HashMap`]. pub struct HashSet where H: BuildHasher, { map: HashMap, } /// [`ConsumableEntry`] is a view into an occupied entry in a [`HashSet`] when iterating over /// entries in it. pub struct ConsumableEntry<'w, K> { consumable: hash_map::ConsumableEntry<'w, K, ()>, } /// [`Reserve`] keeps the capacity of the associated [`HashSet`] higher than a certain level. /// /// The [`HashSet`] does not shrink the capacity below the reserved capacity. pub type Reserve<'h, K, H = RandomState> = super::hash_map::Reserve<'h, K, (), H>; impl HashSet where H: BuildHasher, { /// Creates an empty [`HashSet`] with the given [`BuildHasher`]. /// /// # Examples /// /// ``` /// use scc::HashSet; /// use std::collections::hash_map::RandomState; /// /// let hashset: HashSet = HashSet::with_hasher(RandomState::new()); /// ``` #[cfg(not(feature = "loom"))] #[inline] pub const fn with_hasher(build_hasher: H) -> Self { Self { map: HashMap::with_hasher(build_hasher), } } /// Creates an empty [`HashSet`] with the given [`BuildHasher`]. #[cfg(feature = "loom")] #[inline] pub fn with_hasher(build_hasher: H) -> Self { Self { map: HashMap::with_hasher(build_hasher), } } /// Creates an empty [`HashSet`] with the specified capacity and [`BuildHasher`]. /// /// The actual capacity is equal to or greater than `capacity` unless it is greater than /// `1 << (usize::BITS - 1)`. /// /// # Examples /// /// ``` /// use scc::HashSet; /// use std::collections::hash_map::RandomState; /// /// let hashset: HashSet = /// HashSet::with_capacity_and_hasher(1000, RandomState::new()); /// /// let result = hashset.capacity(); /// assert_eq!(result, 1024); /// ``` #[inline] pub fn with_capacity_and_hasher(capacity: usize, build_hasher: H) -> Self { Self { map: HashMap::with_capacity_and_hasher(capacity, build_hasher), } } } impl HashSet where K: Eq + Hash, H: BuildHasher, { /// Temporarily increases the minimum capacity of the [`HashSet`]. /// /// A [`Reserve`] is returned if the [`HashSet`] could increase the minimum capacity while the /// increased capacity is not exclusively owned by the returned [`Reserve`], allowing others to /// benefit from it. The memory for the additional space may not be immediately allocated if /// the [`HashSet`] is empty or currently being resized, however once the memory is reserved /// eventually, the capacity will not shrink below the additional capacity until the returned /// [`Reserve`] is dropped. /// /// # Errors /// /// Returns `None` if a too large number is given. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::with_capacity(1000); /// assert_eq!(hashset.capacity(), 1024); /// /// let reserved = hashset.reserve(10000); /// assert!(reserved.is_some()); /// assert_eq!(hashset.capacity(), 16384); /// /// assert!(hashset.reserve(usize::MAX).is_none()); /// assert_eq!(hashset.capacity(), 16384); /// /// for i in 0..16 { /// assert!(hashset.insert_sync(i).is_ok()); /// } /// drop(reserved); /// /// assert_eq!(hashset.capacity(), 1024); /// ``` #[inline] pub fn reserve(&self, capacity: usize) -> Option> { self.map.reserve(capacity) } /// Inserts a key into the [`HashSet`]. /// /// # Errors /// /// Returns an error along with the supplied key if the key exists. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// let future_insert = hashset.insert_async(11); /// ``` #[inline] pub async fn insert_async(&self, key: K) -> Result<(), K> { self.map.insert_async(key, ()).await.map_err(|(k, ())| k) } /// Inserts a key into the [`HashSet`]. /// /// # Errors /// /// Returns an error along with the supplied key if the key exists. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// assert_eq!(hashset.insert_sync(1).unwrap_err(), 1); /// ``` #[inline] pub fn insert_sync(&self, key: K) -> Result<(), K> { if let Err((k, ())) = self.map.insert_sync(key, ()) { return Err(k); } Ok(()) } /// Adds a key to the set, replacing the existing key, if any, that is equal to the given one. /// /// Returns the replaced key, if any. /// /// # Examples /// /// ``` /// use std::cmp::{Eq, PartialEq}; /// use std::hash::{Hash, Hasher}; /// /// use scc::HashSet; /// /// #[derive(Debug)] /// struct MaybeEqual(u64, u64); /// /// impl Eq for MaybeEqual {} /// /// impl Hash for MaybeEqual { /// fn hash(&self, state: &mut H) { /// // Do not read `self.1`. /// self.0.hash(state); /// } /// } /// /// impl PartialEq for MaybeEqual { /// fn eq(&self, other: &Self) -> bool { /// // Do not compare `self.1`. /// self.0 == other.0 /// } /// } /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.replace_sync(MaybeEqual(11, 7)).is_none()); /// assert_eq!(hashset.replace_sync(MaybeEqual(11, 11)), Some(MaybeEqual(11, 7))); /// ``` #[inline] pub async fn replace_async(&self, mut key: K) -> Option { let hash = self.map.hash(&key); let mut locked_bucket = self.map.writer_async(hash).await; let mut entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { let k = &mut locked_bucket.entry_mut(&mut entry_ptr).0; swap(k, &mut key); Some(key) } else { locked_bucket.insert(hash, (key, ())); None } } /// Adds a key to the set, replacing the existing key, if any, that is equal to the given one. /// /// Returns the replaced key, if any. /// /// # Examples /// /// ``` /// use std::cmp::{Eq, PartialEq}; /// use std::hash::{Hash, Hasher}; /// /// use scc::HashSet; /// /// #[derive(Debug)] /// struct MaybeEqual(u64, u64); /// /// impl Eq for MaybeEqual {} /// /// impl Hash for MaybeEqual { /// fn hash(&self, state: &mut H) { /// // Do not read `self.1`. /// self.0.hash(state); /// } /// } /// /// impl PartialEq for MaybeEqual { /// fn eq(&self, other: &Self) -> bool { /// // Do not compare `self.1`. /// self.0 == other.0 /// } /// } /// /// let hashset: HashSet = HashSet::default(); /// /// async { /// assert!(hashset.replace_async(MaybeEqual(11, 7)).await.is_none()); /// assert_eq!(hashset.replace_async(MaybeEqual(11, 11)).await, Some(MaybeEqual(11, 7))); /// }; /// ``` #[inline] pub fn replace_sync(&self, mut key: K) -> Option { let hash = self.map.hash(&key); let mut locked_bucket = self.map.writer_sync(hash); let mut entry_ptr = locked_bucket.search(&key, hash); if entry_ptr.is_valid() { let k = &mut locked_bucket.entry_mut(&mut entry_ptr).0; swap(k, &mut key); Some(key) } else { locked_bucket.insert(hash, (key, ())); None } } /// Removes a key if the key exists. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// let future_insert = hashset.insert_async(11); /// let future_remove = hashset.remove_async(&11); /// ``` #[inline] pub async fn remove_async(&self, key: &Q) -> Option where Q: Equivalent + Hash + ?Sized, { self.map .remove_if_async(key, |()| true) .await .map(|(k, ())| k) } /// Removes a key if the key exists. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.remove_sync(&1).is_none()); /// assert!(hashset.insert_sync(1).is_ok()); /// assert_eq!(hashset.remove_sync(&1).unwrap(), 1); /// ``` #[inline] pub fn remove_sync(&self, key: &Q) -> Option where Q: Equivalent + Hash + ?Sized, { self.map.remove_sync(key).map(|(k, ())| k) } /// Removes a key if the key exists and the given condition is met. /// /// Returns `None` if the key does not exist or the condition was not met. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// let future_insert = hashset.insert_async(11); /// let future_remove = hashset.remove_if_async(&11, || true); /// ``` #[inline] pub async fn remove_if_async bool>(&self, key: &Q, condition: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.map .remove_if_async(key, |()| condition()) .await .map(|(k, ())| k) } /// Removes a key if the key exists and the given condition is met. /// /// Returns `None` if the key does not exist or the condition was not met. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(hashset.remove_if_sync(&1, || false).is_none()); /// assert_eq!(hashset.remove_if_sync(&1, || true).unwrap(), 1); /// ``` #[inline] pub fn remove_if_sync bool>(&self, key: &Q, condition: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.map .remove_if_sync(key, |()| condition()) .map(|(k, ())| k) } /// Reads a key. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// let future_insert = hashset.insert_async(11); /// let future_read = hashset.read_async(&11, |k| *k); /// ``` #[inline] pub async fn read_async R>(&self, key: &Q, reader: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.map.read_async(key, |k, ()| reader(k)).await } /// Reads a key. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.read_sync(&1, |_| true).is_none()); /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(hashset.read_sync(&1, |_| true).unwrap()); /// ``` #[inline] pub fn read_sync R>(&self, key: &Q, reader: F) -> Option where Q: Equivalent + Hash + ?Sized, { self.map.read_sync(key, |k, ()| reader(k)) } /// Returns `true` if the [`HashSet`] contains the specified key. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// let future_contains = hashset.contains_async(&1); /// ``` #[inline] pub async fn contains_async(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.map.contains_async(key).await } /// Returns `true` if the [`HashSet`] contains the specified key. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(!hashset.contains_sync(&1)); /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(hashset.contains_sync(&1)); /// ``` #[inline] pub fn contains_sync(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.read_sync(key, |_| ()).is_some() } /// Iterates over entries asynchronously for reading. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// /// async { /// let result = hashset.iter_async(|k| { /// true /// }).await; /// assert!(result); /// }; /// ``` #[inline] pub async fn iter_async bool>(&self, mut f: F) -> bool { self.map.iter_async(|k, ()| f(k)).await } /// Iterates over entries synchronously for reading. /// /// Stops iterating when the closure returns `false`, and this method also returns `false`. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(hashset.insert_sync(2).is_ok()); /// /// let mut acc = 0; /// let result = hashset.iter_sync(|k| { /// acc += *k; /// true /// }); /// /// assert!(result); /// assert_eq!(acc, 3); /// ``` #[inline] pub fn iter_sync bool>(&self, mut f: F) -> bool { self.map.iter_sync(|k, ()| f(k)) } /// Iterates over entries asynchronously for modification. /// /// This method stops iterating when the closure returns `false`, and also returns `false` in /// that case. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(hashset.insert_sync(2).is_ok()); /// /// async { /// let result = hashset.iter_mut_async(|entry| { /// if *entry == 1 { /// entry.consume(); /// return false; /// } /// true /// }).await; /// /// assert!(!result); /// assert_eq!(hashset.len(), 1); /// }; /// ``` #[inline] pub async fn iter_mut_async) -> bool>(&self, mut f: F) -> bool { self.map .iter_mut_async(|consumable| f(ConsumableEntry { consumable })) .await } /// Iterates over entries synchronously for modification. /// /// This method stops iterating when the closure returns `false`, and also returns `false` in /// that case. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(hashset.insert_sync(2).is_ok()); /// assert!(hashset.insert_sync(3).is_ok()); /// /// let result = hashset.iter_mut_sync(|entry| { /// if *entry == 1 { /// entry.consume(); /// return false; /// } /// true /// }); /// /// assert!(!result); /// assert!(!hashset.contains_sync(&1)); /// assert_eq!(hashset.len(), 2); /// ``` #[inline] pub fn iter_mut_sync) -> bool>(&self, mut f: F) -> bool { self.map .iter_mut_sync(|consumable| f(ConsumableEntry { consumable })) } /// Retains keys that satisfy the given predicate. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// let future_insert = hashset.insert_async(1); /// let future_retain = hashset.retain_async(|k| *k == 1); /// ``` #[inline] pub async fn retain_async bool>(&self, mut filter: F) { self.map.retain_async(|k, ()| filter(k)).await; } /// Retains keys that satisfy the given predicate. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(hashset.insert_sync(2).is_ok()); /// assert!(hashset.insert_sync(3).is_ok()); /// /// hashset.retain_sync(|k| *k == 1); /// /// assert!(hashset.contains_sync(&1)); /// assert!(!hashset.contains_sync(&2)); /// assert!(!hashset.contains_sync(&3)); /// ``` #[inline] pub fn retain_sync bool>(&self, mut pred: F) { self.iter_mut_sync(|e| { if !pred(&e) { drop(e.consume()); } true }); } /// Clears the [`HashSet`] by removing all keys. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// let future_insert = hashset.insert_async(1); /// let future_clear = hashset.clear_async(); /// ``` #[inline] pub async fn clear_async(&self) { self.map.clear_async().await; } /// Clears the [`HashSet`] by removing all keys. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// hashset.clear_sync(); /// /// assert!(!hashset.contains_sync(&1)); /// ``` #[inline] pub fn clear_sync(&self) { self.map.clear_sync(); } /// Returns the number of entries in the [`HashSet`]. /// /// It reads the entire metadata area of the bucket array to calculate the number of valid /// entries, making its time complexity `O(N)`. Furthermore, it may overcount entries if an old /// bucket array has yet to be dropped. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// assert_eq!(hashset.len(), 1); /// ``` #[inline] pub fn len(&self) -> usize { self.map.len() } /// Returns `true` if the [`HashSet`] is empty. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.is_empty()); /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(!hashset.is_empty()); /// ``` #[inline] pub fn is_empty(&self) -> bool { self.map.is_empty() } /// Returns the capacity of the [`HashSet`]. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset_default: HashSet = HashSet::default(); /// assert_eq!(hashset_default.capacity(), 0); /// /// assert!(hashset_default.insert_sync(1).is_ok()); /// assert_eq!(hashset_default.capacity(), 64); /// /// let hashset: HashSet = HashSet::with_capacity(1000); /// assert_eq!(hashset.capacity(), 1024); /// ``` #[inline] pub fn capacity(&self) -> usize { self.map.capacity() } /// Returns the current capacity range of the [`HashSet`]. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert_eq!(hashset.capacity_range(), 0..=(1_usize << (usize::BITS - 2))); /// /// let reserved = hashset.reserve(1000); /// assert_eq!(hashset.capacity_range(), 1000..=(1_usize << (usize::BITS - 2))); /// ``` #[inline] pub fn capacity_range(&self) -> RangeInclusive { self.map.capacity_range() } /// Returns the index of the bucket that may contain the key. /// /// The method returns the index of the bucket associated with the key. The number of buckets /// can be calculated by dividing the capacity by `32`. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::with_capacity(1024); /// /// let bucket_index = hashset.bucket_index(&11); /// assert!(bucket_index < hashset.capacity() / 32); /// ``` #[inline] pub fn bucket_index(&self, key: &Q) -> usize where Q: Equivalent + Hash + ?Sized, { self.map.bucket_index(key) } } impl Clone for HashSet where K: Clone + Eq + Hash, H: BuildHasher + Clone, { #[inline] fn clone(&self) -> Self { Self { map: self.map.clone(), } } } impl Debug for HashSet where K: Debug + Eq + Hash, H: BuildHasher, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut d = f.debug_set(); self.iter_sync(|k| { d.entry(k); true }); d.finish() } } impl HashSet { /// Creates an empty default [`HashSet`]. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::new(); /// /// let result = hashset.capacity(); /// assert_eq!(result, 0); /// ``` #[inline] #[must_use] pub fn new() -> Self { Self::default() } /// Creates an empty [`HashSet`] with the specified capacity. /// /// The actual capacity is equal to or greater than `capacity` unless it is greater than /// `1 << (usize::BITS - 1)`. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::with_capacity(1000); /// /// let result = hashset.capacity(); /// assert_eq!(result, 1024); /// ``` #[inline] #[must_use] pub fn with_capacity(capacity: usize) -> Self { Self { map: HashMap::with_capacity(capacity), } } } impl Default for HashSet where H: BuildHasher + Default, { /// Creates an empty default [`HashSet`]. /// /// The default capacity is `0`. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// let result = hashset.capacity(); /// assert_eq!(result, 0); /// ``` #[inline] fn default() -> Self { Self { map: HashMap::default(), } } } impl FromIterator for HashSet where K: Eq + Hash, H: BuildHasher + Default, { #[inline] fn from_iter>(iter: T) -> Self { let into_iter = iter.into_iter(); let hashset = Self::with_capacity_and_hasher( HashMap::::capacity_from_size_hint(into_iter.size_hint()), H::default(), ); into_iter.for_each(|k| { let _result = hashset.insert_sync(k); }); hashset } } impl PartialEq for HashSet where K: Eq + Hash, H: BuildHasher, { /// Compares two [`HashSet`] instances. /// /// ### Locking behavior /// /// Shared locks on buckets are acquired when comparing two instances of [`HashSet`], therefore /// it may lead to a deadlock if the instances are being modified by another thread. #[inline] fn eq(&self, other: &Self) -> bool { if self.iter_sync(|k| other.contains_sync(k)) { return other.iter_sync(|k| self.contains_sync(k)); } false } } impl ConsumableEntry<'_, K> { /// Consumes the entry by moving out the key. /// /// # Examples /// /// ``` /// use scc::HashSet; /// /// let hashset: HashSet = HashSet::default(); /// /// assert!(hashset.insert_sync(1).is_ok()); /// assert!(hashset.insert_sync(2).is_ok()); /// assert!(hashset.insert_sync(3).is_ok()); /// /// let mut consumed = None; /// /// hashset.iter_mut_sync(|entry| { /// if *entry == 1 { /// consumed.replace(entry.consume()); /// } /// true /// }); /// /// assert!(!hashset.contains_sync(&1)); /// assert_eq!(consumed, Some(1)); /// ``` #[inline] #[must_use] pub fn consume(self) -> K { self.consumable.consume().0 } } impl Deref for ConsumableEntry<'_, K> { type Target = K; #[inline] fn deref(&self) -> &Self::Target { &self.consumable.0 } } scc-3.4.8/src/hash_table/bucket.rs000064400000000000000000001554151046102023000151270ustar 00000000000000use std::cell::UnsafeCell; use std::fmt::{self, Debug}; use std::mem::{MaybeUninit, forget, needs_drop}; use std::ops::{Deref, Index}; use std::ptr::{self, NonNull, from_ref}; use std::sync::atomic::Ordering::{Acquire, Relaxed, Release}; use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; use saa::Lock; use sdd::{AtomicShared, Epoch, Guard, Shared, Tag}; use crate::Equivalent; use crate::async_helper::AsyncGuard; /// [`Bucket`] is a lock-protected fixed-size entry array. /// /// In case the fixed-size entry array overflows, additional entries can be stored in a linked list /// of [`LinkedBucket`]. #[repr(align(64))] pub struct Bucket { /// Number of entries in the [`Bucket`]. len: AtomicUsize, /// [`Bucket`] metadata. metadata: Metadata, /// Reader-writer lock. rw_lock: Lock, /// The LRU list of the [`Bucket`]. lru_list: L, } /// The type of [`Bucket`] that only allows sequential access to it. pub const MAP: char = 'S'; /// The type of [`Bucket`] that allows lock-free read access. pub const INDEX: char = 'O'; /// The type of [`Bucket`] that acts as an LRU cache. pub const CACHE: char = 'C'; /// The size of the fixed-size entry array in a [`Bucket`]. pub const BUCKET_LEN: usize = u32::BITS as usize; /// [`DataBlock`] is a type alias of a raw memory chunk of entries. pub struct DataBlock([UnsafeCell>; LEN]); /// [`Writer`] holds an exclusive lock on a [`Bucket`]. #[derive(Debug)] pub struct Writer { bucket_ptr: NonNull>, } /// [`Reader`] holds a shared lock on a [`Bucket`]. #[derive(Debug)] pub struct Reader { bucket_ptr: NonNull>, } /// [`EntryPtr`] points to an entry slot in a [`Bucket`]. pub struct EntryPtr { /// Pointer to a [`LinkedBucket`]. link_ptr: *const LinkedBucket, /// Index of the entry. index: usize, } /// Doubly-linked list interfaces to efficiently manage least-recently-used entries. pub trait LruList: 'static + Default { /// Evicts an entry. #[inline] fn evict(&self, _tail: u32) -> Option<(u8, u32)> { None } /// Removes an entry. #[inline] fn remove(&self, _tail: u32, _entry: u8) -> Option { None } /// Promotes the entry. #[inline] fn promote(&self, _tail: u32, _entry: u8) -> Option { None } } /// [`DoublyLinkedList`] is an array of `(u8, u8)` implementing [`LruList`]. #[derive(Debug, Default)] pub struct DoublyLinkedList([UnsafeCell<(u8, u8)>; BUCKET_LEN]); /// [`Metadata`] is a collection of metadata fields of [`Bucket`] and [`LinkedBucket`]. struct Metadata { /// Linked list of entries. link: AtomicShared>, /// Occupied slot bitmap. occupied_bitmap: AtomicU32, /// Removed slot bitmap, or the 1-based index of the most recently used entry if /// `TYPE = CACHE` where `0` represents `nil`. removed_bitmap: AtomicU32, /// Partial hash array for fast hash lookup, or the epoch when the corresponding entry was /// removed if `TYPE = INDEX`. partial_hash_array: [UnsafeCell; LEN], } /// The size of the linked data block. const LINKED_BUCKET_LEN: usize = BUCKET_LEN / 4; /// [`LinkedBucket`] is a smaller [`Bucket`] that is attached to a [`Bucket`] as a linked list. #[repr(align(128))] struct LinkedBucket { /// [`LinkedBucket`] metadata. metadata: Metadata, /// Own data block. data_block: DataBlock, /// Previous [`LinkedBucket`]. prev_link: AtomicPtr>, } impl Bucket { /// Creates a new [`Bucket`]. #[cfg(any(test, feature = "loom"))] pub fn new() -> Self { Self { len: AtomicUsize::new(0), rw_lock: Lock::default(), metadata: Metadata { link: AtomicShared::default(), occupied_bitmap: AtomicU32::default(), removed_bitmap: AtomicU32::default(), partial_hash_array: Default::default(), }, lru_list: L::default(), } } /// Returns the number of occupied and reachable slots in the [`Bucket`]. #[inline] pub(crate) fn len(&self) -> usize { self.len.load(Relaxed) } /// Reserves memory for insertion and then constructs the key-value pair in-place. #[inline] pub(crate) fn insert( &self, data_block: NonNull>, hash: u64, entry: (K, V), ) -> EntryPtr { let occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); let occupied_bitmap = if TYPE == INDEX && occupied_bitmap == u32::MAX && self.metadata.removed_bitmap.load(Relaxed) != 0 { self.clear_unreachable_entries(data_block_ref(data_block)) } else { occupied_bitmap }; let free_index = occupied_bitmap.trailing_ones() as usize; if free_index == BUCKET_LEN { self.insert_overflow(hash, entry) } else { self.insert_entry( &self.metadata, data_block_ref(data_block), free_index, occupied_bitmap, hash, entry, ); self.len.store(self.len.load(Relaxed) + 1, Relaxed); EntryPtr { link_ptr: ptr::null(), index: free_index, } } } /// Removes the entry pointed to by the supplied [`EntryPtr`]. #[inline] pub(crate) fn remove( &self, data_block: NonNull>, entry_ptr: &mut EntryPtr, ) -> (K, V) { debug_assert_ne!(TYPE, INDEX); debug_assert_ne!(entry_ptr.index, usize::MAX); debug_assert_ne!(entry_ptr.index, BUCKET_LEN); self.len.store(self.len.load(Relaxed) - 1, Relaxed); if let Some(link) = link_ref(entry_ptr.link_ptr) { let mut occupied_bitmap = link.metadata.occupied_bitmap.load(Relaxed); debug_assert_ne!(occupied_bitmap & (1_u32 << entry_ptr.index), 0); occupied_bitmap &= !(1_u32 << entry_ptr.index); link.metadata .occupied_bitmap .store(occupied_bitmap, Relaxed); let removed = Self::read_data_block(&link.data_block, entry_ptr.index); if occupied_bitmap == 0 && TYPE != INDEX { entry_ptr.unlink(&self.metadata.link, link); } removed } else { let occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); debug_assert_ne!(occupied_bitmap & (1_u32 << entry_ptr.index), 0); if TYPE == CACHE { self.remove_from_lru_list(entry_ptr); } self.metadata .occupied_bitmap .store(occupied_bitmap & !(1_u32 << entry_ptr.index), Relaxed); Self::read_data_block(data_block_ref(data_block), entry_ptr.index) } } /// Marks the entry removed without dropping the entry. #[inline] pub(crate) fn mark_removed(&self, entry_ptr: &mut EntryPtr, guard: &Guard) { debug_assert_eq!(TYPE, INDEX); debug_assert_ne!(entry_ptr.index, usize::MAX); debug_assert_ne!(entry_ptr.index, BUCKET_LEN); self.len.store(self.len.load(Relaxed) - 1, Relaxed); if let Some(link) = link_ref(entry_ptr.link_ptr) { self.write_cell(&link.metadata.partial_hash_array[entry_ptr.index], |h| { *h = u8::from(guard.epoch()); }); let mut removed_bitmap = link.metadata.removed_bitmap.load(Relaxed); debug_assert_eq!(removed_bitmap & (1_u32 << entry_ptr.index), 0); removed_bitmap |= 1_u32 << entry_ptr.index; link.metadata.removed_bitmap.store(removed_bitmap, Release); } else { self.write_cell(&self.metadata.partial_hash_array[entry_ptr.index], |h| { *h = u8::from(guard.epoch()); }); let mut removed_bitmap = self.metadata.removed_bitmap.load(Relaxed); debug_assert_eq!(removed_bitmap & (1_u32 << entry_ptr.index), 0); removed_bitmap |= 1_u32 << entry_ptr.index; self.metadata.removed_bitmap.store(removed_bitmap, Release); } } /// Evicts the least recently used entry if the [`Bucket`] is full. #[inline] pub(crate) fn evict_lru_head( &self, data_block: NonNull>, ) -> Option<(K, V)> { debug_assert_eq!(TYPE, CACHE); let occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); if occupied_bitmap == 0b1111_1111_1111_1111_1111_1111_1111_1111 { self.len.store(self.len.load(Relaxed) - 1, Relaxed); let tail = self.metadata.removed_bitmap.load(Relaxed); let evicted = if let Some((evicted, new_tail)) = self.lru_list.evict(tail) { self.metadata.removed_bitmap.store(new_tail, Relaxed); evicted as usize } else { // Evict the first occupied entry. 0 }; debug_assert_ne!(occupied_bitmap & (1_u32 << evicted), 0); self.metadata .occupied_bitmap .store(occupied_bitmap & !(1_u32 << evicted), Relaxed); return Some(Self::read_data_block(data_block_ref(data_block), evicted)); } None } /// Sets the entry as having been just accessed. #[inline] pub(crate) fn update_lru_tail(&self, entry_ptr: &EntryPtr) { debug_assert_eq!(TYPE, CACHE); debug_assert_ne!(entry_ptr.index, usize::MAX); debug_assert_ne!(entry_ptr.index, BUCKET_LEN); if entry_ptr.link_ptr.is_null() { #[allow(clippy::cast_possible_truncation)] let entry = entry_ptr.index as u8; let tail = self.metadata.removed_bitmap.load(Relaxed); if let Some(new_tail) = self.lru_list.promote(tail, entry) { self.metadata.removed_bitmap.store(new_tail, Relaxed); } } } /// Reserves memory for additional entries. #[inline] pub(crate) fn reserve_slots(&self, additional: usize) { debug_assert!(self.rw_lock.is_locked(Relaxed)); let mut capacity = BUCKET_LEN - self.metadata.occupied_bitmap.load(Relaxed).count_ones() as usize; if capacity < additional { let mut link_ptr = self.metadata.load_link(); while let Some(link) = link_ref(link_ptr) { capacity += LINKED_BUCKET_LEN - link.metadata.occupied_bitmap.load(Relaxed).count_ones() as usize; if capacity >= additional { return; } let mut next_link_ptr = link.metadata.load_link(); if next_link_ptr.is_null() { let new_link = unsafe { Shared::new_unchecked(LinkedBucket::new(None)) }; new_link.prev_link.store(link_ptr.cast_mut(), Relaxed); next_link_ptr = new_link.as_ptr(); link.metadata .link .swap((Some(new_link), Tag::None), Release); } link_ptr = next_link_ptr; } } } /// Extracts an entry from the given bucket and inserts the entry into itself. #[inline] pub(crate) fn extract_from( &self, data_block: NonNull>, hash: u64, from_writer: &Writer, from_data_block: NonNull>, from_entry_ptr: &mut EntryPtr, ) { debug_assert!(self.rw_lock.is_locked(Relaxed)); let entry = if let Some(link) = link_ref(from_entry_ptr.link_ptr) { Self::read_data_block(&link.data_block, from_entry_ptr.index) } else { Self::read_data_block(data_block_ref(from_data_block), from_entry_ptr.index) }; self.insert(data_block, hash, entry); let mo = if TYPE == INDEX { Release } else { Relaxed }; if let Some(link) = link_ref(from_entry_ptr.link_ptr) { let occupied_bitmap = link.metadata.occupied_bitmap.load(Relaxed); debug_assert_ne!(occupied_bitmap & (1_u32 << from_entry_ptr.index), 0); link.metadata .occupied_bitmap .store(occupied_bitmap & !(1_u32 << from_entry_ptr.index), mo); } else { let occupied_bitmap = from_writer.metadata.occupied_bitmap.load(Relaxed); debug_assert_ne!(occupied_bitmap & (1_u32 << from_entry_ptr.index), 0); from_writer .metadata .occupied_bitmap .store(occupied_bitmap & !(1_u32 << from_entry_ptr.index), mo); } let from_len = from_writer.len.load(Relaxed); from_writer.len.store(from_len - 1, Relaxed); } /// Drops entries in the [`DataBlock`] when the bucket array is being dropped. /// /// The [`Bucket`] and the [`DataBlock`] should never be used afterward. pub(super) fn drop_entries(&self, data_block: NonNull>) { if !self.metadata.link.is_null(Relaxed) { let mut next = self.metadata.link.swap((None, Tag::None), Acquire); while let Some(current) = next.0 { next = current.metadata.link.swap((None, Tag::None), Acquire); let dropped = unsafe { current.drop_in_place() }; debug_assert!(dropped); } } if needs_drop::<(K, V)>() { let mut occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); while occupied_bitmap != 0 { let index = occupied_bitmap.trailing_zeros(); Self::drop_entry(data_block_ref(data_block), index as usize); occupied_bitmap -= 1_u32 << index; } } } /// Inserts an entry into an overflow bucket. fn insert_overflow(&self, hash: u64, entry: (K, V)) -> EntryPtr { let mut link_ptr = self.metadata.load_link(); while let Some(link) = link_ref(link_ptr) { let occupied_bitmap = link.metadata.occupied_bitmap.load(Relaxed); let free_index = occupied_bitmap.trailing_ones() as usize; if free_index != LINKED_BUCKET_LEN { debug_assert!(free_index < LINKED_BUCKET_LEN); self.insert_entry( &link.metadata, &link.data_block, free_index, occupied_bitmap, hash, entry, ); self.len.store(self.len.load(Relaxed) + 1, Relaxed); return EntryPtr { link_ptr, index: free_index, }; } link_ptr = link.metadata.load_link(); } // Insert a new `LinkedBucket` at the linked list head. let head = self.metadata.link.get_shared(Relaxed, fake_ref(self)); let link = unsafe { Shared::new_unchecked(LinkedBucket::new(head)) }; self.write_cell(&link.data_block[0], |block| unsafe { block.as_mut_ptr().write(entry); }); self.write_cell(&link.metadata.partial_hash_array[0], |h| { *h = Self::partial_hash(hash); }); link.metadata.occupied_bitmap.store(1, Relaxed); if let Some(head) = link_ref(link.metadata.load_link()) { head.prev_link.store(link.as_ptr().cast_mut(), Relaxed); } let link_ptr = link.as_ptr(); self.metadata.link.swap((Some(link), Tag::None), Release); self.len.store(self.len.load(Relaxed) + 1, Relaxed); EntryPtr { link_ptr, index: 0 } } /// Inserts a key-value pair in the slot. #[inline] fn insert_entry( &self, metadata: &Metadata, data_block: &DataBlock, index: usize, occupied_bitmap: u32, hash: u64, entry: (K, V), ) { debug_assert!(index < LEN); debug_assert_eq!(metadata.occupied_bitmap.load(Relaxed) & (1_u32 << index), 0); self.write_cell(&data_block[index], |block| unsafe { block.as_mut_ptr().write(entry); }); self.write_cell(&metadata.partial_hash_array[index], |h| { *h = Self::partial_hash(hash); }); metadata.occupied_bitmap.store( occupied_bitmap | (1_u32 << index), if TYPE == INDEX { Release } else { Relaxed }, ); } /// Clears unreachable entries. fn clear_unreachable_entries(&self, data_block: &DataBlock) -> u32 { debug_assert_eq!(TYPE, INDEX); let guard = Guard::new(); let mut link_ptr = self.metadata.load_link(); while let Some(link) = link_ref(link_ptr) { let mut next_link_ptr = link.metadata.load_link(); if next_link_ptr.is_null() { while let Some(link) = link_ref(link_ptr) { let prev_link_ptr = link.prev_link.load(Acquire); if Self::drop_unreachable_entries(&link.metadata, &link.data_block, &guard) == 0 && next_link_ptr.is_null() { debug_assert!(link.metadata.link.is_null(Relaxed)); let unlinked = if let Some(prev) = unsafe { prev_link_ptr.as_ref() } { prev.metadata.link.swap((None, Tag::None), Acquire).0 } else { self.metadata.link.swap((None, Tag::None), Acquire).0 }; debug_assert!(unlinked.is_some_and(Shared::release)); } else { next_link_ptr = link_ptr; } link_ptr = prev_link_ptr; } break; } link_ptr = next_link_ptr; } Self::drop_unreachable_entries(&self.metadata, data_block, &guard) } /// Drops unreachable entries. fn drop_unreachable_entries( metadata: &Metadata, data_block: &DataBlock, guard: &Guard, ) -> u32 { debug_assert_eq!(TYPE, INDEX); let mut dropped_bitmap = metadata.removed_bitmap.load(Relaxed); let current_epoch = guard.epoch(); for i in 0..LEN { if Epoch::try_from(*Self::read_cell(&metadata.partial_hash_array[i])) .is_ok_and(|e| e.in_same_generation(current_epoch)) { dropped_bitmap &= !(1_u32 << i); } } // Store ordering: `occupied_bitmap` -> `release` -> `removed_bitmap`. let occupied_bitmap = metadata.occupied_bitmap.load(Relaxed) & !dropped_bitmap; metadata.occupied_bitmap.store(occupied_bitmap, Release); let removed_bitmap = metadata.removed_bitmap.load(Relaxed) & !dropped_bitmap; metadata.removed_bitmap.store(removed_bitmap, Release); if removed_bitmap != 0 { guard.set_has_garbage(); } if needs_drop::<(K, V)>() { while dropped_bitmap != 0 { let index = dropped_bitmap.trailing_zeros(); Self::drop_entry(data_block, index as usize); dropped_bitmap -= 1_u32 << index; } } occupied_bitmap } /// Drops the data in place. #[inline] fn drop_entry(data_block: &DataBlock, index: usize) { unsafe { (*data_block[index].get()).as_mut_ptr().drop_in_place(); } } /// Removes the entry from the LRU linked list. #[inline] fn remove_from_lru_list(&self, entry_ptr: &EntryPtr) { debug_assert_eq!(TYPE, CACHE); debug_assert_ne!(entry_ptr.index, usize::MAX); debug_assert_ne!(entry_ptr.index, BUCKET_LEN); if entry_ptr.link_ptr.is_null() { #[allow(clippy::cast_possible_truncation)] let entry = entry_ptr.index as u8; let tail = self.metadata.removed_bitmap.load(Relaxed); if let Some(new_tail) = self.lru_list.remove(tail, entry) { self.metadata.removed_bitmap.store(new_tail, Relaxed); } } } /// Returns the partial hash value of the given hash. #[allow(clippy::cast_possible_truncation)] #[inline] const fn partial_hash(hash: u64) -> u8 { hash as u8 } /// Reads the data at the given index. #[inline] const fn read_data_block( data_block: &DataBlock, index: usize, ) -> (K, V) { unsafe { (*data_block.0[index].get()).as_ptr().read() } } /// Returns a pointer to the slot in the [`DataBlock`]. #[inline] const fn entry_ptr( data_block: &DataBlock, index: usize, ) -> *const (K, V) { Self::read_cell(&data_block.0[index]).as_ptr() } /// Returns a mutable pointer to the slot in the [`DataBlock`]. #[inline] const fn entry_mut_ptr( data_block: &DataBlock, index: usize, ) -> *mut (K, V) { unsafe { (*data_block.0[index].get()).as_mut_ptr() } } /// Reads the cell. #[inline] const fn read_cell(cell: &UnsafeCell) -> &T { unsafe { &*cell.get() } } /// Writes the cell. #[inline] fn write_cell R>(&self, cell: &UnsafeCell, f: F) -> R { debug_assert!(self.rw_lock.is_locked(Relaxed)); unsafe { f(&mut *cell.get()) } } } impl Bucket { /// Searches for an entry containing the key. /// /// Returns `None` if the key is not present. #[inline] pub(super) fn search_entry<'g, Q>( &self, data_block: NonNull>, key: &Q, hash: u64, ) -> Option<&'g (K, V)> where Q: Equivalent + ?Sized, { if self.len() != 0 { if let Some((entry, _)) = Self::search_data_block(&self.metadata, data_block_ref(data_block), key, hash) { return Some(entry); } let mut link_ptr = self.metadata.load_link(); while let Some(link) = link_ref(link_ptr) { if let Some((entry, _)) = Self::search_data_block(&link.metadata, &link.data_block, key, hash) { return Some(entry); } link_ptr = link.metadata.load_link(); } } None } /// Gets an [`EntryPtr`] pointing to the slot containing the key. /// /// Returns an invalid [`EntryPtr`] if the key is not present. #[inline] pub(crate) fn get_entry_ptr( &self, data_block: NonNull>, key: &Q, hash: u64, ) -> EntryPtr where Q: Equivalent + ?Sized, { if self.len() != 0 { if let Some((_, index)) = Self::search_data_block(&self.metadata, data_block_ref(data_block), key, hash) { return EntryPtr { link_ptr: ptr::null(), index, }; } let mut current_link_ptr = self.metadata.load_link(); while let Some(link) = link_ref(current_link_ptr) { if let Some((_, index)) = Self::search_data_block(&link.metadata, &link.data_block, key, hash) { return EntryPtr { link_ptr: current_link_ptr, index, }; } current_link_ptr = link.metadata.load_link(); } } EntryPtr::null() } /// Searches the supplied data block for the entry containing the key. #[inline] fn search_data_block<'g, Q, const LEN: usize>( metadata: &Metadata, data_block: &'g DataBlock, key: &Q, hash: u64, ) -> Option<(&'g (K, V), usize)> where Q: Equivalent + ?Sized, { let mut bitmap = if TYPE == INDEX { // Load ordering: `removed_bitmap` -> `acquire` -> `occupied_bitmap`. (!metadata.removed_bitmap.load(Acquire)) & metadata.occupied_bitmap.load(Acquire) } else { metadata.occupied_bitmap.load(Relaxed) }; // Expect that the loop is vectorized by the compiler. for i in 0..LEN { if *Self::read_cell(&metadata.partial_hash_array[i]) != Self::partial_hash(hash) { bitmap &= !(1_u32 << i); } } let mut offset = bitmap.trailing_zeros(); while offset != u32::BITS { let entry = unsafe { &*Self::entry_ptr(data_block, offset as usize) }; if key.equivalent(&entry.0) { return Some((entry, offset as usize)); } bitmap &= !(1_u32 << offset); offset = bitmap.trailing_zeros(); } None } } unsafe impl Send for Bucket {} unsafe impl Sync for Bucket { } impl Index for DataBlock { type Output = UnsafeCell>; #[inline] fn index(&self, index: usize) -> &Self::Output { &self.0[index] } } unsafe impl Send for DataBlock {} unsafe impl Sync for DataBlock {} impl Writer { /// Locks the [`Bucket`] asynchronously. #[inline] pub(crate) async fn lock_async<'g>( bucket: &'g Bucket, async_guard: &'g AsyncGuard, ) -> Option> { if bucket.rw_lock.lock_async_with(|| async_guard.reset()).await { // The `bucket` was not killed, and will not be killed until the `Writer` is dropped. // This guarantees that the `BucketArray` will survive as long as the `Writer` is alive. Some(Self::from_bucket(bucket)) } else { None } } /// Locks the [`Bucket`] synchronously. #[inline] pub(crate) fn lock_sync(bucket: &Bucket) -> Option> { if bucket.rw_lock.lock_sync() { Some(Self::from_bucket(bucket)) } else { None } } /// Tries to lock the [`Bucket`]. #[inline] pub(crate) fn try_lock( bucket: &Bucket, ) -> Result>, ()> { if bucket.rw_lock.try_lock() { Ok(Some(Self::from_bucket(bucket))) } else if bucket.rw_lock.is_poisoned(Relaxed) { Ok(None) } else { Err(()) } } /// Creates a new [`Writer`] from a [`Bucket`]. #[inline] pub(crate) const fn from_bucket(bucket: &Bucket) -> Writer { Writer { bucket_ptr: bucket_ptr(bucket), } } /// Marks the [`Bucket`] killed by poisoning the lock. #[inline] pub(super) fn kill(self) { debug_assert_eq!(self.len(), 0); debug_assert!(self.rw_lock.is_locked(Relaxed)); debug_assert!( TYPE != INDEX || self.metadata.removed_bitmap.load(Relaxed) == self.metadata.occupied_bitmap.load(Relaxed) ); let poisoned = self.rw_lock.poison_lock(); debug_assert!(poisoned); if (TYPE != INDEX || !needs_drop::<(K, V)>()) && !self.metadata.link.is_null(Relaxed) { // In case `TYPE == INDEX`, overflow buckets may contain removed entries that may be // still accessible to readers; those `(K, V)` that need `drop` should be dropped in // `drop_entries`. let mut link = self.metadata.link.swap((None, Tag::None), Acquire).0; while let Some(current) = link { link = current.metadata.link.swap((None, Tag::None), Acquire).0; let released = if TYPE == INDEX { current.release() } else { unsafe { current.drop_in_place() } }; debug_assert!(released); } } forget(self); } } impl Deref for Writer { type Target = Bucket; #[inline] fn deref(&self) -> &Self::Target { unsafe { self.bucket_ptr.as_ref() } } } impl Drop for Writer { #[inline] fn drop(&mut self) { self.rw_lock.release_lock(); } } unsafe impl Send for Writer {} unsafe impl Sync for Writer { } impl<'g, K, V, L: LruList, const TYPE: char> Reader { /// Locks the [`Bucket`] asynchronously. #[inline] pub(crate) async fn lock_async( bucket: &'g Bucket, async_guard: &AsyncGuard, ) -> Option> { if bucket .rw_lock .share_async_with(|| async_guard.reset()) .await { // The `bucket` was not killed, and will not be killed until the `Reader` is dropped. // This guarantees that the `BucketArray` will survive as long as the `Reader` is alive. Some(Reader { bucket_ptr: bucket_ptr(bucket), }) } else { None } } /// Locks the [`Bucket`] synchronously. /// /// Returns `None` if the [`Bucket`] has been killed or is empty. #[inline] pub(crate) fn lock_sync(bucket: &Bucket) -> Option> { if bucket.rw_lock.share_sync() { Some(Reader { bucket_ptr: bucket_ptr(bucket), }) } else { None } } } impl Deref for Reader { type Target = Bucket; #[inline] fn deref(&self) -> &Self::Target { unsafe { self.bucket_ptr.as_ref() } } } impl Drop for Reader { #[inline] fn drop(&mut self) { self.rw_lock.release_share(); } } unsafe impl Send for Reader {} unsafe impl Sync for Reader { } impl EntryPtr { /// Creates a new invalid [`EntryPtr`]. #[inline] pub(crate) const fn null() -> Self { Self { link_ptr: ptr::null(), index: BUCKET_LEN, } } /// Returns `true` if the [`EntryPtr`] points to, or has pointed to, an occupied entry. #[inline] pub(crate) const fn is_valid(&self) -> bool { self.index != BUCKET_LEN } /// Gets the partial hash value of the entry. /// /// The [`EntryPtr`] must point to a valid entry. #[inline] pub(crate) const fn partial_hash(&self, bucket: &Bucket) -> u8 { if let Some(link) = link_ref(self.link_ptr) { *Bucket::::read_cell(&link.metadata.partial_hash_array[self.index]) } else { *Bucket::::read_cell(&bucket.metadata.partial_hash_array[self.index]) } } /// Gets a reference to the entry. /// /// The [`EntryPtr`] must point to a valid entry. #[inline] pub(crate) const fn get<'e>( &self, data_block: NonNull>, ) -> &'e (K, V) { let entry_ptr = if let Some(link) = link_ref(self.link_ptr) { Bucket::::entry_ptr(&link.data_block, self.index) } else { Bucket::::entry_ptr(data_block_ref(data_block), self.index) }; unsafe { &(*entry_ptr) } } /// Gets a mutable reference to the entry. /// /// The associated [`Bucket`] must be locked, and the [`EntryPtr`] must point to a valid entry. #[inline] pub(crate) const fn get_mut( &mut self, data_block: NonNull>, _writer: &Writer, ) -> &mut (K, V) { let entry_ptr = if let Some(link) = link_ref(self.link_ptr) { Bucket::::entry_mut_ptr(&link.data_block, self.index) } else { Bucket::::entry_mut_ptr(data_block_ref(data_block), self.index) }; unsafe { &mut (*entry_ptr) } } /// Moves the [`EntryPtr`] to point to the next occupied entry. /// /// Returns `true` if it successfully found the next occupied entry. #[inline] pub(crate) fn move_to_next(&mut self, bucket: &Bucket) -> bool { if self.index != usize::MAX { if self.link_ptr.is_null() && self.next_entry::(&bucket.metadata) { return true; } while let Some(link) = link_ref(self.link_ptr) { if self.next_entry::(&link.metadata) { return true; } } // Fuse itself. self.index = usize::MAX; } false } /// Unlinks the [`LinkedBucket`] currently pointed to by this [`EntryPtr`] from the linked list. /// /// The associated [`Bucket`] must be locked. fn unlink(&mut self, link_head: &AtomicShared>, link: &LinkedBucket) { debug_assert_ne!(TYPE, INDEX); let prev_link_ptr = link.prev_link.load(Relaxed); let next = link.metadata.link.swap((None, Tag::None), Relaxed).0; if let Some(next) = next.as_ref() { // Go to the next `Link`. next.prev_link.store(prev_link_ptr, Relaxed); self.link_ptr = next.as_ptr(); self.index = LINKED_BUCKET_LEN; } else { // Fuse the `EntryPtr`. self.link_ptr = ptr::null(); self.index = usize::MAX; } let unlinked = if let Some(prev) = unsafe { prev_link_ptr.as_ref() } { prev.metadata.link.swap((next, Tag::None), Acquire).0 } else { link_head.swap((next, Tag::None), Acquire).0 }; if let Some(link) = unlinked { let dropped = unsafe { link.drop_in_place() }; debug_assert!(dropped); } } /// Moves this [`EntryPtr`] to the next occupied entry in the [`Bucket`]. /// /// Returns `false` if this currently points to the last entry. #[inline] fn next_entry(&mut self, metadata: &Metadata) -> bool { // Search for the next occupied entry. let current_index = if self.index == LEN { 0 } else { self.index + 1 }; if current_index < LEN { let bitmap = if TYPE == INDEX { // Load order: `removed_bitmap` -> `acquire` -> `occupied_bitmap`. (!metadata.removed_bitmap.load(Acquire) & metadata.occupied_bitmap.load(Acquire)) & (!((1_u32 << current_index) - 1)) } else { metadata.occupied_bitmap.load(Relaxed) & (!((1_u32 << current_index) - 1)) }; let next_index = bitmap.trailing_zeros() as usize; if next_index < LEN { self.index = next_index; return true; } } self.link_ptr = metadata.load_link(); self.index = LINKED_BUCKET_LEN; false } } impl Clone for EntryPtr { #[inline] fn clone(&self) -> Self { Self { link_ptr: self.link_ptr, index: self.index, } } } impl Debug for EntryPtr { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("EntryPtr") .field("link_ptr", &self.link_ptr) .field("index", &self.index) .finish() } } unsafe impl Send for EntryPtr {} unsafe impl Sync for EntryPtr {} impl LruList for () {} impl DoublyLinkedList { /// Reads the slot. #[inline] const fn read(&self, index: usize) -> (u8, u8) { unsafe { *self.0[index].get() } } /// Writes the slot. #[inline] fn write R>(&self, index: usize, f: F) -> R { unsafe { f(&mut *self.0[index].get()) } } } impl LruList for DoublyLinkedList { #[inline] fn evict(&self, tail: u32) -> Option<(u8, u32)> { if tail == 0 { None } else { let lru = self.read(tail as usize - 1).0; let new_tail = if tail - 1 == u32::from(lru) { // Reset the linked list. 0 } else { let new_lru = self.read(lru as usize).0; { #![allow(clippy::cast_possible_truncation)] self.write(new_lru as usize, |v| { v.1 = tail as u8 - 1; }); } self.write(tail as usize - 1, |v| { v.0 = new_lru; }); tail }; self.write(lru as usize, |v| { *v = (0, 0); }); Some((lru, new_tail)) } } #[inline] fn remove(&self, tail: u32, entry: u8) -> Option { if tail == 0 || (self.read(entry as usize) == (0, 0) && (self.read(0) != (entry, entry) || (tail != 1 && tail != u32::from(entry) + 1))) { // The linked list is empty, or the entry is not a part of the linked list. return None; } if self.read(entry as usize).0 == entry { // It is the head and the only entry of the linked list. debug_assert_eq!(tail, u32::from(entry) + 1); self.write(entry as usize, |v| { *v = (0, 0); }); return Some(0); } // Adjust `prev -> current`. let (prev, next) = self.read(entry as usize); debug_assert_eq!(self.read(prev as usize).1, entry); self.write(prev as usize, |v| { v.1 = next; }); // Adjust `next -> current`. debug_assert_eq!(self.read(next as usize).0, entry); self.write(next as usize, |v| { v.0 = prev; }); let new_tail = if tail == u32::from(entry) + 1 { // Update `head`. Some(u32::from(next) + 1) } else { None }; self.write(entry as usize, |v| { *v = (0, 0); }); new_tail } #[inline] fn promote(&self, tail: u32, entry: u8) -> Option { if tail == u32::from(entry) + 1 { // Nothing to do. return None; } else if tail == 0 { // The linked list is empty. self.write(entry as usize, |v| { *v = (entry, entry); }); return Some(u32::from(entry) + 1); } // Remove the entry from the linked list only if it is a part of it. if self.read(entry as usize) != (0, 0) || (self.read(0) == (entry, entry) && tail == 1) { // Adjust `prev -> current`. let (prev, next) = self.read(entry as usize); debug_assert_eq!(self.read(prev as usize).1, entry); self.write(prev as usize, |v| { v.1 = next; }); // Adjust `next -> current`. debug_assert_eq!(self.read(next as usize).0, entry); self.write(next as usize, |v| { v.0 = prev; }); } // Adjust `oldest -> head`. let oldest = self.read(tail as usize - 1).0; debug_assert_eq!(u32::from(self.read(oldest as usize).1) + 1, tail); self.write(oldest as usize, |v| { v.1 = entry; }); self.write(entry as usize, |v| { v.0 = oldest; }); // Adjust `head -> new head` self.write(tail as usize - 1, |v| { v.0 = entry; }); { #![allow(clippy::cast_possible_truncation)] self.write(entry as usize, |v| { v.1 = tail as u8 - 1; }); } // Update `head`. Some(u32::from(entry) + 1) } } unsafe impl Send for DoublyLinkedList {} unsafe impl Sync for DoublyLinkedList {} impl Metadata { /// Loads the linked bucket pointer. #[inline] fn load_link(&self) -> *const LinkedBucket { unsafe { self.link.load(Acquire, fake_ref(&self)).as_ptr_unchecked() } } } unsafe impl Send for Metadata {} unsafe impl Sync for Metadata {} impl LinkedBucket { /// Creates an empty [`LinkedBucket`]. #[inline] fn new(next: Option>>) -> Self { let mut bucket = Self { metadata: Metadata { link: AtomicShared::default(), occupied_bitmap: AtomicU32::default(), removed_bitmap: AtomicU32::default(), partial_hash_array: Default::default(), }, data_block: unsafe { #[allow(clippy::uninit_assumed_init)] MaybeUninit::uninit().assume_init() }, prev_link: AtomicPtr::default(), }; if let Some(next) = next { bucket.metadata.link = AtomicShared::from(next); } bucket } } impl Debug for LinkedBucket { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("LinkedBucket").finish() } } impl Drop for LinkedBucket { #[inline] fn drop(&mut self) { if needs_drop::<(K, V)>() { let mut occupied_bitmap = self.metadata.occupied_bitmap.load(Relaxed); while occupied_bitmap != 0 { let index = occupied_bitmap.trailing_zeros(); Bucket::::drop_entry(&self.data_block, index as usize); occupied_bitmap -= 1_u32 << index; } } } } /// Returns a pointer to a bucket. #[inline] const fn bucket_ptr( bucket: &Bucket, ) -> NonNull> { unsafe { NonNull::new_unchecked(from_ref(bucket).cast_mut()) } } /// Returns a reference to the data block. #[inline] const fn data_block_ref<'l, K, V, const LEN: usize>( data_block_ptr: NonNull>, ) -> &'l DataBlock { unsafe { data_block_ptr.as_ref() } } /// Returns a reference to the linked bucket that the pointer might point to. #[inline] const fn link_ref<'l, K, V>(ptr: *const LinkedBucket) -> Option<&'l LinkedBucket> { unsafe { ptr.as_ref() } } /// Returns a fake reference for passing a reference to `U` when it is ensured that the returned /// reference is never used. #[inline] const fn fake_ref<'l, T, U>(v: &T) -> &'l U { unsafe { &*ptr::from_ref(v).cast::() } } #[cfg(not(feature = "loom"))] #[cfg(test)] mod test { use super::*; use std::mem::MaybeUninit; use std::sync::atomic::AtomicPtr; use std::sync::atomic::Ordering::Relaxed; use proptest::prelude::*; use sdd::Shared; use tokio::sync::Barrier; #[cfg(not(miri))] static_assertions::assert_eq_size!(Bucket, [u8; BUCKET_LEN * 2]); #[cfg(not(miri))] static_assertions::assert_eq_size!(Bucket, [u8; BUCKET_LEN * 4]); proptest! { #[cfg_attr(miri, ignore)] #[test] fn evict_untracked(xs in 0..BUCKET_LEN * 2) { let data_block: DataBlock = unsafe { MaybeUninit::uninit().assume_init() }; let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let bucket: Bucket = Bucket::new(); for v in 0..xs { let writer = Writer::lock_sync(&bucket).unwrap(); let evicted = writer.evict_lru_head(data_block_ptr); assert_eq!(v >= BUCKET_LEN, evicted.is_some()); writer.insert(data_block_ptr, 0, (v, v)); assert_eq!(writer.metadata.removed_bitmap.load(Relaxed), 0); } } #[cfg_attr(miri, ignore)] #[test] fn evict_overflowed(xs in 1..BUCKET_LEN * 2) { let data_block: DataBlock = unsafe { MaybeUninit::uninit().assume_init() }; let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let bucket: Bucket = Bucket::new(); let writer = Writer::lock_sync(&bucket).unwrap(); for _ in 0..3 { for v in 0..xs { let entry_ptr = writer.insert(data_block_ptr, 0, (v, v)); writer.update_lru_tail(&entry_ptr); if v < BUCKET_LEN { assert_eq!( writer.metadata.removed_bitmap.load(Relaxed) as usize, v + 1 ); } assert_eq!( writer.lru_list.read (writer.metadata.removed_bitmap.load(Relaxed) as usize - 1) .0, 0 ); } let mut evicted_key = None; if xs >= BUCKET_LEN { let evicted = writer.evict_lru_head(data_block_ptr); assert!(evicted.is_some()); evicted_key = evicted.map(|(k, _)| k); } assert_ne!(writer.metadata.removed_bitmap.load(Relaxed), 0); for v in 0..xs { let mut entry_ptr = writer.get_entry_ptr(data_block_ptr, &v, 0); if entry_ptr.is_valid() { let _erased = writer.remove(data_block_ptr, &mut entry_ptr); } else { assert_eq!(v, evicted_key.unwrap()); } } assert_eq!(writer.metadata.removed_bitmap.load(Relaxed), 0); } } #[cfg_attr(miri, ignore)] #[test] fn evict_tracked(xs in 0..BUCKET_LEN * 2) { let data_block: DataBlock = unsafe { MaybeUninit::uninit().assume_init() }; let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let bucket: Bucket = Bucket::new(); for v in 0..xs { let writer = Writer::lock_sync(&bucket).unwrap(); let evicted = writer.evict_lru_head(data_block_ptr); assert_eq!(v >= BUCKET_LEN, evicted.is_some()); let mut entry_ptr = writer.insert(data_block_ptr, 0, (v, v)); writer.update_lru_tail(&entry_ptr); assert_eq!( writer.metadata.removed_bitmap.load(Relaxed) as usize, entry_ptr.index + 1 ); if v >= BUCKET_LEN { entry_ptr.index = xs % BUCKET_LEN; writer.update_lru_tail(&entry_ptr); assert_eq!( writer.metadata.removed_bitmap.load(Relaxed) as usize, entry_ptr.index + 1 ); let mut iterated = 1; let mut i = writer.lru_list.read(entry_ptr.index).1 as usize; while i != entry_ptr.index { iterated += 1; i = writer.lru_list.read(i).1 as usize; } assert_eq!(iterated, BUCKET_LEN); iterated = 1; i = writer.lru_list.read(entry_ptr.index).0 as usize; while i != entry_ptr.index { iterated += 1; i = writer.lru_list.read(i).0 as usize; } assert_eq!(iterated, BUCKET_LEN); } } } #[cfg_attr(miri, ignore)] #[test] fn removed(xs in 0..BUCKET_LEN) { let data_block: DataBlock = unsafe { MaybeUninit::uninit().assume_init() }; let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let bucket: Bucket = Bucket::new(); for v in 0..xs { let writer = Writer::lock_sync(&bucket).unwrap(); let entry_ptr = writer.insert(data_block_ptr, 0, (v, v)); writer.update_lru_tail(&entry_ptr); let mut iterated = 1; let mut i = writer.lru_list.read(entry_ptr.index).1 as usize; while i != entry_ptr.index { iterated += 1; i = writer.lru_list.read(i).1 as usize; } assert_eq!(iterated, v + 1); } for v in 0..xs { let writer = Writer::lock_sync(&bucket).unwrap(); let data_block_ptr = unsafe { NonNull::new_unchecked(from_ref(&data_block).cast_mut()) }; let entry_ptr = writer.get_entry_ptr(data_block_ptr, &v, 0); let mut iterated = 1; let mut i = writer.lru_list.read(entry_ptr.index).1 as usize; while i != entry_ptr.index { iterated += 1; i = writer.lru_list.read(i).1 as usize; } assert_eq!(iterated, xs - v); writer.remove_from_lru_list(&entry_ptr); } assert_eq!(bucket.metadata.removed_bitmap.load(Relaxed), 0); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn bucket_lock_sync() { let num_tasks = BUCKET_LEN + 2; let barrier = Shared::new(Barrier::new(num_tasks)); let data_block: Shared> = Shared::new(unsafe { MaybeUninit::uninit().assume_init() }); let mut bucket: Shared> = Shared::new(Bucket::new()); let mut data: [u64; 128] = [0; 128]; let mut task_handles = Vec::with_capacity(num_tasks); for task_id in 0..num_tasks { let barrier_clone = barrier.clone(); let data_block_clone = data_block.clone(); let bucket_clone = bucket.clone(); let data_ptr = AtomicPtr::new(&raw mut data); task_handles.push(tokio::spawn(async move { barrier_clone.wait().await; let partial_hash = (task_id % BUCKET_LEN).try_into().unwrap(); for i in 0..2048 { let writer = Writer::lock_sync(&bucket_clone).unwrap(); let mut sum: u64 = 0; for j in 0..128 { unsafe { sum += (*data_ptr.load(Relaxed))[j]; (*data_ptr.load(Relaxed))[j] = if i % 4 == 0 { 2 } else { 4 } }; } assert_eq!(sum % 256, 0); let data_block_ptr = unsafe { NonNull::new_unchecked(data_block_clone.as_ptr().cast_mut()) }; if i == 0 { assert!( writer .insert(data_block_ptr, partial_hash, (task_id, 0)) .is_valid() ); } else { assert_eq!( writer .search_entry(data_block_ptr, &task_id, partial_hash) .unwrap(), &(task_id, 0_usize) ); } drop(writer); let reader = Reader::lock_sync(&*bucket_clone).unwrap(); assert_eq!( reader .search_entry(data_block_ptr, &task_id, partial_hash) .unwrap(), &(task_id, 0_usize) ); } })); } for r in futures::future::join_all(task_handles).await { assert!(r.is_ok()); } let sum: u64 = data.iter().sum(); assert_eq!(sum % 256, 0); assert_eq!(bucket.len(), num_tasks); let data_block_ptr = unsafe { NonNull::new_unchecked(data_block.as_ptr().cast_mut()) }; for task_id in 0..num_tasks { assert_eq!( bucket.search_entry( data_block_ptr, &task_id, (task_id % BUCKET_LEN).try_into().unwrap(), ), Some(&(task_id, 0)) ); } let mut count = 0; let mut entry_ptr = EntryPtr::null(); while entry_ptr.move_to_next(&bucket) { count += 1; } assert_eq!(bucket.len(), count); entry_ptr = EntryPtr::null(); let writer = Writer::lock_sync(&bucket).unwrap(); while entry_ptr.move_to_next(&writer) { writer.remove( unsafe { NonNull::new_unchecked(data_block.as_ptr().cast_mut()) }, &mut entry_ptr, ); } assert_eq!(writer.len(), 0); writer.kill(); assert_eq!(bucket.len(), 0); assert!(Writer::lock_sync(unsafe { bucket.get_mut().unwrap() }).is_none()); } } scc-3.4.8/src/hash_table/bucket_array.rs000064400000000000000000000216771046102023000163270ustar 00000000000000use std::alloc::{Layout, alloc, alloc_zeroed, dealloc}; use std::mem::{align_of, needs_drop, size_of}; use std::panic::UnwindSafe; use std::ptr::NonNull; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::{Acquire, Relaxed}; use sdd::{AtomicShared, Guard, Tag}; use super::bucket::{BUCKET_LEN, Bucket, DataBlock, INDEX, LruList}; use crate::exit_guard::ExitGuard; /// [`BucketArray`] is a special purpose array to manage [`Bucket`] and [`DataBlock`]. pub struct BucketArray { buckets: NonNull>, data_blocks: NonNull>, array_len: usize, hash_offset: u8, sample_size: u8, bucket_ptr_offset: u16, linked_array: AtomicShared>, num_cleared_buckets: AtomicUsize, } impl BucketArray { /// Creates a new [`BucketArray`] of the given capacity. /// /// `capacity` is the desired number of entries, not the length of the bucket array. pub(crate) fn new( capacity: usize, linked_array: AtomicShared>, ) -> Self { let adjusted_capacity = capacity .min(1_usize << (usize::BITS - 2)) .next_power_of_two() .max(Self::minimum_capacity()); let array_len = adjusted_capacity / BUCKET_LEN; let log2_array_len = u8::try_from(array_len.trailing_zeros()).unwrap_or(0); assert_eq!(1_usize << log2_array_len, array_len); // `2 -> 1`, `4 -> 2`, `8 -> 4`, `1024 -> 16`, and `2^58 -> 64`. let sample_size = log2_array_len.next_power_of_two(); let alignment = align_of::>(); let layout = Self::bucket_array_layout(array_len); unsafe { let Some(unaligned_bucket_array_ptr) = NonNull::new(alloc_zeroed(layout)) else { panic!("memory allocation failure: {layout:?}"); }; let bucket_array_ptr_offset = unaligned_bucket_array_ptr.align_offset(alignment); assert_eq!( (unaligned_bucket_array_ptr.addr().get() + bucket_array_ptr_offset) % alignment, 0 ); #[allow(clippy::cast_ptr_alignment)] // The alignment was just asserted. let buckets = unaligned_bucket_array_ptr .add(bucket_array_ptr_offset) .cast::>(); let bucket_array_ptr_offset = u16::try_from(bucket_array_ptr_offset).unwrap_or(0); let data_block_array_layout = Layout::from_size_align( size_of::>() * array_len, align_of::<[DataBlock; 0]>(), ) .unwrap(); // In case the below data block allocation fails, deallocate the bucket array. let alloc_guard = ExitGuard::new((), |()| { dealloc(unaligned_bucket_array_ptr.cast::().as_ptr(), layout); }); let Some(data_blocks) = NonNull::new(alloc(data_block_array_layout).cast::>()) else { panic!("memory allocation failure: {data_block_array_layout:?}"); }; alloc_guard.forget(); #[cfg(feature = "loom")] for i in 0..array_len { // `loom` types need proper initialization. buckets.add(i).write(Bucket::new()); } Self { buckets, data_blocks, array_len, hash_offset: u8::try_from(u64::BITS).unwrap_or(64) - log2_array_len, sample_size, bucket_ptr_offset: bucket_array_ptr_offset, linked_array, num_cleared_buckets: AtomicUsize::new(0), } } } /// Returns the number of [`Bucket`] instances in the [`BucketArray`]. #[inline] pub(crate) const fn len(&self) -> usize { self.array_len } /// Returns the number of entry slots in the bucket array. #[inline] pub(crate) const fn num_slots(&self) -> usize { self.array_len * BUCKET_LEN } /// Calculates the [`Bucket`] index for the hash value. #[allow(clippy::cast_possible_truncation)] // Intended truncation. #[inline] pub(crate) const fn bucket_index(&self, hash: u64) -> usize { // Take the upper n-bits to make sure that a single bucket is spread across a few adjacent // buckets when the hash table is resized. (hash >> self.hash_offset) as usize } /// Returns the minimum capacity. #[inline] pub(crate) const fn minimum_capacity() -> usize { BUCKET_LEN << 1 } /// Returns a reference to its rehashing metadata. #[inline] pub(crate) const fn rehashing_metadata(&self) -> &AtomicUsize { &self.num_cleared_buckets } /// Checks if the key is eligible to initiate sampling. #[allow(clippy::cast_possible_truncation)] // Intended truncation. #[inline] pub(crate) const fn initiate_sampling(&self, hash: u64) -> bool { (hash as u8 & (self.sample_size - 1)) == 0 } /// Returns the recommended sampling size. #[inline] pub(crate) const fn sample_size(&self) -> usize { // `Log2(array_len)`: if `array_len` is sufficiently large, expected error of size // estimation is `~3%`. self.sample_size as usize } /// Returns a reference to a [`Bucket`] at the given position. #[inline] pub(crate) const fn bucket(&self, index: usize) -> &Bucket { debug_assert!(index < self.len()); unsafe { self.buckets.add(index).as_ref() } } /// Returns a pointer to a [`DataBlock`] at the given position. #[inline] pub(crate) const fn data_block(&self, index: usize) -> NonNull> { debug_assert!(index < self.len()); unsafe { self.data_blocks.add(index) } } /// Returns `true` if an linked bucket array exists. #[inline] pub(crate) fn has_linked_array(&self) -> bool { !self.linked_array.is_null(Acquire) } /// Returns a reference to the linked bucket array pointer. #[inline] pub(crate) const fn linked_array_var(&self) -> &AtomicShared> { &self.linked_array } /// Returns a reference to the linked bucket array. #[inline] pub(crate) fn linked_array<'g>( &self, guard: &'g Guard, ) -> Option<&'g BucketArray> { unsafe { self.linked_array.load(Acquire, guard).as_ref_unchecked() } } /// Calculates the layout of the memory block for an array of `T`. #[inline] const fn bucket_array_layout(array_len: usize) -> Layout { let size_of_t = size_of::>(); let allocation_size = (array_len + 1) * size_of_t; // Intentionally misaligned in order to take full advantage of demand paging. unsafe { Layout::from_size_align_unchecked(allocation_size, 1) } } } impl Drop for BucketArray { fn drop(&mut self) { if !self.linked_array.is_null(Relaxed) { unsafe { self.linked_array .swap((None, Tag::None), Relaxed) .0 .map(|a| a.drop_in_place()); } } let num_cleared_buckets = if TYPE == INDEX && needs_drop::<(K, V)>() { // Removed entries in non-overflow buckets are neither relocated nor dropped. 0 } else { self.num_cleared_buckets.load(Relaxed) }; if num_cleared_buckets < self.array_len { for index in num_cleared_buckets..self.array_len { self.bucket(index).drop_entries(self.data_block(index)); } } #[cfg(feature = "loom")] for i in 0..self.array_len { // `loom` types need proper cleanup. drop(unsafe { self.buckets.add(i).read() }); } unsafe { dealloc( self.buckets .cast::() .sub(self.bucket_ptr_offset as usize) .as_ptr(), Self::bucket_array_layout(self.array_len), ); dealloc( self.data_blocks.cast::().as_ptr(), Layout::from_size_align( size_of::>() * self.array_len, align_of::<[DataBlock; 0]>(), ) .unwrap(), ); } } } unsafe impl Send for BucketArray {} unsafe impl Sync for BucketArray { } impl UnwindSafe for BucketArray { } scc-3.4.8/src/hash_table.rs000064400000000000000000001722471046102023000136540ustar 00000000000000pub mod bucket; pub mod bucket_array; use std::hash::{BuildHasher, Hash}; use std::mem::forget; use std::ops::Deref; use std::ptr::{self, NonNull, from_ref}; #[cfg(not(feature = "loom"))] use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use bucket::{BUCKET_LEN, CACHE, DataBlock, EntryPtr, INDEX, LruList, Reader, Writer}; use bucket_array::BucketArray; #[cfg(feature = "loom")] use loom::sync::atomic::AtomicUsize; use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; use super::Equivalent; use super::async_helper::AsyncGuard; use super::exit_guard::ExitGuard; use super::hash_table::bucket::Bucket; /// `HashTable` defines common functions for hash table implementations. pub(super) trait HashTable where K: Eq + Hash, H: BuildHasher, { /// Returns the hash value of the key. #[inline] fn hash(&self, key: &Q) -> u64 where Q: Equivalent + Hash + ?Sized, { self.hasher().hash_one(key) } /// Returns its [`BuildHasher`]. fn hasher(&self) -> &H; /// Returns a reference to the [`BucketArray`] pointer. fn bucket_array_var(&self) -> &AtomicShared>; /// Returns a reference to the current [`BucketArray`]. fn bucket_array<'g>(&self, guard: &'g Guard) -> Option<&'g BucketArray> { unsafe { self.bucket_array_var() .load(Acquire, guard) .as_ref_unchecked() } } /// Passes the bucket array to the garbage collector associated with the hash table type. #[inline] fn defer_reclaim(&self, bucket_array: Shared>, _guard: &Guard) { drop(bucket_array); } /// Calculates the bucket index from the supplied key. #[inline] fn calculate_bucket_index(&self, key: &Q) -> usize where Q: Equivalent + Hash + ?Sized, { unsafe { self.bucket_array_var() .load(Acquire, &Guard::new()) .as_ref_unchecked() .map_or(0, |a| a.bucket_index(self.hash(key))) } } /// Returns a reference to a variable containing the minimum allowed capacity. fn minimum_capacity_var(&self) -> &AtomicUsize; /// Returns the current minimum allowed capacity. fn minimum_capacity(&self) -> usize { self.minimum_capacity_var().load(Relaxed) & (!RESIZING) } /// Returns the maximum capacity. /// /// The maximum capacity must be a power of `2`. fn maximum_capacity(&self) -> usize { MAXIMUM_CAPACITY_LIMIT } /// Reserves the specified capacity. /// /// Returns the actually allocated capacity. Return `0` if the sum of the current minimum /// capacity and the additional capacity exceeds [`Self::maximum_capacity`]. fn reserve_capacity(&self, additional_capacity: usize) -> usize { let mut current_minimum_capacity = self.minimum_capacity_var().load(Relaxed); loop { if additional_capacity > self.maximum_capacity() - (current_minimum_capacity & (!RESIZING)) { return 0; } match self.minimum_capacity_var().compare_exchange_weak( current_minimum_capacity, additional_capacity + current_minimum_capacity, Relaxed, Relaxed, ) { Ok(_) => { let guard = Guard::new(); if let Some(current_array) = self.bucket_array(&guard) { if !current_array.has_linked_array() { self.try_resize(current_array, 0, &guard); } } return additional_capacity; } Err(actual) => current_minimum_capacity = actual, } } } /// Returns a reference to the bucket array. /// /// Allocates a new one if no bucket array has been allocated. #[inline] fn get_or_create_bucket_array<'g>(&self, guard: &'g Guard) -> &'g BucketArray { if let Some(current_array) = self.bucket_array(guard) { current_array } else { self.allocate_bucket_array(guard) } } /// Allocates a new bucket array. fn allocate_bucket_array<'g>(&self, guard: &'g Guard) -> &'g BucketArray { unsafe { let capacity = self.minimum_capacity(); let allocated = Shared::new_unchecked(BucketArray::new(capacity, AtomicShared::null())); match self.bucket_array_var().compare_exchange( Ptr::null(), (Some(allocated), Tag::None), AcqRel, Acquire, guard, ) { Ok((_, ptr)) | Err((_, ptr)) => ptr.as_ref_unchecked().unwrap_unchecked(), } } } /// Returns the number of entry slots. #[inline] fn num_slots(&self, guard: &Guard) -> usize { if let Some(current_array) = self.bucket_array(guard) { current_array.num_slots() } else { 0 } } /// Returns the number of entries. /// /// In case there are more than `usize::MAX` entries, it returns `usize::MAX`. fn num_entries(&self, guard: &Guard) -> usize { let mut num_entries: usize = 0; if let Some(current_array) = self.bucket_array(guard) { if let Some(old_array) = current_array.linked_array(guard) { self.incremental_rehash_sync::(current_array, guard); for i in 0..old_array.len() { num_entries = num_entries.saturating_add(old_array.bucket(i).len()); } } for i in 0..current_array.len() { num_entries = num_entries.saturating_add(current_array.bucket(i).len()); } if num_entries == 0 && self.minimum_capacity() == 0 { self.try_resize(current_array, 0, guard); } } num_entries } /// Returns `true` if a valid entry is found. fn has_entry(&self, guard: &Guard) -> bool { if let Some(current_array) = self.bucket_array(guard) { if let Some(old_array) = current_array.linked_array(guard) { self.incremental_rehash_sync::(current_array, guard); for i in 0..old_array.len() { if old_array.bucket(i).len() != 0 { return true; } } } for i in 0..current_array.len() { if current_array.bucket(i).len() != 0 { return true; } } if self.minimum_capacity() == 0 { self.try_resize(current_array, 0, guard); } } false } /// Estimates the number of entries by sampling buckets. #[inline] fn sample(current_array: &BucketArray, sampling_index: usize) -> usize { let sample_size = current_array.sample_size(); let sample_1 = sampling_index & (!(sample_size - 1)); let sample_2 = if sample_1 == 0 { current_array.len() - sample_size } else { 0 }; let mut num_entries = 0; for i in (sample_1..sample_1 + sample_size).chain(sample_2..(sample_2 + sample_size)) { num_entries += current_array.bucket(i).len(); } num_entries * (current_array.len() / (sample_size * 2)) } /// Peeks an entry from the [`HashTable`]. #[inline] fn peek_entry<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<&'g (K, V)> where Q: Equivalent + Hash + ?Sized, { debug_assert_eq!(TYPE, INDEX); let hash = self.hash(key); let mut current_array_ptr = self.bucket_array_var().load(Acquire, guard); while let Some(current_array) = unsafe { current_array_ptr.as_ref_unchecked() } { if let Some(old_array) = current_array.linked_array(guard) { self.incremental_rehash_sync::(current_array, guard); let index = old_array.bucket_index(hash); if let Some(entry) = old_array .bucket(index) .search_entry(old_array.data_block(index), key, hash) { return Some(entry); } } let index = current_array.bucket_index(hash); if let Some(entry) = current_array .bucket(index) .search_entry(current_array.data_block(index), key, hash) { return Some(entry); } let new_current_array_ptr = self.bucket_array_var().load(Acquire, guard); if current_array_ptr == new_current_array_ptr { break; } current_array_ptr = new_current_array_ptr; } None } /// Reads an entry asynchronously from the [`HashTable`] with a shared lock acquired on the /// bucket. #[inline] async fn reader_async R>(&self, key: &Q, f: F) -> Option where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let async_guard = AsyncGuard::default(); while let Some(current_array) = async_guard.load_unchecked(self.bucket_array_var(), Acquire) { if current_array.has_linked_array() { self.incremental_rehash_async(current_array, &async_guard) .await; if !self .dedup_bucket_async( current_array, current_array.bucket_index(hash), &async_guard, ) .await { continue; } } let bucket_index = current_array.bucket_index(hash); let bucket = current_array.bucket(bucket_index); if let Some(reader) = Reader::lock_async(bucket, &async_guard).await { if let Some(entry) = reader.search_entry(current_array.data_block(bucket_index), key, hash) { return Some(f(&entry.0, &entry.1)); } break; } } None } /// Reads an entry synchronously from the [`HashTable`] with a shared lock acquired on the /// bucket. #[inline] fn reader_sync R>(&self, key: &Q, f: F) -> Option where Q: Equivalent + Hash + ?Sized, { let hash = self.hash(key); let guard = Guard::new(); while let Some(current_array) = self.bucket_array(&guard) { let index = current_array.bucket_index(hash); if let Some(old_array) = current_array.linked_array(&guard) { self.incremental_rehash_sync::(current_array, &guard); self.dedup_bucket_sync::(current_array, old_array, index); } let bucket = current_array.bucket(index); if let Some(reader) = Reader::lock_sync(bucket) { if let Some(entry) = reader.search_entry(current_array.data_block(index), key, hash) { return Some(f(&entry.0, &entry.1)); } break; } } None } /// Returns a [`LockedBucket`] for writing an entry asynchronously. /// /// If the container is empty, a new bucket array is allocated. #[inline] async fn writer_async(&self, hash: u64) -> LockedBucket { let async_guard = AsyncGuard::default(); if let Some(locked_bucket) = self.try_optional_writer::(hash, async_guard.guard()) { return locked_bucket; } loop { let current_array = self.get_or_create_bucket_array(async_guard.guard()); if current_array.has_linked_array() { self.incremental_rehash_async(current_array, &async_guard) .await; if !self .dedup_bucket_async( current_array, current_array.bucket_index(hash), &async_guard, ) .await { continue; } } let bucket_index = current_array.bucket_index(hash); let bucket = current_array.bucket(bucket_index); if (TYPE != CACHE || current_array.num_slots() < self.maximum_capacity()) && bucket.len() >= BUCKET_LEN - 1 && current_array.initiate_sampling(hash) { self.try_enlarge( current_array, bucket_index, bucket.len(), async_guard.guard(), ); } if let Some(writer) = Writer::lock_async(bucket, &async_guard).await { return LockedBucket { writer, data_block: current_array.data_block(bucket_index), bucket_index, bucket_array: into_non_null(current_array), }; } } } /// Returns a [`LockedBucket`] for writing an entry synchronously. /// /// If the container is empty, a new bucket array is allocated. #[inline] fn writer_sync(&self, hash: u64) -> LockedBucket { let guard = Guard::new(); loop { let current_array = self.get_or_create_bucket_array(&guard); let bucket_index = current_array.bucket_index(hash); if let Some(old_array) = current_array.linked_array(&guard) { self.incremental_rehash_sync::(current_array, &guard); self.dedup_bucket_sync::(current_array, old_array, bucket_index); } let bucket = current_array.bucket(bucket_index); if (TYPE != CACHE || current_array.num_slots() < self.maximum_capacity()) && bucket.len() >= BUCKET_LEN - 1 && current_array.initiate_sampling(hash) { self.try_enlarge(current_array, bucket_index, bucket.len(), &guard); } if let Some(writer) = Writer::lock_sync(bucket) { return LockedBucket { writer, data_block: current_array.data_block(bucket_index), bucket_index, bucket_array: into_non_null(current_array), }; } } } /// Returns a [`LockedBucket`] for writing an entry asynchronously. /// /// If the container is empty, `None` is returned. #[inline] async fn optional_writer_async(&self, hash: u64) -> Option> { let async_guard = AsyncGuard::default(); if let Some(locked_bucket) = self.try_optional_writer::(hash, async_guard.guard()) { return Some(locked_bucket); } while let Some(current_array) = async_guard.load_unchecked(self.bucket_array_var(), Acquire) { if current_array.has_linked_array() { self.incremental_rehash_async(current_array, &async_guard) .await; if !self .dedup_bucket_async( current_array, current_array.bucket_index(hash), &async_guard, ) .await { continue; } } let bucket_index = current_array.bucket_index(hash); let bucket = current_array.bucket(bucket_index); if let Some(writer) = Writer::lock_async(bucket, &async_guard).await { return Some(LockedBucket { writer, data_block: current_array.data_block(bucket_index), bucket_index, bucket_array: into_non_null(current_array), }); } } None } /// Returns a [`LockedBucket`] for writing an entry synchronously. /// /// If the container is empty, `None` is returned. #[inline] fn optional_writer_sync(&self, hash: u64) -> Option> { let guard = Guard::new(); while let Some(current_array) = self.bucket_array(&guard) { let bucket_index = current_array.bucket_index(hash); if let Some(old_array) = current_array.linked_array(&guard) { self.incremental_rehash_sync::(current_array, &guard); self.dedup_bucket_sync::(current_array, old_array, bucket_index); } let bucket = current_array.bucket(bucket_index); if let Some(writer) = Writer::lock_sync(bucket) { return Some(LockedBucket { writer, data_block: current_array.data_block(bucket_index), bucket_index, bucket_array: into_non_null(current_array), }); } } None } /// Tries to returns a [`LockedBucket`] for writing an entry. #[inline] fn try_optional_writer( &self, hash: u64, guard: &Guard, ) -> Option> { if let Some(current_array) = self.bucket_array(guard) { if current_array.has_linked_array() { return None; } let bucket_index = current_array.bucket_index(hash); let bucket = current_array.bucket(bucket_index); if CHECK_SIZE && bucket.len() >= BUCKET_LEN { return None; } if let Ok(Some(writer)) = Writer::try_lock(bucket) { return Some(LockedBucket { writer, data_block: current_array.data_block(bucket_index), bucket_index, bucket_array: into_non_null(current_array), }); } } None } /// Iterates over all the buckets in the [`HashTable`] asynchronously. /// /// This method stops iterating when the closure returns `false`. #[inline] async fn for_each_reader_async(&self, mut f: F) where F: FnMut(Reader, NonNull>) -> bool, { let async_guard = AsyncGuard::default(); let mut start_index = 0; let mut prev_len = 0; while let Some(current_array) = async_guard.load_unchecked(self.bucket_array_var(), Acquire) { // In case the method is repeating the routine, iterate over entries from the middle of // the array. start_index = if prev_len == 0 || prev_len == current_array.len() { start_index } else { from_index_to_range(prev_len, current_array.len(), start_index).0 }; prev_len = current_array.len(); while start_index < current_array.len() { if current_array.has_linked_array() { self.incremental_rehash_async(current_array, &async_guard) .await; if !self .dedup_bucket_async(current_array, start_index, &async_guard) .await { // Retry the operation since there is a possibility that the current bucket // array was replaced by a new one. break; } } let bucket = current_array.bucket(start_index); if let Some(reader) = Reader::lock_async(bucket, &async_guard).await { if !async_guard.check_ref(self.bucket_array_var(), current_array, Acquire) { // `current_array` is no longer the current one. break; } let data_block = current_array.data_block(start_index); if !f(reader, data_block) { return; } } else { // `current_array` is no longer the current one. break; } start_index += 1; } if start_index == current_array.len() { break; } } } /// Iterates over all the buckets in the [`HashTable`] synchronously. /// /// This method stops iterating when the closure returns `false`. #[inline] fn for_each_reader_sync(&self, guard: &Guard, mut f: F) where F: FnMut(Reader, NonNull>) -> bool, { let mut start_index = 0; let mut prev_len = 0; while let Some(current_array) = self.bucket_array(guard) { // In case the method is repeating the routine, iterate over entries from the middle of // the array. start_index = if prev_len == 0 || prev_len == current_array.len() { start_index } else { from_index_to_range(prev_len, current_array.len(), start_index).0 }; prev_len = current_array.len(); while start_index < current_array.len() { let index = start_index; if let Some(old_array) = current_array.linked_array(guard) { self.incremental_rehash_sync::(current_array, guard); self.dedup_bucket_sync::(current_array, old_array, index); } let bucket = current_array.bucket(index); if let Some(reader) = Reader::lock_sync(bucket) { let data_block = current_array.data_block(index); if !f(reader, data_block) { return; } } else { // `current_array` is no longer the current one. break; } start_index += 1; } if start_index == current_array.len() { break; } } } /// Iterates over all the buckets in the [`HashTable`]. /// /// This method stops iterating when the closure returns `true`. #[inline] async fn for_each_writer_async( &self, mut start_index: usize, expected_array_len: usize, mut f: F, ) where F: FnMut(LockedBucket, &mut bool) -> bool, { let async_guard = AsyncGuard::default(); let mut prev_len = expected_array_len; let mut removed = false; while let Some(current_array) = async_guard.load_unchecked(self.bucket_array_var(), Acquire) { // In case the method is repeating the routine, iterate over entries from the middle of // the array. let current_array_len = current_array.len(); start_index = if prev_len == 0 || prev_len == current_array_len { start_index } else { from_index_to_range(prev_len, current_array_len, start_index).0 }; prev_len = current_array_len; while start_index < current_array_len { let bucket_index = start_index; if current_array.has_linked_array() { self.incremental_rehash_async(current_array, &async_guard) .await; if !self .dedup_bucket_async(current_array, bucket_index, &async_guard) .await { // Retry the operation since there is a possibility that the current bucket // array was replaced by a new one. break; } } let bucket = current_array.bucket(bucket_index); if let Some(writer) = Writer::lock_async(bucket, &async_guard).await { if !async_guard.check_ref(self.bucket_array_var(), current_array, Acquire) { // `current_array` is no longer the current one. break; } let locked_bucket = LockedBucket { writer, data_block: current_array.data_block(bucket_index), bucket_index, bucket_array: into_non_null(current_array), }; let stop = f(locked_bucket, &mut removed); if stop { // Stop iterating over buckets. start_index = current_array_len; break; } } else { // `current_array` is no longer the current one. break; } start_index += 1; } if start_index == current_array_len { break; } } if removed { if TYPE == INDEX { async_guard.guard().accelerate(); } if let Some(current_array) = self.bucket_array(async_guard.guard()) { self.try_shrink(current_array, 0, async_guard.guard()); } } } /// Iterates over all the buckets in the [`HashTable`]. /// /// This methods stops iterating when the closure returns `true`. #[inline] fn for_each_writer_sync( &self, mut start_index: usize, expected_array_len: usize, guard: &Guard, mut f: F, ) where F: FnMut(LockedBucket, &mut bool) -> bool, { let mut prev_len = expected_array_len; let mut removed = false; while let Some(current_array) = self.bucket_array(guard) { // In case the method is repeating the routine, iterate over entries from the middle of // the array. let current_array_len = current_array.len(); start_index = if prev_len == 0 || prev_len == current_array_len { start_index } else { from_index_to_range(prev_len, current_array_len, start_index).0 }; prev_len = current_array_len; while start_index < current_array_len { let bucket_index = start_index; if let Some(old_array) = current_array.linked_array(guard) { self.incremental_rehash_sync::(current_array, guard); self.dedup_bucket_sync::(current_array, old_array, bucket_index); } let bucket = current_array.bucket(bucket_index); if let Some(writer) = Writer::lock_sync(bucket) { let locked_bucket = LockedBucket { writer, data_block: current_array.data_block(bucket_index), bucket_index, bucket_array: into_non_null(current_array), }; let stop = f(locked_bucket, &mut removed); if stop { // Stop iterating over buckets. start_index = current_array_len; break; } } else { // `current_array` is no longer the current one. break; } start_index += 1; } if start_index == current_array_len { break; } } if removed { if TYPE == INDEX { guard.accelerate(); } if let Some(current_array) = self.bucket_array(guard) { self.try_shrink(current_array, 0, guard); } } } /// Tries to reserve a [`Bucket`] and returns a [`LockedBucket`]. #[inline] fn try_reserve_bucket(&self, hash: u64, guard: &Guard) -> Option> { loop { let current_array = self.get_or_create_bucket_array(guard); let bucket_index = current_array.bucket_index(hash); if let Some(old_array) = current_array.linked_array(guard) { self.incremental_rehash_sync::(current_array, guard); if !self.dedup_bucket_sync::(current_array, old_array, bucket_index) { return None; } } let mut bucket = current_array.bucket(bucket_index); if (TYPE != CACHE || current_array.num_slots() < self.maximum_capacity()) && bucket.len() >= BUCKET_LEN - 1 && current_array.initiate_sampling(hash) { self.try_enlarge(current_array, bucket_index, bucket.len(), guard); bucket = current_array.bucket(bucket_index); } let Ok(writer) = Writer::try_lock(bucket) else { return None; }; if let Some(writer) = writer { return Some(LockedBucket { writer, data_block: current_array.data_block(bucket_index), bucket_index, bucket_array: into_non_null(current_array), }); } } } /// Deduplicates buckets that may share the same hash values asynchronously. /// /// Returns `false` if the old buckets may remain in the old bucket array, or the whole /// operation has to be retried due to an ABA problem. /// /// # Note /// /// There is a possibility of an ABA problem where the bucket array was deallocated and a new /// bucket array of a different size has been allocated in the same memory region. To avoid /// this problem, the method returns `false` if it finds a killed bucket and the task was /// suspended. async fn dedup_bucket_async<'g>( &self, current_array: &'g BucketArray, index: usize, async_guard: &'g AsyncGuard, ) -> bool { if !async_guard.check_ref(self.bucket_array_var(), current_array, Acquire) { // A new bucket array was created in the meantime. return false; } if let Some(old_array) = async_guard.load_unchecked(current_array.linked_array_var(), Acquire) { let range = from_index_to_range(current_array.len(), old_array.len(), index); for old_index in range.0..range.1 { let bucket = old_array.bucket(old_index); let writer = Writer::lock_async(bucket, async_guard).await; if let Some(writer) = writer { self.relocate_bucket_async( current_array, old_array, old_index, writer, async_guard, ) .await; } else if !async_guard.has_guard() { // The bucket was killed and the guard has been invalidated. Validating the // reference is not sufficient in this case since the current bucket array could // have been replaced with a new one. return false; } // The old bucket array was removed, no point in trying to move entries from it. if !current_array.has_linked_array() { break; } } } true } /// Deduplicates buckets that may share the same hash values synchronously. /// /// Returns `true` if the corresponding entries were successfully moved. fn dedup_bucket_sync<'g, const TRY_LOCK: bool>( &self, current_array: &'g BucketArray, old_array: &'g BucketArray, index: usize, ) -> bool { let range = from_index_to_range(current_array.len(), old_array.len(), index); for old_index in range.0..range.1 { let bucket = old_array.bucket(old_index); let writer = if TRY_LOCK { let Ok(writer) = Writer::try_lock(bucket) else { return false; }; writer } else { Writer::lock_sync(bucket) }; if let Some(writer) = writer { if !self.relocate_bucket_sync::( current_array, old_array, old_index, writer, ) { return false; } } } true } /// Relocates the bucket to the current bucket array. async fn relocate_bucket_async<'g>( &self, current_array: &'g BucketArray, old_array: &'g BucketArray, old_index: usize, old_writer: Writer, async_guard: &'g AsyncGuard, ) { if old_writer.len() == 0 { // Instantiate a guard while the lock is held to ensure that the bucket arrays are not // dropped. async_guard.guard(); old_writer.kill(); return; } // Lock the target buckets. let (target_index, end_target_index) = from_index_to_range(old_array.len(), current_array.len(), old_index); for i in target_index..end_target_index { let writer = unsafe { Writer::lock_async(current_array.bucket(i), async_guard) .await .unwrap_unchecked() }; forget(writer); } // It may seem inefficient to reevaluate the same values, but it is beneficial for reducing // the `Future` size. let Some(old_array) = current_array.linked_array(async_guard.guard()) else { return; }; let (target_index, end_target_index) = from_index_to_range(old_array.len(), current_array.len(), old_index); let unlock = ExitGuard::new( (current_array, target_index, end_target_index), |(current_array, target_index, end_target_index)| { for i in target_index..end_target_index { let writer = Writer::from_bucket(current_array.bucket(i)); drop(writer); } }, ); self.relocate_bucket(unlock.0, unlock.1, old_array, old_index, &old_writer); drop(unlock); // Instantiate a guard while the lock is held to ensure that the bucket arrays are not // dropped. async_guard.guard(); old_writer.kill(); } /// Relocates the bucket to the current bucket array. /// /// Returns `false` if locking failed. fn relocate_bucket_sync<'g, const TRY_LOCK: bool>( &self, current_array: &'g BucketArray, old_array: &'g BucketArray, old_index: usize, old_writer: Writer, ) -> bool { if old_writer.len() == 0 { old_writer.kill(); return true; } let (target_index, end_target_index) = from_index_to_range(old_array.len(), current_array.len(), old_index); // Lock the target buckets. for i in target_index..end_target_index { let writer = if TRY_LOCK { let Ok(Some(writer)) = Writer::try_lock(current_array.bucket(i)) else { for j in target_index..i { let writer = Writer::from_bucket(current_array.bucket(j)); drop(writer); } return false; }; writer } else { unsafe { Writer::lock_sync(current_array.bucket(i)).unwrap_unchecked() } }; forget(writer); } let unlock = ExitGuard::new((), |()| { for i in target_index..end_target_index { let writer = Writer::from_bucket(current_array.bucket(i)); drop(writer); } }); self.relocate_bucket( current_array, target_index, old_array, old_index, &old_writer, ); drop(unlock); old_writer.kill(); true } /// Relocates entries from the old bucket to the corresponding buckets in the current bucket /// array. /// /// This assumes that all the target buckets are locked. fn relocate_bucket( &self, current_array: &BucketArray, target_index: usize, old_array: &BucketArray, old_index: usize, old_writer: &Writer, ) { // Need to pre-allocate slots if the container is shrinking or the old bucket overflows, // because incomplete relocation of entries may result in duplicate key problems. let pre_allocate_slots = old_array.len() > current_array.len() || old_writer.len() > BUCKET_LEN; let old_data_block = old_array.data_block(old_index); let mut entry_ptr = EntryPtr::null(); let mut position = 0; let mut dist = [0_u32; 8]; let mut extended_dist: Vec = Vec::new(); let mut hash_data = [0_u64; BUCKET_LEN]; // Collect data for relocation. while entry_ptr.move_to_next(old_writer) { let (offset, hash) = if old_array.len() >= current_array.len() { (0, u64::from(entry_ptr.partial_hash(&**old_writer))) } else { let hash = self.hash(&entry_ptr.get_mut(old_data_block, old_writer).0); let new_index = current_array.bucket_index(hash); debug_assert!(new_index - target_index < (current_array.len() / old_array.len())); (new_index - target_index, hash) }; if pre_allocate_slots { if position != BUCKET_LEN { hash_data[position] = hash; position += 1; } if offset < 8 { dist[offset] += 1; } else { if extended_dist.len() < offset - 7 { extended_dist.resize(offset - 7, 0); } extended_dist[offset - 8] += 1; } } else { current_array.bucket(target_index + offset).extract_from( current_array.data_block(target_index + offset), hash, old_writer, old_data_block, &mut entry_ptr, ); } } if !pre_allocate_slots { return; } // Allocate memory. for (i, d) in dist.iter().chain(extended_dist.iter()).enumerate() { if *d != 0 { let bucket = current_array.bucket(target_index + i); bucket.reserve_slots((*d) as usize); } } // Relocate entries; it is infallible. entry_ptr = EntryPtr::null(); position = 0; while entry_ptr.move_to_next(old_writer) { let hash = if old_array.len() >= current_array.len() { u64::from(entry_ptr.partial_hash(&**old_writer)) } else if position == BUCKET_LEN { self.hash(&entry_ptr.get(old_data_block).0) } else { position += 1; hash_data[position - 1] }; let index = if old_array.len() >= current_array.len() { target_index } else { current_array.bucket_index(hash) }; current_array.bucket(index).extract_from( current_array.data_block(index), hash, old_writer, old_data_block, &mut entry_ptr, ); } } /// Starts incremental rehashing. #[inline] fn start_incremental_rehash(old_array: &BucketArray) -> Option { // Assign itself a range of `Bucket` instances to rehash. // // Aside from the range, it increments the implicit reference counting field in // `old_array.rehashing`. let rehashing_metadata = old_array.rehashing_metadata(); let mut current = rehashing_metadata.load(Relaxed); loop { if current >= old_array.len() || (current & (BUCKET_LEN - 1)) == BUCKET_LEN - 1 { // Only `BUCKET_LEN` threads are allowed to rehash a `Bucket` at a moment. return None; } match rehashing_metadata.compare_exchange_weak( current, current + BUCKET_LEN + 1, Acquire, Relaxed, ) { Ok(_) => { current &= !(BUCKET_LEN - 1); return Some(current); } Err(result) => current = result, } } } /// Ends incremental rehashing. #[inline] fn end_incremental_rehash( old_array: &BucketArray, prev: usize, success: bool, ) -> bool { let rehashing_metadata = old_array.rehashing_metadata(); if success { // Keep the index as it is. let metadata = rehashing_metadata.fetch_sub(1, Release) - 1; (metadata & (BUCKET_LEN - 1) == 0) && metadata >= old_array.len() } else { // On failure, `rehashing` reverts to its previous state. let mut current = rehashing_metadata.load(Relaxed); loop { let new = if current <= prev { current - 1 } else { let refs = current & (BUCKET_LEN - 1); prev | (refs - 1) }; match rehashing_metadata.compare_exchange_weak(current, new, Release, Relaxed) { Ok(_) => break, Err(actual) => current = actual, } } false } } /// Relocates a fixed number of buckets from the old bucket array to the current array /// asynchronously. /// /// Once this methods successfully started rehashing, there is no possibility that the bucket /// array is deallocated. async fn incremental_rehash_async<'g>( &self, current_array: &'g BucketArray, async_guard: &'g AsyncGuard, ) { if let Some(old_array) = async_guard.load_unchecked(current_array.linked_array_var(), Acquire) { if let Some(current) = Self::start_incremental_rehash(old_array) { let rehashing_guard = ExitGuard::new((old_array, current), |(old_array, prev)| { Self::end_incremental_rehash(old_array, prev, false); }); for bucket_index in rehashing_guard.1..(rehashing_guard.1 + BUCKET_LEN).min(old_array.len()) { let old_bucket = rehashing_guard.0.bucket(bucket_index); let writer = Writer::lock_async(old_bucket, async_guard).await; if let Some(writer) = writer { self.relocate_bucket_async( current_array, rehashing_guard.0, bucket_index, writer, async_guard, ) .await; } debug_assert!(current_array.has_linked_array()); } if Self::end_incremental_rehash(rehashing_guard.0, rehashing_guard.1, true) { if let Some(bucket_array) = current_array .linked_array_var() .swap((None, Tag::None), Release) .0 { self.defer_reclaim(bucket_array, async_guard.guard()); } } rehashing_guard.forget(); } } } /// Relocates a fixed number of buckets from the old array to the current array synchronously. /// /// Returns `true` if `old_array` is null. fn incremental_rehash_sync<'g, const TRY_LOCK: bool>( &self, current_array: &'g BucketArray, guard: &'g Guard, ) { if let Some(old_array) = current_array.linked_array(guard) { if let Some(current) = Self::start_incremental_rehash(old_array) { let rehashing_guard = ExitGuard::new((old_array, current), |(old_array, prev)| { Self::end_incremental_rehash(old_array, prev, false); }); for bucket_index in rehashing_guard.1..(rehashing_guard.1 + BUCKET_LEN).min(old_array.len()) { let old_bucket = rehashing_guard.0.bucket(bucket_index); let writer = if TRY_LOCK { let Ok(writer) = Writer::try_lock(old_bucket) else { return; }; writer } else { Writer::lock_sync(old_bucket) }; if let Some(writer) = writer { if !self.relocate_bucket_sync::( current_array, rehashing_guard.0, bucket_index, writer, ) { return; } } } if Self::end_incremental_rehash(rehashing_guard.0, rehashing_guard.1, true) { if let Some(bucket_array) = current_array .linked_array_var() .swap((None, Tag::None), Release) .0 { self.defer_reclaim(bucket_array, guard); } } rehashing_guard.forget(); } } } /// Tries to enlarge [`HashTable`] if the estimated load factor is greater than `7/8`. fn try_enlarge( &self, current_array: &BucketArray, index: usize, mut num_entries: usize, guard: &Guard, ) { if !current_array.has_linked_array() { // Try to grow if the estimated load factor is greater than `13/16`. let threshold = current_array.sample_size() * (BUCKET_LEN / 16) * 13; if num_entries > threshold || (1..current_array.sample_size()).any(|i| { num_entries += current_array .bucket((index + i) % current_array.len()) .len(); num_entries > threshold }) { self.try_resize(current_array, index, guard); } } } /// Tries to shrink the [`HashTable`] to fit the estimated number of entries it to optimize the /// storage. fn try_shrink(&self, current_array: &BucketArray, index: usize, guard: &Guard) { if !current_array.has_linked_array() { let minimum_capacity = self.minimum_capacity(); if current_array.num_slots() > minimum_capacity { // Try to shrink if the estimated load factor is less than `1/8`. let shrink_threshold = current_array.sample_size() * BUCKET_LEN / 8; let mut num_entries = 0; for i in 0..current_array.sample_size() { let bucket = current_array.bucket((index + i) % current_array.len()); num_entries += bucket.len(); if num_entries >= shrink_threshold { // Early exit. return; } } if num_entries <= shrink_threshold { self.try_resize(current_array, index, guard); } } } } /// Tries to resize the array. fn try_resize( &self, sampled_array: &BucketArray, sampling_index: usize, guard: &Guard, ) { let current_array_ptr = self.bucket_array_var().load(Acquire, guard); let Some(current_array) = (unsafe { current_array_ptr.as_ref_unchecked() }) else { // The hash table is empty. return; }; if !ptr::eq(current_array, sampled_array) { // The preliminary sampling result cannot be trusted anymore. return; } else if current_array.has_linked_array() { // Cannot resize with a bucket array linked to the current bucket array. return; } if self .minimum_capacity_var() .fetch_update(AcqRel, Acquire, |lock_state| { if lock_state >= RESIZING { None } else { Some(lock_state + RESIZING) } }) .is_err() { // The bucket array is being replaced with a new one. return; } let _lock_guard = ExitGuard::new((), |()| { self.minimum_capacity_var().fetch_sub(RESIZING, Release); }); if self.bucket_array_var().load(Acquire, guard) != current_array_ptr { // Resized in the meantime. return; } let minimum_capacity = self.minimum_capacity(); let capacity = current_array.num_slots(); let estimated_num_entries = Self::sample(current_array, sampling_index); let new_capacity = if capacity < minimum_capacity || estimated_num_entries >= (capacity / 16) * 13 { // Double the capacity if the estimated load factor is equal to or greater than // `13/16`; `~10%` of buckets are expected to have overflow buckets. if capacity == self.maximum_capacity() { // Do not resize if the capacity cannot be increased. capacity } else { // Double `new_capacity` until the expected load factor becomes `~0.4`. let mut new_capacity = minimum_capacity.next_power_of_two().max(capacity); while new_capacity / 2 < estimated_num_entries { if new_capacity >= self.maximum_capacity() { break; } new_capacity *= 2; } new_capacity } } else if estimated_num_entries <= capacity / 8 { // Shrink to fit if the estimated load factor is equal to or less than `1/8`. (estimated_num_entries * 2) .max(minimum_capacity) .max(BucketArray::::minimum_capacity()) .next_power_of_two() } else { capacity }; let try_resize = new_capacity != capacity; let try_drop_table = estimated_num_entries == 0 && minimum_capacity == 0; if !try_resize && !try_drop_table { // Nothing to do. return; } if try_drop_table { // Try to drop the hash table with all the buckets locked. let mut writer_guard = ExitGuard::new((0, false), |(len, success): (usize, bool)| { for i in 0..len { let writer = Writer::from_bucket(current_array.bucket(i)); if success { debug_assert_eq!(writer.len(), 0); writer.kill(); } } }); if !(0..current_array.len()).any(|i| { if let Ok(Some(writer)) = Writer::try_lock(current_array.bucket(i)) { if writer.len() == 0 { // The bucket will be unlocked later. writer_guard.0 = i + 1; forget(writer); return false; } } true }) { // All the buckets are empty and locked. writer_guard.1 = true; if let Some(bucket_array) = self.bucket_array_var().swap((None, Tag::None), Release).0 { self.defer_reclaim(bucket_array, guard); } } } else if try_resize { let new_bucket_array = unsafe { Shared::new_unchecked(BucketArray::::new( new_capacity, (*self.bucket_array_var()).clone(Relaxed, guard), )) }; self.bucket_array_var() .swap((Some(new_bucket_array), Tag::None), Release); } } /// Returns an estimated required size of the container based on the size hint. #[inline] fn capacity_from_size_hint(size_hint: (usize, Option)) -> usize { // A resize can be triggered when the load factor reaches ~80%. (size_hint .1 .unwrap_or(size_hint.0) .min(1_usize << (usize::BITS - 2)) / 4) * 5 } } /// Hard limit of the maximum capacity of each container type. pub(super) const MAXIMUM_CAPACITY_LIMIT: usize = 1_usize << (usize::BITS - 2); /// Denotes a state where a thread is resizing the container. pub(super) const RESIZING: usize = 1_usize << (usize::BITS - 1); /// [`LockedBucket`] has exclusive access to a [`Bucket`]. #[derive(Debug)] pub(crate) struct LockedBucket { /// Holds an exclusive lock on the [`Bucket`]. pub writer: Writer, /// Corresponding [`DataBlock`]. pub data_block: NonNull>, /// The index of the [`Bucket`] within the [`BucketArray`]. pub bucket_index: usize, /// Corresponding [`BucketArray`]. /// /// The [`BucketArray`] is not dropped as long as it holds an exclusive lock on the [`Bucket`]. pub bucket_array: NonNull>, } impl LockedBucket { /// Returns a reference to the [`BucketArray`] that contains this [`LockedBucket`]. #[inline] pub(crate) const fn bucket_array(&self) -> &BucketArray { unsafe { self.bucket_array.as_ref() } } /// Gets a mutable reference to the entry. #[inline] pub(crate) fn entry(&self, entry_ptr: &EntryPtr) -> &(K, V) { entry_ptr.get(self.data_block) } /// Gets a mutable reference to the entry. #[inline] pub(crate) fn entry_mut<'b>( &'b mut self, entry_ptr: &'b mut EntryPtr, ) -> &'b mut (K, V) { entry_ptr.get_mut(self.data_block, &self.writer) } /// Inserts a new entry with the supplied constructor function. #[inline] pub(crate) fn insert(&self, hash: u64, entry: (K, V)) -> EntryPtr { self.writer.insert(self.data_block, hash, entry) } } impl LockedBucket { /// Searches for an entry with the given key. #[inline] pub(crate) fn search(&self, key: &Q, hash: u64) -> EntryPtr where Q: Equivalent + ?Sized, { (*self.writer).get_entry_ptr(self.data_block, key, hash) } /// Removes the entry and tries to shrink the container. #[inline] pub(crate) fn remove>( self, hash_table: &T, entry_ptr: &mut EntryPtr, ) -> (K, V) where H: BuildHasher, { let removed = self.writer.remove(self.data_block, entry_ptr); if self.len() == 0 { self.try_shrink(hash_table, &Guard::new()); } removed } /// Removes the entry and tries to shrink the container. #[inline] pub(crate) fn mark_removed>( self, hash_table: &T, entry_ptr: &mut EntryPtr, ) where H: BuildHasher, { debug_assert_eq!(TYPE, INDEX); let guard = Guard::new(); self.writer.mark_removed(entry_ptr, &guard); self.set_has_garbage(&guard); if self.writer.len() == 0 { self.try_shrink(hash_table, &guard); } } /// Sets that there can be a garbage entry in the bucket so the epoch should be advanced. #[inline] pub(crate) const fn set_has_garbage(&self, guard: &Guard) { let sample_size = self.bucket_array().sample_size(); if self.bucket_index % (sample_size * sample_size) == 0 { guard.set_has_garbage(); } } /// Tries to shrink the container. #[inline] pub(crate) fn try_shrink>(self, hash_table: &T, guard: &Guard) where H: BuildHasher, { if let Some(current_array) = hash_table.bucket_array(guard) { if ptr::eq(current_array, self.bucket_array()) { let bucket_index = self.bucket_index; drop(self); // Tries to shrink the container after unlocking the bucket. hash_table.try_shrink(current_array, bucket_index, guard); } } } /// Returns a [`LockedBucket`] owning the next bucket asynchronously. #[inline] pub(super) async fn next_async>( self, hash_table: &T, entry_ptr: &mut EntryPtr, ) -> Option> where H: BuildHasher, { if entry_ptr.move_to_next(&self.writer) { return Some(self); } let next_index = self.bucket_index + 1; let len = self.bucket_array().len(); if self.writer.len() == 0 { self.try_shrink(hash_table, &Guard::new()); } else { drop(self); } if next_index == len { return None; } let mut next_entry = None; hash_table .for_each_writer_async(next_index, len, |locked_bucket, _| { *entry_ptr = EntryPtr::null(); if entry_ptr.move_to_next(&locked_bucket.writer) { next_entry = Some(locked_bucket); return true; } false }) .await; next_entry } /// Returns a [`LockedBucket`] owning the next bucket synchronously. #[inline] pub(super) fn next_sync>( self, hash_table: &T, entry_ptr: &mut EntryPtr, ) -> Option where H: BuildHasher, { if entry_ptr.move_to_next(&self.writer) { return Some(self); } let next_index = self.bucket_index + 1; let len = self.bucket_array().len(); if self.writer.len() == 0 { self.try_shrink(hash_table, &Guard::new()); } else { drop(self); } if next_index == len { return None; } let mut next_entry = None; hash_table.for_each_writer_sync(next_index, len, &Guard::new(), |locked_bucket, _| { *entry_ptr = EntryPtr::null(); if entry_ptr.move_to_next(&locked_bucket.writer) { next_entry = Some(locked_bucket); return true; } false }); next_entry } } impl Deref for LockedBucket { type Target = Bucket; #[inline] fn deref(&self) -> &Self::Target { &self.writer } } unsafe impl Send for LockedBucket {} unsafe impl Sync for LockedBucket { } /// For the given index in the current array, calculate the respective range in the old array. #[inline] const fn from_index_to_range(from_len: usize, to_len: usize, from_index: usize) -> (usize, usize) { debug_assert!(from_len.is_power_of_two() && to_len.is_power_of_two()); if from_len < to_len { let ratio = to_len / from_len; let start_index = from_index * ratio; debug_assert!(start_index + ratio <= to_len,); (start_index, start_index + ratio) } else { let ratio = from_len / to_len; let start_index = from_index / ratio; debug_assert!(start_index < to_len,); (start_index, start_index + 1) } } /// Turns a reference into a [`NonNull`] pointer. const fn into_non_null(t: &T) -> NonNull { unsafe { NonNull::new_unchecked(from_ref(t).cast_mut()) } } scc-3.4.8/src/lib.rs000064400000000000000000000014131046102023000123120ustar 00000000000000#![deny(missing_docs, warnings, clippy::all, clippy::pedantic)] #![doc = include_str!("../README.md")] // Re-export [`sdd`](https://crates.io/crates/sdd) modules for backward compatibility. pub use sdd::{AtomicShared, Bag, Guard, LinkedEntry, LinkedList, Queue, Shared, Stack, Tag}; pub use sdd::{bag, queue, stack}; #[cfg(not(feature = "equivalent"))] mod equivalent; pub use equivalent::{Comparable, Equivalent}; pub mod hash_cache; pub use hash_cache::HashCache; pub mod hash_index; pub use hash_index::HashIndex; pub mod hash_map; pub use hash_map::HashMap; pub mod hash_set; pub use hash_set::HashSet; pub mod tree_index; pub use tree_index::TreeIndex; mod async_helper; mod exit_guard; mod hash_table; #[cfg(feature = "serde")] mod serde; #[cfg(test)] mod tests; scc-3.4.8/src/serde.rs000064400000000000000000000254321046102023000126550ustar 00000000000000//! This module implements helper types and traits for `serde` serialization and deserialization. #![deny(unsafe_code)] use std::fmt; use std::hash::{BuildHasher, Hash}; use std::marker::PhantomData; use serde::Deserializer; use serde::de::{Deserialize, MapAccess, SeqAccess, Visitor}; use serde::ser::{Serialize, SerializeMap, SerializeSeq, Serializer}; use super::{Guard, HashCache, HashIndex, HashMap, HashSet, TreeIndex}; /// Helper type to allow `serde` to access [`HashMap`] entries. pub struct HashMapVisitor { #[allow(clippy::type_complexity)] marker: PhantomData HashMap>, } /// The maximum initial capacity for containers. const MAX_CAPACITY: usize = 1 << 24; impl HashMapVisitor where K: Eq + Hash, H: BuildHasher, { fn new() -> Self { HashMapVisitor { marker: PhantomData, } } } impl<'d, K, V, H> Visitor<'d> for HashMapVisitor where K: Deserialize<'d> + Eq + Hash, V: Deserialize<'d>, H: BuildHasher + Default, { type Value = HashMap; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("HashMap") } fn visit_map(self, mut access: M) -> Result where M: MapAccess<'d>, { let hashmap = HashMap::with_capacity_and_hasher( access.size_hint().unwrap_or(0).min(MAX_CAPACITY), H::default(), ); while let Some((key, val)) = access.next_entry()? { hashmap.upsert_sync(key, val); } Ok(hashmap) } } impl<'d, K, V, H> Deserialize<'d> for HashMap where K: Deserialize<'d> + Eq + Hash, V: Deserialize<'d>, H: BuildHasher + Default, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'d>, { deserializer.deserialize_map(HashMapVisitor::::new()) } } impl Serialize for HashMap where K: Eq + Hash + Serialize, V: Serialize, H: BuildHasher, { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut map = serializer.serialize_map(Some(self.len()))?; let mut error = None; self.iter_sync(|k, v| { if error.is_none() { if let Err(e) = map.serialize_entry(k, v) { error.replace(e); } } true }); if let Some(e) = error { return Err(e); } map.end() } } /// Helper type to allow `serde` to access [`HashSet`] entries. pub struct HashSetVisitor { marker: PhantomData HashSet>, } impl HashSetVisitor where K: Eq + Hash, H: BuildHasher, { fn new() -> Self { HashSetVisitor { marker: PhantomData, } } } impl<'d, K, H> Visitor<'d> for HashSetVisitor where K: Deserialize<'d> + Eq + Hash, H: BuildHasher + Default, { type Value = HashSet; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("HashSet") } fn visit_seq(self, mut access: M) -> Result where M: SeqAccess<'d>, { let hashset = HashSet::with_capacity_and_hasher( access.size_hint().unwrap_or(0).min(MAX_CAPACITY), H::default(), ); while let Some(key) = access.next_element()? { let _result = hashset.insert_sync(key); } Ok(hashset) } } impl<'d, K, H> Deserialize<'d> for HashSet where K: Deserialize<'d> + Eq + Hash, H: BuildHasher + Default, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'d>, { deserializer.deserialize_seq(HashSetVisitor::::new()) } } impl Serialize for HashSet where K: Eq + Hash + Serialize, H: BuildHasher, { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut seq = serializer.serialize_seq(Some(self.len()))?; let mut error = None; self.iter_sync(|k| { if error.is_none() { if let Err(e) = seq.serialize_element(k) { error.replace(e); } } true }); if let Some(e) = error { return Err(e); } seq.end() } } /// Helper type to allow `serde` to access [`HashIndex`] entries. pub struct HashIndexVisitor { #[allow(clippy::type_complexity)] marker: PhantomData HashIndex>, } impl HashIndexVisitor where K: Eq + Hash, H: BuildHasher, { fn new() -> Self { HashIndexVisitor { marker: PhantomData, } } } impl<'d, K, V, H> Visitor<'d> for HashIndexVisitor where K: Deserialize<'d> + Eq + Hash, V: Deserialize<'d>, H: BuildHasher + Default, { type Value = HashIndex; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("HashIndex") } fn visit_map(self, mut access: M) -> Result where M: MapAccess<'d>, { let hashindex = HashIndex::with_capacity_and_hasher( access.size_hint().unwrap_or(0).min(MAX_CAPACITY), H::default(), ); while let Some((key, val)) = access.next_entry()? { let _result = hashindex.insert_sync(key, val); } Ok(hashindex) } } impl<'d, K, V, H> Deserialize<'d> for HashIndex where K: Deserialize<'d> + Eq + Hash, V: Deserialize<'d>, H: BuildHasher + Default, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'d>, { deserializer.deserialize_map(HashIndexVisitor::::new()) } } impl Serialize for HashIndex where K: Eq + Hash + Serialize, V: Serialize, H: BuildHasher, { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut map = serializer.serialize_map(Some(self.len()))?; let mut error = None; self.iter(&Guard::new()).any(|(k, v)| { if let Err(e) = map.serialize_entry(k, v) { error.replace(e); true } else { false } }); if let Some(e) = error { return Err(e); } map.end() } } /// Helper type to allow `serde` to access [`HashCache`] entries. pub struct HashCacheVisitor { #[allow(clippy::type_complexity)] marker: PhantomData HashCache>, } impl HashCacheVisitor where K: Eq + Hash, H: BuildHasher, { fn new() -> Self { HashCacheVisitor { marker: PhantomData, } } } impl<'d, K, V, H> Visitor<'d> for HashCacheVisitor where K: Deserialize<'d> + Eq + Hash, V: Deserialize<'d>, H: BuildHasher + Default, { type Value = HashCache; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("HashCache") } fn visit_map(self, mut access: M) -> Result where M: MapAccess<'d>, { let capacity = access.size_hint().unwrap_or(0).min(MAX_CAPACITY); let hashcache = HashCache::with_capacity_and_hasher(0, capacity, H::default()); while let Some((key, val)) = access.next_entry()? { let _result = hashcache.put_sync(key, val); } Ok(hashcache) } } impl<'d, K, V, H> Deserialize<'d> for HashCache where K: Deserialize<'d> + Eq + Hash, V: Deserialize<'d>, H: BuildHasher + Default, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'d>, { deserializer.deserialize_map(HashCacheVisitor::::new()) } } impl Serialize for HashCache where K: Eq + Hash + Serialize, V: Serialize, H: BuildHasher, { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let capacity_range = self.capacity_range(); let mut map = serializer.serialize_map(Some(*capacity_range.end()))?; let mut error = None; self.iter_sync(|k, v| { if error.is_none() { if let Err(e) = map.serialize_entry(k, v) { error.replace(e); } } true }); if let Some(e) = error { return Err(e); } map.end() } } /// Helper type to allow `serde` to access [`TreeIndex`] entries. pub struct TreeIndexVisitor { #[allow(clippy::type_complexity)] marker: PhantomData TreeIndex>, } impl TreeIndexVisitor where K: Clone + Ord, V: Clone, { fn new() -> Self { TreeIndexVisitor { marker: PhantomData, } } } impl<'d, K, V> Visitor<'d> for TreeIndexVisitor where K: 'static + Clone + Deserialize<'d> + Ord, V: 'static + Clone + Deserialize<'d>, { type Value = TreeIndex; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("TreeIndex") } fn visit_map(self, mut access: M) -> Result where M: MapAccess<'d>, { let treeindex = TreeIndex::default(); while let Some((key, val)) = access.next_entry()? { let _result = treeindex.insert_sync(key, val); } Ok(treeindex) } } impl<'d, K, V> Deserialize<'d> for TreeIndex where K: 'static + Clone + Deserialize<'d> + Ord, V: 'static + Clone + Deserialize<'d>, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'d>, { deserializer.deserialize_map(TreeIndexVisitor::::new()) } } impl Serialize for TreeIndex where K: 'static + Clone + Ord + Serialize, V: 'static + Clone + Serialize, { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut map = serializer.serialize_map(Some(self.len()))?; let mut error = None; self.iter(&Guard::new()).any(|(k, v)| { if let Err(e) = map.serialize_entry(k, v) { error.replace(e); true } else { false } }); if let Some(e) = error { return Err(e); } map.end() } } scc-3.4.8/src/tests/benchmarks.rs000064400000000000000000000627461046102023000150430ustar 00000000000000#![allow(clippy::inline_always)] mod common { use std::hash::{BuildHasher, Hash}; use std::ptr::addr_of; use sdd::Guard; use crate::{HashIndex, HashMap, TreeIndex}; #[allow(clippy::struct_excessive_bools)] #[derive(Clone, Copy, Debug, Default)] pub struct Workload { pub size: usize, pub insert_local: bool, pub insert_remote: bool, pub scan: bool, pub read_local: bool, pub read_remote: bool, pub remove_local: bool, pub remove_remote: bool, } impl Workload { pub fn new(workload_size: usize) -> Self { let default = Self::default(); Self { size: workload_size, ..default } } pub const fn mixed(self) -> Self { Self { insert_local: true, insert_remote: true, read_local: true, read_remote: true, remove_local: true, remove_remote: true, ..self } } pub const fn insert_local(self) -> Self { Self { insert_local: true, ..self } } pub const fn insert_remote(self) -> Self { Self { insert_remote: true, ..self } } pub const fn scan(self) -> Self { Self { scan: true, ..self } } pub const fn read_local(self) -> Self { Self { read_local: true, ..self } } pub const fn remove_local(self) -> Self { Self { remove_local: true, ..self } } pub const fn remove_remote(self) -> Self { Self { remove_remote: true, ..self } } pub const fn has_remote_op(&self) -> bool { self.insert_remote || self.read_remote || self.remove_remote } } pub trait BenchmarkOperation: 'static + Send + Sync where K: 'static + Clone + Eq + Hash + Ord + Send + Sync, V: 'static + Clone + Send + Sync, H: 'static + BuildHasher + Send + Sync, { fn insert_async(&self, k: K, v: V) -> impl Future + Send; fn read_async(&self, k: &K) -> impl Future + Send; fn scan_async(&self) -> impl Future + Send; fn remove_async(&self, k: &K) -> impl Future + Send; fn insert_sync(&self, k: K, v: V) -> bool; fn read_sync(&self, k: &K) -> bool; fn scan_sync(&self) -> usize; fn remove_sync(&self, k: &K) -> bool; } impl BenchmarkOperation for HashMap where K: 'static + Clone + Eq + Hash + Ord + Send + Sync, V: 'static + Clone + Send + Sync, H: 'static + BuildHasher + Send + Sync, { #[inline(always)] async fn insert_async(&self, k: K, v: V) -> bool { self.insert_async(k, v).await.is_ok() } #[inline(always)] async fn read_async(&self, k: &K) -> bool { self.read_async(k, |_, _| ()).await.is_some() } #[inline(always)] async fn scan_async(&self) -> usize { let mut scanned = 0; self.iter_async(|_, v| { if addr_of!(v) as usize != 0 { scanned += 1; } true }) .await; scanned } #[inline(always)] async fn remove_async(&self, k: &K) -> bool { self.remove_async(k).await.is_some() } #[inline(always)] fn insert_sync(&self, k: K, v: V) -> bool { self.insert_sync(k, v).is_ok() } #[inline(always)] fn read_sync(&self, k: &K) -> bool { self.read_sync(k, |_, _| ()).is_some() } #[inline(always)] fn scan_sync(&self) -> usize { let mut scanned = 0; self.iter_sync(|_, v| { if addr_of!(v) as usize != 0 { scanned += 1; } true }); scanned } #[inline(always)] fn remove_sync(&self, k: &K) -> bool { self.remove_sync(k).is_some() } } impl BenchmarkOperation for HashIndex where K: 'static + Clone + Eq + Hash + Ord + Send + Sync, V: 'static + Clone + Send + Sync, H: 'static + BuildHasher + Send + Sync, { #[inline(always)] async fn insert_async(&self, k: K, v: V) -> bool { self.insert_async(k, v).await.is_ok() } #[inline(always)] async fn read_async(&self, k: &K) -> bool { self.peek_with(k, |_, _| ()).is_some() } #[inline(always)] async fn scan_async(&self) -> usize { let mut scanned = 0; self.iter_async(|_, v| { if addr_of!(v) as usize != 0 { scanned += 1; } true }) .await; scanned } #[inline(always)] async fn remove_async(&self, k: &K) -> bool { self.remove_async(k).await } #[inline(always)] fn insert_sync(&self, k: K, v: V) -> bool { self.insert_sync(k, v).is_ok() } #[inline(always)] fn read_sync(&self, k: &K) -> bool { self.peek_with(k, |_, _| ()).is_some() } #[inline(always)] fn scan_sync(&self) -> usize { let guard = Guard::new(); self.iter(&guard) .filter(|(_, v)| addr_of!(v) as usize != 0) .count() } #[inline(always)] fn remove_sync(&self, k: &K) -> bool { self.remove_sync(k) } } impl BenchmarkOperation for TreeIndex where K: 'static + Clone + Eq + Hash + Ord + Send + Sync, V: 'static + Clone + Send + Sync, H: 'static + BuildHasher + Send + Sync, { #[inline(always)] async fn insert_async(&self, k: K, v: V) -> bool { self.insert_async(k, v).await.is_ok() } #[inline(always)] async fn read_async(&self, k: &K) -> bool { self.peek_with(k, |_, _| ()).is_some() } #[inline(always)] async fn scan_async(&self) -> usize { let guard = Guard::new(); self.iter(&guard) .filter(|(_, v)| addr_of!(v) as usize != 0) .count() } #[inline(always)] async fn remove_async(&self, k: &K) -> bool { self.remove_async(k).await } #[inline(always)] fn insert_sync(&self, k: K, v: V) -> bool { self.insert_sync(k, v).is_ok() } #[inline(always)] fn read_sync(&self, k: &K) -> bool { self.peek_with(k, |_, _| ()).is_some() } #[inline(always)] fn scan_sync(&self) -> usize { let guard = Guard::new(); self.iter(&guard) .filter(|(_, v)| addr_of!(v) as usize != 0) .count() } #[inline(always)] fn remove_sync(&self, k: &K) -> bool { self.remove_sync(k) } } pub trait ConvertFromUsize { fn convert(from: usize) -> Self; } impl ConvertFromUsize for usize { #[inline(always)] fn convert(from: usize) -> usize { from } } impl ConvertFromUsize for String { #[inline(always)] fn convert(from: usize) -> String { from.to_string() } } } mod async_benchmarks { use std::collections::hash_map::RandomState; use std::hash::Hash; use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use std::time::{Duration, Instant}; use sdd::Guard; use tokio::sync::Barrier; use super::common::{BenchmarkOperation, ConvertFromUsize, Workload}; use crate::{HashIndex, HashMap, TreeIndex}; async fn perform( num_tasks: usize, start_index: usize, container: &Arc, workload: Workload, ) -> (Duration, usize) where K: 'static + Clone + ConvertFromUsize + Eq + Hash + Ord + Send + Sync, V: 'static + Clone + ConvertFromUsize + Send + Sync, C: 'static + BenchmarkOperation, { for _ in 0..64 { Guard::new().accelerate(); } let total_ops = Arc::new(AtomicUsize::new(0)); let mut task_handles = Vec::with_capacity(num_tasks); let barrier = Arc::new(Barrier::new(num_tasks + 1)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let container = container.clone(); let total_ops = total_ops.clone(); task_handles.push(tokio::task::spawn(async move { let mut num_operations = 0; barrier.wait().await; if workload.scan { num_operations += container.scan_async().await; } for i in 0..workload.size { let remote_task_id = if num_tasks < 2 { 0 } else { (task_id + 1 + i % (num_tasks - 1)) % num_tasks }; assert!(num_tasks < 2 || task_id != remote_task_id); if workload.insert_local { let local_index = task_id * workload.size + i + start_index; let result = container .insert_async(K::convert(local_index), V::convert(i)) .await; assert!(result || workload.has_remote_op()); num_operations += 1; } if workload.insert_remote { let remote_index = remote_task_id * workload.size + i + start_index; container .insert_async(K::convert(remote_index), V::convert(i)) .await; num_operations += 1; } if workload.read_local { let local_index = task_id * workload.size + i + start_index; let result = container.read_async(&K::convert(local_index)).await; assert!(result || workload.has_remote_op()); num_operations += 1; } if workload.read_remote { let remote_index = remote_task_id * workload.size + i + start_index; container.read_async(&K::convert(remote_index)).await; num_operations += 1; } if workload.remove_local { let local_index = task_id * workload.size + i + start_index; let result = container.remove_async(&K::convert(local_index)).await; assert!(result || workload.has_remote_op()); num_operations += 1; } if workload.remove_remote { let remote_index = remote_task_id * workload.size + i + start_index; container.remove_async(&K::convert(remote_index)).await; num_operations += 1; } } barrier.wait().await; total_ops.fetch_add(num_operations, Relaxed); })); } barrier.wait().await; let start_time = Instant::now(); barrier.wait().await; let end_time = Instant::now(); for r in futures::future::join_all(task_handles).await { assert!(r.is_ok()); } ( end_time.saturating_duration_since(start_time), total_ops.load(Relaxed), ) } macro_rules! scenario { ($n:ident, $c:expr, $s:expr, $t:expr) => { let $n = $c; // 1. insert-local let insert = Workload::new($s).insert_local(); let (duration, total_ops) = perform($t, 0, &$n, insert).await; println!( "{}-insert-local-async: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), $s * $t); // 2. scan let scan = Workload::new($s).scan(); let (duration, total_ops) = perform($t, 0, &$n, scan).await; println!( "{}-scan-async: {}, {duration:?}, {total_ops}", stringify!($n), $t ); // 3. read-local let read = Workload::new($s).read_local(); let (duration, total_ops) = perform($t, 0, &$n, read).await; println!( "{}-read-local-async: {}, {duration:?}, {total_ops}", stringify!($n), $t ); // 4. remove-local let remove = Workload::new($s).remove_local(); let (duration, total_ops) = perform($t, 0, &$n, remove).await; println!( "{}-remove-local-async: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), 0); // 5. insert-local-remote let insert = Workload::new($s).insert_local().insert_remote(); let (duration, total_ops) = perform($t, 0, &$n, insert).await; println!( "{}-insert-local-remote-async: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), $s * $t); // 6. mixed let mixed = Workload::new($s).mixed(); let (duration, total_ops) = perform($t, $s * $t, &$n, mixed).await; println!( "{}-mixed-async: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), $s * $t); // 7. remove-local-remote let remove = Workload::new($s).remove_local().remove_remote(); let (duration, total_ops) = perform($t, 0, &$n, remove).await; println!( "{}-remove-local-remote-async: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), 0); }; } async fn hashmap_benchmark(workload_size: usize, num_tasks_list: Vec) where T: 'static + ConvertFromUsize + Clone + Hash + Ord + Send + Sync, { for num_tasks in num_tasks_list { scenario!( hashmap, Arc::new(HashMap::::default()), workload_size, num_tasks ); } } async fn hashindex_benchmark(workload_size: usize, num_tasks_list: Vec) where T: 'static + ConvertFromUsize + Clone + Hash + Ord + Send + Sync, { for num_tasks in num_tasks_list { scenario!( hashmap, Arc::new(HashIndex::::default()), workload_size, num_tasks ); } } async fn treeindex_benchmark(workload_size: usize, num_tasks_list: Vec) where T: 'static + ConvertFromUsize + Clone + Hash + Ord + Send + Sync, { for num_tasks in num_tasks_list { scenario!( hashmap, Arc::new(TreeIndex::::default()), workload_size, num_tasks ); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn benchmarks_async() { hashmap_benchmark::(65536, vec![4, 8]).await; hashindex_benchmark::(65536, vec![4, 8]).await; treeindex_benchmark::(65536, vec![4, 8]).await; } #[ignore = "too long"] #[tokio::test(flavor = "multi_thread", worker_threads = 128)] async fn full_scale_benchmarks_async() { hashmap_benchmark::( 1024 * 1024 * 16, vec![1, 1, 1, 4, 4, 4, 16, 16, 16, 64, 64, 64], ) .await; println!("----"); hashindex_benchmark::( 1024 * 1024 * 16, vec![1, 1, 1, 4, 4, 4, 16, 16, 16, 64, 64, 64], ) .await; println!("----"); treeindex_benchmark::( 1024 * 1024 * 16, vec![1, 1, 1, 4, 4, 4, 16, 16, 16, 64, 64, 64], ) .await; } } mod sync_benchmarks { use std::collections::hash_map::RandomState; use std::hash::Hash; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use std::sync::{Arc, Barrier}; use std::thread; use std::time::{Duration, Instant}; use sdd::Guard; use super::common::{BenchmarkOperation, ConvertFromUsize, Workload}; use crate::{HashIndex, HashMap, TreeIndex}; fn perform( num_threads: usize, start_index: usize, container: &Arc, workload: Workload, ) -> (Duration, usize) where K: 'static + Clone + ConvertFromUsize + Eq + Hash + Ord + Send + Sync, V: 'static + Clone + ConvertFromUsize + Send + Sync, C: BenchmarkOperation + 'static + Send + Sync, { for _ in 0..64 { Guard::new().accelerate(); } let barrier = Arc::new(Barrier::new(num_threads + 1)); let total_ops = Arc::new(AtomicUsize::new(0)); let mut thread_handles = Vec::with_capacity(num_threads); for thread_id in 0..num_threads { let container = container.clone(); let barrier = barrier.clone(); let total_ops = total_ops.clone(); thread_handles.push(thread::spawn(move || { let mut num_operations = 0; barrier.wait(); if workload.scan { num_operations += container.scan_sync(); } for i in 0..workload.size { let remote_thread_id = if num_threads < 2 { 0 } else { (thread_id + 1 + i % (num_threads - 1)) % num_threads }; assert!(num_threads < 2 || thread_id != remote_thread_id); if workload.insert_local { let local_index = thread_id * workload.size + i + start_index; let result = container.insert_sync(K::convert(local_index), V::convert(i)); assert!(result || workload.has_remote_op()); num_operations += 1; } if workload.insert_remote { let remote_index = remote_thread_id * workload.size + i + start_index; container.insert_sync(K::convert(remote_index), V::convert(i)); num_operations += 1; } if workload.read_local { let local_index = thread_id * workload.size + i + start_index; let result = container.read_sync(&K::convert(local_index)); assert!(result || workload.has_remote_op()); num_operations += 1; } if workload.read_remote { let remote_index = remote_thread_id * workload.size + i + start_index; container.read_sync(&K::convert(remote_index)); num_operations += 1; } if workload.remove_local { let local_index = thread_id * workload.size + i + start_index; let result = container.remove_sync(&K::convert(local_index)); assert!(result || workload.has_remote_op()); num_operations += 1; } if workload.remove_remote { let remote_index = remote_thread_id * workload.size + i + start_index; container.remove_sync(&K::convert(remote_index)); num_operations += 1; } } barrier.wait(); total_ops.fetch_add(num_operations, Relaxed); })); } barrier.wait(); let start_time = Instant::now(); barrier.wait(); let end_time = Instant::now(); for handle in thread_handles { handle.join().unwrap(); } ( end_time.saturating_duration_since(start_time), total_ops.load(Relaxed), ) } macro_rules! scenario { ($n:ident, $c:expr, $s:expr, $t:expr) => { let $n = $c; // 1. insert-local let insert = Workload::new($s).insert_local(); let (duration, total_ops) = perform($t, 0, &$n, insert); println!( "{}-insert-local-sync: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), $s * $t); // 2. scan let scan = Workload::new($s).scan(); let (duration, total_ops) = perform($t, 0, &$n, scan); println!( "{}-scan-sync: {}, {duration:?}, {total_ops}", stringify!($n), $t ); // 3. read-local let read = Workload::new($s).read_local(); let (duration, total_ops) = perform($t, 0, &$n, read); println!( "{}-read-local-sync: {}, {duration:?}, {total_ops}", stringify!($n), $t ); // 4. remove-local let remove = Workload::new($s).remove_local(); let (duration, total_ops) = perform($t, 0, &$n, remove); println!( "{}-remove-local-sync: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), 0); // 5. insert-local-remote let insert = Workload::new($s).insert_local().insert_remote(); let (duration, total_ops) = perform($t, 0, &$n, insert); println!( "{}-insert-local-remote-sync: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), $s * $t); // 6. mixed let mixed = Workload::new($s).mixed(); let (duration, total_ops) = perform($t, $s * $t, &$n, mixed); println!( "{}-mixed-sync: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), $s * $t); // 7. remove-local-remote let remove = Workload::new($s).remove_local().remove_remote(); let (duration, total_ops) = perform($t, 0, &$n, remove); println!( "{}-remove-local-remote-sync: {}, {duration:?}, {total_ops}", stringify!($n), $t ); assert_eq!($n.len(), 0); }; } fn hashmap_benchmark(workload_size: usize, num_threads: Vec) where T: 'static + ConvertFromUsize + Clone + Eq + Hash + Ord + Send + Sync, { for num_threads in num_threads { scenario!( hashmap, Arc::new(HashMap::::default()), workload_size, num_threads ); } } fn hashindex_benchmark(workload_size: usize, num_threads: Vec) where T: 'static + ConvertFromUsize + Clone + Eq + Hash + Ord + Send + Sync, { for num_threads in num_threads { scenario!( hashindex, Arc::new(HashIndex::::default()), workload_size, num_threads ); } } fn treeindex_benchmark(workload_size: usize, num_threads: Vec) where T: 'static + ConvertFromUsize + Clone + Hash + Ord + Send + Sync, { for num_threads in num_threads { scenario!( treeindex, Arc::new(TreeIndex::::default()), workload_size, num_threads ); } } #[cfg_attr(miri, ignore)] #[test] fn benchmarks_sync() { hashmap_benchmark::(65536, vec![4, 8]); hashindex_benchmark::(65536, vec![4, 8]); treeindex_benchmark::(65536, vec![4, 8]); } #[ignore = "too long"] #[test] fn full_scale_benchmarks_sync() { hashmap_benchmark::( 1024 * 1024 * 16, vec![1, 1, 1, 4, 4, 4, 16, 16, 16, 64, 64, 64], ); println!("----"); hashindex_benchmark::( 1024 * 1024 * 16, vec![1, 1, 1, 4, 4, 4, 16, 16, 16, 64, 64, 64], ); println!("----"); treeindex_benchmark::( 1024 * 1024 * 16, vec![1, 1, 1, 4, 4, 4, 16, 16, 16, 64, 64, 64], ); } } scc-3.4.8/src/tests/models.rs000064400000000000000000000302161046102023000141740ustar 00000000000000use sdd::Guard; use std::borrow::Borrow; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use std::sync::{Arc, Mutex}; use fnv::FnvBuildHasher; use loom::model::Builder; use loom::thread::{spawn, yield_now}; use crate::{HashIndex, HashMap, TreeIndex}; #[derive(Debug)] struct A(usize, Arc); impl A { fn new(d: usize, c: Arc) -> Self { c.fetch_add(1, Relaxed); Self(d, c) } } impl Clone for A { fn clone(&self) -> Self { self.1.fetch_add(1, Relaxed); Self(self.0, self.1.clone()) } } impl Drop for A { fn drop(&mut self) { self.1.fetch_sub(1, Relaxed); } } static SERIALIZER: Mutex<()> = Mutex::new(()); // Checks if the key is visible when another thread is inserting a key. #[test] fn hashmap_key_visibility() { let _guard = SERIALIZER.lock().unwrap(); for max_key in 42..72 { let mut model_builder_key_visibility = Builder::new(); model_builder_key_visibility.max_threads = 2; model_builder_key_visibility.max_branches = 1_048_576; model_builder_key_visibility.check(move || { let hashmap: Arc> = Arc::new( HashMap::with_capacity_and_hasher(64, FnvBuildHasher::default()), ); assert_eq!(hashmap.capacity(), 64); for k in 0..max_key { assert!(hashmap.insert_sync(k, k).is_ok()); } let hashmap_clone = hashmap.clone(); let thread_insert = spawn(move || { assert!(hashmap_clone.insert_sync(usize::MAX, usize::MAX).is_ok()); assert!(hashmap_clone.read_sync(&0, |_, _| ()).is_some()); drop(hashmap_clone); loop { let guard = Guard::new(); if !guard.has_garbage() { break; } guard.accelerate(); yield_now(); } }); assert!(hashmap.read_sync(&0, |_, _| ()).is_some()); assert!(thread_insert.join().is_ok()); for k in 0..max_key { assert!(hashmap.read_sync(&k, |_, _| ()).is_some()); } assert!(hashmap.read_sync(&usize::MAX, |_, _| ()).is_some()); assert_eq!(hashmap.len(), max_key + 1); drop(hashmap); loop { let guard = Guard::new(); if !guard.has_garbage() { break; } guard.accelerate(); yield_now(); } }); } } // Checks if the same key cannot be inserted twice. #[test] fn hashmap_key_uniqueness() { let _guard = SERIALIZER.lock().unwrap(); for max_key in 42..72 { let mut model_builder_key_uniqueness = Builder::new(); model_builder_key_uniqueness.max_threads = 2; model_builder_key_uniqueness.max_branches = 1_048_576; model_builder_key_uniqueness.check(move || { let hashmap: Arc> = Arc::new( HashMap::with_capacity_and_hasher(64, FnvBuildHasher::default()), ); let check = Arc::new(AtomicUsize::new(0)); assert_eq!(hashmap.capacity(), 64); for k in 0..max_key { assert!(hashmap.insert_sync(k, k).is_ok()); } let hashmap_clone = hashmap.clone(); let check_clone = check.clone(); let thread_insert = spawn(move || { if hashmap_clone.insert_sync(usize::MAX, usize::MAX).is_ok() { check_clone.fetch_add(1, Relaxed); } drop(hashmap_clone); loop { let guard = Guard::new(); if !guard.has_garbage() { break; } guard.accelerate(); yield_now(); } }); if hashmap.insert_sync(usize::MAX, usize::MAX).is_ok() { check.fetch_add(1, Relaxed); } assert!(thread_insert.join().is_ok()); for k in 0..max_key { assert!(hashmap.read_sync(&k, |_, _| ()).is_some(), "{k} {max_key}"); } assert!(hashmap.read_sync(&usize::MAX, |_, _| ()).is_some()); assert_eq!(hashmap.len(), max_key + 1); assert_eq!(check.load(Relaxed), 1); drop(hashmap); loop { let guard = Guard::new(); if !guard.has_garbage() { break; } guard.accelerate(); yield_now(); } }); } } // Checks if the key is visible when another thread is inserting a key. #[test] fn hashindex_key_visibility() { let _guard = SERIALIZER.lock().unwrap(); for max_key in 42..72 { let mut model_builder_key_visibility = Builder::new(); model_builder_key_visibility.max_threads = 2; model_builder_key_visibility.max_branches = 1_048_576; model_builder_key_visibility.check(move || { let hashindex: Arc> = Arc::new( HashIndex::with_capacity_and_hasher(64, FnvBuildHasher::default()), ); assert_eq!(hashindex.capacity(), 64); for k in 0..max_key { assert!(hashindex.insert_sync(k, k).is_ok()); } let hashindex_clone = hashindex.clone(); let thread_insert = spawn(move || { assert!(hashindex_clone.insert_sync(usize::MAX, usize::MAX).is_ok()); assert!(hashindex_clone.peek_with(&0, |_, _| ()).is_some()); drop(hashindex_clone); loop { let guard = Guard::new(); if !guard.has_garbage() { break; } guard.accelerate(); yield_now(); } }); assert!(hashindex.peek_with(&0, |_, _| ()).is_some()); assert!(thread_insert.join().is_ok()); for k in 0..max_key { assert!(hashindex.peek_with(&k, |_, _| ()).is_some()); } assert!(hashindex.peek_with(&usize::MAX, |_, _| ()).is_some()); assert_eq!(hashindex.len(), max_key + 1); drop(hashindex); loop { let guard = Guard::new(); if !guard.has_garbage() { break; } guard.accelerate(); yield_now(); } }); } } // Checks if keys are visible while the leaf node is being split. #[test] fn tree_index_split_leaf_node() { let _guard = SERIALIZER.lock().unwrap(); let keys = 14; let key_to_remove = 0; let mut model_builder_leaf_node = Builder::new(); model_builder_leaf_node.max_branches = 1_048_576; model_builder_leaf_node.check(move || { let cnt = Arc::new(AtomicUsize::new(0)); let tree_index = Arc::new(TreeIndex::::default()); for k in 0..keys { assert!(tree_index.insert_sync(k, A::new(k, cnt.clone())).is_ok()); } let cnt_clone = cnt.clone(); let tree_index_clone = tree_index.clone(); let thread_insert = spawn(move || { assert!( tree_index_clone .insert_sync(keys, A::new(keys, cnt_clone)) .is_ok() ); }); let thread_remove = spawn(move || { let key: usize = key_to_remove; assert_eq!( tree_index .peek_with(key.borrow(), |_key, value| value.0) .unwrap(), key ); }); assert!(thread_insert.join().is_ok()); assert!(thread_remove.join().is_ok()); while cnt.load(Relaxed) != 0 { Guard::new().accelerate(); yield_now(); } }); } // Checks if keys are visible while the internal node is being split. #[test] fn tree_index_split_internal_node() { let _guard = SERIALIZER.lock().unwrap(); let keys = 365; let key_to_remove = 0; let mut model_builder_new_internal_node = Builder::new(); model_builder_new_internal_node.max_branches = 1_048_576 * 16; model_builder_new_internal_node.check(move || { let cnt = Arc::new(AtomicUsize::new(0)); let tree_index = Arc::new(TreeIndex::::default()); for k in 0..keys { assert!(tree_index.insert_sync(k, A::new(k, cnt.clone())).is_ok()); } let cnt_clone = cnt.clone(); let tree_index_clone = tree_index.clone(); let thread_insert = spawn(move || { assert!( tree_index_clone .insert_sync(keys, A::new(keys, cnt_clone)) .is_ok() ); }); let thread_remove = spawn(move || { let key: usize = key_to_remove; assert_eq!( tree_index .peek_with(key.borrow(), |_key, value| value.0) .unwrap(), key ); assert!(tree_index.remove_sync(key.borrow())); }); assert!(thread_insert.join().is_ok()); assert!(thread_remove.join().is_ok()); while cnt.load(Relaxed) != 0 { Guard::new().accelerate(); yield_now(); } }); } // Checks if keys are visible while a leaf node is being removed. #[test] fn tree_index_remove_leaf_node() { let _guard = SERIALIZER.lock().unwrap(); let keys = 15; let key_to_remove = 14; let mut model_builder_remove_leaf = Builder::new(); model_builder_remove_leaf.max_branches = 1_048_576 * 16; model_builder_remove_leaf.check(move || { let cnt = Arc::new(AtomicUsize::new(0)); let tree_index = Arc::new(TreeIndex::::default()); for k in 0..keys { assert!(tree_index.insert_sync(k, A::new(k, cnt.clone())).is_ok()); } for k in 0..keys - 3 { assert!(tree_index.remove_sync(k.borrow())); } let tree_index_clone = tree_index.clone(); let thread_remove = spawn(move || { let key_to_remove = keys - 2; assert!(tree_index_clone.remove_sync(key_to_remove.borrow())); }); let thread_read = spawn(move || { let key = key_to_remove; assert_eq!( tree_index.peek_with(&key, |_key, value| value.0).unwrap(), key ); }); assert!(thread_remove.join().is_ok()); assert!(thread_read.join().is_ok()); while cnt.load(Relaxed) != 0 { Guard::new().accelerate(); yield_now(); } }); } // Check if keys are visible while a node is being deallocated. #[test] fn tree_index_remove_internal_node() { let _guard = SERIALIZER.lock().unwrap(); let keys = 366; let key_to_remove = 338; let mut model_builder_remove_node = Builder::new(); model_builder_remove_node.max_branches = 1_048_576 * 16; model_builder_remove_node.check(move || { let cnt = Arc::new(AtomicUsize::new(0)); let tree_index = Arc::new(TreeIndex::::default()); for k in 0..keys { assert!(tree_index.insert_sync(k, A::new(k, cnt.clone())).is_ok()); } for k in key_to_remove + 1..keys { assert!(tree_index.remove_sync(&k)); } let tree_index_clone = tree_index.clone(); let thread_read = spawn(move || { assert_eq!( tree_index_clone .peek_with(&0, |_key, value| value.0) .unwrap(), 0 ); }); let thread_remove = spawn(move || { assert!(tree_index.remove_sync(key_to_remove.borrow())); }); assert!(thread_read.join().is_ok()); assert!(thread_remove.join().is_ok()); while cnt.load(Relaxed) != 0 { Guard::new().accelerate(); yield_now(); } }); } scc-3.4.8/src/tests/unit_tests.rs000064400000000000000000003616021046102023000151200ustar 00000000000000mod common { use std::hash::{Hash, Hasher}; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use crate::Equivalent; pub struct R(pub &'static AtomicUsize); impl R { pub fn new(cnt: &'static AtomicUsize) -> R { cnt.fetch_add(1, Relaxed); R(cnt) } } impl Clone for R { fn clone(&self) -> Self { self.0.fetch_add(1, Relaxed); R(self.0) } } impl Drop for R { fn drop(&mut self) { self.0.fetch_sub(1, Relaxed); } } #[derive(Debug)] pub struct MaybeEq(pub u64, pub u64); impl Eq for MaybeEq {} impl Hash for MaybeEq { fn hash(&self, state: &mut H) { self.0.hash(state); } } impl PartialEq for MaybeEq { fn eq(&self, other: &Self) -> bool { self.0 == other.0 } } #[derive(Debug, Eq, PartialEq)] pub struct EqTest(pub String, pub usize); impl Equivalent for str { fn equivalent(&self, key: &EqTest) -> bool { key.0.eq(self) } } impl Hash for EqTest { fn hash(&self, state: &mut H) { self.0.hash(state); } } } mod hashmap { use std::collections::BTreeSet; use std::hash::{Hash, Hasher}; use std::panic::{RefUnwindSafe, UnwindSafe}; use std::rc::Rc; use std::sync::atomic::Ordering::{Acquire, Relaxed, Release}; use std::sync::atomic::{AtomicU64, AtomicUsize}; use std::sync::{Arc, Barrier}; use std::thread; use std::time::Duration; use futures::future::join_all; use proptest::prelude::*; use proptest::strategy::ValueTree; use proptest::test_runner::TestRunner; use tokio::sync::Barrier as AsyncBarrier; use super::common::{EqTest, MaybeEq, R}; use crate::HashMap; use crate::async_helper::AsyncGuard; use crate::hash_map::{self, Entry, ReplaceResult, Reserve}; use crate::hash_table::bucket::{MAP, Writer}; static_assertions::assert_eq_size!(Option>, usize); static_assertions::assert_impl_all!(AsyncGuard: Send, Sync); static_assertions::assert_eq_size!(AsyncGuard, usize); static_assertions::assert_not_impl_any!(HashMap, Rc>: Send, Sync); static_assertions::assert_not_impl_any!(hash_map::Entry, Rc>: Send, Sync); static_assertions::assert_impl_all!(HashMap: Send, Sync, RefUnwindSafe, UnwindSafe); static_assertions::assert_impl_all!(Reserve: Send, Sync, UnwindSafe); static_assertions::assert_not_impl_any!(HashMap: Send, Sync); static_assertions::assert_not_impl_any!(Reserve: Send, Sync); static_assertions::assert_impl_all!(hash_map::OccupiedEntry: Send, Sync); static_assertions::assert_not_impl_any!(hash_map::OccupiedEntry: Send, Sync, RefUnwindSafe, UnwindSafe); static_assertions::assert_impl_all!(hash_map::VacantEntry: Send, Sync); static_assertions::assert_not_impl_any!(hash_map::VacantEntry: Send, Sync, RefUnwindSafe, UnwindSafe); struct Data { data: usize, checker: Arc, } impl Data { fn new(data: usize, checker: Arc) -> Data { checker.fetch_add(1, Relaxed); Data { data, checker } } } impl Clone for Data { fn clone(&self) -> Self { Data::new(self.data, self.checker.clone()) } } impl Drop for Data { fn drop(&mut self) { self.checker.fetch_sub(1, Relaxed); } } impl Eq for Data {} impl Hash for Data { fn hash(&self, state: &mut H) { self.data.hash(state); } } impl PartialEq for Data { fn eq(&self, other: &Self) -> bool { self.data == other.data } } #[test] fn equivalent() { let hashmap: HashMap = HashMap::default(); assert!( hashmap .insert_sync(EqTest("HELLO".to_owned(), 1), 1) .is_ok() ); assert!(!hashmap.contains_sync("NO")); assert!(hashmap.contains_sync("HELLO")); } #[test] fn future_size() { // Small type. { let limit = 488; // In v2, 104. let hashmap: HashMap<(), ()> = HashMap::default(); let get_size = size_of_val(&hashmap.get_async(&())); assert!(get_size <= limit + 24, "{get_size}"); let contains_size = size_of_val(&hashmap.contains_async(&())); assert!(contains_size <= limit + 16, "{contains_size}"); let insert_size = size_of_val(&hashmap.insert_async((), ())); assert!(insert_size <= limit, "{insert_size}"); let entry_size = size_of_val(&hashmap.entry_async(())); assert!(entry_size <= limit + 8, "{entry_size}"); let read_size = size_of_val(&hashmap.read_async(&(), |(), ()| {})); assert!(read_size <= limit + 16, "{read_size}"); let remove_size = size_of_val(&hashmap.remove_async(&())); assert!(remove_size <= limit + 48, "{remove_size}"); let iter_size = size_of_val(&hashmap.iter_async(|(), ()| true)); assert!(iter_size <= limit + 40, "{iter_size}"); } // Medium type. { let limit = 480 + 2 * size_of::<(u64, u64)>(); // In v2, 104 + 2 * size_of::<(u64, u64)>. let hashmap: HashMap = HashMap::default(); let get_size = size_of_val(&hashmap.get_async(&0)); assert!(get_size <= limit, "{get_size}"); let contains_size = size_of_val(&hashmap.contains_async(&0)); assert!(contains_size <= limit, "{contains_size}"); let insert_size = size_of_val(&hashmap.insert_async(0, 0)); assert!(insert_size <= limit + 8, "{insert_size}"); let entry_size = size_of_val(&hashmap.entry_async(0)); assert!(entry_size <= limit, "{entry_size}"); let read_size = size_of_val(&hashmap.read_async(&0, |_, _| {})); assert!(read_size <= limit, "{read_size}"); let remove_size = size_of_val(&hashmap.remove_async(&0)); assert!(remove_size <= limit + 24, "{remove_size}"); let iter_size = size_of_val(&hashmap.iter_async(|_, _| true)); assert!(iter_size <= limit + 8, "{iter_size}"); } { type Large = [u64; 32]; let limit = 480 + 2 * size_of::<(Vec, Large)>(); // In v2, 104 + 2 * size_of::<(Vec, Large)>. let hashmap: HashMap, Large> = HashMap::default(); let get_size = size_of_val(&hashmap.get_async(&vec![])); assert!(get_size <= limit, "{get_size}"); let contains_size = size_of_val(&hashmap.contains_async(&vec![])); assert!(contains_size <= limit + 16, "{contains_size}"); let insert_size = size_of_val(&hashmap.insert_async(vec![], [0; 32])); assert!(insert_size <= limit + 8, "{insert_size}"); let entry_size = size_of_val(&hashmap.entry_async(vec![])); assert!(entry_size <= limit, "{entry_size}"); let read_size = size_of_val(&hashmap.read_async(&vec![], |_, _| {})); assert!(read_size <= limit, "{read_size}"); let remove_size = size_of_val(&hashmap.remove_async(&vec![])); assert!(remove_size <= limit, "{remove_size}"); let iter_size = size_of_val(&hashmap.iter_async(|_, _| true)); assert!(iter_size <= limit, "{iter_size}"); } } #[cfg_attr(miri, ignore)] #[tokio::test] async fn capacity_async() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashmap: HashMap = HashMap::default(); let workload_size = 1_048_576; for k in 0..workload_size { assert!(hashmap.insert_async(k, R::new(&INST_CNT)).await.is_ok()); } assert_eq!(INST_CNT.load(Relaxed), workload_size); assert_eq!(hashmap.len(), workload_size); assert!(hashmap.capacity() >= workload_size); drop(hashmap); assert_eq!(INST_CNT.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[test] fn capacity_sync() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashmap: HashMap = HashMap::default(); assert_eq!(hashmap.capacity(), 0); assert!( hashmap .reserve((1_usize << (usize::BITS - 1)) + 1) .is_none() ); let reserved_capacity = 1_048_576_usize; let reserve = hashmap.reserve(reserved_capacity); assert!(reserve.is_some()); for k in 0..256 { assert!(hashmap.insert_sync(k, R::new(&INST_CNT)).is_ok()); } assert_eq!(hashmap.capacity(), reserved_capacity); assert_eq!(hashmap.len(), 256); for k in 0..256 { assert!(hashmap.insert_sync(k, R::new(&INST_CNT)).is_err()); } drop(reserve); hashmap.clear_sync(); assert!(hashmap.capacity() < reserved_capacity); assert_eq!(hashmap.len(), 0); assert_eq!(INST_CNT.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test] async fn insert_drop_async() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashmap: HashMap = HashMap::default(); let workload_size = 1024; for k in 0..workload_size { assert!(hashmap.insert_async(k, R::new(&INST_CNT)).await.is_ok()); } assert_eq!(INST_CNT.load(Relaxed), workload_size); assert_eq!(hashmap.len(), workload_size); drop(hashmap); assert_eq!(INST_CNT.load(Relaxed), 0); } #[test] fn insert_drop_sync() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashmap: HashMap = HashMap::default(); let workload_size = 256; for k in 0..workload_size { assert!(hashmap.insert_sync(k, R::new(&INST_CNT)).is_ok()); } assert_eq!(INST_CNT.load(Relaxed), workload_size); assert_eq!(hashmap.len(), workload_size); drop(hashmap); assert_eq!(INST_CNT.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test] async fn replace_async() { let hashmap: HashMap = HashMap::default(); let workload_size = 256; for k in 0..workload_size { let ReplaceResult::NotReplaced(v) = hashmap.replace_async(MaybeEq(k, 0)).await else { unreachable!(); }; v.insert_entry(k); } for k in 0..workload_size { let ReplaceResult::Replaced(o, p) = hashmap.replace_async(MaybeEq(k, 1)).await else { unreachable!(); }; assert_eq!(p.1, 0); assert_eq!(o.get(), &k); } assert!(hashmap.any_async(|k, _| k.1 == 0).await.is_none()); } #[test] fn replace_sync() { let hashmap: HashMap = HashMap::default(); let workload_size = 256; for k in 0..workload_size { let ReplaceResult::NotReplaced(v) = hashmap.replace_sync(MaybeEq(k, 0)) else { unreachable!(); }; v.insert_entry(k); } for k in 0..workload_size { let ReplaceResult::Replaced(o, p) = hashmap.replace_sync(MaybeEq(k, 1)) else { unreachable!(); }; assert_eq!(p.1, 0); assert_eq!(o.get(), &k); } assert!(hashmap.any_sync(|k, _| k.1 == 0).is_none()); } #[cfg_attr(miri, ignore)] #[tokio::test] async fn clear_async() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashmap: HashMap = HashMap::default(); let workload_size = 1_usize << 18; for _ in 0..2 { for k in 0..workload_size { assert!(hashmap.insert_async(k, R::new(&INST_CNT)).await.is_ok()); } assert_eq!(INST_CNT.load(Relaxed), workload_size); assert_eq!(hashmap.len(), workload_size); hashmap.clear_async().await; assert_eq!(INST_CNT.load(Relaxed), 0); } } #[test] fn clear_sync() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashmap: HashMap = HashMap::default(); for workload_size in 2..64 { for k in 0..workload_size { assert!(hashmap.insert_sync(k, R::new(&INST_CNT)).is_ok()); } assert_eq!(INST_CNT.load(Relaxed), workload_size); assert_eq!(hashmap.len(), workload_size); hashmap.clear_sync(); assert_eq!(INST_CNT.load(Relaxed), 0); } } #[test] fn read_remove_sync() { let hashmap = Arc::new(HashMap::>::new()); let barrier = Arc::new(Barrier::new(2)); hashmap.insert_sync("first".into(), vec![123]).unwrap(); let hashmap_clone = hashmap.clone(); let barrier_clone = barrier.clone(); let task = thread::spawn(move || { hashmap_clone.read_sync("first", |_key, value| { { let first_item = value.first(); assert_eq!(first_item.unwrap(), &123_u8); } barrier_clone.wait(); thread::sleep(Duration::from_millis(16)); { let first_item = value.first(); assert_eq!(first_item.unwrap(), &123_u8); } }); }); barrier.wait(); assert!(hashmap.remove_sync("first").is_some()); assert!(task.join().is_ok()); } #[test] fn from_iter() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let workload_size = 256; let hashmap = (0..workload_size) .map(|k| (k / 2, R::new(&INST_CNT))) .collect::>(); assert_eq!(hashmap.len(), workload_size / 2); hashmap.clear_sync(); assert_eq!(INST_CNT.load(Relaxed), 0); } #[test] fn clone() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashmap: HashMap = HashMap::default(); let workload_size = 256; for k in 0..workload_size { assert!(hashmap.insert_sync(k, R::new(&INST_CNT)).is_ok()); } let hashmap_clone = hashmap.clone(); hashmap.clear_sync(); for k in 0..workload_size { assert!(hashmap_clone.read_sync(&k, |_, _| ()).is_some()); } hashmap_clone.clear_sync(); assert_eq!(INST_CNT.load(Relaxed), 0); } #[test] fn compare() { let hashmap1: HashMap = HashMap::new(); let hashmap2: HashMap = HashMap::new(); assert_eq!(hashmap1, hashmap2); assert!(hashmap1.insert_sync("Hi".to_string(), 1).is_ok()); assert_ne!(hashmap1, hashmap2); assert!(hashmap2.insert_sync("Hello".to_string(), 2).is_ok()); assert_ne!(hashmap1, hashmap2); assert!(hashmap1.insert_sync("Hello".to_string(), 2).is_ok()); assert_ne!(hashmap1, hashmap2); assert!(hashmap2.insert_sync("Hi".to_string(), 1).is_ok()); assert_eq!(hashmap1, hashmap2); assert!(hashmap1.remove_sync("Hi").is_some()); assert_ne!(hashmap1, hashmap2); } #[test] fn string_key() { let num_iter = if cfg!(miri) { 4 } else { 4096 }; let hashmap1: HashMap = HashMap::default(); let hashmap2: HashMap = HashMap::default(); let mut checker1 = BTreeSet::new(); let mut checker2 = BTreeSet::new(); let mut runner = TestRunner::default(); for i in 0..num_iter { let prop_str = "[a-z]{1,16}".new_tree(&mut runner).unwrap(); let str_val = prop_str.current(); if hashmap1.insert_sync(str_val.clone(), i).is_ok() { checker1.insert((str_val.clone(), i)); } let str_borrowed = str_val.as_str(); assert!(hashmap1.contains_sync(str_borrowed)); assert!(hashmap1.read_sync(str_borrowed, |_, _| ()).is_some()); if hashmap2.insert_sync(i, str_val.clone()).is_ok() { checker2.insert((i, str_val.clone())); } } assert_eq!(hashmap1.len(), checker1.len()); assert_eq!(hashmap2.len(), checker2.len()); for iter in checker1 { let v = hashmap1.remove_sync(iter.0.as_str()); assert_eq!(v.unwrap().1, iter.1); } for iter in checker2 { let e = hashmap2.entry_sync(iter.0); match e { Entry::Occupied(o) => assert_eq!(o.remove(), iter.1), Entry::Vacant(_) => unreachable!(), } } assert_eq!(hashmap1.len(), 0); assert_eq!(hashmap2.len(), 0); } #[test] fn local_ref() { struct L<'a>(&'a AtomicUsize); impl<'a> L<'a> { fn new(cnt: &'a AtomicUsize) -> Self { cnt.fetch_add(1, Relaxed); L(cnt) } } impl Drop for L<'_> { fn drop(&mut self) { self.0.fetch_sub(1, Relaxed); } } let workload_size = 256; let cnt = AtomicUsize::new(0); let hashmap: HashMap = HashMap::default(); for k in 0..workload_size { assert!(hashmap.insert_sync(k, L::new(&cnt)).is_ok()); } hashmap.retain_sync(|k, _| { assert!(*k < workload_size); true }); assert_eq!(cnt.load(Relaxed), workload_size); for k in 0..workload_size / 2 { assert!(hashmap.remove_sync(&k).is_some()); } hashmap.retain_sync(|k, _| { assert!(*k >= workload_size / 2); true }); assert_eq!(cnt.load(Relaxed), workload_size / 2); drop(hashmap); assert_eq!(cnt.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn insert_async() { for _ in 0..64 { let hashmap: Arc> = Arc::new(HashMap::default()); let num_tasks = 8; let workload_size = 256; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashmap = hashmap.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { let result = hashmap.insert_async(id, id).await; assert!(result.is_ok()); } for id in range.clone() { let result = hashmap.insert_async(id, id).await; assert_eq!(result, Err((id, id))); } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashmap.len(), num_tasks * workload_size); } } #[test] fn insert_sync() { let num_threads = if cfg!(miri) { 2 } else { 8 }; let num_iters = if cfg!(miri) { 1 } else { 64 }; for _ in 0..num_iters { let hashmap: Arc> = Arc::new(HashMap::default()); let workload_size = 256; let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for thread_id in 0..num_threads { let barrier = barrier.clone(); let hashmap = hashmap.clone(); threads.push(thread::spawn(move || { barrier.wait(); let range = (thread_id * workload_size)..((thread_id + 1) * workload_size); for id in range.clone() { let result = hashmap.insert_sync(id, id); assert!(result.is_ok()); } for id in range.clone() { let result = hashmap.insert_sync(id, id); assert_eq!(result, Err((id, id))); } })); } for thread in threads { assert!(thread.join().is_ok()); } assert_eq!(hashmap.len(), num_threads * workload_size); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn insert_duplicate_async() { let num_tasks = 8; let num_iters = 64; for _ in 0..num_iters { let hashmap: Arc> = Arc::new(HashMap::default()); let workload_size = 16 + num_iters; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashmap = hashmap.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { assert!(hashmap.insert_async(id, id).await.is_ok()); tokio::task::yield_now().await; assert!(hashmap.insert_async(id, id).await.is_err()); let _result: Result<(), (usize, usize)> = hashmap.insert_async(num_tasks * workload_size, 0).await; } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashmap.len(), num_tasks * workload_size + 1); } } #[test] fn insert_duplicate_sync() { let num_threads = if cfg!(miri) { 2 } else { 8 }; let num_iters = if cfg!(miri) { 1 } else { 64 }; for _ in 0..num_iters { let hashmap: Arc> = Arc::new(HashMap::default()); let workload_size = 16 + num_iters; let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for thread_id in 0..num_threads { let barrier = barrier.clone(); let hashmap = hashmap.clone(); threads.push(thread::spawn(move || { barrier.wait(); let range = (thread_id * workload_size)..((thread_id + 1) * workload_size); for id in range.clone() { assert!(hashmap.insert_sync(id, id).is_ok()); thread::yield_now(); assert!(hashmap.insert_sync(id, id).is_err()); let _result: Result<(), (usize, usize)> = hashmap.insert_sync(num_threads * workload_size, 0); } })); } for thread in threads { assert!(thread.join().is_ok()); } assert_eq!(hashmap.len(), num_threads * workload_size + 1); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn insert_update_read_remove_async() { let hashmap: Arc> = Arc::new(HashMap::default()); let num_tasks = 8; let workload_size = 256; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashmap = hashmap.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { let result = hashmap.update_async(&id, |_, _| 1).await; assert!(result.is_none()); } for id in range.clone() { if id % 10 == 0 { hashmap.entry_async(id).await.or_insert(id); } else if id % 3 == 0 { let entry = hashmap.entry_async(id).await; let o = match entry { Entry::Occupied(mut o) => { *o.get_mut() = id; o } Entry::Vacant(v) => v.insert_entry(id), }; assert_eq!(*o.get(), id); } else { let result = hashmap.insert_async(id, id).await; assert!(result.is_ok()); } } for id in range.clone() { if id % 7 == 4 { let entry = hashmap.entry_async(id).await; match entry { Entry::Occupied(mut o) => { *o.get_mut() += 1; } Entry::Vacant(v) => { v.insert_entry(id); } } } else { let result = hashmap .update_async(&id, |_, v| { *v += 1; *v }) .await; assert_eq!(result, Some(id + 1)); } } for id in range.clone() { let result = hashmap.read_async(&id, |_, v| *v).await; assert_eq!(result, Some(id + 1)); assert_eq!(*hashmap.get_async(&id).await.unwrap().get(), id + 1); } for id in range.clone() { let result = hashmap.remove_if_async(&id, |v| *v == id + 1).await; assert_eq!(result, Some((id, id + 1))); assert!(hashmap.read_async(&id, |_, v| *v).await.is_none()); assert!(hashmap.get_async(&id).await.is_none()); } for id in range { let result = hashmap.remove_if_async(&id, |v| *v == id + 1).await; assert_eq!(result, None); } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashmap.len(), 0); } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn entry_read_next_async() { let hashmap: Arc> = Arc::new(HashMap::default()); for _ in 0..64 { let num_tasks = 8; let workload_size = 512; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashmap = hashmap.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { let result = hashmap.insert_async(id, id).await; assert!(result.is_ok()); } for id in range.clone() { assert!(hashmap.read_async(&id, |_, _| ()).await.is_some()); } let mut in_range = 0; let mut entry = hashmap.begin_async().await; while let Some(current_entry) = entry.take() { if range.contains(current_entry.key()) { in_range += 1; } entry = current_entry.next_async().await; } assert!(in_range >= workload_size, "{in_range} {workload_size}"); let mut removed = 0; hashmap .retain_async(|k, _| { if range.contains(k) { removed += 1; false } else { true } }) .await; assert_eq!(removed, workload_size); let mut entry = hashmap.begin_async().await; while let Some(current_entry) = entry.take() { assert!(!range.contains(current_entry.key())); entry = current_entry.next_async().await; } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashmap.len(), 0); } } #[test] fn entry_read_next_sync() { let num_iter = if cfg!(miri) { 1 } else { 64 }; let hashmap: Arc> = Arc::new(HashMap::default()); for _ in 0..num_iter { let num_threads = if cfg!(miri) { 2 } else { 8 }; let workload_size = 256; let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for thread_id in 0..num_threads { let barrier = barrier.clone(); let hashmap = hashmap.clone(); threads.push(thread::spawn(move || { barrier.wait(); let range = (thread_id * workload_size)..((thread_id + 1) * workload_size); for id in range.clone() { let result = hashmap.insert_sync(id, id); assert!(result.is_ok()); } for id in range.clone() { assert!(hashmap.read_sync(&id, |_, _| ()).is_some()); } let mut in_range = 0; let mut entry = hashmap.begin_sync(); while let Some(current_entry) = entry.take() { if range.contains(current_entry.key()) { in_range += 1; } entry = current_entry.next_sync(); } assert!(in_range >= workload_size, "{in_range} {workload_size}"); let mut removed = 0; hashmap.retain_sync(|k, _| { if range.contains(k) { removed += 1; false } else { true } }); assert_eq!(removed, workload_size); let mut entry = hashmap.begin_sync(); while let Some(current_entry) = entry.take() { assert!(!range.contains(current_entry.key())); entry = current_entry.next_sync(); } })); } for thread in threads { assert!(thread.join().is_ok()); } assert_eq!(hashmap.len(), 0); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn insert_any_async() { let hashmap: Arc> = Arc::new(HashMap::default()); for _ in 0..64 { let num_tasks = 8; let workload_size = 512; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashmap = hashmap.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { let result = hashmap.insert_async(id, id).await; assert!(result.is_ok()); } for id in range.clone() { let result = hashmap.insert_async(id, id).await; assert_eq!(result, Err((id, id))); } let mut iterated = 0; hashmap .retain_async(|k, _| { if range.contains(k) { iterated += 1; } true }) .await; assert!(iterated >= workload_size); let mut removed = 0; hashmap .retain_async(|k, _| { if range.contains(k) { removed += 1; false } else { true } }) .await; assert_eq!(removed, workload_size); assert!(hashmap.iter_async(|k, _| !range.contains(k)).await); })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashmap.len(), 0); } } #[test] fn insert_any_sync() { let num_iter = if cfg!(miri) { 2 } else { 64 }; let hashmap: Arc> = Arc::new(HashMap::default()); for _ in 0..num_iter { let num_threads = if cfg!(miri) { 2 } else { 8 }; let workload_size = 256; let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for thread_id in 0..num_threads { let barrier = barrier.clone(); let hashmap = hashmap.clone(); threads.push(thread::spawn(move || { barrier.wait(); let range = (thread_id * workload_size)..((thread_id + 1) * workload_size); for id in range.clone() { let result = hashmap.insert_sync(id, id); assert!(result.is_ok()); } for id in range.clone() { let result = hashmap.insert_sync(id, id); assert_eq!(result, Err((id, id))); } let mut iterated = 0; hashmap.retain_sync(|k, _| { if range.contains(k) { iterated += 1; } true }); assert!(iterated >= workload_size); let mut removed = 0; hashmap.retain_sync(|k, _| { if range.contains(k) { removed += 1; false } else { true } }); assert_eq!(removed, workload_size); assert!(hashmap.iter_sync(|k, _| !range.contains(k))); })); } for thread in threads { assert!(thread.join().is_ok()); } assert_eq!(hashmap.len(), 0); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn insert_retain_remove_async() { let data_size = 4096; for _ in 0..16 { let hashmap: Arc> = Arc::new(HashMap::default()); let hashmap_clone = hashmap.clone(); let barrier = Arc::new(AsyncBarrier::new(2)); let barrier_clone = barrier.clone(); let inserted = Arc::new(AtomicU64::new(0)); let inserted_clone = inserted.clone(); let removed = Arc::new(AtomicU64::new(data_size)); let removed_clone = removed.clone(); let task = tokio::task::spawn(async move { // test insert barrier_clone.wait().await; let mut scanned = 0; let mut checker = BTreeSet::new(); let mut max = inserted_clone.load(Acquire); hashmap_clone.retain_sync(|k, _| { scanned += 1; checker.insert(*k); true }); for key in 0..max { assert!(checker.contains(&key)); } barrier_clone.wait().await; scanned = 0; checker = BTreeSet::new(); max = inserted_clone.load(Acquire); hashmap_clone .retain_async(|k, _| { scanned += 1; checker.insert(*k); true }) .await; for key in 0..max { assert!(checker.contains(&key)); } // test remove barrier_clone.wait().await; scanned = 0; max = removed_clone.load(Acquire); hashmap_clone.retain_sync(|k, _| { scanned += 1; assert!(*k < max); true }); barrier_clone.wait().await; scanned = 0; max = removed_clone.load(Acquire); hashmap_clone .retain_async(|k, _| { scanned += 1; assert!(*k < max); true }) .await; }); // insert barrier.wait().await; for i in 0..data_size { if i == data_size / 2 { barrier.wait().await; } assert!(hashmap.insert_sync(i, i).is_ok()); inserted.store(i, Release); } // remove barrier.wait().await; for i in (0..data_size).rev() { if i == data_size / 2 { barrier.wait().await; } assert!(hashmap.remove_sync(&i).is_some()); removed.store(i, Release); } assert!(task.await.is_ok()); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn insert_prune_any_async() { let hashmap: Arc> = Arc::new(HashMap::default()); for _ in 0..256 { let num_tasks = 8; let workload_size = 256; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashmap = hashmap.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { let result = hashmap.insert_async(id, id).await; assert!(result.is_ok()); } let mut removed = 0; hashmap .iter_mut_async(|entry| { if range.contains(&entry.0) { let (k, v) = entry.consume(); assert_eq!(k, v); removed += 1; } true }) .await; assert_eq!(removed, workload_size); assert!(hashmap.iter_async(|k, _| !range.contains(k)).await); })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashmap.len(), 0); } } #[test] fn insert_prune_any_sync() { let num_threads = if cfg!(miri) { 2 } else { 8 }; let num_iters = if cfg!(miri) { 1 } else { 256 }; let hashmap: Arc> = Arc::new(HashMap::default()); for _ in 0..num_iters { let workload_size = 256; let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for thread_id in 0..num_threads { let barrier = barrier.clone(); let hashmap = hashmap.clone(); threads.push(thread::spawn(move || { barrier.wait(); let range = (thread_id * workload_size)..((thread_id + 1) * workload_size); for id in range.clone() { let result = hashmap.insert_sync(id, id); assert!(result.is_ok()); } let mut removed = 0; hashmap.iter_mut_sync(|entry| { if range.contains(&entry.0) { let (k, v) = entry.consume(); assert_eq!(k, v); removed += 1; } true }); assert_eq!(removed, workload_size); assert!(hashmap.iter_sync(|k, _| !range.contains(k))); })); } for thread in threads { assert!(thread.join().is_ok()); } assert_eq!(hashmap.len(), 0); } } proptest! { #[cfg_attr(miri, ignore)] #[test] fn insert_prop(key in 0_usize..16) { let range = 256; let checker = Arc::new(AtomicUsize::new(0)); let hashmap: HashMap = HashMap::default(); for d in key..(key + range) { assert!( hashmap .insert_sync(Data::new(d, checker.clone()), Data::new(d, checker.clone())) .is_ok() ); *hashmap .entry_sync(Data::new(d, checker.clone())) .or_insert(Data::new(d + 1, checker.clone())) .get_mut() = Data::new(d + 2, checker.clone()); } for d in (key + range)..(key + range + range) { assert!( hashmap .insert_sync(Data::new(d, checker.clone()), Data::new(d, checker.clone())) .is_ok() ); *hashmap .entry_sync(Data::new(d, checker.clone())) .or_insert(Data::new(d + 1, checker.clone())) .get_mut() = Data::new(d + 2, checker.clone()); } let mut removed = 0; hashmap.retain_sync(|k, _| { if k.data >= key + range { removed += 1; false } else { true } }); assert_eq!(removed, range); assert_eq!(hashmap.len(), range); let mut found_keys = 0; hashmap.retain_sync(|k, v| { assert!(k.data < key + range); assert!(v.data >= key); found_keys += 1; true }); assert_eq!(found_keys, range); assert_eq!(checker.load(Relaxed), range * 2); for d in key..(key + range) { assert!(hashmap.contains_sync(&Data::new(d, checker.clone()))); } for d in key..(key + range) { assert!(hashmap.remove_sync(&Data::new(d, checker.clone())).is_some()); } assert_eq!(checker.load(Relaxed), 0); for d in key..(key + range) { assert!( hashmap .insert_sync(Data::new(d, checker.clone()), Data::new(d, checker.clone())) .is_ok() ); *hashmap .entry_sync(Data::new(d, checker.clone())) .or_insert(Data::new(d + 1, checker.clone())) .get_mut() = Data::new(d + 2, checker.clone()); } hashmap.clear_sync(); assert_eq!(checker.load(Relaxed), 0); for d in key..(key + range) { assert!( hashmap .insert_sync(Data::new(d, checker.clone()), Data::new(d, checker.clone())) .is_ok() ); *hashmap .entry_sync(Data::new(d, checker.clone())) .or_insert(Data::new(d + 1, checker.clone())) .get_mut() = Data::new(d + 2, checker.clone()); } assert_eq!(checker.load(Relaxed), range * 2); drop(hashmap); assert_eq!(checker.load(Relaxed), 0); } } } mod hashindex { use std::collections::BTreeSet; use std::panic::UnwindSafe; use std::rc::Rc; use std::sync::atomic::Ordering::{Acquire, Relaxed, Release}; use std::sync::atomic::{AtomicU64, AtomicUsize, fence}; use std::sync::{Arc, Barrier}; use std::thread; use futures::future::join_all; use proptest::strategy::{Strategy, ValueTree}; use proptest::test_runner::TestRunner; use sdd::Guard; use tokio::sync::Barrier as AsyncBarrier; use super::common::{EqTest, R}; use crate::HashIndex; use crate::hash_index::{self, Iter}; static_assertions::assert_not_impl_any!(HashIndex, Rc>: Send, Sync); static_assertions::assert_not_impl_any!(hash_index::Entry, Rc>: Send, Sync); static_assertions::assert_impl_all!(HashIndex: Send, Sync, UnwindSafe); static_assertions::assert_impl_all!(Iter<'static, String, String>: UnwindSafe); static_assertions::assert_not_impl_any!(HashIndex: Send, Sync); static_assertions::assert_not_impl_any!(Iter<'static, String, *const String>: Send, Sync); #[test] fn equivalent() { let hashindex: HashIndex = HashIndex::default(); assert!( hashindex .insert_sync(EqTest("HELLO".to_owned(), 1), 1) .is_ok() ); assert!(!hashindex.contains("NO")); assert!(hashindex.contains("HELLO")); } #[test] fn compare() { let hashindex1: HashIndex = HashIndex::new(); let hashindex2: HashIndex = HashIndex::new(); assert_eq!(hashindex1, hashindex2); assert!(hashindex1.insert_sync("Hi".to_string(), 1).is_ok()); assert_ne!(hashindex1, hashindex2); assert!(hashindex2.insert_sync("Hello".to_string(), 2).is_ok()); assert_ne!(hashindex1, hashindex2); assert!(hashindex1.insert_sync("Hello".to_string(), 2).is_ok()); assert_ne!(hashindex1, hashindex2); assert!(hashindex2.insert_sync("Hi".to_string(), 1).is_ok()); assert_eq!(hashindex1, hashindex2); assert!(hashindex1.remove_sync("Hi")); assert_ne!(hashindex1, hashindex2); } #[cfg_attr(miri, ignore)] #[tokio::test] async fn clear_async() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let workload_size = ((1_usize << 18) / 16) * 15; for _ in 0..2 { let hashindex: HashIndex = HashIndex::default(); for k in 0..workload_size { assert!(hashindex.insert_async(k, R::new(&INST_CNT)).await.is_ok()); } assert!(INST_CNT.load(Relaxed) >= workload_size); assert_eq!(hashindex.len(), workload_size); hashindex.clear_async().await; drop(hashindex); assert_eq!(INST_CNT.load(Relaxed), 0); } } #[test] fn clear_sync() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); for workload_size in 2..64 { let hashindex: HashIndex = HashIndex::default(); for k in 0..workload_size { assert!(hashindex.insert_sync(k, R::new(&INST_CNT)).is_ok()); } assert!(INST_CNT.load(Relaxed) >= workload_size); assert_eq!(hashindex.len(), workload_size); hashindex.clear_sync(); drop(hashindex); assert_eq!(INST_CNT.load(Relaxed), 0); } } #[test] fn from_iter() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let workload_size = 256; let hashindex = (0..workload_size) .map(|k| (k / 2, R::new(&INST_CNT))) .collect::>(); assert_eq!(hashindex.len(), workload_size / 2); drop(hashindex); assert_eq!(INST_CNT.load(Relaxed), 0); } #[test] fn clone() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashindex: HashIndex = HashIndex::with_capacity(1024); let workload_size = 256; for k in 0..workload_size { assert!(hashindex.insert_sync(k, R::new(&INST_CNT)).is_ok()); } let hashindex_clone = hashindex.clone(); drop(hashindex); for k in 0..workload_size { assert!(hashindex_clone.peek_with(&k, |_, _| ()).is_some()); } drop(hashindex_clone); assert_eq!(INST_CNT.load(Relaxed), 0); } #[test] fn string_key() { let num_iter = if cfg!(miri) { 4 } else { 4096 }; let hashindex1: HashIndex = HashIndex::default(); let hashindex2: HashIndex = HashIndex::default(); let mut checker1 = BTreeSet::new(); let mut checker2 = BTreeSet::new(); let mut runner = TestRunner::default(); for i in 0..num_iter { let prop_str = "[a-z]{1,16}".new_tree(&mut runner).unwrap(); let str_val = prop_str.current(); if hashindex1.insert_sync(str_val.clone(), i).is_ok() { checker1.insert((str_val.clone(), i)); } let str_borrowed = str_val.as_str(); if !cfg!(miri) { // `Miri` complains about concurrent access to `partial_hash`. assert!(hashindex1.peek_with(str_borrowed, |_, _| ()).is_some()); } if hashindex2.insert_sync(i, str_val.clone()).is_ok() { checker2.insert((i, str_val.clone())); } } assert_eq!(hashindex1.len(), checker1.len()); assert_eq!(hashindex2.len(), checker2.len()); for iter in checker1 { assert!(hashindex1.remove_sync(iter.0.as_str())); } for iter in checker2 { assert!(hashindex2.remove_sync(&iter.0)); } assert_eq!(hashindex1.len(), 0); assert_eq!(hashindex2.len(), 0); } #[test] fn local_ref() { struct L<'a>(&'a AtomicUsize); impl<'a> L<'a> { fn new(cnt: &'a AtomicUsize) -> Self { cnt.fetch_add(1, Relaxed); L(cnt) } } impl Drop for L<'_> { fn drop(&mut self) { self.0.fetch_sub(1, Relaxed); } } let workload_size = 256; let cnt = AtomicUsize::new(0); let hashindex: HashIndex = HashIndex::default(); for k in 0..workload_size { assert!(hashindex.insert_sync(k, L::new(&cnt)).is_ok()); } hashindex.retain_sync(|k, _| { assert!(*k < workload_size); true }); assert_eq!(cnt.load(Relaxed), workload_size); for k in 0..workload_size / 2 { assert!(hashindex.remove_sync(&k)); } hashindex.retain_sync(|k, _| { assert!(*k >= workload_size / 2); true }); drop(hashindex); assert_eq!(cnt.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn insert_peek_remove_async() { let hashindex: Arc> = Arc::new(HashIndex::default()); let num_tasks = 4; let workload_size = 1024 * 1024; for k in 0..num_tasks { assert!(hashindex.insert_async(k, k).await.is_ok()); } let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashindex = hashindex.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; if task_id == 0 { for k in num_tasks..workload_size { assert!(hashindex.insert_async(k, k).await.is_ok()); } for k in num_tasks..workload_size { assert!(hashindex.remove_async(&k).await); } } else { for k in 0..num_tasks { assert!(hashindex.peek_with(&k, |_, _| ()).is_some()); } } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } } #[test] fn insert_peek_remove_sync() { let hashindex: Arc> = Arc::new(HashIndex::default()); let num_threads = if cfg!(miri) { 2 } else { 4 }; let workload_size = if cfg!(miri) { 1024 } else { 1024 * 1024 }; for k in 0..num_threads { assert!(hashindex.insert_sync(k, k).is_ok()); } let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for task_id in 0..num_threads { let barrier = barrier.clone(); let hashindex = hashindex.clone(); threads.push(thread::spawn(move || { barrier.wait(); if task_id == 0 { for k in num_threads..workload_size { assert!(hashindex.insert_sync(k, k).is_ok()); } for k in num_threads..workload_size { assert!(hashindex.remove_sync(&k)); } } else if !cfg!(miri) { // See notes about `Miri` in `peek*`. for k in 0..num_threads { assert!(hashindex.peek_with(&k, |_, _| ()).is_some()); } } })); } for thread in threads { assert!(thread.join().is_ok()); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn peek_remove_insert_async() { let hashindex: Arc> = Arc::new(HashIndex::default()); let num_tasks = 4; let num_iter = 64; let workload_size = 256; let str = "HOW ARE YOU HOW ARE YOU"; for k in 0..num_tasks * workload_size { assert!(hashindex.insert_async(k, str.to_string()).await.is_ok()); } let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashindex = hashindex.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); if task_id == 0 { for _ in 0..num_iter { let v = { let guard = Guard::new(); let v = hashindex.peek(&(task_id * workload_size), &guard).unwrap(); assert_eq!(str, v); v.to_owned() }; fence(Acquire); for id in range.clone() { assert!(hashindex.remove_async(&id).await); assert!(hashindex.insert_async(id, str.to_string()).await.is_ok()); } fence(Acquire); assert_eq!(str, v); } } else { for _ in 0..num_iter { for id in range.clone() { assert!(hashindex.remove_async(&id).await); assert!(hashindex.insert_async(id, str.to_string()).await.is_ok()); } } } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashindex.len(), num_tasks * workload_size); } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn rebuild_async() { let hashindex: Arc> = Arc::new(HashIndex::default()); let num_tasks = 4; let num_iter = 64; let workload_size = 256; for k in 0..num_tasks * workload_size { assert!(hashindex.insert_sync(k, k).is_ok()); } let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashindex = hashindex.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for _ in 0..num_iter { for id in range.clone() { assert!(hashindex.remove_async(&id).await); assert!(hashindex.insert_async(id, id).await.is_ok()); } } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashindex.len(), num_tasks * workload_size); } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn entry_read_next_async() { let hashindex: Arc> = Arc::new(HashIndex::default()); for _ in 0..64 { let num_tasks = 8; let workload_size = 512; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashindex = hashindex.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { let result = hashindex.insert_async(id, id).await; assert!(result.is_ok()); } for id in range.clone() { assert!(hashindex.peek_with(&id, |_, _| ()).is_some()); } let mut in_range = 0; let mut entry = hashindex.begin_async().await; while let Some(current_entry) = entry.take() { if range.contains(current_entry.key()) { in_range += 1; } entry = current_entry.next_async().await; } assert!(in_range >= workload_size, "{in_range} {workload_size}"); let mut removed = 0; hashindex .retain_async(|k, _| { if range.contains(k) { removed += 1; false } else { true } }) .await; assert_eq!(removed, workload_size); let mut entry = hashindex.begin_async().await; while let Some(current_entry) = entry.take() { assert!(!range.contains(current_entry.key())); entry = current_entry.next_async().await; } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashindex.len(), 0); } } #[test] fn entry_read_next_sync() { let num_iter = if cfg!(miri) { 2 } else { 64 }; let hashindex: Arc> = Arc::new(HashIndex::default()); for _ in 0..num_iter { let num_threads = if cfg!(miri) { 3 } else { 8 }; let workload_size = 256; let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for thread_id in 0..num_threads { let barrier = barrier.clone(); let hashindex = hashindex.clone(); threads.push(thread::spawn(move || { barrier.wait(); let range = (thread_id * workload_size)..((thread_id + 1) * workload_size); for id in range.clone() { let result = hashindex.insert_sync(id, id); assert!(result.is_ok()); } for id in range.clone() { if !cfg!(miri) { // See notes about `Miri` in `peek*`. assert!(hashindex.peek_with(&id, |_, _| ()).is_some()); } } let mut in_range = 0; let mut entry = hashindex.begin_sync(); while let Some(current_entry) = entry.take() { if range.contains(current_entry.key()) { in_range += 1; } entry = current_entry.next_sync(); } assert!(in_range >= workload_size, "{in_range} {workload_size}"); let mut removed = 0; hashindex.retain_sync(|k, _| { if range.contains(k) { removed += 1; false } else { true } }); assert_eq!(removed, workload_size); let mut entry = hashindex.begin_sync(); while let Some(current_entry) = entry.take() { assert!(!range.contains(current_entry.key())); entry = current_entry.next_sync(); } })); } for thread in threads { assert!(thread.join().is_ok()); } assert_eq!(hashindex.len(), 0); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn insert_retain_remove_async() { let data_size = 4096; for _ in 0..16 { let hashindex: Arc> = Arc::new(HashIndex::default()); let hashindex_clone = hashindex.clone(); let barrier = Arc::new(AsyncBarrier::new(2)); let barrier_clone = barrier.clone(); let inserted = Arc::new(AtomicU64::new(0)); let inserted_clone = inserted.clone(); let removed = Arc::new(AtomicU64::new(data_size)); let removed_clone = removed.clone(); let task = tokio::task::spawn(async move { // test insert barrier_clone.wait().await; let mut scanned = 0; let mut checker = BTreeSet::new(); let mut max = inserted_clone.load(Acquire); hashindex_clone.retain_sync(|k, _| { scanned += 1; checker.insert(*k); true }); for key in 0..max { assert!(checker.contains(&key)); } barrier_clone.wait().await; scanned = 0; checker = BTreeSet::new(); max = inserted_clone.load(Acquire); hashindex_clone .retain_async(|k, _| { scanned += 1; checker.insert(*k); true }) .await; for key in 0..max { assert!(checker.contains(&key)); } // test remove barrier_clone.wait().await; scanned = 0; max = removed_clone.load(Acquire); hashindex_clone.retain_sync(|k, _| { scanned += 1; assert!(*k < max); true }); barrier_clone.wait().await; scanned = 0; max = removed_clone.load(Acquire); hashindex_clone .retain_async(|k, _| { scanned += 1; assert!(*k < max); true }) .await; }); // insert barrier.wait().await; for i in 0..data_size { if i == data_size / 2 { barrier.wait().await; } assert!(hashindex.insert_sync(i, i).is_ok()); inserted.store(i, Release); } // remove barrier.wait().await; for i in (0..data_size).rev() { if i == data_size / 2 { barrier.wait().await; } assert!(hashindex.remove_sync(&i)); removed.store(i, Release); } assert!(task.await.is_ok()); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn update_get_async() { let hashindex: Arc> = Arc::new(HashIndex::default()); for _ in 0..256 { let num_tasks = 8; let workload_size = 256; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashindex = hashindex.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { let result = hashindex.insert_async(id, id).await; assert!(result.is_ok()); let entry = hashindex.get_async(&id).await.unwrap(); assert_eq!(*entry.get(), id); } for id in range.clone() { hashindex.peek_with(&id, |k, v| assert_eq!(k, v)); let mut entry = hashindex.get_async(&id).await.unwrap(); assert_eq!(*entry.get(), id); entry.update(usize::MAX); } for id in range.clone() { hashindex.peek_with(&id, |_, v| assert_eq!(*v, usize::MAX)); let entry = hashindex.get_async(&id).await.unwrap(); assert_eq!(*entry.get(), usize::MAX); entry.remove_entry(); } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashindex.len(), 0); } } #[test] fn update_get_sync() { let num_threads = if cfg!(miri) { 2 } else { 8 }; let num_iters = if cfg!(miri) { 2 } else { 256 }; let hashindex: Arc> = Arc::new(HashIndex::default()); for _ in 0..num_iters { let workload_size = 256; let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for thread_id in 0..num_threads { let barrier = barrier.clone(); let hashindex = hashindex.clone(); threads.push(thread::spawn(move || { barrier.wait(); let range = (thread_id * workload_size)..((thread_id + 1) * workload_size); for id in range.clone() { let result = hashindex.insert_sync(id, id); assert!(result.is_ok()); let entry = hashindex.get_sync(&id).unwrap(); assert_eq!(*entry.get(), id); } for id in range.clone() { if !cfg!(miri) { // See notes about `Miri` in `peek*`. hashindex.peek_with(&id, |k, v| assert_eq!(k, v)); } let mut entry = hashindex.get_sync(&id).unwrap(); assert_eq!(*entry.get(), id); entry.update(usize::MAX); } for id in range.clone() { if !cfg!(miri) { // See notes about `Miri` in `peek*`. hashindex.peek_with(&id, |_, v| assert_eq!(*v, usize::MAX)); } let entry = hashindex.get_sync(&id).unwrap(); assert_eq!(*entry.get(), usize::MAX); entry.remove_entry(); } })); } for thread in threads { assert!(thread.join().is_ok()); } assert_eq!(hashindex.len(), 0); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn insert_retain_async() { let hashindex: Arc> = Arc::new(HashIndex::default()); for _ in 0..256 { let num_tasks = 8; let workload_size = 256; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashindex = hashindex.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { let result = hashindex.insert_async(id, id).await; assert!(result.is_ok()); } for id in range.clone() { let result = hashindex.insert_async(id, id).await; assert_eq!(result, Err((id, id))); } let mut iterated = 0; hashindex.iter(&Guard::new()).for_each(|(k, _)| { if range.contains(k) { iterated += 1; } }); assert!(iterated >= workload_size); assert!( hashindex .iter(&Guard::new()) .any(|(k, _)| range.contains(k)) ); let mut removed = 0; hashindex .retain_async(|k, _| { if range.contains(k) { removed += 1; false } else { true } }) .await; assert_eq!(removed, workload_size); assert!( !hashindex .iter(&Guard::new()) .any(|(k, _)| range.contains(k)) ); })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashindex.len(), 0); } } } mod hashset { use std::panic::UnwindSafe; use std::rc::Rc; use super::common::EqTest; use crate::HashSet; static_assertions::assert_not_impl_any!(HashSet>: Send, Sync); static_assertions::assert_impl_all!(HashSet: Send, Sync, UnwindSafe); static_assertions::assert_not_impl_any!(HashSet<*const String>: Send, Sync); #[test] fn equivalent() { let hashset: HashSet = HashSet::default(); assert!(hashset.insert_sync(EqTest("HELLO".to_owned(), 1)).is_ok()); assert!(!hashset.contains_sync("NO")); assert!(hashset.contains_sync("HELLO")); } #[test] fn from_iter() { let workload_size = 256; let hashset = (0..workload_size) .map(|k| k / 2) .collect::>(); assert_eq!(hashset.len(), workload_size / 2); } #[test] fn compare() { let hashset1: HashSet = HashSet::new(); let hashset2: HashSet = HashSet::new(); assert_eq!(hashset1, hashset2); assert!(hashset1.insert_sync("Hi".to_string()).is_ok()); assert_ne!(hashset1, hashset2); assert!(hashset2.insert_sync("Hello".to_string()).is_ok()); assert_ne!(hashset1, hashset2); assert!(hashset1.insert_sync("Hello".to_string()).is_ok()); assert_ne!(hashset1, hashset2); assert!(hashset2.insert_sync("Hi".to_string()).is_ok()); assert_eq!(hashset1, hashset2); assert!(hashset1.remove_sync("Hi").is_some()); assert_ne!(hashset1, hashset2); } } mod hashcache { use futures::future::join_all; use std::panic::UnwindSafe; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use tokio::sync::Barrier as AsyncBarrier; use proptest::prelude::*; use super::common::{EqTest, MaybeEq, R}; use crate::HashCache; use crate::hash_cache::{self, ReplaceResult}; static_assertions::assert_not_impl_any!(HashCache, Rc>: Send, Sync); static_assertions::assert_not_impl_any!(hash_cache::Entry, Rc>: Send, Sync); static_assertions::assert_impl_all!(HashCache: Send, Sync, UnwindSafe); static_assertions::assert_not_impl_any!(HashCache: Send, Sync); static_assertions::assert_impl_all!(hash_cache::OccupiedEntry: Send, Sync); static_assertions::assert_not_impl_any!(hash_cache::OccupiedEntry: Send, Sync, UnwindSafe); static_assertions::assert_impl_all!(hash_cache::VacantEntry: Send, Sync); static_assertions::assert_not_impl_any!(hash_cache::VacantEntry: Send, Sync, UnwindSafe); #[test] fn equivalent() { let hashcache: HashCache = HashCache::default(); assert!(hashcache.put_sync(EqTest("HELLO".to_owned(), 1), 1).is_ok()); assert!(!hashcache.contains_sync("NO")); assert!(hashcache.contains_sync("HELLO")); } #[test] fn put_drop() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashcache: HashCache = HashCache::default(); let workload_size = 256; for k in 0..workload_size { assert!(hashcache.put_sync(k, R::new(&INST_CNT)).is_ok()); } assert!(INST_CNT.load(Relaxed) <= hashcache.capacity()); drop(hashcache); assert_eq!(INST_CNT.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test] async fn put_drop_async() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashcache: HashCache = HashCache::default(); let workload_size = 1024; for k in 0..workload_size { assert!(hashcache.put_async(k, R::new(&INST_CNT)).await.is_ok()); } assert!(INST_CNT.load(Relaxed) <= hashcache.capacity()); drop(hashcache); assert_eq!(INST_CNT.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test] async fn replace_async() { let hashcache: HashCache = HashCache::default(); let workload_size = 256; for k in 0..workload_size { let ReplaceResult::NotReplaced(v) = hashcache.replace_async(MaybeEq(k, 0)).await else { unreachable!(); }; v.put_entry(k); } for k in 0..workload_size { let ReplaceResult::Replaced(o, p) = hashcache.replace_async(MaybeEq(k, 1)).await else { continue; }; assert_eq!(p.1, 0); assert_eq!(o.get(), &k); } assert!(hashcache.iter_async(|k, _| k.1 == 1).await); } #[test] fn replace_sync() { let hashcache: HashCache = HashCache::default(); let workload_size = 256; for k in 0..workload_size { let ReplaceResult::NotReplaced(v) = hashcache.replace_sync(MaybeEq(k, 0)) else { unreachable!(); }; v.put_entry(k); } for k in 0..workload_size { let ReplaceResult::Replaced(o, p) = hashcache.replace_sync(MaybeEq(k, 1)) else { continue; }; assert_eq!(p.1, 0); assert_eq!(o.get(), &k); } assert!(hashcache.iter_sync(|k, _| k.1 == 1)); } #[test] fn put_full_clear_put() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let capacity = 256; let hashcache: HashCache = HashCache::with_capacity(capacity, capacity); let mut max_key = 0; for k in 0..=capacity { if let Ok(Some(_)) = hashcache.put_sync(k, R::new(&INST_CNT)) { max_key = k; break; } } hashcache.clear_sync(); for k in 0..=capacity { if let Ok(Some(_)) = hashcache.put_sync(k, R::new(&INST_CNT)) { assert_eq!(max_key, k); break; } } assert!(INST_CNT.load(Relaxed) <= hashcache.capacity()); drop(hashcache); assert_eq!(INST_CNT.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test] async fn put_full_clear_put_async() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let capacity = 256; let hashcache: HashCache = HashCache::with_capacity(capacity, capacity); let mut max_key = 0; for k in 0..=capacity { if let Ok(Some(_)) = hashcache.put_async(k, R::new(&INST_CNT)).await { max_key = k; break; } } for i in 0..4 { if i % 2 == 0 { hashcache.clear_async().await; } else { hashcache.clear_sync(); } for k in 0..=capacity { if let Ok(Some(_)) = hashcache.put_async(k, R::new(&INST_CNT)).await { assert_eq!(max_key, k); break; } } } assert!(INST_CNT.load(Relaxed) <= hashcache.capacity()); drop(hashcache); assert_eq!(INST_CNT.load(Relaxed), 0); } #[test] fn put_retain_get() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let workload_size = 256; let retain_limit = 64; let hashcache: HashCache = HashCache::with_capacity(0, 256); for k in 0..workload_size { assert!(hashcache.put_sync(k, R::new(&INST_CNT)).is_ok()); } hashcache.retain_sync(|k, _| *k <= retain_limit); for k in 0..workload_size { if hashcache.get_sync(&k).is_some() { assert!(k <= retain_limit); } } for k in 0..workload_size { if hashcache.put_sync(k, R::new(&INST_CNT)).is_err() { assert!(k <= retain_limit); } } hashcache.retain_sync(|k, _| *k > retain_limit); for k in 0..workload_size { if hashcache.get_sync(&k).is_some() { assert!(k > retain_limit); } } for k in 0..workload_size { if hashcache.put_sync(k, R::new(&INST_CNT)).is_err() { assert!(k > retain_limit); } } assert!(INST_CNT.load(Relaxed) <= hashcache.capacity()); drop(hashcache); assert_eq!(INST_CNT.load(Relaxed), 0); } #[test] fn sparse_cache() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashcache = HashCache::::with_capacity(64, 64); for s in 0..16 { for k in s * 4..s * 4 + 4 { assert!(hashcache.put_sync(k, R::new(&INST_CNT)).is_ok()); } hashcache.retain_sync(|k, _| *k % 2 == 0); for k in s * 4..s * 4 + 4 { if hashcache.put_sync(k, R::new(&INST_CNT)).is_err() { assert!(k % 2 == 0); } } hashcache.clear_sync(); } drop(hashcache); assert_eq!(INST_CNT.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn put_get_remove() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let hashcache: Arc> = Arc::new(HashCache::default()); for _ in 0..256 { let num_tasks = 8; let workload_size = 256; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let hashcache = hashcache.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { if id % 4 == 0 { hashcache.entry_async(id).await.or_put(R::new(&INST_CNT)); } else { assert!(hashcache.put_async(id, R::new(&INST_CNT)).await.is_ok()); } } let mut hit_count = 0; for id in range.clone() { let hit = hashcache.get_async(&id).await.is_some(); if hit { hit_count += 1; } } assert!(hit_count <= *hashcache.capacity_range().end()); let mut remove_count = 0; for id in range.clone() { let removed = hashcache.remove_async(&id).await.is_some(); if removed { remove_count += 1; } } assert!(remove_count <= hit_count); })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(hashcache.len(), 0); } drop(hashcache); assert_eq!(INST_CNT.load(Relaxed), 0); } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn put_remove_maintain() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); for _ in 0..64 { let hashcache: Arc> = Arc::new(HashCache::with_capacity(256, 1024)); let num_tasks = 8; let workload_size = 2048; let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); let evicted = Arc::new(AtomicUsize::new(0)); let removed = Arc::new(AtomicUsize::new(0)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let evicted = evicted.clone(); let removed = removed.clone(); let hashcache = hashcache.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); let mut cnt = 0; for key in range.clone() { let result = hashcache.put_async(key, R::new(&INST_CNT)).await; match result { Ok(Some(_)) => cnt += 1, Ok(_) => (), Err(_) => unreachable!(), } } evicted.fetch_add(cnt, Relaxed); cnt = 0; for key in range.clone() { let result = hashcache.remove_async(&key).await; if result.is_some() { cnt += 1; } } removed.fetch_add(cnt, Relaxed); })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!( evicted.load(Relaxed) + removed.load(Relaxed), workload_size * num_tasks ); assert_eq!(hashcache.len(), 0); assert_eq!(INST_CNT.load(Relaxed), 0); } } proptest! { #[cfg_attr(miri, ignore)] #[test] fn capacity(xs in 0_usize..256) { let hashcache: HashCache = HashCache::with_capacity(0, 64); for k in 0..xs { assert!(hashcache.put_sync(k, k).is_ok()); } assert!(hashcache.capacity() <= 64); let hashcache: HashCache = HashCache::with_capacity(xs, xs * 2); for k in 0..xs { assert!(hashcache.put_sync(k, k).is_ok()); } if xs == 0 { assert_eq!(hashcache.capacity_range(), 0..=64); } else { assert_eq!(hashcache.capacity_range(), xs.next_power_of_two().max(64)..=(xs * 2).next_power_of_two().max(64)); } } } } mod treeindex { use std::borrow::Borrow; use std::cmp::Ordering; use std::collections::BTreeSet; use std::ops::RangeInclusive; use std::panic::UnwindSafe; use std::rc::Rc; use std::sync::atomic::Ordering::{Acquire, Relaxed, Release}; use std::sync::atomic::{AtomicBool, AtomicUsize}; use std::sync::{Arc, Barrier}; use std::thread; use futures::future::join_all; use proptest::prelude::*; use proptest::strategy::ValueTree; use proptest::test_runner::TestRunner; use sdd::Guard; use sdd::suspend; use tokio::sync::Barrier as AsyncBarrier; use tokio::task; use super::common::R; use crate::tree_index::{Iter, Range}; use crate::{Comparable, Equivalent, TreeIndex}; static_assertions::assert_not_impl_any!(TreeIndex, Rc>: Send, Sync); static_assertions::assert_impl_all!(TreeIndex: Send, Sync, UnwindSafe); static_assertions::assert_impl_all!(Iter<'static, 'static, String, String>: UnwindSafe); static_assertions::assert_impl_all!(Range<'static, 'static, String, String, String, RangeInclusive>: UnwindSafe); static_assertions::assert_not_impl_any!(TreeIndex: Send, Sync); static_assertions::assert_not_impl_any!(Iter<'static, 'static, String, *const String>: Send, Sync); static_assertions::assert_not_impl_any!(Range<'static, 'static, String, *const String, String, RangeInclusive>: Send, Sync); #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] struct CmpTest(String, usize); impl Comparable for &str { fn compare(&self, key: &CmpTest) -> Ordering { self.cmp(&key.0.borrow()) } } impl Equivalent for &str { fn equivalent(&self, key: &CmpTest) -> bool { key.0.eq(self) } } #[test] fn comparable() { let tree: TreeIndex = TreeIndex::default(); assert!(tree.insert_sync(CmpTest("A".to_owned(), 0), 0).is_ok()); assert!(tree.peek_with(&"A", |_, _| true).unwrap()); assert!(tree.insert_sync(CmpTest("B".to_owned(), 0), 0).is_ok()); assert!(tree.peek_with(&"B", |_, _| true).unwrap()); assert!(tree.insert_sync(CmpTest("C".to_owned(), 0), 0).is_ok()); assert!(tree.peek_with(&"C", |_, _| true).unwrap()); assert!(tree.insert_sync(CmpTest("Z".to_owned(), 0), 0).is_ok()); assert!(tree.peek_with(&"Z", |_, _| true).unwrap()); assert!(tree.insert_sync(CmpTest("Y".to_owned(), 0), 0).is_ok()); assert!(tree.peek_with(&"Y", |_, _| true).unwrap()); assert!(tree.insert_sync(CmpTest("X".to_owned(), 0), 0).is_ok()); assert!(tree.peek_with(&"X", |_, _| true).unwrap()); let guard = Guard::new(); let mut range = tree.range("C".."Y", &guard); assert_eq!(range.next().unwrap().0.0, "C"); assert_eq!(range.next().unwrap().0.0, "X"); assert!(range.next().is_none()); tree.remove_range_sync("C".."Y"); assert!(tree.peek_with(&"A", |_, _| true).unwrap()); assert!(tree.peek_with(&"B", |_, _| true).unwrap()); assert!(tree.peek_with(&"C", |_, _| true).is_none()); assert!(tree.peek_with(&"X", |_, _| true).is_none()); assert!(tree.peek_with(&"Y", |_, _| true).unwrap()); assert!(tree.peek_with(&"Z", |_, _| true).unwrap()); } #[test] fn insert_drop() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let tree: TreeIndex = TreeIndex::default(); let workload_size = 256; for k in 0..workload_size { assert!(tree.insert_sync(k, R::new(&INST_CNT)).is_ok()); } assert!(INST_CNT.load(Relaxed) >= workload_size); assert_eq!(tree.len(), workload_size); drop(tree); while INST_CNT.load(Relaxed) != 0 { Guard::new().accelerate(); thread::yield_now(); } } #[cfg_attr(miri, ignore)] #[tokio::test] async fn insert_drop_async() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let tree: TreeIndex = TreeIndex::default(); let workload_size = 1024; for k in 0..workload_size { assert!(tree.insert_async(k, R::new(&INST_CNT)).await.is_ok()); } assert!(INST_CNT.load(Relaxed) >= workload_size); assert_eq!(tree.len(), workload_size); drop(tree); while INST_CNT.load(Relaxed) != 0 { Guard::new().accelerate(); tokio::task::yield_now().await; } } #[test] fn insert_remove() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let tree: TreeIndex = TreeIndex::default(); let workload_size = 256; for k in 0..workload_size { assert!(tree.insert_sync(k, R::new(&INST_CNT)).is_ok()); } assert!(INST_CNT.load(Relaxed) >= workload_size); assert_eq!(tree.len(), workload_size); for k in 0..workload_size { assert!(tree.remove_sync(&k)); } assert_eq!(tree.len(), 0); while INST_CNT.load(Relaxed) != 0 { Guard::new().accelerate(); thread::yield_now(); } } #[test] fn insert_remove_clear() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let num_threads = 3; let num_iter = if cfg!(miri) { 1 } else { 16 }; let workload_size = if cfg!(miri) { 32 } else { 1024 }; let tree: Arc> = Arc::new(TreeIndex::default()); let mut threads = Vec::with_capacity(num_threads); let barrier = Arc::new(Barrier::new(num_threads)); for task_id in 0..num_threads { let barrier = barrier.clone(); let tree = tree.clone(); threads.push(thread::spawn(move || { for _ in 0..num_iter { barrier.wait(); match task_id { 0 => { for k in 0..workload_size { assert!(tree.insert_sync(k, R::new(&INST_CNT)).is_ok()); } } 1 => { for k in 0..workload_size / 8 { tree.remove_sync(&(k * 4)); } } _ => { for _ in 0..workload_size / 64 { if tree.len() >= workload_size / 4 { tree.clear(); } } } } tree.clear(); assert!(suspend()); } drop(tree); })); } for thread in threads { assert!(thread.join().is_ok()); } drop(tree); while INST_CNT.load(Relaxed) != 0 { Guard::new().accelerate(); thread::yield_now(); } } #[test] fn clear() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let tree: TreeIndex = TreeIndex::default(); let workload_size = if cfg!(miri) { 256 } else { 1024 * 1024 }; for k in 0..workload_size { assert!(tree.insert_sync(k, R::new(&INST_CNT)).is_ok()); } assert!(INST_CNT.load(Relaxed) >= workload_size); assert_eq!(tree.len(), workload_size); tree.clear(); while INST_CNT.load(Relaxed) != 0 { Guard::new().accelerate(); thread::yield_now(); } } #[test] fn reclaim() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); struct R(usize); impl R { fn new() -> R { R(INST_CNT.fetch_add(1, Relaxed)) } } impl Clone for R { fn clone(&self) -> Self { INST_CNT.fetch_add(1, Relaxed); R(self.0) } } impl Drop for R { fn drop(&mut self) { INST_CNT.fetch_sub(1, Relaxed); } } let data_size = 256; let tree: TreeIndex = TreeIndex::new(); for k in 0..data_size { assert!(tree.insert_sync(k, R::new()).is_ok()); } for k in (0..data_size).rev() { assert!(tree.remove_sync(&k)); } let mut cnt = 0; while INST_CNT.load(Relaxed) > 0 { Guard::new().accelerate(); cnt += 1; } assert!(cnt >= INST_CNT.load(Relaxed)); let tree: TreeIndex = TreeIndex::new(); for k in 0..(data_size / 16) { assert!(tree.insert_sync(k, R::new()).is_ok()); } tree.clear(); while INST_CNT.load(Relaxed) > 0 { Guard::new().accelerate(); } } #[test] fn clone() { static INST_CNT: AtomicUsize = AtomicUsize::new(0); let tree: TreeIndex = TreeIndex::default(); let workload_size = 256; for k in 0..workload_size { assert!(tree.insert_sync(k, R::new(&INST_CNT)).is_ok()); } let tree_clone = tree.clone(); tree.clear(); for k in 0..workload_size { assert!(tree_clone.peek_with(&k, |_, _| ()).is_some()); } tree_clone.clear(); while INST_CNT.load(Relaxed) != 0 { Guard::new().accelerate(); thread::yield_now(); } } #[test] fn compare() { let tree1: TreeIndex = TreeIndex::new(); let tree2: TreeIndex = TreeIndex::new(); assert_eq!(tree1, tree2); assert!(tree1.insert_sync("Hi".to_string(), 1).is_ok()); assert_ne!(tree1, tree2); assert!(tree2.insert_sync("Hello".to_string(), 2).is_ok()); assert_ne!(tree1, tree2); assert!(tree1.insert_sync("Hello".to_string(), 2).is_ok()); assert_ne!(tree1, tree2); assert!(tree2.insert_sync("Hi".to_string(), 1).is_ok()); assert_eq!(tree1, tree2); assert!(tree1.remove_sync("Hi")); assert_ne!(tree1, tree2); } #[test] fn insert_peek() { let num_iter = if cfg!(miri) { 2 } else { 1024 }; let tree1: TreeIndex = TreeIndex::default(); let tree2: TreeIndex = TreeIndex::default(); let mut checker1 = BTreeSet::new(); let mut checker2 = BTreeSet::new(); let mut runner = TestRunner::default(); for i in 0..num_iter { let prop_str = "[a-z]{1,16}".new_tree(&mut runner).unwrap(); let str_val = prop_str.current(); if tree1.insert_sync(str_val.clone(), i).is_ok() { checker1.insert((str_val.clone(), i)); } let str_borrowed = str_val.as_str(); assert!(tree1.peek_with(str_borrowed, |_, _| ()).is_some()); if tree2.insert_sync(i, str_val.clone()).is_ok() { checker2.insert((i, str_val.clone())); } } for iter in &checker1 { let v = tree1.peek_with(iter.0.as_str(), |_, v| *v); assert_eq!(v.unwrap(), iter.1); } for iter in &checker2 { let v = tree2.peek_with(&iter.0, |_, v| v.clone()); assert_eq!(v.unwrap(), iter.1); } } #[test] fn range() { let tree: TreeIndex = TreeIndex::default(); assert!(tree.insert_sync("Ape".to_owned(), 0).is_ok()); assert!(tree.insert_sync("Apple".to_owned(), 1).is_ok()); assert!(tree.insert_sync("Banana".to_owned(), 3).is_ok()); assert!(tree.insert_sync("Badezimmer".to_owned(), 2).is_ok()); assert_eq!(tree.range(..="Ball".to_owned(), &Guard::new()).count(), 3); assert_eq!( tree.range("Ape".to_owned()..="Ball".to_owned(), &Guard::new()) .count(), 3 ); assert_eq!( tree.range("Apex".to_owned()..="Ball".to_owned(), &Guard::new()) .count(), 2 ); assert_eq!( tree.range("Ace".to_owned()..="Ball".to_owned(), &Guard::new()) .count(), 3 ); assert_eq!(tree.range(..="Z".to_owned(), &Guard::new()).count(), 4); assert_eq!( tree.range("Ape".to_owned()..="Z".to_owned(), &Guard::new()) .count(), 4 ); assert_eq!( tree.range("Apex".to_owned()..="Z".to_owned(), &Guard::new()) .count(), 3 ); assert_eq!( tree.range("Ace".to_owned()..="Z".to_owned(), &Guard::new()) .count(), 4 ); assert_eq!(tree.range(.."Banana".to_owned(), &Guard::new()).count(), 3); assert_eq!( tree.range("Ape".to_owned().."Banana".to_owned(), &Guard::new()) .count(), 3 ); assert_eq!( tree.range("Apex".to_owned().."Banana".to_owned(), &Guard::new()) .count(), 2 ); assert_eq!( tree.range("Ace".to_owned().."Banana".to_owned(), &Guard::new()) .count(), 3 ); } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn insert_peek_remove_async() { let num_tasks = 8; let workload_size = 256; for _ in 0..256 { let tree: Arc> = Arc::new(TreeIndex::default()); let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); for task_id in 0..num_tasks { let barrier = barrier.clone(); let tree = tree.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { assert!(tree.insert_async(id, id).await.is_ok()); assert!(tree.insert_async(id, id).await.is_err()); } for id in range.clone() { let result = tree.peek_with(&id, |_, v| *v); assert_eq!(result, Some(id)); } for id in range.clone() { assert!(tree.remove_if_async(&id, |v| *v == id).await); } for id in range { assert!(!tree.remove_if_async(&id, |v| *v == id).await); } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } assert_eq!(tree.len(), 0); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn remove_range_async() { let num_tasks = 2; let workload_size = 4096; for _ in 0..16 { let tree: Arc> = Arc::new(TreeIndex::default()); let mut tasks = Vec::with_capacity(num_tasks); let barrier = Arc::new(AsyncBarrier::new(num_tasks)); let data = Arc::new(AtomicUsize::default()); for task_id in 0..num_tasks { let barrier = barrier.clone(); let data = data.clone(); let tree = tree.clone(); tasks.push(tokio::task::spawn(async move { barrier.wait().await; if task_id == 0 { for k in 1..=workload_size { assert!(tree.insert_async(k, k).await.is_ok()); assert!(tree.peek(&k, &Guard::new()).is_some()); data.store(k, Release); } } else { loop { let end_bound = data.load(Acquire); if end_bound == workload_size { break; } else if end_bound <= 1 { task::yield_now().await; continue; } if end_bound % 2 == 0 { let keys = tree .range(..end_bound, &Guard::new()) .map(|(k, v)| (*k, *v)) .collect::>(); for (k, _) in keys { tree.remove_async(&k).await; } } else { tree.remove_range_async(..end_bound).await; } if end_bound % 5 == 0 { for (k, v) in tree.iter(&Guard::new()) { assert_eq!(k, v); assert!(!(..end_bound).contains(k), "{k}"); } } else { assert!( tree.peek(&(end_bound - 1), &Guard::new()).is_none(), "{end_bound} {}", data.load(Relaxed) ); assert!(tree.peek(&end_bound, &Guard::new()).is_some()); } // Give the scheduler a chance to run the other task. task::yield_now().await; } } })); } for task in join_all(tasks).await { assert!(task.is_ok()); } tree.remove_range_sync(..workload_size); assert!(tree.peek(&(workload_size - 1), &Guard::new()).is_none()); assert!(tree.peek(&workload_size, &Guard::new()).is_some()); assert_eq!(tree.len(), 1); assert_eq!(tree.depth(), 1); } } #[test] fn insert_peek_iter_sync() { let range = if cfg!(miri) { 64 } else { 4096 }; let num_threads = if cfg!(miri) { 2 } else { 16 }; let tree: Arc> = Arc::new(TreeIndex::new()); let barrier = Arc::new(Barrier::new(num_threads)); let mut threads = Vec::with_capacity(num_threads); for thread_id in 0..num_threads { let tree = tree.clone(); let barrier = barrier.clone(); threads.push(thread::spawn(move || { let first_key = thread_id * range; barrier.wait(); for key in first_key..(first_key + range / 2) { assert!(tree.insert_sync(key, key).is_ok()); } for key in first_key..(first_key + range / 2) { assert!( tree.peek_with(&key, |key, val| assert_eq!(key, val)) .is_some() ); } for key in (first_key + range / 2)..(first_key + range) { assert!(tree.insert_sync(key, key).is_ok()); } for key in (first_key + range / 2)..(first_key + range) { assert!( tree.peek_with(&key, |key, val| assert_eq!(key, val)) .is_some() ); } })); } for thread in threads { assert!(thread.join().is_ok()); } let mut found = 0; for key in 0..num_threads * range { if tree .peek_with(&key, |key, val| assert_eq!(key, val)) .is_some() { found += 1; } } assert_eq!(found, num_threads * range); for key in 0..num_threads * range { assert!( tree.peek_with(&key, |key, val| assert_eq!(key, val)) .is_some() ); } let guard = Guard::new(); let scanner = tree.iter(&guard); let mut prev = 0; for entry in scanner { assert!(prev == 0 || prev < *entry.0); assert_eq!(*entry.0, *entry.1); prev = *entry.0; } } #[test] fn insert_peek_remove_range_sync() { let range = if cfg!(miri) { 8 } else { 4096 }; let num_threads = if cfg!(miri) { 2 } else { 8 }; let tree: Arc> = Arc::new(TreeIndex::new()); for t in 0..num_threads { // insert markers assert!(tree.insert_sync(t * range, t * range).is_ok()); } let stopped: Arc = Arc::new(AtomicBool::new(false)); let barrier = Arc::new(Barrier::new(num_threads + 1)); let mut threads = Vec::with_capacity(num_threads); for thread_id in 0..num_threads { let tree = tree.clone(); let stopped = stopped.clone(); let barrier = barrier.clone(); threads.push(thread::spawn(move || { let first_key = thread_id * range; barrier.wait(); while !stopped.load(Relaxed) { for key in (first_key + 1)..(first_key + range) { assert!(tree.insert_sync(key, key).is_ok()); } for key in (first_key + 1)..(first_key + range) { assert!( tree.peek_with(&key, |key, val| assert_eq!(key, val)) .is_some() ); } { let guard = Guard::new(); let mut range_scanner = tree.range(first_key.., &guard); let mut entry = range_scanner.next().unwrap(); assert_eq!(entry, (&first_key, &first_key)); entry = range_scanner.next().unwrap(); assert_eq!(entry, (&(first_key + 1), &(first_key + 1))); entry = range_scanner.next().unwrap(); assert_eq!(entry, (&(first_key + 2), &(first_key + 2))); entry = range_scanner.next().unwrap(); assert_eq!(entry, (&(first_key + 3), &(first_key + 3))); } let key_at_halfway = first_key + range / 2; for key in (first_key + 1)..(first_key + range) { if key == key_at_halfway { let guard = Guard::new(); let mut range_scanner = tree.range((first_key + 1).., &guard); let entry = range_scanner.next().unwrap(); assert_eq!(entry, (&key_at_halfway, &key_at_halfway)); let entry = range_scanner.next().unwrap(); assert_eq!(entry, (&(key_at_halfway + 1), &(key_at_halfway + 1))); } assert!(tree.remove_sync(&key)); assert!(!tree.remove_sync(&key)); assert!(tree.peek_with(&(first_key + 1), |_, _| ()).is_none()); assert!(tree.peek_with(&key, |_, _| ()).is_none()); } for key in (first_key + 1)..(first_key + range) { assert!( tree.peek_with(&key, |key, val| assert_eq!(key, val)) .is_none() ); } } })); } barrier.wait(); let iteration = if cfg!(miri) { 16 } else { 512 }; for _ in 0..iteration { let mut found_0 = false; let mut found_markers = 0; let mut prev_marker = 0; let mut prev = 0; let guard = Guard::new(); for iter in tree.iter(&guard) { let current = *iter.0; if current % range == 0 { found_markers += 1; if current == 0 { found_0 = true; } if current > 0 { assert_eq!(prev_marker + range, current); } prev_marker = current; } assert!(prev == 0 || prev < current); prev = current; } assert!(found_0); assert_eq!(found_markers, num_threads); } stopped.store(true, Release); for thread in threads { assert!(thread.join().is_ok()); } } #[test] fn insert_peek_remove_sync() { let num_threads = if cfg!(miri) { 2 } else { 16 }; let tree: Arc> = Arc::new(TreeIndex::new()); let barrier = Arc::new(Barrier::new(num_threads)); let mut threads = Vec::with_capacity(num_threads); for thread_id in 0..num_threads { let tree = tree.clone(); let barrier = barrier.clone(); threads.push(thread::spawn(move || { barrier.wait(); let data_size = if cfg!(miri) { 16 } else { 4096 }; for _ in 0..data_size { let range = 0..32; let inserted = range .clone() .filter(|i| tree.insert_sync(*i, thread_id).is_ok()) .count(); let found = range .clone() .filter(|i| tree.peek_with(i, |_, v| *v == thread_id).is_some_and(|t| t)) .count(); let removed = range .clone() .filter(|i| tree.remove_if_sync(i, |v| *v == thread_id)) .count(); let removed_again = range .clone() .filter(|i| tree.remove_if_sync(i, |v| *v == thread_id)) .count(); assert_eq!(removed_again, 0); assert_eq!(found, removed, "{inserted} {found} {removed}"); assert_eq!(inserted, found, "{inserted} {found} {removed}"); } })); } for thread in threads { assert!(thread.join().is_ok()); } assert_eq!(tree.len(), 0); } #[test] fn insert_remove_iter_sync() { let num_iter = if cfg!(miri) { 4 } else { 64 }; let data_size = if cfg!(miri) { 128 } else { 4096 }; for _ in 0..num_iter { let tree: Arc> = Arc::new(TreeIndex::default()); let barrier = Arc::new(Barrier::new(3)); let inserted = Arc::new(AtomicUsize::new(0)); let removed = Arc::new(AtomicUsize::new(data_size)); let mut threads = Vec::new(); for _ in 0..2 { let tree = tree.clone(); let barrier = barrier.clone(); let inserted = inserted.clone(); let removed = removed.clone(); let thread = thread::spawn(move || { // test insert for _ in 0..2 { barrier.wait(); let max = inserted.load(Acquire); let mut prev = 0; let mut iterated = 0; let guard = Guard::new(); for iter in tree.iter(&guard) { assert!( prev == 0 || (*iter.0 <= max && prev + 1 == *iter.0) || *iter.0 > prev ); prev = *iter.0; iterated += 1; } assert!(iterated >= max); } // test remove for _ in 0..2 { barrier.wait(); let mut prev = 0; let max = removed.load(Acquire); let guard = Guard::new(); for iter in tree.iter(&guard) { let current = *iter.0; assert!(current < max); assert!(prev + 1 == current || prev == 0); prev = current; } } }); threads.push(thread); } // insert barrier.wait(); for i in 0..data_size { if i == data_size / 2 { barrier.wait(); } assert!(tree.insert_sync(i, 0).is_ok()); inserted.store(i, Release); } // remove barrier.wait(); for i in (0..data_size).rev() { if i == data_size / 2 { barrier.wait(); } assert!(tree.remove_sync(&i)); removed.store(i, Release); } for thread in threads { assert!(thread.join().is_ok()); } } } proptest! { #[cfg_attr(miri, ignore)] #[test] fn remove_range_prop(lower in 0_usize..4096_usize, range in 0_usize..4096_usize) { let remove_range = lower..lower + range; let insert_range = (256_usize, 4095_usize); let tree = TreeIndex::default(); for k in insert_range.0..=insert_range.1 { prop_assert!(tree.insert_sync(k, k).is_ok()); } if usize::BITS == 32 { prop_assert_eq!(tree.depth(), 4); } else { prop_assert_eq!(tree.depth(), 3); } tree.remove_range_sync(remove_range.clone()); if remove_range.contains(&insert_range.0) && remove_range.contains(&insert_range.1) { prop_assert!(tree.is_empty()); } for (k, v) in tree.iter(&Guard::new()) { prop_assert_eq!(k, v); prop_assert!(!remove_range.contains(k), "{k}"); } for k in 0_usize..4096_usize { if tree.peek_with(&k, |_, _| ()).is_some() { prop_assert!(!remove_range.contains(&k), "{k}"); } } for k in remove_range.clone() { prop_assert!(tree.insert_sync(k, k).is_ok()); } let mut cnt = 0; for (k, v) in tree.iter(&Guard::new()) { prop_assert_eq!(k, v); if remove_range.contains(k) { cnt += 1; } } assert_eq!(cnt, range); } } } #[cfg(feature = "serde")] mod serde { use serde_test::{Token, assert_tokens}; use crate::{HashCache, HashIndex, HashMap, HashSet, TreeIndex}; #[test] fn hashmap() { let hashmap: HashMap = HashMap::new(); assert!(hashmap.insert_sync(2, -6).is_ok()); assert_tokens( &hashmap, &[ Token::Map { len: Some(1) }, Token::U64(2), Token::I16(-6), Token::MapEnd, ], ); } #[test] fn hashset() { let hashset: HashSet = HashSet::new(); assert!(hashset.insert_sync(2).is_ok()); assert_tokens( &hashset, &[Token::Seq { len: Some(1) }, Token::U64(2), Token::SeqEnd], ); } #[test] fn hashindex() { let hashindex: HashIndex = HashIndex::new(); assert!(hashindex.insert_sync(2, -6).is_ok()); assert_tokens( &hashindex, &[ Token::Map { len: Some(1) }, Token::U64(2), Token::I16(-6), Token::MapEnd, ], ); } #[test] fn hashcache() { let hashcache: HashCache = HashCache::new(); let capacity_range = hashcache.capacity_range(); assert!(hashcache.put_sync(2, -6).is_ok()); assert_tokens( &hashcache, &[ Token::Map { len: Some(*capacity_range.end()), }, Token::U64(2), Token::I16(-6), Token::MapEnd, ], ); } #[test] fn treeindex() { let treeindex: TreeIndex = TreeIndex::new(); assert!(treeindex.insert_sync(4, -4).is_ok()); assert!(treeindex.insert_sync(2, -6).is_ok()); assert!(treeindex.insert_sync(3, -5).is_ok()); assert_tokens( &treeindex, &[ Token::Map { len: Some(3) }, Token::U64(2), Token::I16(-6), Token::U64(3), Token::I16(-5), Token::U64(4), Token::I16(-4), Token::MapEnd, ], ); } } scc-3.4.8/src/tests.rs000064400000000000000000000002031046102023000127020ustar 00000000000000#[cfg(feature = "loom")] mod models; #[cfg(not(feature = "loom"))] mod benchmarks; #[cfg(not(feature = "loom"))] mod unit_tests; scc-3.4.8/src/tree_index/internal_node.rs000064400000000000000000001537471046102023000165350ustar 00000000000000use std::cmp::Ordering::{Equal, Greater, Less}; use std::mem::forget; use std::ops::RangeBounds; use std::ptr; use std::sync::atomic::AtomicPtr; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use saa::Lock; use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; use super::leaf::{DIMENSION, InsertResult, Leaf, RemoveResult, Scanner}; use super::leaf_node::RETIRED; use super::leaf_node::RemoveRangeState; use super::node::Node; use crate::Comparable; use crate::async_helper::TryWait; use crate::exit_guard::ExitGuard; /// Internal node. /// /// The layout of an internal node: `|ptr(children)/max(child keys)|...|ptr(children)|`. pub struct InternalNode { /// Children of the [`InternalNode`]. pub(super) children: Leaf>>, /// A child [`Node`] that has no upper key bound. /// /// It stores the maximum key in the node, and key-value pairs are first pushed to this [`Node`] /// until it splits. pub(super) unbounded_child: AtomicShared>, /// Ongoing split operation. split_op: StructuralChange, /// [`Lock`] to protect the [`InternalNode`]. pub(super) lock: Lock, } /// [`Locker`] holds exclusive ownership of an [`InternalNode`]. pub(super) struct Locker<'n, K, V> { internal_node: &'n InternalNode, } /// [`StructuralChange`] stores intermediate results during a split operation. /// /// `AtomicPtr` members may point to values under the protection of the [`Guard`] used for the /// split operation. struct StructuralChange { origin_node_key: AtomicPtr, origin_node: AtomicShared>, low_key_node: AtomicShared>, middle_key: AtomicPtr, high_key_node: AtomicShared>, } impl InternalNode { /// Creates a new empty internal node. #[inline] pub(super) fn new() -> InternalNode { InternalNode { children: Leaf::new(), unbounded_child: AtomicShared::null(), split_op: StructuralChange::default(), lock: Lock::default(), } } /// Clears the internal node. #[inline] pub(super) fn clear(&self, guard: &Guard) { let scanner = Scanner::new(&self.children); for (_, child) in scanner { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { child.clear(guard); } } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { unbounded.clear(guard); } } /// Returns the depth of the node. #[inline] pub(super) fn depth(&self, depth: usize, guard: &Guard) -> usize { let unbounded_ptr = self.unbounded_child.load(Relaxed, guard); if let Some(unbounded_ref) = unbounded_ptr.as_ref() { return unbounded_ref.depth(depth + 1, guard); } depth } /// Returns `true` if the [`InternalNode`] has retired. #[inline] pub(super) fn retired(&self) -> bool { self.unbounded_child.tag(Acquire) == RETIRED } } impl InternalNode where K: 'static + Clone + Ord, V: 'static + Clone, { /// Searches for an entry containing the specified key. #[inline] pub(super) fn search_entry<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<(&'g K, &'g V)> where K: 'g, Q: Comparable + ?Sized, { loop { let (child, metadata) = self.children.min_greater_equal(key); if let Some((_, child)) = child { if let Some(child) = child.load(Acquire, guard).as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. return child.search_entry(key, guard); } } } else { let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if self.children.validate(metadata) { return unbounded.search_entry(key, guard); } } else { return None; } } } } /// Searches for the value associated with the specified key. #[inline] pub(super) fn search_value<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<&'g V> where K: 'g, Q: Comparable + ?Sized, { loop { let (child, metadata) = self.children.min_greater_equal(key); if let Some((_, child)) = child { if let Some(child) = child.load(Acquire, guard).as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. return child.search_value(key, guard); } } } else { let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if self.children.validate(metadata) { return unbounded.search_value(key, guard); } } else { return None; } } } } /// Returns the minimum key entry. #[inline] pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { loop { let mut retry = false; let scanner = Scanner::new(&self.children); let metadata = scanner.metadata(); for (_, child) in scanner { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. if let Some(scanner) = child.min(guard) { return Some(scanner); } continue; } } // It is not a hot loop - see `LeafNode::search_entry`. retry = true; break; } if retry { continue; } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if self.children.validate(metadata) { return unbounded.min(guard); } continue; } return None; } } /// Returns a [`Scanner`] pointing to an entry that is close enough to the entry with the /// maximum key among those keys that are smaller than or equal to the given key. /// /// Returns `None` if all keys in the [`InternalNode`] are greater than the given key. #[inline] pub(super) fn max_le_appr<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option> where K: 'g, Q: Comparable + ?Sized, { loop { if let Some(scanner) = Scanner::max_less(&self.children, key) { if let Some((_, child)) = scanner.get() { if let Some(child) = child.load(Acquire, guard).as_ref() { if self.children.validate(scanner.metadata()) { // Data race resolution - see `LeafNode::search_entry`. if let Some(scanner) = child.max_le_appr(key, guard) { return Some(scanner); } // Fallback. break; } } // It is not a hot loop - see `LeafNode::search_entry`. continue; } } // Fallback. break; } // Starts scanning from the minimum key. let mut min_scanner = self.min(guard)?; min_scanner.next(); loop { if let Some((k, _)) = min_scanner.get() { if key.compare(k).is_ge() { return Some(min_scanner); } break; } min_scanner = min_scanner.jump(None, guard)?; } None } /// Inserts a key-value pair. #[inline] pub(super) fn insert( &self, mut key: K, mut val: V, async_wait: &mut W, guard: &Guard, ) -> Result, (K, V)> { loop { let (child, metadata) = self.children.min_greater_equal(&key); if let Some((child_key, child)) = child { let child_ptr = child.load(Acquire, guard); if let Some(child_ref) = child_ptr.as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. let insert_result = child_ref.insert(key, val, async_wait, guard)?; match insert_result { InsertResult::Success | InsertResult::Duplicate(..) | InsertResult::Frozen(..) => return Ok(insert_result), InsertResult::Full(k, v) => { let split_result = self.split_node( (k, v), Some(child_key), child_ptr, child, false, async_wait, guard, )?; if let InsertResult::Retry(k, v) = split_result { key = k; val = v; continue; } return Ok(split_result); } InsertResult::Retired(k, v) => { debug_assert!(child_ref.retired()); if self.coalesce(guard) == RemoveResult::Retired { debug_assert!(self.retired()); return Ok(InsertResult::Retired(k, v)); } return Err((k, v)); } InsertResult::Retry(k, v) => { // `child` has been split, therefore it can be retried. if self.cleanup_link(&k, false, guard) { key = k; val = v; continue; } return Ok(InsertResult::Retry(k, v)); } }; } } // It is not a hot loop - see `LeafNode::search_entry`. continue; } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { debug_assert!(unbounded_ptr.tag() == Tag::None); if !self.children.validate(metadata) { continue; } let insert_result = unbounded.insert(key, val, async_wait, guard)?; match insert_result { InsertResult::Success | InsertResult::Duplicate(..) | InsertResult::Frozen(..) => return Ok(insert_result), InsertResult::Full(k, v) => { let split_result = self.split_node( (k, v), None, unbounded_ptr, &self.unbounded_child, false, async_wait, guard, )?; if let InsertResult::Retry(k, v) = split_result { key = k; val = v; continue; } return Ok(split_result); } InsertResult::Retired(k, v) => { debug_assert!(unbounded.retired()); if self.coalesce(guard) == RemoveResult::Retired { debug_assert!(self.retired()); return Ok(InsertResult::Retired(k, v)); } return Err((k, v)); } InsertResult::Retry(k, v) => { if self.cleanup_link(&k, false, guard) { key = k; val = v; continue; } return Ok(InsertResult::Retry(k, v)); } }; } debug_assert!(unbounded_ptr.tag() == RETIRED); return Ok(InsertResult::Retired(key, val)); } } /// Removes an entry associated with the given key. /// /// # Errors /// /// Returns an error if a retry is required. #[inline] pub(super) fn remove_if bool, W>( &self, key: &Q, condition: &mut F, async_wait: &mut W, guard: &Guard, ) -> Result where Q: Comparable + ?Sized, W: TryWait, { loop { let (child, metadata) = self.children.min_greater_equal(key); if let Some((_, child)) = child { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. let result = child.remove_if::<_, _, _>(key, condition, async_wait, guard)?; if result == RemoveResult::Cleanup { if self.cleanup_link(key, false, guard) { return Ok(RemoveResult::Success); } return Ok(RemoveResult::Cleanup); } if result == RemoveResult::Retired { return Ok(self.coalesce(guard)); } return Ok(result); } } // It is not a hot loop - see `LeafNode::search_entry`. continue; } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { debug_assert!(unbounded_ptr.tag() == Tag::None); if !self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. continue; } let result = unbounded.remove_if::<_, _, _>(key, condition, async_wait, guard)?; if result == RemoveResult::Cleanup { if self.cleanup_link(key, false, guard) { return Ok(RemoveResult::Success); } return Ok(RemoveResult::Cleanup); } if result == RemoveResult::Retired { return Ok(self.coalesce(guard)); } return Ok(result); } return Ok(RemoveResult::Fail); } } /// Removes a range of entries. /// /// Returns the number of remaining children. #[allow(clippy::too_many_lines)] #[inline] pub(super) fn remove_range<'g, Q, R: RangeBounds, W: TryWait>( &self, range: &R, start_unbounded: bool, valid_lower_max_leaf: Option<&'g Leaf>, valid_upper_min_node: Option<&'g Node>, async_wait: &mut W, guard: &'g Guard, ) -> Result where Q: Comparable + ?Sized, { debug_assert!(valid_lower_max_leaf.is_none() || start_unbounded); debug_assert!(valid_lower_max_leaf.is_none() || valid_upper_min_node.is_none()); let Some(_lock) = Locker::try_lock(self) else { async_wait.try_wait(&self.lock); return Err(()); }; let mut current_state = RemoveRangeState::Below; let mut num_children = 1; let mut lower_border = None; let mut upper_border = None; for (key, node) in Scanner::new(&self.children) { current_state = current_state.next(key, range, start_unbounded); match current_state { RemoveRangeState::Below => { num_children += 1; } RemoveRangeState::MaybeBelow => { debug_assert!(!start_unbounded); num_children += 1; lower_border.replace((Some(key), node)); } RemoveRangeState::FullyContained => { // There can be another thread inserting keys into the node, and this may // render those concurrent operations completely ineffective. self.children.remove_if(key, &mut |_| true); node.swap((None, Tag::None), AcqRel); } RemoveRangeState::MaybeAbove => { if valid_upper_min_node.is_some() { // `valid_upper_min_node` is not in this sub-tree. self.children.remove_if(key, &mut |_| true); node.swap((None, Tag::None), AcqRel); } else { num_children += 1; upper_border.replace(node); } break; } } } // Now, examine the unbounded child. match current_state { RemoveRangeState::Below => { // The unbounded child is the only child, or all the children are below the range. debug_assert!(lower_border.is_none() && upper_border.is_none()); if valid_upper_min_node.is_some() { lower_border.replace((None, &self.unbounded_child)); } else { upper_border.replace(&self.unbounded_child); } } RemoveRangeState::MaybeBelow => { debug_assert!(!start_unbounded); debug_assert!(lower_border.is_some() && upper_border.is_none()); upper_border.replace(&self.unbounded_child); } RemoveRangeState::FullyContained => { debug_assert!(upper_border.is_none()); upper_border.replace(&self.unbounded_child); } RemoveRangeState::MaybeAbove => { debug_assert!(upper_border.is_some()); } } if let Some(lower_leaf) = valid_lower_max_leaf { // It is currently in the middle of a recursive call: pass `lower_leaf` to connect leaves. debug_assert!(start_unbounded && lower_border.is_none() && upper_border.is_some()); if let Some(upper_node) = upper_border.and_then(|n| n.load(Acquire, guard).as_ref()) { upper_node.remove_range(range, true, Some(lower_leaf), None, async_wait, guard)?; } } else if let Some(upper_node) = valid_upper_min_node { // Pass `upper_node` to the lower leaf to connect leaves, so that this method can be // recursively invoked on `upper_node`. debug_assert!(lower_border.is_some()); if let Some((Some(key), lower_node)) = lower_border { self.children.remove_if(key, &mut |_| true); self.unbounded_child .swap((lower_node.get_shared(Acquire, guard), Tag::None), AcqRel); lower_node.swap((None, Tag::None), Release); } if let Some(lower_node) = self.unbounded_child.load(Acquire, guard).as_ref() { lower_node.remove_range( range, start_unbounded, None, Some(upper_node), async_wait, guard, )?; } } else { let lower_node = lower_border.and_then(|n| n.1.load(Acquire, guard).as_ref()); let upper_node = upper_border.and_then(|n| n.load(Acquire, guard).as_ref()); match (lower_node, upper_node) { (_, None) => (), (None, Some(upper_node)) => { upper_node.remove_range( range, start_unbounded, None, None, async_wait, guard, )?; } (Some(lower_node), Some(upper_node)) => { debug_assert!(!ptr::eq(lower_node, upper_node)); lower_node.remove_range( range, start_unbounded, None, Some(upper_node), async_wait, guard, )?; } } } Ok(num_children) } /// Splits a full node. /// /// # Errors /// /// Returns an error if a retry is required. #[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(super) fn split_node( &self, entry: (K, V), full_node_key: Option<&K>, full_node_ptr: Ptr>, full_node: &AtomicShared>, root_split: bool, async_wait: &mut W, guard: &Guard, ) -> Result, (K, V)> { let target = full_node_ptr.as_ref().unwrap(); if !self.lock.try_lock() { target.rollback(guard); async_wait.try_wait(&self.lock); return Err(entry); } debug_assert!(!self.retired()); if full_node_ptr != full_node.load(Relaxed, guard) { self.lock.release_lock(); target.rollback(guard); return Err(entry); } let prev = self .split_op .origin_node .swap((full_node.get_shared(Relaxed, guard), Tag::None), Relaxed) .0; debug_assert!(prev.is_none()); if let Some(full_node_key) = full_node_key { self.split_op .origin_node_key .store(ptr::from_ref(full_node_key).cast_mut(), Relaxed); } let exit_guard = ExitGuard::new((), |()| { self.rollback(guard); }); match target { Node::Internal(full_internal_node) => { // Copies nodes except for the known full node to the newly allocated internal node entries. let internal_nodes = ( Shared::new(Node::new_internal_node()), Shared::new(Node::new_internal_node()), ); let Node::Internal(low_key_nodes) = internal_nodes.0.as_ref() else { unreachable!() }; let Node::Internal(high_key_nodes) = internal_nodes.1.as_ref() else { unreachable!() }; // Builds a list of valid nodes. #[allow(clippy::type_complexity)] let mut entry_array: [Option<( Option<&K>, AtomicShared>, )>; DIMENSION.num_entries + 2] = Default::default(); let mut num_entries = 0; let scanner = Scanner::new(&full_internal_node.children); let recommended_boundary = Leaf::::optimal_boundary(scanner.metadata()); for entry in scanner { if unsafe { full_internal_node .split_op .origin_node_key .load(Relaxed) .as_ref() .map_or_else(|| false, |key| entry.0 == key) } { let low_key_node_ptr = full_internal_node .split_op .low_key_node .load(Relaxed, guard); if !low_key_node_ptr.is_null() { entry_array[num_entries].replace(( Some(unsafe { full_internal_node .split_op .middle_key .load(Relaxed) .as_ref() .unwrap() }), full_internal_node .split_op .low_key_node .clone(Relaxed, guard), )); num_entries += 1; } let high_key_node_ptr = full_internal_node .split_op .high_key_node .load(Relaxed, guard); if !high_key_node_ptr.is_null() { entry_array[num_entries].replace(( Some(entry.0), full_internal_node .split_op .high_key_node .clone(Relaxed, guard), )); num_entries += 1; } } else { entry_array[num_entries] .replace((Some(entry.0), entry.1.clone(Acquire, guard))); num_entries += 1; } } if full_internal_node .split_op .origin_node_key .load(Relaxed) .is_null() { // If the origin is an unbounded node, assign the high key node to the high key // node's unbounded. let low_key_node_ptr = full_internal_node .split_op .low_key_node .load(Relaxed, guard); if !low_key_node_ptr.is_null() { entry_array[num_entries].replace(( Some(unsafe { full_internal_node .split_op .middle_key .load(Relaxed) .as_ref() .unwrap() }), full_internal_node .split_op .low_key_node .clone(Relaxed, guard), )); num_entries += 1; } let high_key_node_ptr = full_internal_node .split_op .high_key_node .load(Relaxed, guard); if !high_key_node_ptr.is_null() { entry_array[num_entries].replace(( None, full_internal_node .split_op .high_key_node .clone(Relaxed, guard), )); num_entries += 1; } } else { // If the origin is a bounded node, assign the unbounded node to the high key // node's unbounded. entry_array[num_entries].replace(( None, full_internal_node.unbounded_child.clone(Relaxed, guard), )); num_entries += 1; } debug_assert!(num_entries >= 2); let low_key_node_array_size = recommended_boundary.min(num_entries - 1); for (i, entry) in entry_array.iter().enumerate() { if let Some((k, v)) = entry { match (i + 1).cmp(&low_key_node_array_size) { Less => { low_key_nodes.children.insert_unchecked( k.unwrap().clone(), v.clone(Relaxed, guard), i, ); } Equal => { if let Some(&k) = k.as_ref() { self.split_op .middle_key .store(ptr::from_ref(k).cast_mut(), Relaxed); } low_key_nodes .unbounded_child .swap((v.get_shared(Relaxed, guard), Tag::None), Relaxed); } Greater => { if let Some(k) = k.cloned() { high_key_nodes.children.insert_unchecked( k, v.clone(Relaxed, guard), i - low_key_node_array_size, ); } else { high_key_nodes .unbounded_child .swap((v.get_shared(Relaxed, guard), Tag::None), Relaxed); } } } } else { break; } } // Turns the new nodes into internal nodes. self.split_op .low_key_node .swap((Some(internal_nodes.0), Tag::None), Relaxed); self.split_op .high_key_node .swap((Some(internal_nodes.1), Tag::None), Relaxed); } Node::Leaf(full_leaf_node) => { // Copies leaves except for the known full leaf to the newly allocated leaf node entries. let leaf_nodes = ( Shared::new(Node::new_leaf_node()), Shared::new(Node::new_leaf_node()), ); let low_key_leaf_node = if let Node::Leaf(low_key_leaf_node) = leaf_nodes.0.as_ref() { Some(low_key_leaf_node) } else { None }; let high_key_leaf_node = if let Node::Leaf(high_key_leaf_node) = &leaf_nodes.1.as_ref() { Some(high_key_leaf_node) } else { None }; self.split_op.middle_key.store( ptr::from_ref(full_leaf_node.split_leaf_node( low_key_leaf_node.unwrap(), high_key_leaf_node.unwrap(), guard, )) .cast_mut(), Relaxed, ); // Turns the new leaves into leaf nodes. self.split_op .low_key_node .swap((Some(leaf_nodes.0), Tag::None), Relaxed); self.split_op .high_key_node .swap((Some(leaf_nodes.1), Tag::None), Relaxed); } } // Inserts the newly allocated internal nodes into the main array. match self.children.insert( unsafe { self.split_op .middle_key .load(Relaxed) .as_ref() .unwrap() .clone() }, self.split_op.low_key_node.clone(Relaxed, guard), ) { InsertResult::Success => (), InsertResult::Duplicate(..) | InsertResult::Frozen(..) | InsertResult::Retry(..) => { unreachable!() } InsertResult::Full(..) | InsertResult::Retired(..) => { // Insertion failed: expects that the parent splits this node. exit_guard.forget(); return Ok(InsertResult::Full(entry.0, entry.1)); } } exit_guard.forget(); // Replace the full node with the high-key node. let unused_node = full_node .swap( ( self.split_op.high_key_node.get_shared(Relaxed, guard), Tag::None, ), Release, ) .0; if root_split { // Return without unlocking it. return Ok(InsertResult::Retry(entry.0, entry.1)); } // Unlock the node. self.finish_split(); // Drop the deprecated nodes. if let Some(unused_node) = unused_node { // Clean up the split operation by committing it. unused_node.commit(guard); let _: bool = unused_node.release(); } // Since a new node has been inserted, the caller can retry. Ok(InsertResult::Retry(entry.0, entry.1)) } /// Finishes splitting the [`InternalNode`]. #[inline] pub(super) fn finish_split(&self) { let origin = self.split_op.reset(); self.lock.release_lock(); origin.map(Shared::release); } /// Commits an ongoing structural change recursively. #[inline] pub(super) fn commit(&self, guard: &Guard) { let origin = self.split_op.reset(); // Prevent further exclusive access to the internal node. self.lock.poison_lock(); if let Some(origin) = origin { origin.commit(guard); let _: bool = origin.release(); } } /// Rolls back the ongoing split operation recursively. #[inline] pub(super) fn rollback(&self, guard: &Guard) { let origin = self.split_op.reset(); self.lock.release_lock(); if let Some(origin) = origin { origin.rollback(guard); let _: bool = origin.release(); } } /// Cleans up logically deleted leaves in the linked list. /// /// Returns `false` if the target leaf node does not exist in the subtree. #[inline] pub(super) fn cleanup_link<'g, Q>(&self, key: &Q, traverse_max: bool, guard: &'g Guard) -> bool where K: 'g, Q: Comparable + ?Sized, { if traverse_max { // It just has to search for the maximum leaf node in the tree. if let Some(unbounded) = self.unbounded_child.load(Acquire, guard).as_ref() { return unbounded.cleanup_link(key, true, guard); } } else if let Some(child_scanner) = Scanner::max_less(&self.children, key) { if let Some((_, child)) = child_scanner.get() { if let Some(child) = child.load(Acquire, guard).as_ref() { return child.cleanup_link(key, true, guard); } } } false } /// Tries to coalesce nodes. fn coalesce(&self, guard: &Guard) -> RemoveResult { let mut node_deleted = false; while let Some(lock) = Locker::try_lock(self) { let mut max_key_entry = None; for (key, node) in Scanner::new(&self.children) { let node_ptr = node.load(Acquire, guard); let node_ref = node_ptr.as_ref().unwrap(); if node_ref.retired() { let result = self.children.remove_if(key, &mut |_| true); debug_assert_ne!(result, RemoveResult::Fail); // Once the key is removed, it is safe to deallocate the node as the validation // loop ensures the absence of readers. if let Some(node) = node.swap((None, Tag::None), Release).0 { let _: bool = node.release(); node_deleted = true; } } else { max_key_entry.replace((key, node)); } } // The unbounded node is replaced with the maximum key node if retired. let unbounded_ptr = self.unbounded_child.load(Acquire, guard); let fully_empty = if let Some(unbounded) = unbounded_ptr.as_ref() { if unbounded.retired() { if let Some((key, max_key_child)) = max_key_entry { if let Some(obsolete_node) = self .unbounded_child .swap( (max_key_child.get_shared(Relaxed, guard), Tag::None), Release, ) .0 { debug_assert!(obsolete_node.retired()); let _: bool = obsolete_node.release(); node_deleted = true; } let result = self.children.remove_if(key, &mut |_| true); debug_assert_ne!(result, RemoveResult::Fail); if let Some(node) = max_key_child.swap((None, Tag::None), Release).0 { let _: bool = node.release(); node_deleted = true; } false } else { if let Some(obsolete_node) = self.unbounded_child.swap((None, RETIRED), Release).0 { debug_assert!(obsolete_node.retired()); let _: bool = obsolete_node.release(); node_deleted = true; } true } } else { false } } else { debug_assert!(unbounded_ptr.tag() == RETIRED); true }; if fully_empty { return RemoveResult::Retired; } drop(lock); if !self.has_retired_node(guard) { break; } } if node_deleted { RemoveResult::Cleanup } else { RemoveResult::Success } } /// Checks if the [`InternalNode`] has a retired [`Node`]. fn has_retired_node(&self, guard: &Guard) -> bool { let mut has_valid_node = false; for entry in Scanner::new(&self.children) { let leaf_ptr = entry.1.load(Relaxed, guard); if let Some(leaf) = leaf_ptr.as_ref() { if leaf.retired() { return true; } has_valid_node = true; } } if !has_valid_node { let unbounded_ptr = self.unbounded_child.load(Relaxed, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if unbounded.retired() { return true; } } } false } } impl<'n, K, V> Locker<'n, K, V> { /// Acquires exclusive lock on the [`InternalNode`]. #[inline] pub(super) fn try_lock(internal_node: &'n InternalNode) -> Option> { if internal_node.lock.try_lock() { Some(Locker { internal_node }) } else { None } } /// Retires the node with the lock released. pub(super) fn unlock_retire(self) { self.internal_node.lock.poison_lock(); forget(self); } } impl Drop for Locker<'_, K, V> { #[inline] fn drop(&mut self) { self.internal_node.lock.release_lock(); } } impl StructuralChange { fn reset(&self) -> Option>> { self.origin_node_key.store(ptr::null_mut(), Relaxed); self.low_key_node.swap((None, Tag::None), Relaxed); self.middle_key.store(ptr::null_mut(), Relaxed); self.high_key_node.swap((None, Tag::None), Relaxed); self.origin_node.swap((None, Tag::None), Relaxed).0 } } impl Default for StructuralChange { #[inline] fn default() -> Self { Self { origin_node_key: AtomicPtr::default(), origin_node: AtomicShared::null(), low_key_node: AtomicShared::null(), middle_key: AtomicPtr::default(), high_key_node: AtomicShared::null(), } } } #[cfg(not(feature = "loom"))] #[cfg(test)] mod test { use super::*; use std::sync::atomic::AtomicBool; use tokio::sync::Barrier; fn new_level_3_node() -> InternalNode { InternalNode { children: Leaf::new(), unbounded_child: AtomicShared::new(Node::Internal(InternalNode { children: Leaf::new(), unbounded_child: AtomicShared::new(Node::new_leaf_node()), split_op: StructuralChange::default(), lock: Lock::default(), })), split_op: StructuralChange::default(), lock: Lock::default(), } } #[test] fn bulk() { let internal_node = new_level_3_node(); let guard = Guard::new(); assert_eq!(internal_node.depth(1, &guard), 3); let data_size = if cfg!(miri) { 256 } else { 8192 }; for k in 0..data_size { match internal_node.insert(k, k, &mut (), &guard) { Ok(result) => match result { InsertResult::Success => { assert_eq!(internal_node.search_entry(&k, &guard), Some((&k, &k))); } InsertResult::Duplicate(..) | InsertResult::Frozen(..) | InsertResult::Retired(..) => unreachable!(), InsertResult::Full(_, _) => { internal_node.rollback(&guard); for j in 0..k { assert_eq!(internal_node.search_entry(&j, &guard), Some((&j, &j))); if j == k - 1 { assert!(matches!( internal_node.remove_if::<_, _, _>( &j, &mut |_| true, &mut (), &guard ), Ok(RemoveResult::Retired) )); } else { assert!( internal_node .remove_if::<_, _, _>(&j, &mut |_| true, &mut (), &guard) .is_ok(), ); } assert_eq!(internal_node.search_entry(&j, &guard), None); } break; } InsertResult::Retry(k, v) => { let result = internal_node.insert(k, v, &mut (), &guard); assert!(result.is_ok()); assert_eq!(internal_node.search_entry(&k, &guard), Some((&k, &k))); } }, Err((k, v)) => { let result = internal_node.insert(k, v, &mut (), &guard); assert!(result.is_ok()); assert_eq!(internal_node.search_entry(&k, &guard), Some((&k, &k))); } } } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn parallel() { let num_tasks = 8; let workload_size = 64; let barrier = Shared::new(Barrier::new(num_tasks)); for _ in 0..64 { let internal_node = Shared::new(new_level_3_node()); assert!( internal_node .insert(usize::MAX, usize::MAX, &mut (), &Guard::new()) .is_ok() ); let mut task_handles = Vec::with_capacity(num_tasks); for task_id in 0..num_tasks { let barrier_clone = barrier.clone(); let internal_node_clone = internal_node.clone(); task_handles.push(tokio::task::spawn(async move { barrier_clone.wait().await; let guard = Guard::new(); let mut max_key = None; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { loop { if let Ok(r) = internal_node_clone.insert(id, id, &mut (), &guard) { match r { InsertResult::Success => { match internal_node_clone.insert(id, id, &mut (), &guard) { Ok(InsertResult::Duplicate(..)) | Err(_) => (), _ => unreachable!(), } break; } InsertResult::Full(..) => { internal_node_clone.rollback(&guard); max_key.replace(id); break; } InsertResult::Frozen(..) | InsertResult::Retry(..) => (), _ => unreachable!(), } } } if max_key.is_some() { break; } } for id in range.clone() { if max_key == Some(id) { break; } assert_eq!( internal_node_clone.search_entry(&id, &guard), Some((&id, &id)) ); } for id in range { if max_key == Some(id) { break; } loop { if let Ok(r) = internal_node_clone.remove_if::<_, _, _>( &id, &mut |_| true, &mut (), &guard, ) { match r { RemoveResult::Success | RemoveResult::Cleanup | RemoveResult::Fail => break, RemoveResult::Frozen | RemoveResult::Retired => unreachable!(), } } } assert!(internal_node_clone.search_entry(&id, &guard).is_none()); if let Ok(RemoveResult::Success) = internal_node_clone.remove_if::<_, _, _>( &id, &mut |_| true, &mut (), &guard, ) { unreachable!() } } })); } for r in futures::future::join_all(task_handles).await { assert!(r.is_ok()); } assert!( internal_node .remove_if::<_, _, _>(&usize::MAX, &mut |_| true, &mut (), &Guard::new()) .is_ok() ); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn durability() { let num_tasks = 8_usize; let num_iterations = 64; let workload_size = 64_usize; for k in 0..64 { let fixed_point = k * 16; for _ in 0..=num_iterations { let barrier = Shared::new(Barrier::new(num_tasks)); let internal_node = Shared::new(new_level_3_node()); let inserted: Shared = Shared::new(AtomicBool::new(false)); let mut task_handles = Vec::with_capacity(num_tasks); for _ in 0..num_tasks { let barrier_clone = barrier.clone(); let internal_node_clone = internal_node.clone(); let inserted_clone = inserted.clone(); task_handles.push(tokio::spawn(async move { { barrier_clone.wait().await; let guard = Guard::new(); match internal_node_clone.insert( fixed_point, fixed_point, &mut (), &guard, ) { Ok(InsertResult::Success) => { assert!(!inserted_clone.swap(true, Relaxed)); } Ok(InsertResult::Full(_, _) | InsertResult::Retired(_, _)) => { internal_node_clone.rollback(&guard); } _ => (), } assert_eq!( internal_node_clone .search_entry(&fixed_point, &guard) .unwrap(), (&fixed_point, &fixed_point) ); } { barrier_clone.wait().await; let guard = Guard::new(); for i in 0..workload_size { if i != fixed_point { if let Ok( InsertResult::Full(_, _) | InsertResult::Retired(_, _), ) = internal_node_clone.insert(i, i, &mut (), &guard) { internal_node_clone.rollback(&guard); } } assert_eq!( internal_node_clone .search_entry(&fixed_point, &guard) .unwrap(), (&fixed_point, &fixed_point) ); } for i in 0..workload_size { let max_scanner = internal_node_clone .max_le_appr(&fixed_point, &guard) .unwrap(); assert!(*max_scanner.get().unwrap().0 <= fixed_point); let mut min_scanner = internal_node_clone.min(&guard).unwrap(); if let Some((f, v)) = min_scanner.next() { assert_eq!(*f, *v); assert!(*f <= fixed_point); } else { let (f, v) = min_scanner.jump(None, &guard).unwrap().get().unwrap(); assert_eq!(*f, *v); assert!(*f <= fixed_point); } let _result = internal_node_clone.remove_if::<_, _, _>( &i, &mut |v| *v != fixed_point, &mut (), &guard, ); assert_eq!( internal_node_clone .search_entry(&fixed_point, &guard) .unwrap(), (&fixed_point, &fixed_point) ); } } })); } for r in futures::future::join_all(task_handles).await { assert!(r.is_ok()); } assert!((*inserted).load(Relaxed)); } } } } scc-3.4.8/src/tree_index/leaf.rs000064400000000000000000001372331046102023000146130ustar 00000000000000use std::cell::UnsafeCell; use std::cmp::Ordering; use std::fmt::{self, Debug}; use std::mem::{MaybeUninit, needs_drop}; use std::ops::Bound::{Excluded, Included, Unbounded}; use std::ops::RangeBounds; #[cfg(not(feature = "loom"))] use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use sdd::{AtomicShared, Guard, Shared}; use crate::Comparable; use crate::LinkedList; #[cfg(feature = "loom")] use loom::sync::atomic::AtomicUsize; /// [`Leaf`] is an ordered array of key-value pairs. /// /// A constructed key-value pair entry is never dropped until the entire [`Leaf`] instance is /// dropped. pub struct Leaf { /// The metadata containing information about the [`Leaf`] and individual entries. /// /// The state of each entry is as follows. /// * `0`: `uninit`. /// * `1-ARRAY_SIZE`: `rank`. /// * `ARRAY_SIZE + 1`: `removed`. /// /// The entry state transitions as follows. /// * `uninit -> removed -> rank -> removed`. metadata: AtomicUsize, /// The array of key-value pairs. entry_array: UnsafeCell>, /// A pointer that points to the next adjacent [`Leaf`]. link: AtomicShared>, } /// The number of entries and number of state bits per entry. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct Dimension { pub num_entries: usize, pub num_bits_per_entry: usize, } /// The result of insertion. pub enum InsertResult { /// Insertion succeeded. Success, /// Duplicate key found. Duplicate(K, V), /// No vacant slot for the key. Full(K, V), /// The [`Leaf`] is frozen. /// /// This is not a terminal state as a frozen [`Leaf`] can be unfrozen. Frozen(K, V), /// Insertion failed as the [`Leaf`] has retired. /// /// It is a terminal state. Retired(K, V), /// The operation can be retried. Retry(K, V), } /// The result of removal. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RemoveResult { /// Remove succeeded. Success, /// Remove succeeded and cleanup required. Cleanup, /// Remove succeeded and the [`Leaf`] has retired without usable entries left. Retired, /// Remove failed. Fail, /// The [`Leaf`] is frozen. Frozen, } /// Emulates `RangeBounds::contains`. pub(crate) fn range_contains>(range: &R, key: &K) -> bool where Q: Comparable + ?Sized, { (match range.start_bound() { Included(start) => start.compare(key).is_le(), Excluded(start) => start.compare(key).is_lt(), Unbounded => true, }) && (match range.end_bound() { Included(end) => end.compare(key).is_ge(), Excluded(end) => end.compare(key).is_gt(), Unbounded => true, }) } impl Leaf { /// Creates a new [`Leaf`]. #[cfg(not(feature = "loom"))] #[inline] pub(super) const fn new() -> Leaf { #[allow(clippy::uninit_assumed_init)] Leaf { metadata: AtomicUsize::new(0), entry_array: UnsafeCell::new(unsafe { MaybeUninit::uninit().assume_init() }), link: AtomicShared::null(), } } #[cfg(feature = "loom")] #[inline] pub(super) fn new() -> Leaf { #[allow(clippy::uninit_assumed_init)] Leaf { metadata: AtomicUsize::new(0), entry_array: UnsafeCell::new(unsafe { MaybeUninit::uninit().assume_init() }), link: AtomicShared::null(), } } /// Thaws the [`Leaf`]. #[inline] pub(super) fn thaw(&self) -> bool { self.metadata .fetch_update(Release, Relaxed, |p| { if Dimension::frozen(p) { Some(Dimension::thaw(p)) } else { None } }) .is_ok() } /// Returns `true` if the [`Leaf`] has retired. #[inline] pub(super) fn is_retired(&self) -> bool { Dimension::retired(self.metadata.load(Acquire)) } /// Returns `true` if the [`Leaf`] has no reachable entry. #[inline] pub(super) fn is_empty(&self) -> bool { let mut mutable_metadata = self.metadata.load(Acquire); for _ in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank != Dimension::uninit_rank() && rank != DIMENSION.removed_rank() { return false; } mutable_metadata >>= DIMENSION.num_bits_per_entry; } true } /// Returns a reference to the max key. #[inline] pub(super) fn max_key(&self) -> Option<&K> { self.max_entry().map(|(k, _)| k) } /// Returns a reference to the max entry. #[inline] pub(super) fn max_entry(&self) -> Option<(&K, &V)> { let mut mutable_metadata = self.metadata.load(Acquire); let mut max_rank = 0; let mut max_index = DIMENSION.num_entries; for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank > max_rank && rank != DIMENSION.removed_rank() { max_rank = rank; max_index = i; } mutable_metadata >>= DIMENSION.num_bits_per_entry; } if max_index != DIMENSION.num_entries { return Some((self.key_at(max_index), self.value_at(max_index))); } None } /// Inserts a key value pair at the specified position without checking the metadata. /// /// `rank` is calculated as `index + 1`. #[inline] pub(super) fn insert_unchecked(&self, key: K, val: V, index: usize) { debug_assert!(index < DIMENSION.num_entries); let metadata = self.metadata.load(Relaxed); let new_metadata = DIMENSION.augment(metadata, index, index + 1); self.write(index, key, val); self.metadata.store(new_metadata, Release); } /// Compares the given metadata value with the current one. #[inline] pub(super) fn validate(&self, metadata: usize) -> bool { // `Relaxed` is sufficient as long as the caller has load-acquired its contents. self.metadata.load(Relaxed) == metadata } /// Freezes the [`Leaf`] temporarily. /// /// A frozen [`Leaf`] cannot store more entries, and on-going insertion is canceled. #[inline] pub(super) fn freeze(&self) -> bool { self.metadata .fetch_update(AcqRel, Acquire, |p| { if Dimension::frozen(p) { None } else { Some(Dimension::freeze(p)) } }) .is_ok() } /// Returns the recommended number of entries that the left-side node should store when a /// [`Leaf`] is split. /// /// Returns a number in `[1, len(leaf))` that represents the recommended number of entries in /// the left-side node. The number is calculated as follows for each adjacent slot: /// - Initial `score = len(leaf)`. /// - Rank increased: `score -= 1`. /// - Rank decreased: `score += 1`. /// - Clamp `score` in `[len(leaf) / 2 + 1, len(leaf) / 2 + len(leaf) - 1)`. /// - Take `score - len(leaf) / 2`. /// /// For instance, when the length of a [`Leaf`] is 7, /// - Returns 6 for `rank = [1, 2, 3, 4, 5, 6, 7]`. /// - Returns 1 for `rank = [7, 6, 5, 4, 3, 2, 1]`. #[inline] pub(super) fn optimal_boundary(mut mutable_metadata: usize) -> usize { let mut boundary: usize = DIMENSION.num_entries; let mut prev_rank = 0; for _ in 0..DIMENSION.num_entries { let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank != 0 && rank != DIMENSION.removed_rank() { if prev_rank >= rank { boundary -= 1; } else if prev_rank != 0 { boundary += 1; } prev_rank = rank; } mutable_metadata >>= DIMENSION.num_bits_per_entry; } boundary.clamp( DIMENSION.num_entries / 2 + 1, DIMENSION.num_entries + DIMENSION.num_entries / 2 - 1, ) - DIMENSION.num_entries / 2 } /// Returns a reference to the key at the given index. const fn key_at(&self, index: usize) -> &K { unsafe { &*(*self.entry_array.get()).0[index].as_ptr() } } /// Returns a reference to the key at the given index. const fn value_at(&self, index: usize) -> &V { unsafe { &*(*self.entry_array.get()).1[index].as_ptr() } } /// Writes the key and value at the given index. const fn write(&self, index: usize, key: K, val: V) { unsafe { (*self.entry_array.get()).0[index].as_mut_ptr().write(key); (*self.entry_array.get()).1[index].as_mut_ptr().write(val); } } /// Takes the key and value at the given index. const fn take(&self, index: usize) -> (K, V) { unsafe { ( (*self.entry_array.get()).0[index].as_ptr().read(), (*self.entry_array.get()).1[index].as_ptr().read(), ) } } /// Rolls back the insertion at the given index. fn rollback(&self, index: usize) -> InsertResult { let (key, val) = self.take(index); let result = self .metadata .fetch_and(!DIMENSION.rank_mask(index), Release) & (!DIMENSION.rank_mask(index)); if Dimension::retired(result) { InsertResult::Retired(key, val) } else if Dimension::frozen(result) { InsertResult::Frozen(key, val) } else { InsertResult::Duplicate(key, val) } } /// Returns the index of the corresponding entry of the next higher ranked entry. fn next(index: usize, mut mutable_metadata: usize) -> usize { debug_assert_ne!(index, usize::MAX); let current_entry_rank = if index == DIMENSION.num_entries { 0 } else { DIMENSION.rank(mutable_metadata, index) }; let mut next_index = DIMENSION.num_entries; if current_entry_rank < DIMENSION.num_entries { let mut next_rank = DIMENSION.removed_rank(); for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } if i != index { let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank != Dimension::uninit_rank() && rank < next_rank { if rank == current_entry_rank + 1 { return i; } else if rank > current_entry_rank { next_rank = rank; next_index = i; } } } mutable_metadata >>= DIMENSION.num_bits_per_entry; } } next_index } } impl Leaf where K: 'static + Clone + Ord, V: 'static + Clone, { /// Inserts a key value pair. #[inline] pub(super) fn insert(&self, key: K, val: V) -> InsertResult { let mut metadata = self.metadata.load(Acquire); 'after_read_metadata: loop { if Dimension::retired(metadata) { return InsertResult::Retired(key, val); } else if Dimension::frozen(metadata) { return InsertResult::Frozen(key, val); } let mut mutable_metadata = metadata; for i in 0..DIMENSION.num_entries { let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank == Dimension::uninit_rank() { let interim_metadata = DIMENSION.augment(metadata, i, DIMENSION.removed_rank()); // Reserve the slot. // // It doesn't have to be a release-store. if let Err(actual) = self.metadata .compare_exchange(metadata, interim_metadata, Acquire, Acquire) { metadata = actual; continue 'after_read_metadata; } self.write(i, key, val); return self.post_insert(i, interim_metadata); } mutable_metadata >>= DIMENSION.num_bits_per_entry; } if self.search_slot(&key, metadata).is_some() { return InsertResult::Duplicate(key, val); } return InsertResult::Full(key, val); } } /// Removes the key if the condition is met. #[inline] pub(super) fn remove_if bool>( &self, key: &Q, condition: &mut F, ) -> RemoveResult where Q: Comparable + ?Sized, { let mut metadata = self.metadata.load(Acquire); if Dimension::frozen(metadata) { return RemoveResult::Frozen; } let mut min_max_rank = DIMENSION.removed_rank(); let mut max_min_rank = 0; let mut mutable_metadata = metadata; for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank < min_max_rank && rank > max_min_rank { match self.compare(i, key) { Ordering::Less => { if max_min_rank < rank { max_min_rank = rank; } } Ordering::Greater => { if min_max_rank > rank { min_max_rank = rank; } } Ordering::Equal => { // Found the key. loop { if !condition(self.value_at(i)) { // The given condition is not met. return RemoveResult::Fail; } let mut empty = true; mutable_metadata = metadata; for j in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } if i != j { let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank != Dimension::uninit_rank() && rank != DIMENSION.removed_rank() { empty = false; break; } } mutable_metadata >>= DIMENSION.num_bits_per_entry; } let mut new_metadata = metadata | DIMENSION.rank_mask(i); if empty { new_metadata = Dimension::retire(new_metadata); } match self.metadata.compare_exchange( metadata, new_metadata, AcqRel, Acquire, ) { Ok(_) => { if empty { return RemoveResult::Retired; } return RemoveResult::Success; } Err(actual) => { if DIMENSION.rank(actual, i) == DIMENSION.removed_rank() { return RemoveResult::Fail; } if Dimension::frozen(actual) { return RemoveResult::Frozen; } metadata = actual; } } } } } } mutable_metadata >>= DIMENSION.num_bits_per_entry; } RemoveResult::Fail } /// Removes a range of entries. /// /// Returns the number of remaining entries. #[inline] pub(super) fn remove_range>(&self, range: &R) where Q: Comparable + ?Sized, { let mut mutable_metadata = self.metadata.load(Acquire); for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank != Dimension::uninit_rank() && rank != DIMENSION.removed_rank() { let k = self.key_at(i); if range_contains(range, k) { self.remove_if(k, &mut |_| true); } } mutable_metadata >>= DIMENSION.num_bits_per_entry; } } /// Returns an entry containing the specified key. #[inline] pub(super) fn search_entry(&self, key: &Q) -> Option<(&K, &V)> where Q: Comparable + ?Sized, { let metadata = self.metadata.load(Acquire); self.search_slot(key, metadata) .map(|i| (self.key_at(i), self.value_at(i))) } /// Returns the value associated with the specified key. #[inline] pub(super) fn search_value(&self, key: &Q) -> Option<&V> where Q: Comparable + ?Sized, { let metadata = self.metadata.load(Acquire); self.search_slot(key, metadata).map(|i| self.value_at(i)) } /// Returns the index of the key-value pair that has a key smaller than the given key. #[inline] pub(super) fn max_less(&self, mut mutable_metadata: usize, key: &Q) -> usize where Q: Comparable + ?Sized, { let mut min_max_rank = DIMENSION.removed_rank(); let mut max_min_rank = 0; let mut max_min_index = DIMENSION.num_entries; for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank < min_max_rank && rank > max_min_rank { match self.compare(i, key) { Ordering::Less => { if max_min_rank < rank { max_min_rank = rank; max_min_index = i; } } Ordering::Greater => { if min_max_rank > rank { min_max_rank = rank; } } Ordering::Equal => { min_max_rank = rank; } } } mutable_metadata >>= DIMENSION.num_bits_per_entry; } max_min_index } /// Returns the minimum entry among those that are not `Ordering::Less` than the given key. /// /// It additionally returns the current version of its metadata so the caller can validate the /// correctness of the result. #[inline] pub(super) fn min_greater_equal(&self, key: &Q) -> (Option<(&K, &V)>, usize) where Q: Comparable + ?Sized, { let metadata = self.metadata.load(Acquire); let mut min_max_rank = DIMENSION.removed_rank(); let mut max_min_rank = 0; let mut min_max_index = DIMENSION.num_entries; let mut mutable_metadata = metadata; for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank < min_max_rank && rank > max_min_rank { let k = self.key_at(i); match key.compare(k) { Ordering::Greater => { if max_min_rank < rank { max_min_rank = rank; } } Ordering::Less => { if min_max_rank > rank { min_max_rank = rank; min_max_index = i; } } Ordering::Equal => { return (Some((k, self.value_at(i))), metadata); } } } mutable_metadata >>= DIMENSION.num_bits_per_entry; } if min_max_index != DIMENSION.num_entries { return ( Some((self.key_at(min_max_index), self.value_at(min_max_index))), metadata, ); } (None, metadata) } /// Freezes the [`Leaf`] and distributes entries to two new leaves. #[inline] pub(super) fn freeze_and_distribute( &self, low_key_leaf: &mut Shared>, high_key_leaf: &mut Option>>, ) { let metadata = unsafe { self.metadata .fetch_update(AcqRel, Acquire, |p| { if Dimension::frozen(p) { None } else { Some(Dimension::freeze(p)) } }) .unwrap_unchecked() }; let boundary = Self::optimal_boundary(metadata); let scanner = Scanner { leaf: self, metadata, entry_index: DIMENSION.num_entries, }; for (i, (k, v)) in scanner.enumerate() { if i < boundary { low_key_leaf.insert_unchecked(k.clone(), v.clone(), i); } else { high_key_leaf .get_or_insert_with(|| Shared::new(Leaf::new())) .insert_unchecked(k.clone(), v.clone(), i - boundary); } } } /// Post-processing after reserving a free slot. fn post_insert(&self, free_slot_index: usize, mut prev_metadata: usize) -> InsertResult { let key = self.key_at(free_slot_index); loop { let mut min_max_rank = DIMENSION.removed_rank(); let mut max_min_rank = 0; let mut new_metadata = prev_metadata; let mut mutable_metadata = prev_metadata; for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank < min_max_rank && rank > max_min_rank { match self.compare(i, key) { Ordering::Less => { if max_min_rank < rank { max_min_rank = rank; } } Ordering::Greater => { if min_max_rank > rank { min_max_rank = rank; } new_metadata = DIMENSION.augment(new_metadata, i, rank + 1); } Ordering::Equal => { // Duplicate key. return self.rollback(free_slot_index); } } } else if rank != DIMENSION.removed_rank() && rank > min_max_rank { new_metadata = DIMENSION.augment(new_metadata, i, rank + 1); } mutable_metadata >>= DIMENSION.num_bits_per_entry; } // Make the newly inserted value reachable. let final_metadata = DIMENSION.augment(new_metadata, free_slot_index, max_min_rank + 1); if let Err(actual) = self.metadata .compare_exchange(prev_metadata, final_metadata, AcqRel, Acquire) { if Dimension::frozen(actual) || Dimension::retired(actual) { return self.rollback(free_slot_index); } prev_metadata = actual; continue; } return InsertResult::Success; } } /// Searches for a slot in which the key is stored. fn search_slot(&self, key: &Q, mut mutable_metadata: usize) -> Option where Q: Comparable + ?Sized, { let mut min_max_rank = DIMENSION.removed_rank(); let mut max_min_rank = 0; for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } let rank = mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry); if rank < min_max_rank && rank > max_min_rank { match self.compare(i, key) { Ordering::Less => { if max_min_rank < rank { max_min_rank = rank; } } Ordering::Greater => { if min_max_rank > rank { min_max_rank = rank; } } Ordering::Equal => { return Some(i); } } } mutable_metadata >>= DIMENSION.num_bits_per_entry; } None } fn compare(&self, index: usize, key: &Q) -> Ordering where Q: Comparable + ?Sized, { key.compare(self.key_at(index)).reverse() } } impl Drop for Leaf { #[inline] fn drop(&mut self) { if needs_drop::<(K, V)>() { let mut mutable_metadata = self.metadata.load(Acquire); for i in 0..DIMENSION.num_entries { if mutable_metadata == 0 { break; } if mutable_metadata % (1_usize << DIMENSION.num_bits_per_entry) != Dimension::uninit_rank() { self.take(i); } mutable_metadata >>= DIMENSION.num_bits_per_entry; } } } } /// [`LinkedList`] implementation for [`Leaf`]. impl LinkedList for Leaf { #[inline] fn link_ref(&self) -> &AtomicShared> { &self.link } } unsafe impl Send for Leaf {} unsafe impl Sync for Leaf {} impl Dimension { /// Checks if the [`Leaf`] is frozen. const fn frozen(metadata: usize) -> bool { metadata & (1_usize << (usize::BITS - 2)) != 0 } /// Makes the metadata represent a frozen state. const fn freeze(metadata: usize) -> usize { metadata | (1_usize << (usize::BITS - 2)) } /// Updates the metadata to represent a non-frozen state. const fn thaw(metadata: usize) -> usize { metadata & (!(1_usize << (usize::BITS - 2))) } /// Checks if the [`Leaf`] is retired. const fn retired(metadata: usize) -> bool { metadata & (1_usize << (usize::BITS - 1)) != 0 } /// Makes the metadata represent a retired state. const fn retire(metadata: usize) -> usize { metadata | (1_usize << (usize::BITS - 1)) } /// Returns a bit mask for an entry. const fn rank_mask(&self, index: usize) -> usize { ((1_usize << self.num_bits_per_entry) - 1) << (index * self.num_bits_per_entry) } /// Returns the rank of an entry. const fn rank(&self, metadata: usize, index: usize) -> usize { (metadata >> (index * self.num_bits_per_entry)) % (1_usize << self.num_bits_per_entry) } /// Returns the uninitialized rank value which is smaller than all the valid rank values. const fn uninit_rank() -> usize { 0 } /// Returns the removed rank value which is greater than all the valid rank values. const fn removed_rank(&self) -> usize { (1_usize << self.num_bits_per_entry) - 1 } /// Augments the rank to the given metadata. const fn augment(&self, metadata: usize, index: usize, rank: usize) -> usize { (metadata & (!self.rank_mask(index))) | (rank << (index * self.num_bits_per_entry)) } } /// The maximum number of entries and the number of metadata bits per entry in a [`Leaf`]. /// /// * `M`: The maximum number of entries. /// * `B`: The minimum number of bits to express the state of an entry. /// * `2`: The number of special states of an entry: uninitialized, removed. /// * `2`: The number of special states of a [`Leaf`]: frozen, retired. /// * `U`: `usize::BITS`. /// * `Eq1 = M + 2 <= 2^B`: `B` bits represent at least `M + 2` states. /// * `Eq2 = B * M + 2 <= U`: `M entries + 2` special state. /// * `Eq3 = Ceil(Log2(M + 2)) * M + 2 <= U`: derived from `Eq1` and `Eq2`. /// /// Therefore, when `U = 64 => M = 14 / B = 4`, and `U = 32 => M = 7 / B = 4`. pub const DIMENSION: Dimension = match usize::BITS / 8 { 1 => Dimension { num_entries: 2, num_bits_per_entry: 2, }, 2 => Dimension { num_entries: 4, num_bits_per_entry: 3, }, 4 => Dimension { num_entries: 7, num_bits_per_entry: 4, }, 8 => Dimension { num_entries: 14, num_bits_per_entry: 4, }, _ => Dimension { num_entries: 25, num_bits_per_entry: 5, }, }; /// Each constructed entry in an `EntryArray` is never dropped until the [`Leaf`] is dropped. pub type EntryArray = ( [MaybeUninit; DIMENSION.num_entries], [MaybeUninit; DIMENSION.num_entries], ); /// Leaf scanner. pub struct Scanner<'l, K, V> { leaf: &'l Leaf, metadata: usize, entry_index: usize, } impl<'l, K, V> Scanner<'l, K, V> { /// Creates a new [`Scanner`]. #[inline] pub(super) fn new(leaf: &'l Leaf) -> Scanner<'l, K, V> { Scanner { leaf, metadata: leaf.metadata.load(Acquire), entry_index: DIMENSION.num_entries, } } /// Returns the metadata that the [`Scanner`] is currently using. #[inline] pub(super) const fn metadata(&self) -> usize { self.metadata } /// Returns a reference to the entry that the scanner is currently pointing to. #[inline] pub(super) const fn get(&self) -> Option<(&'l K, &'l V)> { if self.entry_index >= DIMENSION.num_entries { return None; } Some(( self.leaf.key_at(self.entry_index), self.leaf.value_at(self.entry_index), )) } /// Returns a reference to the max key. #[inline] pub(super) fn max_key(&self) -> Option<&'l K> { self.leaf.max_key() } /// Traverses the linked list. #[inline] pub(super) fn jump<'g>( &self, min_allowed_key: Option<&K>, guard: &'g Guard, ) -> Option> where K: Ord, { let mut next_leaf_ptr = self.leaf.next_ptr(Acquire, guard); while let Some(next_leaf_ref) = next_leaf_ptr.as_ref() { let mut leaf_scanner = Scanner::new(next_leaf_ref); if let Some(key) = min_allowed_key { if !self.leaf.is_clear(Relaxed) { // Data race resolution: compare keys if the current leaf has been deleted. // // There is a chance that the current leaf has been deleted, and smaller // keys have been inserted into the next leaf. while let Some((k, _)) = leaf_scanner.next() { if key.cmp(k) == Ordering::Less { return Some(leaf_scanner); } } next_leaf_ptr = next_leaf_ref.next_ptr(Acquire, guard); continue; } } if leaf_scanner.next().is_some() { return Some(leaf_scanner); } next_leaf_ptr = next_leaf_ref.next_ptr(Acquire, guard); } None } /// Proceeds to the next entry. fn proceed(&mut self) { if self.entry_index == usize::MAX { return; } let index = Leaf::::next(self.entry_index, self.metadata); if index == DIMENSION.num_entries { // Fuse the iterator. self.entry_index = usize::MAX; } else { self.entry_index = index; } } } impl<'l, K, V> Scanner<'l, K, V> where K: 'static + Clone + Ord, V: 'static + Clone, { /// Returns a [`Scanner`] pointing to the max-less entry if there is one. #[inline] pub(super) fn max_less(leaf: &'l Leaf, key: &Q) -> Option> where Q: Comparable + ?Sized, { let metadata = leaf.metadata.load(Acquire); let index = leaf.max_less(metadata, key); if index == DIMENSION.num_entries { None } else { Some(Scanner { leaf, metadata, entry_index: index, }) } } } impl Debug for Scanner<'_, K, V> { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Scanner") .field("metadata", &self.metadata) .field("entry_index", &self.entry_index) .finish() } } impl<'l, K, V> Iterator for Scanner<'l, K, V> { type Item = (&'l K, &'l V); #[inline] fn next(&mut self) -> Option { self.proceed(); self.get() } } #[cfg(not(feature = "loom"))] #[cfg(test)] mod test { use super::*; use proptest::prelude::*; use std::sync::atomic::AtomicBool; use tokio::sync::Barrier; #[test] fn basic() { let leaf: Leaf = Leaf::new(); assert!(matches!( leaf.insert("MY GOODNESS!".to_owned(), "OH MY GOD!!".to_owned()), InsertResult::Success )); assert!(matches!( leaf.insert("GOOD DAY".to_owned(), "OH MY GOD!!".to_owned()), InsertResult::Success )); assert_eq!(leaf.search_entry("MY GOODNESS!").unwrap().1, "OH MY GOD!!"); assert_eq!(leaf.search_entry("GOOD DAY").unwrap().1, "OH MY GOD!!"); for i in 0..DIMENSION.num_entries { if let InsertResult::Full(k, v) = leaf.insert(i.to_string(), i.to_string()) { assert_eq!(i + 2, DIMENSION.num_entries); assert_eq!(k, i.to_string()); assert_eq!(v, i.to_string()); break; } assert_eq!( leaf.search_entry(&i.to_string()).unwrap(), (&i.to_string(), &i.to_string()) ); } for i in 0..DIMENSION.num_entries { let result = leaf.remove_if(&i.to_string(), &mut |_| i >= 10); if i >= 10 && i + 2 < DIMENSION.num_entries { assert_eq!(result, RemoveResult::Success); } else { assert_eq!(result, RemoveResult::Fail); } } assert_eq!( leaf.remove_if("GOOD DAY", &mut |v| v == "OH MY"), RemoveResult::Fail ); assert_eq!( leaf.remove_if("GOOD DAY", &mut |v| v == "OH MY GOD!!"), RemoveResult::Success ); assert!(leaf.search_entry("GOOD DAY").is_none()); assert_eq!( leaf.remove_if("MY GOODNESS!", &mut |_| true), RemoveResult::Success ); assert!(leaf.search_entry("MY GOODNESS!").is_none()); assert!(leaf.search_entry("1").is_some()); assert!(matches!( leaf.insert("1".to_owned(), "1".to_owned()), InsertResult::Duplicate(..) )); assert!(matches!( leaf.insert("100".to_owned(), "100".to_owned()), InsertResult::Full(..) )); let mut scanner = Scanner::new(&leaf); for i in 0..DIMENSION.num_entries { if let Some(e) = scanner.next() { assert_eq!(e.0, &i.to_string()); assert_eq!(e.1, &i.to_string()); assert_ne!( leaf.remove_if(&i.to_string(), &mut |_| true), RemoveResult::Fail ); } else { break; } } assert!(matches!( leaf.insert("200".to_owned(), "200".to_owned()), InsertResult::Retired(..) )); } #[test] fn calculate_boundary() { let leaf: Leaf = Leaf::new(); for i in 0..DIMENSION.num_entries { assert!(matches!(leaf.insert(i, i), InsertResult::Success)); } assert_eq!( Leaf::::optimal_boundary(leaf.metadata.load(Relaxed)), DIMENSION.num_entries - 1 ); let leaf: Leaf = Leaf::new(); for i in (0..DIMENSION.num_entries).rev() { assert!(matches!(leaf.insert(i, i), InsertResult::Success)); } assert_eq!( Leaf::::optimal_boundary(leaf.metadata.load(Relaxed)), 1 ); let leaf: Leaf = Leaf::new(); for i in 0..DIMENSION.num_entries { if i < DIMENSION.num_entries / 2 { assert!(matches!( leaf.insert(usize::MAX - i, usize::MAX - i), InsertResult::Success )); } else { assert!(matches!(leaf.insert(i, i), InsertResult::Success)); } } if usize::BITS == 32 { assert_eq!( Leaf::::optimal_boundary(leaf.metadata.load(Relaxed)), 4 ); } else { assert_eq!( Leaf::::optimal_boundary(leaf.metadata.load(Relaxed)), 6 ); } } #[test] fn special() { let leaf: Leaf = Leaf::new(); assert!(matches!(leaf.insert(11, 17), InsertResult::Success)); assert!(matches!(leaf.insert(17, 11), InsertResult::Success)); let mut leaf1 = Shared::new(Leaf::new()); let mut leaf2 = None; leaf.freeze_and_distribute(&mut leaf1, &mut leaf2); assert_eq!(leaf1.search_entry(&11), Some((&11, &17))); assert_eq!(leaf1.search_entry(&17), Some((&17, &11))); assert!(leaf2.is_none()); assert!(matches!(leaf.insert(1, 7), InsertResult::Frozen(..))); assert_eq!(leaf.remove_if(&17, &mut |_| true), RemoveResult::Frozen); assert!(matches!(leaf.insert(3, 5), InsertResult::Frozen(..))); assert!(leaf.thaw()); assert!(matches!(leaf.insert(1, 7), InsertResult::Success)); assert_eq!(leaf.remove_if(&1, &mut |_| true), RemoveResult::Success); assert_eq!(leaf.remove_if(&17, &mut |_| true), RemoveResult::Success); assert_eq!(leaf.remove_if(&11, &mut |_| true), RemoveResult::Retired); assert!(matches!(leaf.insert(5, 3), InsertResult::Retired(..))); } proptest! { #[cfg_attr(miri, ignore)] #[test] fn general(insert in 0_usize..DIMENSION.num_entries, remove in 0_usize..DIMENSION.num_entries) { let leaf: Leaf = Leaf::new(); assert!(leaf.is_empty()); for i in 0..insert { assert!(matches!(leaf.insert(i, i), InsertResult::Success)); if i != 0 { let result = leaf.max_less(leaf.metadata.load(Relaxed), &i); assert_eq!(*leaf.key_at(result), i - 1); assert_eq!(*leaf.value_at(result), i - 1); } } if insert == 0 { assert_eq!(leaf.max_key(), None); assert!(leaf.is_empty()); } else { assert_eq!(leaf.max_key(), Some(&(insert - 1))); assert!(!leaf.is_empty()); } for i in 0..insert { assert!(matches!(leaf.insert(i, i), InsertResult::Duplicate(..))); assert!(!leaf.is_empty()); let result = leaf.min_greater_equal(&i); assert_eq!(result.0, Some((&i, &i))); } for i in 0..insert { assert_eq!(leaf.search_entry(&i).unwrap(), (&i, &i)); } if insert == DIMENSION.num_entries { assert!(matches!(leaf.insert(usize::MAX, usize::MAX), InsertResult::Full(..))); } for i in 0..remove { if i < insert { if i == insert - 1 { assert!(matches!(leaf.remove_if(&i, &mut |_| true), RemoveResult::Retired)); for i in 0..insert { assert!(matches!(leaf.insert(i, i), InsertResult::Retired(..))); } } else { assert!(matches!(leaf.remove_if(&i, &mut |_| true), RemoveResult::Success)); } } else { assert!(matches!(leaf.remove_if(&i, &mut |_| true), RemoveResult::Fail)); assert!(leaf.is_empty()); } } } #[cfg_attr(miri, ignore)] #[test] fn range(start in 0_usize..DIMENSION.num_entries, end in 0_usize..DIMENSION.num_entries) { let leaf: Leaf = Leaf::new(); for i in 1..DIMENSION.num_entries - 1 { prop_assert!(matches!(leaf.insert(i, i), InsertResult::Success)); } leaf.remove_range(&(start..end)); for i in 1..DIMENSION.num_entries - 1 { prop_assert!(leaf.search_entry(&i).is_none() == (start..end).contains(&i)); } prop_assert!(leaf.search_entry(&0).is_none()); prop_assert!(leaf.search_entry(&(DIMENSION.num_entries - 1)).is_none()); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn update() { let num_excess = 3; let num_tasks = DIMENSION.num_entries + num_excess; for _ in 0..256 { let barrier = Shared::new(Barrier::new(num_tasks)); let leaf: Shared> = Shared::new(Leaf::new()); let full: Shared = Shared::new(AtomicUsize::new(0)); let retire: Shared = Shared::new(AtomicUsize::new(0)); let mut task_handles = Vec::with_capacity(num_tasks); for t in 1..=num_tasks { let barrier_clone = barrier.clone(); let leaf_clone = leaf.clone(); let full_clone = full.clone(); let retire_clone = retire.clone(); task_handles.push(tokio::spawn(async move { barrier_clone.wait().await; let inserted = match leaf_clone.insert(t, t) { InsertResult::Success => { assert_eq!(leaf_clone.search_entry(&t).unwrap(), (&t, &t)); true } InsertResult::Duplicate(_, _) | InsertResult::Frozen(_, _) | InsertResult::Retired(_, _) | InsertResult::Retry(_, _) => { unreachable!(); } InsertResult::Full(k, v) => { assert_eq!(k, v); assert_eq!(k, t); full_clone.fetch_add(1, Relaxed); false } }; { let mut prev = 0; let mut scanner = Scanner::new(&leaf_clone); for e in scanner.by_ref() { assert_eq!(e.0, e.1); assert!(*e.0 > prev); prev = *e.0; } } barrier_clone.wait().await; assert_eq!((*full_clone).load(Relaxed), num_excess); if inserted { assert_eq!(leaf_clone.search_entry(&t).unwrap(), (&t, &t)); } { let scanner = Scanner::new(&leaf_clone); assert_eq!(scanner.count(), DIMENSION.num_entries); } barrier_clone.wait().await; match leaf_clone.remove_if(&t, &mut |_| true) { RemoveResult::Success => assert!(inserted), RemoveResult::Fail => assert!(!inserted), RemoveResult::Frozen | RemoveResult::Cleanup => unreachable!(), RemoveResult::Retired => { assert!(inserted); assert_eq!(retire_clone.swap(1, Relaxed), 0); } } })); } for r in futures::future::join_all(task_handles).await { assert!(r.is_ok()); } } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn durability() { let num_tasks = 16_usize; let workload_size = 8_usize; for _ in 0..16 { for k in 0..=workload_size { let barrier = Shared::new(Barrier::new(num_tasks)); let leaf: Shared> = Shared::new(Leaf::new()); let inserted: Shared = Shared::new(AtomicBool::new(false)); let mut task_handles = Vec::with_capacity(num_tasks); for _ in 0..num_tasks { let barrier_clone = barrier.clone(); let leaf_clone = leaf.clone(); let inserted_clone = inserted.clone(); task_handles.push(tokio::spawn(async move { { barrier_clone.wait().await; if let InsertResult::Success = leaf_clone.insert(k, k) { assert!(!inserted_clone.swap(true, Relaxed)); } } { barrier_clone.wait().await; for i in 0..workload_size { if i != k { let _result = leaf_clone.insert(i, i); } assert!(!leaf_clone.is_retired()); assert_eq!(leaf_clone.search_entry(&k).unwrap(), (&k, &k)); } for i in 0..workload_size { let _result = leaf_clone.remove_if(&i, &mut |v| *v != k); assert_eq!(leaf_clone.search_entry(&k).unwrap(), (&k, &k)); } } })); } for r in futures::future::join_all(task_handles).await { assert!(r.is_ok()); } assert!((*inserted).load(Relaxed)); } } } } scc-3.4.8/src/tree_index/leaf_node.rs000064400000000000000000001527121046102023000156170ustar 00000000000000use std::cmp::Ordering::{Equal, Greater, Less}; use std::ops::{Bound, RangeBounds}; use std::ptr; use std::sync::atomic::AtomicPtr; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use saa::Lock; use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; use super::Leaf; use super::leaf::{DIMENSION, InsertResult, RemoveResult, Scanner, range_contains}; use super::node::Node; use crate::Comparable; use crate::LinkedList; use crate::async_helper::TryWait; use crate::exit_guard::ExitGuard; /// [`Tag::First`] indicates that the corresponding node has retired. pub const RETIRED: Tag = Tag::First; /// [`LeafNode`] contains a list of instances of `K, V` [`Leaf`]. /// /// The layout of a leaf node: `|ptr(entry array)/max(child keys)|...|ptr(entry array)|` pub struct LeafNode { /// Children of the [`LeafNode`]. pub(super) children: Leaf>>, /// A child [`Leaf`] that has no upper key bound. /// /// It stores the maximum key in the node, and key-value pairs are first pushed to this /// [`Leaf`] until it splits. unbounded_child: AtomicShared>, /// Ongoing split operation. split_op: StructuralChange, /// [`Lock`] to protecct the [`LeafNode`]. pub(super) lock: Lock, } /// [`Locker`] holds exclusive ownership of a [`LeafNode`]. pub(super) struct Locker<'n, K, V> { leaf_node: &'n LeafNode, } /// A state machine to keep track of the progress of a bulk removal operation. #[derive(Clone, Copy, Eq, PartialEq)] pub(super) enum RemoveRangeState { /// The maximum key of the node is less than the start bound of the range. Below, /// The maximum key of the node is contained in the range, but it is not clear whether the /// minimum key of the node is contained in the range. MaybeBelow, /// The maximum key and the minimum key of the node are contained in the range. FullyContained, /// The maximum key of the node is not contained in the range, but it is not clear whether /// the minimum key of the node is contained in the range. MaybeAbove, } /// [`StructuralChange`] stores intermediate results during a split operation. /// /// `AtomicPtr` members may point to values under the protection of the [`Guard`] used for the /// split operation. pub(super) struct StructuralChange { origin_leaf_key: AtomicPtr, origin_leaf: AtomicShared>, low_key_leaf: AtomicShared>, high_key_leaf: AtomicShared>, low_key_leaf_node: AtomicPtr>, high_key_leaf_node: AtomicPtr>, } impl LeafNode { /// Creates a new empty [`LeafNode`]. #[inline] pub(super) fn new() -> LeafNode { LeafNode { children: Leaf::new(), unbounded_child: AtomicShared::null(), split_op: StructuralChange::default(), lock: Lock::default(), } } /// Clears the leaf node by unlinking all the leaves. #[inline] pub(super) fn clear(&self, guard: &Guard) { let scanner = Scanner::new(&self.children); for (_, child) in scanner { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { child.link_ref().swap((None, Tag::None), Acquire); } } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { unbounded.link_ref().swap((None, Tag::None), Acquire); } } /// Returns `true` if the [`LeafNode`] has retired. #[inline] pub(super) fn retired(&self) -> bool { self.unbounded_child.tag(Acquire) == RETIRED } } impl LeafNode where K: 'static + Clone + Ord, V: 'static + Clone, { /// Searches for an entry containing the specified key. #[inline] pub(super) fn search_entry<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<(&'g K, &'g V)> where K: 'g, Q: Comparable + ?Sized, { loop { let (child, metadata) = self.children.min_greater_equal(key); if let Some((_, child)) = child { if let Some(child) = child.load(Acquire, guard).as_ref() { if self.children.validate(metadata) { // Data race with split. // - Writer: start to insert an intermediate low key leaf. // - Reader: read the metadata not including the intermediate low key leaf. // - Writer: insert the intermediate low key leaf. // - Writer: replace the high key leaf pointer. // - Reader: read the new high key leaf pointer // Consequently, the reader may miss keys in the low key leaf. // // Resolution: metadata validation. return child.search_entry(key); } } // The child leaf must have been just removed. // // The `LeafNode` metadata is updated before a leaf is removed. This implies that // the new `metadata` will be different from the current `metadata`. } else { let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if self.children.validate(metadata) { return unbounded.search_entry(key); } } else { return None; } } } } /// Searches for the value associated with the specified key. #[inline] pub(super) fn search_value<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<&'g V> where K: 'g, Q: Comparable + ?Sized, { loop { let (child, metadata) = self.children.min_greater_equal(key); if let Some((_, child)) = child { if let Some(child) = child.load(Acquire, guard).as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. return child.search_value(key); } } // Data race resolution - see `LeafNode::search_entry`. } else { let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if self.children.validate(metadata) { return unbounded.search_value(key); } } else { return None; } } } } /// Returns the minimum key entry. #[inline] pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { loop { let mut scanner = Scanner::new(&self.children); let metadata = scanner.metadata(); if let Some((_, child)) = scanner.next() { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. return Some(Scanner::new(child)); } } // It is not a hot loop - see `LeafNode::search_entry`. continue; } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if self.children.validate(metadata) { return Some(Scanner::new(unbounded)); } continue; } return None; } } /// Returns a [`Scanner`] pointing to an entry that is close enough to the entry with the /// maximum key among those keys that are smaller than or equal to the given key. /// /// Returns `None` if all keys in the [`LeafNode`] are greater than the given key. #[inline] pub(super) fn max_le_appr<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option> where K: 'g, Q: Comparable + ?Sized, { loop { if let Some(scanner) = Scanner::max_less(&self.children, key) { if let Some((_, child)) = scanner.get() { if let Some(child) = child.load(Acquire, guard).as_ref() { if self.children.validate(scanner.metadata()) { // Data race resolution - see `LeafNode::search_entry`. if let Some(scanner) = Scanner::max_less(child, key) { return Some(scanner); } // Fallback. break; } } // It is not a hot loop - see `LeafNode::search_entry`. continue; } } // Fallback. break; } // Starts scanning from the minimum key. let mut min_scanner = self.min(guard)?; min_scanner.next(); loop { if let Some((k, _)) = min_scanner.get() { if key.compare(k).is_ge() { return Some(min_scanner); } break; } min_scanner = min_scanner.jump(None, guard)?; } None } /// Inserts a key-value pair. /// /// # Errors /// /// Returns an error if a retry is required. #[inline] pub(super) fn insert( &self, mut key: K, mut val: V, async_wait: &mut W, guard: &Guard, ) -> Result, (K, V)> { loop { let (child, metadata) = self.children.min_greater_equal(&key); if let Some((child_key, child)) = child { let child_ptr = child.load(Acquire, guard); if let Some(child_ref) = child_ptr.as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. let insert_result = child_ref.insert(key, val); match insert_result { InsertResult::Success | InsertResult::Duplicate(..) | InsertResult::Retry(..) => return Ok(insert_result), InsertResult::Full(k, v) | InsertResult::Retired(k, v) => { let split_result = self.split_leaf( (k, v), Some(child_key), child_ptr, child, async_wait, guard, )?; if let InsertResult::Retry(k, v) = split_result { key = k; val = v; continue; } return Ok(split_result); } InsertResult::Frozen(k, v) => { // The `Leaf` is being split: retry. async_wait.try_wait(&self.lock); return Err((k, v)); } }; } } // It is not a hot loop - see `LeafNode::search_entry`. continue; } let mut unbounded_ptr = self.unbounded_child.load(Acquire, guard); if unbounded_ptr.is_null() { match self.unbounded_child.compare_exchange( Ptr::null(), (Some(Shared::new(Leaf::new())), Tag::None), AcqRel, Acquire, guard, ) { Ok((_, ptr)) => { unbounded_ptr = ptr; } Err((_, actual)) => { unbounded_ptr = actual; } } } if let Some(unbounded) = unbounded_ptr.as_ref() { debug_assert!(unbounded_ptr.tag() == Tag::None); if !self.children.validate(metadata) { continue; } let insert_result = unbounded.insert(key, val); match insert_result { InsertResult::Success | InsertResult::Duplicate(..) | InsertResult::Retry(..) => { return Ok(insert_result); } InsertResult::Full(k, v) | InsertResult::Retired(k, v) => { let split_result = self.split_leaf( (k, v), None, unbounded_ptr, &self.unbounded_child, async_wait, guard, )?; if let InsertResult::Retry(k, v) = split_result { key = k; val = v; continue; } return Ok(split_result); } InsertResult::Frozen(k, v) => { async_wait.try_wait(&self.lock); return Err((k, v)); } }; } return Ok(InsertResult::Retired(key, val)); } } /// Removes an entry associated with the given key. /// /// # Errors /// /// Returns an error if a retry is required. #[inline] pub(super) fn remove_if bool, W: TryWait>( &self, key: &Q, condition: &mut F, async_wait: &mut W, guard: &Guard, ) -> Result where Q: Comparable + ?Sized, { loop { let (child, metadata) = self.children.min_greater_equal(key); if let Some((_, child)) = child { let child_ptr = child.load(Acquire, guard); if let Some(child) = child_ptr.as_ref() { if self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. let result = child.remove_if(key, condition); if result == RemoveResult::Frozen { // When a `Leaf` is frozen, its entries may be being copied to new // `Leaves`. async_wait.try_wait(&self.lock); return Err(()); } else if result == RemoveResult::Retired { return Ok(self.coalesce(guard)); } return Ok(result); } } // It is not a hot loop - see `LeafNode::search_entry`. continue; } let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { debug_assert!(unbounded_ptr.tag() == Tag::None); if !self.children.validate(metadata) { // Data race resolution - see `LeafNode::search_entry`. continue; } let result = unbounded.remove_if(key, condition); if result == RemoveResult::Frozen { async_wait.try_wait(&self.lock); return Err(()); } else if result == RemoveResult::Retired { return Ok(self.coalesce(guard)); } return Ok(result); } return Ok(RemoveResult::Fail); } } /// Removes a range of entries. /// /// Returns the number of remaining children. #[inline] pub(super) fn remove_range<'g, Q, R: RangeBounds, W: TryWait>( &self, range: &R, start_unbounded: bool, valid_lower_max_leaf: Option<&'g Leaf>, valid_upper_min_node: Option<&'g Node>, async_wait: &mut W, guard: &'g Guard, ) -> Result where Q: Comparable + ?Sized, { debug_assert!(valid_lower_max_leaf.is_none() || start_unbounded); debug_assert!(valid_lower_max_leaf.is_none() || valid_upper_min_node.is_none()); let Some(_lock) = Locker::try_lock(self) else { async_wait.try_wait(&self.lock); return Err(()); }; let mut current_state = RemoveRangeState::Below; let mut num_leaves = 1; let mut first_valid_leaf = None; for (key, leaf) in Scanner::new(&self.children) { current_state = current_state.next(key, range, start_unbounded); match current_state { RemoveRangeState::Below | RemoveRangeState::MaybeBelow => { if let Some(leaf) = leaf.load(Acquire, guard).as_ref() { leaf.remove_range(range); } num_leaves += 1; if first_valid_leaf.is_none() { first_valid_leaf.replace(leaf); } } RemoveRangeState::FullyContained => { // There can be another thread inserting keys into the leaf, and this may // render those operations completely ineffective. self.children.remove_if(key, &mut |_| true); if let Some(leaf) = leaf.swap((None, Tag::None), AcqRel).0 { leaf.delete_self(Release); } } RemoveRangeState::MaybeAbove => { if let Some(leaf) = leaf.load(Acquire, guard).as_ref() { leaf.remove_range(range); } num_leaves += 1; if first_valid_leaf.is_none() { first_valid_leaf.replace(leaf); } break; } } } if let Some(unbounded) = self.unbounded_child.load(Acquire, guard).as_ref() { unbounded.remove_range(range); } if let Some(valid_lower_max_leaf) = valid_lower_max_leaf { // Connect the specified leaf with the first valid leaf. if first_valid_leaf.is_none() { first_valid_leaf.replace(&self.unbounded_child); } valid_lower_max_leaf.link_ref().swap( ( first_valid_leaf.and_then(|l| l.get_shared(Acquire, guard)), Tag::None, ), AcqRel, ); } else if let Some(valid_upper_min_node) = valid_upper_min_node { // Connect the unbounded child with the minimum valid leaf in the node. valid_upper_min_node.remove_range( range, true, self.unbounded_child.load(Acquire, guard).as_ref(), None, async_wait, guard, )?; } Ok(num_leaves) } /// Splits itself into the given leaf nodes, and returns the middle key value. #[allow(clippy::too_many_lines)] pub(super) fn split_leaf_node<'g>( &'g self, low_key_leaf_node: &LeafNode, high_key_leaf_node: &LeafNode, guard: &'g Guard, ) -> &'g K { let mut middle_key = None; low_key_leaf_node.lock.lock_sync(); high_key_leaf_node.lock.lock_sync(); // It is safe to keep the pointers to the new leaf nodes in this full leaf node since the // whole split operation is protected under a single `ebr::Guard`, and the pointers are // only dereferenced during the operation. self.split_op .low_key_leaf_node .swap(ptr::from_ref(low_key_leaf_node).cast_mut(), Relaxed); self.split_op .high_key_leaf_node .swap(ptr::from_ref(high_key_leaf_node).cast_mut(), Relaxed); // Builds a list of valid leaves #[allow(clippy::type_complexity)] let mut entry_array: [Option<(Option<&K>, AtomicShared>)>; DIMENSION.num_entries + 2] = Default::default(); let mut num_entries = 0; let low_key_leaf_ref = self .split_op .low_key_leaf .load(Relaxed, guard) .as_ref() .unwrap(); let middle_key_ref = low_key_leaf_ref.max_key().unwrap(); let scanner = Scanner::new(&self.children); let recommended_boundary = Leaf::::optimal_boundary(scanner.metadata()); for entry in scanner { if unsafe { self.split_op .origin_leaf_key .load(Relaxed) .as_ref() .map_or_else(|| false, |key| entry.0 == key) } { entry_array[num_entries].replace(( Some(middle_key_ref), self.split_op.low_key_leaf.clone(Relaxed, guard), )); num_entries += 1; if !self.split_op.high_key_leaf.is_null(Relaxed) { entry_array[num_entries].replace(( Some(entry.0), self.split_op.high_key_leaf.clone(Relaxed, guard), )); num_entries += 1; } } else { entry_array[num_entries].replace((Some(entry.0), entry.1.clone(Acquire, guard))); num_entries += 1; } } if self.split_op.origin_leaf_key.load(Relaxed).is_null() { // If the origin is an unbounded node, assign the high key node to the high key node's // unbounded. entry_array[num_entries].replace(( Some(middle_key_ref), self.split_op.low_key_leaf.clone(Relaxed, guard), )); num_entries += 1; if !self.split_op.high_key_leaf.is_null(Relaxed) { entry_array[num_entries] .replace((None, self.split_op.high_key_leaf.clone(Relaxed, guard))); num_entries += 1; } } else { // If the origin is a bounded node, assign the unbounded leaf as the high key node's // unbounded. entry_array[num_entries].replace((None, self.unbounded_child.clone(Relaxed, guard))); num_entries += 1; } debug_assert!(num_entries >= 2); let low_key_leaf_array_size = recommended_boundary.min(num_entries - 1); for (i, entry) in entry_array.iter().enumerate() { if let Some((k, v)) = entry { match (i + 1).cmp(&low_key_leaf_array_size) { Less => { low_key_leaf_node.children.insert_unchecked( k.unwrap().clone(), v.clone(Relaxed, guard), i, ); } Equal => { middle_key.replace(k.unwrap()); low_key_leaf_node .unbounded_child .swap((v.get_shared(Relaxed, guard), Tag::None), Relaxed); } Greater => { if let Some(k) = k.cloned() { high_key_leaf_node.children.insert_unchecked( k, v.clone(Relaxed, guard), i - low_key_leaf_array_size, ); } else { high_key_leaf_node .unbounded_child .swap((v.get_shared(Relaxed, guard), Tag::None), Relaxed); } } } } else { break; } } middle_key.unwrap() } /// Commits an ongoing structural change. #[inline] pub(super) fn commit(&self, guard: &Guard) { // Unfreeze both leaves. if let Some(origin_leaf) = self.split_op.origin_leaf.swap((None, Tag::None), Relaxed).0 { // Make the origin leaf unreachable before making the new leaves updatable. origin_leaf.delete_self(Release); let _: bool = origin_leaf.release(); } let low_key_leaf = self .split_op .low_key_leaf .load(Relaxed, guard) .as_ref() .unwrap(); let high_key_leaf = self .split_op .high_key_leaf .load(Relaxed, guard) .as_ref() .unwrap(); let unfrozen = low_key_leaf.thaw(); debug_assert!(unfrozen); let unfrozen = high_key_leaf.thaw(); debug_assert!(unfrozen); // It is safe to dereference those pointers since the whole split operation is under a // single `ebr::Guard`. if let Some(low_key_leaf_node) = unsafe { self.split_op.low_key_leaf_node.load(Relaxed).as_ref() } { let origin = low_key_leaf_node.split_op.reset(); low_key_leaf_node.lock.release_lock(); origin.map(Shared::release); } if let Some(high_key_leaf_node) = unsafe { self.split_op.high_key_leaf_node.load(Relaxed).as_ref() } { let origin = high_key_leaf_node.split_op.reset(); high_key_leaf_node.lock.release_lock(); origin.map(Shared::release); } let origin = self.split_op.reset(); // The leaf node can no longer be locked. self.lock.poison_lock(); origin.map(Shared::release); } /// Rolls back the ongoing split operation. #[inline] pub(super) fn rollback(&self, guard: &Guard) { let low_key_leaf = self .split_op .low_key_leaf .load(Relaxed, guard) .as_ref() .unwrap(); let high_key_leaf = self .split_op .high_key_leaf .load(Relaxed, guard) .as_ref() .unwrap(); // Roll back the linked list state. // // `high_key_leaf` must be deleted first in order for `Scanners` not to omit entries. let deleted = high_key_leaf.delete_self(Release); debug_assert!(deleted); let deleted = low_key_leaf.delete_self(Release); debug_assert!(deleted); if let Some(origin_leaf) = self.split_op.origin_leaf.swap((None, Tag::None), Relaxed).0 { // Unfreeze the origin. let unfrozen = origin_leaf.thaw(); debug_assert!(unfrozen); // Remove the mark from the full leaf node. // // `clear` clears the tag, so there is no guarantee that the tag has been kept. origin_leaf.unmark(Release); } let origin = self.split_op.reset(); self.lock.release_lock(); origin.map(Shared::release); } /// Cleans up logically deleted leaf instances in the linked list. /// /// Returns `false` if the target leaf does not exist in the [`LeafNode`]. #[inline] pub(super) fn cleanup_link<'g, Q>(&self, key: &Q, traverse_max: bool, guard: &'g Guard) -> bool where K: 'g, Q: Comparable + ?Sized, { let scanner = if traverse_max { if let Some(unbounded) = self.unbounded_child.load(Acquire, guard).as_ref() { Scanner::new(unbounded) } else { return false; } } else if let Some(leaf_scanner) = Scanner::max_less(&self.children, key) { if let Some((_, child)) = leaf_scanner.get() { if let Some(child) = child.load(Acquire, guard).as_ref() { Scanner::new(child) } else { return false; } } else { return false; } } else { return false; }; // It *would* be the maximum leaf node among those containing keys smaller than the // target key. Hopefully, two jumps will be sufficient. scanner.jump(None, guard).map(|s| s.jump(None, guard)); true } /// Splits a full leaf. /// /// # Errors /// /// Returns an error if a retry is required. #[allow(clippy::too_many_lines)] fn split_leaf( &self, entry: (K, V), full_leaf_key: Option<&K>, full_leaf_ptr: Ptr>, full_leaf: &AtomicShared>, async_wait: &mut W, guard: &Guard, ) -> Result, (K, V)> { if !self.lock.try_lock() { async_wait.try_wait(&self.lock); return Err(entry); } else if self.retired() { self.lock.release_lock(); return Ok(InsertResult::Retired(entry.0, entry.1)); } else if full_leaf_ptr != full_leaf.load(Relaxed, guard) { self.lock.release_lock(); return Err(entry); } let prev = self .split_op .origin_leaf .swap((full_leaf.get_shared(Relaxed, guard), Tag::None), Relaxed) .0; debug_assert!(prev.is_none()); if let Some(full_leaf_key) = full_leaf_key { self.split_op .origin_leaf_key .store(ptr::from_ref(full_leaf_key).cast_mut(), Relaxed); } let target = full_leaf_ptr.as_ref().unwrap(); // Distribute entries to two leaves after make the target retired. let exit_guard = ExitGuard::new((), |()| { target.thaw(); self.split_op.reset(); self.lock.release_lock(); }); let mut low_key_leaf_shared = Shared::new(Leaf::new()); let mut high_key_leaf_shared = None; target.freeze_and_distribute(&mut low_key_leaf_shared, &mut high_key_leaf_shared); self.split_op .low_key_leaf .swap((Some(low_key_leaf_shared), Tag::None), Relaxed); self.split_op .high_key_leaf .swap((high_key_leaf_shared.take(), Tag::None), Relaxed); // When a new leaf is added to the linked list, the leaf is marked to let `Scanners` // acknowledge that the newly added leaf may contain keys that are smaller than those // having been `scanned`. let low_key_leaf_ptr = self.split_op.low_key_leaf.load(Relaxed, guard); let high_key_leaf_ptr = self.split_op.high_key_leaf.load(Relaxed, guard); let unused_leaf = if let Some(high_key_leaf) = high_key_leaf_ptr.as_ref() { // From here, `Scanners` can reach the new leaves. let result = target.push_back( high_key_leaf_ptr.get_shared().unwrap(), true, Release, guard, ); debug_assert!(result.is_ok()); let result = target.push_back(low_key_leaf_ptr.get_shared().unwrap(), true, Release, guard); debug_assert!(result.is_ok()); // Take the max key value stored in the low key leaf as the leaf key. let low_key_leaf = low_key_leaf_ptr.as_ref().unwrap(); // Need to freeze the leaf before trying to make it reachable. let frozen = low_key_leaf.freeze(); debug_assert!(frozen); match self.children.insert( low_key_leaf.max_key().unwrap().clone(), self.split_op.low_key_leaf.clone(Relaxed, guard), ) { InsertResult::Success => (), InsertResult::Duplicate(..) | InsertResult::Frozen(..) | InsertResult::Retry(..) => unreachable!(), InsertResult::Full(_, _) | InsertResult::Retired(_, _) => { // Need to freeze the other leaf. let frozen = high_key_leaf.freeze(); debug_assert!(frozen); exit_guard.forget(); return Ok(InsertResult::Full(entry.0, entry.1)); } } // Mark the full leaf deleted before making the new one reachable and updatable. // // If the order is reversed, there emerges a possibility that entries were removed from // the replaced leaf node whereas those entries still remain in `unused_leaf`; if that // happens, iterators may see the removed entries momentarily. let deleted = target.delete_self(Release); debug_assert!(deleted); // Unfreeze the leaf. let unfrozen = low_key_leaf.thaw(); debug_assert!(unfrozen); // Replace the full leaf with the high-key leaf. let high_key_leaf = self .split_op .high_key_leaf .swap((None, Tag::None), Relaxed) .0; full_leaf.swap((high_key_leaf, Tag::None), Release).0 } else { // From here, `Scanner` can reach the new leaf. // // The full leaf is marked so that readers know that the next leaves may contain // smaller keys. let result = target.push_back(low_key_leaf_ptr.get_shared().unwrap(), true, Release, guard); debug_assert!(result.is_ok()); // Mark the full leaf deleted before making the new one reachable and updatable. let deleted = target.delete_self(Release); debug_assert!(deleted); full_leaf .swap( self.split_op.low_key_leaf.swap((None, Tag::None), Release), Release, ) .0 }; exit_guard.forget(); let origin = self.split_op.reset(); self.lock.release_lock(); origin.map(Shared::release); unused_leaf.map(Shared::release); // Since a new leaf has been inserted, the caller can retry. Ok(InsertResult::Retry(entry.0, entry.1)) } /// Tries to coalesce empty or obsolete leaves after a successful removal of an entry. fn coalesce(&self, guard: &Guard) -> RemoveResult { let mut uncleaned_leaf = false; let mut prev_valid_leaf = None; while let Some(lock) = Locker::try_lock(self) { prev_valid_leaf.take(); for entry in Scanner::new(&self.children) { let leaf_ptr = entry.1.load(Acquire, guard); let leaf = leaf_ptr.as_ref().unwrap(); if leaf.is_retired() { let deleted = leaf.delete_self(Release); debug_assert!(deleted); let result = self.children.remove_if(entry.0, &mut |_| true); debug_assert_ne!(result, RemoveResult::Fail); // The pointer is nullified after the metadata of `self.children` is updated so // that readers are able to retry when they find it being `null`. if let Some(leaf) = entry.1.swap((None, Tag::None), Release).0 { let _: bool = leaf.release(); if let Some(prev_leaf) = prev_valid_leaf.as_ref() { // One jump is sufficient. Scanner::new(*prev_leaf).jump(None, guard); } else { uncleaned_leaf = true; } } } else { prev_valid_leaf.replace(leaf); } } // The unbounded leaf becomes unreachable after all the other leaves are gone. let fully_empty = if prev_valid_leaf.is_some() { false } else { let unbounded_ptr = self.unbounded_child.load(Acquire, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if unbounded.is_retired() { let deleted = unbounded.delete_self(Release); debug_assert!(deleted); // It has to mark the pointer in order to prevent `LeafNode::insert` from // replacing it with a new `Leaf`. if let Some(obsolete_leaf) = self.unbounded_child.swap((None, RETIRED), Release).0 { let _: bool = obsolete_leaf.release(); uncleaned_leaf = true; } true } else { false } } else { debug_assert!(unbounded_ptr.tag() == RETIRED); true } }; if fully_empty { return RemoveResult::Retired; } drop(lock); if !self.has_retired_leaf(guard) { break; } } if uncleaned_leaf { RemoveResult::Cleanup } else { RemoveResult::Success } } /// Checks if the [`LeafNode`] has a retired [`Leaf`]. fn has_retired_leaf(&self, guard: &Guard) -> bool { let mut has_valid_leaf = false; for entry in Scanner::new(&self.children) { let leaf_ptr = entry.1.load(Relaxed, guard); if let Some(leaf) = leaf_ptr.as_ref() { if leaf.is_retired() { return true; } has_valid_leaf = true; } } if !has_valid_leaf { let unbounded_ptr = self.unbounded_child.load(Relaxed, guard); if let Some(unbounded) = unbounded_ptr.as_ref() { if unbounded.is_retired() { return true; } } } false } } impl<'n, K, V> Locker<'n, K, V> { /// Acquires exclusive lock on the [`LeafNode`]. #[inline] pub(super) fn try_lock(leaf_node: &'n LeafNode) -> Option> { if leaf_node.lock.try_lock() { Some(Locker { leaf_node }) } else { None } } } impl Drop for Locker<'_, K, V> { #[inline] fn drop(&mut self) { self.leaf_node.lock.release_lock(); } } impl RemoveRangeState { /// Returns the next state. pub(super) fn next>( self, key: &K, range: &R, start_unbounded: bool, ) -> Self where Q: Comparable + ?Sized, { if range_contains(range, key) { match self { RemoveRangeState::Below => { if start_unbounded { RemoveRangeState::FullyContained } else { RemoveRangeState::MaybeBelow } } RemoveRangeState::MaybeBelow | RemoveRangeState::FullyContained => { RemoveRangeState::FullyContained } RemoveRangeState::MaybeAbove => unreachable!(), } } else { match self { RemoveRangeState::Below => match range.start_bound() { Bound::Included(k) => match k.compare(key) { Less | Equal => RemoveRangeState::MaybeAbove, Greater => RemoveRangeState::Below, }, Bound::Excluded(k) => match k.compare(key) { Less => RemoveRangeState::MaybeAbove, Greater | Equal => RemoveRangeState::Below, }, Bound::Unbounded => RemoveRangeState::MaybeAbove, }, RemoveRangeState::MaybeBelow | RemoveRangeState::FullyContained => { RemoveRangeState::MaybeAbove } RemoveRangeState::MaybeAbove => unreachable!(), } } } } impl StructuralChange { fn reset(&self) -> Option>> { self.origin_leaf_key.store(ptr::null_mut(), Relaxed); self.low_key_leaf.swap((None, Tag::None), Relaxed); self.high_key_leaf.swap((None, Tag::None), Relaxed); self.low_key_leaf_node.store(ptr::null_mut(), Relaxed); self.high_key_leaf_node.store(ptr::null_mut(), Relaxed); self.origin_leaf.swap((None, Tag::None), Relaxed).0 } } impl Default for StructuralChange { #[inline] fn default() -> Self { Self { origin_leaf_key: AtomicPtr::default(), origin_leaf: AtomicShared::null(), low_key_leaf: AtomicShared::null(), high_key_leaf: AtomicShared::null(), low_key_leaf_node: AtomicPtr::default(), high_key_leaf_node: AtomicPtr::default(), } } } #[cfg(not(feature = "loom"))] #[cfg(test)] mod test { use super::*; use std::sync::atomic::AtomicBool; use tokio::sync::Barrier; #[test] fn basic() { let guard = Guard::new(); let leaf_node: LeafNode = LeafNode::new(); assert!(matches!( leaf_node.insert( "MY GOODNESS!".to_owned(), "OH MY GOD!!".to_owned(), &mut (), &guard ), Ok(InsertResult::Success) )); assert!(matches!( leaf_node.insert( "GOOD DAY".to_owned(), "OH MY GOD!!".to_owned(), &mut (), &guard ), Ok(InsertResult::Success) )); assert_eq!( leaf_node.search_entry("MY GOODNESS!", &guard).unwrap().1, "OH MY GOD!!" ); assert_eq!( leaf_node.search_entry("GOOD DAY", &guard).unwrap().1, "OH MY GOD!!" ); assert!(matches!( leaf_node.remove_if::<_, _, _>("GOOD DAY", &mut |v| v == "OH MY", &mut (), &guard), Ok(RemoveResult::Fail) )); assert!(matches!( leaf_node.remove_if::<_, _, _>( "GOOD DAY", &mut |v| v == "OH MY GOD!!", &mut (), &guard ), Ok(RemoveResult::Success) )); assert!(matches!( leaf_node.remove_if::<_, _, _>("GOOD", &mut |v| v == "OH MY", &mut (), &guard), Ok(RemoveResult::Fail) )); assert!(matches!( leaf_node.remove_if::<_, _, _>("MY GOODNESS!", &mut |_| true, &mut (), &guard), Ok(RemoveResult::Retired) )); assert!(matches!( leaf_node.insert("HI".to_owned(), "HO".to_owned(), &mut (), &guard), Ok(InsertResult::Retired(..)) )); } #[test] fn bulk() { let guard = Guard::new(); let leaf_node: LeafNode = LeafNode::new(); for k in 0..1024 { let mut result = leaf_node.insert(k, k, &mut (), &guard); if result.is_err() { result = leaf_node.insert(k, k, &mut (), &guard); } match result.unwrap() { InsertResult::Success => { assert_eq!(leaf_node.search_entry(&k, &guard), Some((&k, &k))); } InsertResult::Duplicate(..) | InsertResult::Frozen(..) | InsertResult::Retired(..) => unreachable!(), InsertResult::Full(_, _) => { leaf_node.rollback(&guard); for r in 0..(k - 1) { assert_eq!(leaf_node.search_entry(&r, &guard), Some((&r, &r))); assert!( leaf_node .remove_if::<_, _, _>(&r, &mut |_| true, &mut (), &guard) .is_ok() ); assert_eq!(leaf_node.search_entry(&r, &guard), None); } assert_eq!( leaf_node.search_entry(&(k - 1), &guard), Some((&(k - 1), &(k - 1))) ); assert_eq!( leaf_node.remove_if::<_, _, _>(&(k - 1), &mut |_| true, &mut (), &guard), Ok(RemoveResult::Retired) ); assert_eq!(leaf_node.search_entry(&(k - 1), &guard), None); break; } InsertResult::Retry(..) => { assert!(leaf_node.insert(k, k, &mut (), &guard).is_ok()); } } } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn parallel() { let num_tasks = 8; let workload_size = 64; let barrier = Shared::new(Barrier::new(num_tasks)); for _ in 0..16 { let leaf_node = Shared::new(LeafNode::new()); assert!( leaf_node .insert(usize::MAX, usize::MAX, &mut (), &Guard::new()) .is_ok() ); let mut task_handles = Vec::with_capacity(num_tasks); for task_id in 0..num_tasks { let barrier_clone = barrier.clone(); let leaf_node_clone = leaf_node.clone(); task_handles.push(tokio::task::spawn(async move { barrier_clone.wait().await; let guard = Guard::new(); let mut max_key = None; let range = (task_id * workload_size)..((task_id + 1) * workload_size); for id in range.clone() { loop { if let Ok(r) = leaf_node_clone.insert(id, id, &mut (), &guard) { match r { InsertResult::Success => { match leaf_node_clone.insert(id, id, &mut (), &guard) { Ok(InsertResult::Duplicate(..)) | Err(_) => (), _ => unreachable!(), } break; } InsertResult::Full(..) => { leaf_node_clone.rollback(&guard); max_key.replace(id); break; } InsertResult::Frozen(..) | InsertResult::Retry(..) => (), _ => unreachable!(), } } } if max_key.is_some() { break; } } for id in range.clone() { if max_key == Some(id) { break; } assert_eq!(leaf_node_clone.search_entry(&id, &guard), Some((&id, &id))); } for id in range { if max_key == Some(id) { break; } loop { if let Ok(r) = leaf_node_clone.remove_if::<_, _, _>( &id, &mut |_| true, &mut (), &guard, ) { match r { RemoveResult::Success | RemoveResult::Cleanup | RemoveResult::Fail => break, RemoveResult::Frozen | RemoveResult::Retired => unreachable!(), } } } assert!( leaf_node_clone.search_entry(&id, &guard).is_none(), "{}", id ); if let Ok(RemoveResult::Success) = leaf_node_clone.remove_if::<_, _, _>( &id, &mut |_| true, &mut (), &guard, ) { unreachable!() } } })); } for r in futures::future::join_all(task_handles).await { assert!(r.is_ok()); } assert!( leaf_node .remove_if::<_, _, _>(&usize::MAX, &mut |_| true, &mut (), &Guard::new()) .is_ok() ); } } #[cfg_attr(miri, ignore)] #[tokio::test(flavor = "multi_thread", worker_threads = 16)] async fn durability() { let num_tasks = 8_usize; let workload_size = 64_usize; for _ in 0..16 { for k in 0..=workload_size { let barrier = Shared::new(Barrier::new(num_tasks)); let leaf_node: Shared> = Shared::new(LeafNode::new()); let inserted: Shared = Shared::new(AtomicBool::new(false)); let mut task_handles = Vec::with_capacity(num_tasks); for _ in 0..num_tasks { let barrier_clone = barrier.clone(); let leaf_node_clone = leaf_node.clone(); let inserted_clone = inserted.clone(); task_handles.push(tokio::spawn(async move { { barrier_clone.wait().await; let guard = Guard::new(); match leaf_node_clone.insert(k, k, &mut (), &guard) { Ok(InsertResult::Success) => { assert!(!inserted_clone.swap(true, Relaxed)); } Ok(InsertResult::Full(_, _) | InsertResult::Retired(_, _)) => { leaf_node_clone.rollback(&guard); } _ => (), } } { barrier_clone.wait().await; let guard = Guard::new(); for i in 0..workload_size { if i != k { if let Ok( InsertResult::Full(_, _) | InsertResult::Retired(_, _), ) = leaf_node_clone.insert(i, i, &mut (), &guard) { leaf_node_clone.rollback(&guard); } } assert_eq!( leaf_node_clone.search_entry(&k, &guard).unwrap(), (&k, &k) ); } for i in 0..workload_size { let max_scanner = leaf_node_clone.max_le_appr(&k, &guard).unwrap(); assert!(*max_scanner.get().unwrap().0 <= k); let mut min_scanner = leaf_node_clone.min(&guard).unwrap(); if let Some((k_ref, v_ref)) = min_scanner.next() { assert_eq!(*k_ref, *v_ref); assert!(*k_ref <= k); } else { let (k_ref, v_ref) = min_scanner.jump(None, &guard).unwrap().get().unwrap(); assert_eq!(*k_ref, *v_ref); assert!(*k_ref <= k); } let _result = leaf_node_clone.remove_if::<_, _, _>( &i, &mut |v| *v != k, &mut (), &guard, ); assert_eq!( leaf_node_clone.search_entry(&k, &guard).unwrap(), (&k, &k) ); } } })); } for r in futures::future::join_all(task_handles).await { assert!(r.is_ok()); } assert!((*inserted).load(Relaxed)); } } } } scc-3.4.8/src/tree_index/node.rs000064400000000000000000000320661046102023000146270ustar 00000000000000use std::fmt::{self, Debug}; use std::ops::RangeBounds; use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed}; use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; use super::internal_node::{self, InternalNode}; use super::leaf::{InsertResult, Leaf, RemoveResult, Scanner}; use super::leaf_node::{self, LeafNode}; use crate::Comparable; use crate::async_helper::TryWait; use crate::exit_guard::ExitGuard; /// [`Node`] is either [`Self::Internal`] or [`Self::Leaf`]. pub enum Node { /// Internal node. Internal(InternalNode), /// Leaf node. Leaf(LeafNode), } impl Node { /// Creates a new [`InternalNode`]. #[inline] pub(super) fn new_internal_node() -> Self { Self::Internal(InternalNode::new()) } /// Creates a new [`LeafNode`]. #[inline] pub(super) fn new_leaf_node() -> Self { Self::Leaf(LeafNode::new()) } /// Clears the node. #[inline] pub(super) fn clear(&self, guard: &Guard) { match &self { Self::Internal(internal_node) => internal_node.clear(guard), Self::Leaf(leaf_node) => leaf_node.clear(guard), } } /// Returns the depth of the node. #[inline] pub(super) fn depth(&self, depth: usize, guard: &Guard) -> usize { match &self { Self::Internal(internal_node) => internal_node.depth(depth, guard), Self::Leaf(_) => depth, } } /// Checks if the node has retired. #[inline] pub(super) fn retired(&self) -> bool { match &self { Self::Internal(internal_node) => internal_node.retired(), Self::Leaf(leaf_node) => leaf_node.retired(), } } } impl Node where K: 'static + Clone + Ord, V: 'static + Clone, { /// Searches for an entry containing the specified key. #[inline] pub(super) fn search_entry<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<(&'g K, &'g V)> where K: 'g, Q: Comparable + ?Sized, { match &self { Self::Internal(internal_node) => internal_node.search_entry(key, guard), Self::Leaf(leaf_node) => leaf_node.search_entry(key, guard), } } /// Searches for the value associated with the specified key. #[inline] pub(super) fn search_value<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<&'g V> where K: 'g, Q: Comparable + ?Sized, { match &self { Self::Internal(internal_node) => internal_node.search_value(key, guard), Self::Leaf(leaf_node) => leaf_node.search_value(key, guard), } } /// Returns the minimum key-value pair. /// /// This method is not linearizable. #[inline] pub(super) fn min<'g>(&self, guard: &'g Guard) -> Option> { match &self { Self::Internal(internal_node) => internal_node.min(guard), Self::Leaf(leaf_node) => leaf_node.min(guard), } } /// Returns a [`Scanner`] pointing to an entry that is close enough to the entry with the /// maximum key among those keys that are smaller than or equal to the given key. /// /// This method is not linearizable. #[inline] pub(super) fn max_le_appr<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option> where K: 'g, Q: Comparable + ?Sized, { match &self { Self::Internal(internal_node) => internal_node.max_le_appr(key, guard), Self::Leaf(leaf_node) => leaf_node.max_le_appr(key, guard), } } /// Inserts a key-value pair. #[inline] pub(super) fn insert( &self, key: K, val: V, async_wait: &mut W, guard: &Guard, ) -> Result, (K, V)> { match &self { Self::Internal(internal_node) => internal_node.insert(key, val, async_wait, guard), Self::Leaf(leaf_node) => leaf_node.insert(key, val, async_wait, guard), } } /// Removes an entry associated with the given key. #[inline] pub(super) fn remove_if bool, W>( &self, key: &Q, condition: &mut F, async_wait: &mut W, guard: &Guard, ) -> Result where Q: Comparable + ?Sized, W: TryWait, { match &self { Self::Internal(internal_node) => { internal_node.remove_if::<_, _, _>(key, condition, async_wait, guard) } Self::Leaf(leaf_node) => { leaf_node.remove_if::<_, _, _>(key, condition, async_wait, guard) } } } /// Removes a range of entries. /// /// Returns the number of remaining children. #[inline] pub(super) fn remove_range<'g, Q, R: RangeBounds, W: TryWait>( &self, range: &R, start_unbounded: bool, valid_lower_max_leaf: Option<&'g Leaf>, valid_upper_min_node: Option<&'g Node>, async_wait: &mut W, guard: &'g Guard, ) -> Result where Q: Comparable + ?Sized, { match &self { Self::Internal(internal_node) => internal_node.remove_range( range, start_unbounded, valid_lower_max_leaf, valid_upper_min_node, async_wait, guard, ), Self::Leaf(leaf_node) => leaf_node.remove_range( range, start_unbounded, valid_lower_max_leaf, valid_upper_min_node, async_wait, guard, ), } } /// Splits the current root node. #[inline] pub(super) fn split_root( root_ptr: Ptr>, root: &AtomicShared>, mut key: K, mut val: V, guard: &Guard, ) -> (K, V) { let mut exit_guard = ExitGuard::new(true, |rollback| { if rollback { if let Some(old_root) = root_ptr.as_ref() { old_root.rollback(guard); } } }); // The fact that the `TreeIndex` calls this function means the root is full and locked. let mut new_root = Shared::new(Node::new_internal_node()); if let (Some(Self::Internal(internal_node)), Some(old_root)) = (unsafe { new_root.get_mut() }, root_ptr.get_shared()) { if root.load(Relaxed, guard) != root_ptr { // The root has changed. return (key, val); } internal_node.unbounded_child = AtomicShared::from(old_root); // Now, the old root is connected to the new root, so the split operation will be rolled // back in the `split_node` method if memory allocation fails. *exit_guard = false; let result = internal_node.split_node( (key, val), None, root_ptr, &internal_node.unbounded_child, true, &mut (), guard, ); let Ok(InsertResult::Retry(k, v)) = result else { unreachable!() }; key = k; val = v; // Updates the pointer before unlocking the root. match root.compare_exchange( root_ptr, (Some(new_root), Tag::None), AcqRel, Acquire, guard, ) { Ok((old_root, new_root_ptr)) => { if let Some(Self::Internal(internal_node)) = new_root_ptr.as_ref() { internal_node.finish_split(); } if let Some(old_root) = old_root { old_root.commit(guard); } } Err((new_root, _)) => { // The root has been changed. if let Some(Self::Internal(internal_node)) = new_root.as_deref() { internal_node.finish_split(); } // The old root needs to rollback the split operation. *exit_guard = true; } } } (key, val) } /// Cleans up or removes the current root node. /// /// If the root is empty, the root is removed from the tree, or if the root has only a single /// child, the root is replaced with the child. /// /// Returns `false` if a conflict is detected. #[inline] pub(super) fn cleanup_root( root: &AtomicShared>, async_wait: &mut W, guard: &Guard, ) -> bool { let mut root_ptr = root.load(Acquire, guard); while let Some(root_ref) = root_ptr.as_ref() { let mut internal_node_locker = None; let mut leaf_node_locker = None; match root_ref { Self::Internal(internal_node) => { if !internal_node.retired() && !internal_node.children.is_empty() { // The internal node is still usable. break; } else if let Some(locker) = internal_node::Locker::try_lock(internal_node) { internal_node_locker.replace(locker); } else { async_wait.try_wait(&internal_node.lock); } } Self::Leaf(leaf_node) => { if !leaf_node.retired() { // The leaf node is still usable. break; } else if let Some(locker) = leaf_node::Locker::try_lock(leaf_node) { leaf_node_locker.replace(locker); } else { async_wait.try_wait(&leaf_node.lock); } } } if internal_node_locker.is_none() && leaf_node_locker.is_none() { // The root node is locked by another thread. return false; } let new_root = match root_ref { Node::Internal(internal_node) => { if internal_node.retired() { // The internal node is empty, therefore the entire tree can be emptied. None } else if internal_node.children.is_empty() { // Replace the root with the unbounded child. internal_node.unbounded_child.get_shared(Acquire, guard) } else { // The internal node is not empty. break; } } Node::Leaf(leaf_node) => { if leaf_node.retired() { // The leaf node is empty, therefore the entire tree can be emptied. None } else { // The leaf node is not empty. break; } } }; match root.compare_exchange(root_ptr, (new_root, Tag::None), AcqRel, Acquire, guard) { Ok((_, new_root_ptr)) => { root_ptr = new_root_ptr; if let Some(internal_node_locker) = internal_node_locker { internal_node_locker.unlock_retire(); } } Err((_, new_root_ptr)) => { // The root node has been changed. root_ptr = new_root_ptr; } } } true } /// Commits an ongoing structural change. #[inline] pub(super) fn commit(&self, guard: &Guard) { match &self { Self::Internal(internal_node) => internal_node.commit(guard), Self::Leaf(leaf_node) => leaf_node.commit(guard), } } /// Rolls back an ongoing structural change. #[inline] pub(super) fn rollback(&self, guard: &Guard) { match &self { Self::Internal(internal_node) => internal_node.rollback(guard), Self::Leaf(leaf_node) => leaf_node.rollback(guard), } } /// Cleans up logically deleted [`LeafNode`] instances in the linked list. /// /// Returns `false` if the target leaf node does not exist in the subtree. #[inline] pub(super) fn cleanup_link<'g, Q>(&self, key: &Q, traverse_max: bool, guard: &'g Guard) -> bool where K: 'g, Q: Comparable + ?Sized, { match &self { Self::Internal(internal_node) => internal_node.cleanup_link(key, traverse_max, guard), Self::Leaf(leaf_node) => leaf_node.cleanup_link(key, traverse_max, guard), } } } impl Debug for Node { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Internal(_) => f.debug_tuple("Internal").finish(), Self::Leaf(_) => f.debug_tuple("Leaf").finish(), } } } scc-3.4.8/src/tree_index.rs000064400000000000000000001056251046102023000137040ustar 00000000000000//! [`TreeIndex`] is a read-optimized concurrent B-plus tree. mod internal_node; mod leaf; mod leaf_node; mod node; use std::fmt::{self, Debug}; use std::iter::FusedIterator; use std::marker::PhantomData; use std::ops::Bound::{Excluded, Included, Unbounded}; use std::ops::RangeBounds; use std::panic::UnwindSafe; use std::pin::pin; use std::sync::atomic::Ordering::{AcqRel, Acquire}; use sdd::{AtomicShared, Guard, Ptr, Shared, Tag}; use crate::Comparable; use crate::async_helper::AsyncWait; use leaf::{InsertResult, Leaf, RemoveResult, Scanner}; use node::Node; /// Scalable concurrent B-plus tree. /// /// [`TreeIndex`] is a concurrent B-plus tree variant optimized for read operations. Read /// operations, such as read iteration over entries, are neither blocked nor interrupted by other /// threads or tasks. Write operations, such as insert and remove, do not block if structural /// changes are not required. /// /// ## Note /// /// [`TreeIndex`] methods are linearizable. However, its iterator methods are not; [`Iter`] and /// [`Range`] are only guaranteed to observe events that happened before the first call to /// [`Iterator::next`]. /// /// ## The key features of [`TreeIndex`] /// /// * Lock-free read: read and scan operations are never blocked and do not modify shared data. /// * Near lock-free write: write operations do not block unless a structural change is needed. /// * No busy waiting: each node has a wait queue to avoid spinning. /// * Immutability: the data in the container is immutable until it becomes unreachable. /// /// ## The key statistics for [`TreeIndex`] /// /// * The maximum number of entries that a leaf can contain: 14. /// * The maximum number of leaves or child nodes a node can contain: 15. /// /// ## Locking behavior /// /// Read access is always lock-free and non-blocking. Write access to an entry is also lock-free and /// non-blocking as long as no structural changes are required. However, when nodes are split or /// merged by a write operation, other write operations on keys in the affected range are blocked. /// /// ### Synchronous methods in an asynchronous code block /// /// It is generally not recommended to use blocking methods, such as [`TreeIndex::insert_sync`], in /// an asynchronous code block or [`poll`](std::future::Future::poll), since it may lead to /// deadlocks or performance degradation. /// /// ### Unwind safety /// /// [`TreeIndex`] is impervious to out-of-memory errors and panics in user-specified code under one /// condition; `K::drop` and `V::drop` must not panic. pub struct TreeIndex { root: AtomicShared>, } /// An iterator over the entries of a [`TreeIndex`]. /// /// An [`Iter`] iterates over all the entries that exist during the lifetime of the [`Iter`] in /// monotonically increasing order. pub struct Iter<'t, 'g, K, V> { root: &'t AtomicShared>, leaf_scanner: Option>, guard: &'g Guard, } /// An iterator over a sub-range of entries in a [`TreeIndex`]. pub struct Range<'t, 'g, K, V, Q: ?Sized, R: RangeBounds> { root: &'t AtomicShared>, leaf_scanner: Option>, bounds: R, check_lower_bound: bool, check_upper_bound: bool, guard: &'g Guard, query: PhantomData Q>, } impl TreeIndex { /// Creates an empty [`TreeIndex`]. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// ``` #[cfg(not(feature = "loom"))] #[inline] #[must_use] pub const fn new() -> Self { Self { root: AtomicShared::null(), } } /// Creates an empty [`TreeIndex`]. #[cfg(feature = "loom")] #[inline] #[must_use] pub fn new() -> Self { Self { root: AtomicShared::null(), } } /// Clears the [`TreeIndex`]. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// treeindex.clear(); /// assert_eq!(treeindex.len(), 0); /// ``` #[inline] pub fn clear(&self) { if let (Some(root), _) = self.root.swap((None, Tag::None), Acquire) { root.clear(&Guard::new()); } } /// Returns the depth of the [`TreeIndex`]. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// assert_eq!(treeindex.depth(), 0); /// ``` #[inline] #[must_use] pub fn depth(&self) -> usize { let guard = Guard::new(); self.root .load(Acquire, &guard) .as_ref() .map_or(0, |root_ref| root_ref.depth(1, &guard)) } } impl TreeIndex where K: 'static + Clone + Ord, V: 'static + Clone, { /// Inserts a key-value pair. /// /// # Errors /// /// Returns an error along with the supplied key-value pair if the key exists. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// let future_insert = treeindex.insert_async(1, 10); /// ``` #[inline] pub async fn insert_async(&self, mut key: K, mut val: V) -> Result<(), (K, V)> { let mut pinned_async_wait = pin!(AsyncWait::default()); let mut new_root = None; loop { let need_await = { let guard = Guard::new(); let root_ptr = self.root.load(Acquire, &guard); if let Some(root_ref) = root_ptr.as_ref() { match root_ref.insert(key, val, &mut pinned_async_wait, &guard) { Ok(r) => match r { InsertResult::Success => return Ok(()), InsertResult::Frozen(k, v) | InsertResult::Retry(k, v) => { key = k; val = v; root_ref.cleanup_link(&key, false, &guard); true } InsertResult::Duplicate(k, v) => return Err((k, v)), InsertResult::Full(k, v) => { let (k, v) = Node::split_root(root_ptr, &self.root, k, v, &guard); key = k; val = v; continue; } InsertResult::Retired(k, v) => { key = k; val = v; !Node::cleanup_root(&self.root, &mut pinned_async_wait, &guard) } }, Err((k, v)) => { key = k; val = v; true } } } else { false } }; if need_await { pinned_async_wait.wait().await; } let node = if let Some(new_root) = new_root.take() { new_root } else { Shared::new(Node::new_leaf_node()) }; if let Err((node, _)) = self.root.compare_exchange( Ptr::null(), (Some(node), Tag::None), AcqRel, Acquire, &Guard::new(), ) { new_root = node; } } } /// Inserts a key-value pair. /// /// # Errors /// /// Returns an error along with the supplied key-value pair if the key exists. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// assert!(treeindex.insert_sync(1, 10).is_ok()); /// assert_eq!(treeindex.insert_sync(1, 11).err().unwrap(), (1, 11)); /// assert_eq!(treeindex.peek_with(&1, |k, v| *v).unwrap(), 10); /// ``` #[inline] pub fn insert_sync(&self, mut key: K, mut val: V) -> Result<(), (K, V)> { let mut new_root = None; loop { let guard = Guard::new(); let root_ptr = self.root.load(Acquire, &guard); if let Some(root_ref) = root_ptr.as_ref() { match root_ref.insert(key, val, &mut (), &guard) { Ok(r) => match r { InsertResult::Success => return Ok(()), InsertResult::Frozen(k, v) | InsertResult::Retry(k, v) => { key = k; val = v; root_ref.cleanup_link(&key, false, &guard); } InsertResult::Duplicate(k, v) => return Err((k, v)), InsertResult::Full(k, v) => { let (k, v) = Node::split_root(root_ptr, &self.root, k, v, &guard); key = k; val = v; continue; } InsertResult::Retired(k, v) => { key = k; val = v; let _result = Node::cleanup_root(&self.root, &mut (), &guard); } }, Err((k, v)) => { key = k; val = v; } } } let node = if let Some(new_root) = new_root.take() { new_root } else { Shared::new(Node::new_leaf_node()) }; if let Err((node, _)) = self.root.compare_exchange( Ptr::null(), (Some(node), Tag::None), AcqRel, Acquire, &guard, ) { new_root = node; } } } /// Removes a key-value pair. /// /// Returns `false` if the key does not exist. Returns `true` if the key existed and the /// condition was met after marking the entry unreachable; the memory will be reclaimed later. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// let future_remove = treeindex.remove_async(&1); /// ``` #[inline] pub async fn remove_async(&self, key: &Q) -> bool where Q: Comparable + ?Sized, { self.remove_if_async(key, |_| true).await } /// Removes a key-value pair. /// /// Returns `false` if the key does not exist. /// /// Returns `true` if the key existed and the condition was met after marking the entry /// unreachable; the memory will be reclaimed later. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// assert!(!treeindex.remove_sync(&1)); /// assert!(treeindex.insert_sync(1, 10).is_ok()); /// assert!(treeindex.remove_sync(&1)); /// ``` #[inline] pub fn remove_sync(&self, key: &Q) -> bool where Q: Comparable + ?Sized, { self.remove_if_sync(key, |_| true) } /// Removes a key-value pair if the given condition is met. /// /// Returns `false` if the key does not exist or the condition was not met. Returns `true` if /// the key existed and the condition was met after marking the entry unreachable; the memory /// will be reclaimed later. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// let future_remove = treeindex.remove_if_async(&1, |v| *v == 0); /// ``` #[inline] pub async fn remove_if_async bool>(&self, key: &Q, mut condition: F) -> bool where Q: Comparable + ?Sized, { let mut pinned_async_wait = pin!(AsyncWait::default()); let mut removed = false; loop { { let guard = Guard::new(); if let Some(root_ref) = self.root.load(Acquire, &guard).as_ref() { if let Ok(result) = root_ref.remove_if::<_, _, _>( key, &mut condition, &mut pinned_async_wait, &guard, ) { if matches!(result, RemoveResult::Cleanup) { root_ref.cleanup_link(key, false, &guard); } match result { RemoveResult::Success => return true, RemoveResult::Cleanup | RemoveResult::Retired => { if Node::cleanup_root(&self.root, &mut pinned_async_wait, &guard) { return true; } removed = true; } RemoveResult::Fail => { if removed { if Node::cleanup_root( &self.root, &mut pinned_async_wait, &guard, ) { return true; } } else { return false; } } RemoveResult::Frozen => (), } } } else { return removed; } } pinned_async_wait.wait().await; } } /// Removes a key-value pair if the given condition is met. /// /// Returns `false` if the key does not exist or the condition was not met. /// /// Returns `true` if the key existed and the condition was met after marking the entry /// unreachable; the memory will be reclaimed later. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// assert!(treeindex.insert_sync(1, 10).is_ok()); /// assert!(!treeindex.remove_if_sync(&1, |v| *v == 0)); /// assert!(treeindex.remove_if_sync(&1, |v| *v == 10)); /// ``` #[inline] pub fn remove_if_sync bool>(&self, key: &Q, mut condition: F) -> bool where Q: Comparable + ?Sized, { let mut removed = false; loop { let guard = Guard::new(); if let Some(root_ref) = self.root.load(Acquire, &guard).as_ref() { if let Ok(result) = root_ref.remove_if::<_, _, _>(key, &mut condition, &mut (), &guard) { if matches!(result, RemoveResult::Cleanup) { root_ref.cleanup_link(key, false, &guard); } match result { RemoveResult::Success => return true, RemoveResult::Cleanup | RemoveResult::Retired => { if Node::cleanup_root(&self.root, &mut (), &guard) { return true; } removed = true; } RemoveResult::Fail => { if removed { if Node::cleanup_root(&self.root, &mut (), &guard) { return true; } } else { return false; } } RemoveResult::Frozen => (), } } } else { return removed; } } } /// Removes keys in the specified range. /// /// This method removes internal nodes that are definitely contained in the specified range /// first, and then removes remaining entries individually. /// /// # Note /// /// Internally, multiple internal node locks need to be acquired, thus making this method /// susceptible to lock starvation. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// for k in 2..8 { /// assert!(treeindex.insert_sync(k, 1).is_ok()); /// } /// /// let future_remove_range = treeindex.remove_range_async(3..8); /// ``` #[inline] pub async fn remove_range_async>(&self, range: R) where Q: Comparable + ?Sized, { let mut pinned_async_wait = pin!(AsyncWait::default()); let start_unbounded = matches!(range.start_bound(), Unbounded); loop { { let guard = Guard::new(); // Remove internal nodes, and individual entries in affected leaves. // // It takes O(N) to traverse sub-trees on the range border. if let Some(root_ref) = self.root.load(Acquire, &guard).as_ref() { if let Ok(num_children) = root_ref.remove_range( &range, start_unbounded, None, None, &mut pinned_async_wait, &guard, ) { if num_children >= 2 || Node::cleanup_root(&self.root, &mut pinned_async_wait, &guard) { // Completed removal and cleaning up the root. return; } } } else { // Nothing to remove. return; } } pinned_async_wait.wait().await; } } /// Removes keys in the specified range. /// /// This method removes internal nodes that are definitely contained in the specified range /// first, and then removes remaining entries individually. /// /// # Note /// /// Internally, multiple internal node locks need to be acquired, thus making this method /// susceptible to lock starvation. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// for k in 2..8 { /// assert!(treeindex.insert_sync(k, 1).is_ok()); /// } /// /// treeindex.remove_range_sync(3..8); /// /// assert!(treeindex.contains(&2)); /// assert!(!treeindex.contains(&3)); /// ``` #[inline] pub fn remove_range_sync>(&self, range: R) where Q: Comparable + ?Sized, { let start_unbounded = matches!(range.start_bound(), Unbounded); let guard = Guard::new(); // Remove internal nodes, and individual entries in affected leaves. // // It takes O(N) to traverse sub-trees on the range border. while let Some(root_ref) = self.root.load(Acquire, &guard).as_ref() { if let Ok(num_children) = root_ref.remove_range(&range, start_unbounded, None, None, &mut (), &guard) { if num_children < 2 && !Node::cleanup_root(&self.root, &mut (), &guard) { continue; } break; } } } /// Returns a guarded reference to the value for the specified key without acquiring locks. /// /// Returns `None` if the key does not exist. The returned reference can survive as long as the /// associated [`Guard`] is alive. /// /// # Examples /// /// ``` /// use std::sync::Arc; /// /// use scc::TreeIndex; /// /// use sdd::Guard; /// /// let treeindex: TreeIndex, u32> = TreeIndex::new(); /// /// let guard = Guard::new(); /// assert!(treeindex.peek("foo", &guard).is_none()); /// /// treeindex.insert_sync("foo".into(), 1).expect("insert in empty TreeIndex"); /// ``` #[inline] pub fn peek<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<&'g V> where Q: Comparable + ?Sized, { if let Some(root_ref) = self.root.load(Acquire, guard).as_ref() { return root_ref.search_value(key, guard); } None } /// Peeks a key-value pair without acquiring locks. /// /// Returns `None` if the key does not exist. /// /// # Examples /// /// ``` /// use std::sync::Arc; /// use scc::TreeIndex; /// /// let treeindex: TreeIndex, u32> = TreeIndex::new(); /// /// assert!(treeindex.peek_with("foo", |k, v| *v).is_none()); /// /// treeindex.insert_sync("foo".into(), 1).expect("insert in empty TreeIndex"); /// /// let key: Arc = treeindex /// .peek_with("foo", |k, _v| Arc::clone(k)) /// .expect("peek_with by borrowed key"); /// ``` #[inline] pub fn peek_with R>(&self, key: &Q, reader: F) -> Option where Q: Comparable + ?Sized, { let guard = Guard::new(); self.peek_entry(key, &guard).map(|(k, v)| reader(k, v)) } /// Returns a guarded reference to the key-value pair for the specified key without acquiring locks. /// /// Returns `None` if the key does not exist. The returned reference can survive as long as the /// associated [`Guard`] is alive. /// /// # Examples /// /// ``` /// use std::sync::Arc; /// /// use scc::TreeIndex; /// /// use sdd::Guard; /// /// let treeindex: TreeIndex, u32> = TreeIndex::new(); /// /// let guard = Guard::new(); /// assert!(treeindex.peek_entry("foo", &guard).is_none()); /// /// treeindex.insert_sync("foo".into(), 1).expect("insert in empty TreeIndex"); /// /// let key: Arc = treeindex /// .peek_entry("foo", &guard) /// .map(|(k, _v)| Arc::clone(k)) /// .expect("peek_entry by borrowed key"); /// ``` #[inline] pub fn peek_entry<'g, Q>(&self, key: &Q, guard: &'g Guard) -> Option<(&'g K, &'g V)> where Q: Comparable + ?Sized, { if let Some(root_ref) = self.root.load(Acquire, guard).as_ref() { return root_ref.search_entry(key, guard); } None } /// Returns `true` if the [`TreeIndex`] contains the key. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::default(); /// /// assert!(!treeindex.contains(&1)); /// assert!(treeindex.insert_sync(1, 0).is_ok()); /// assert!(treeindex.contains(&1)); /// ``` #[inline] pub fn contains(&self, key: &Q) -> bool where Q: Comparable + ?Sized, { self.peek(key, &Guard::new()).is_some() } /// Returns the size of the [`TreeIndex`]. /// /// It internally scans all the leaf nodes, and therefore the time complexity is O(N). /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// assert_eq!(treeindex.len(), 0); /// ``` #[inline] pub fn len(&self) -> usize { let guard = Guard::new(); self.iter(&guard).count() } /// Returns `true` if the [`TreeIndex`] is empty. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// assert!(treeindex.is_empty()); /// ``` #[inline] pub fn is_empty(&self) -> bool { let guard = Guard::new(); !self.iter(&guard).any(|_| true) } /// Returns an [`Iter`]. /// /// The returned [`Iter`] starts scanning from the minimum key-value pair. Key-value pairs /// are scanned in ascending order, and key-value pairs that have existed since the invocation /// of the method are guaranteed to be visited if they are not removed. However, it is possible /// to visit removed key-value pairs momentarily. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// use sdd::Guard; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// let guard = Guard::new(); /// let mut iter = treeindex.iter(&guard); /// assert!(iter.next().is_none()); /// ``` #[inline] pub fn iter<'t, 'g>(&'t self, guard: &'g Guard) -> Iter<'t, 'g, K, V> { Iter::new(&self.root, guard) } /// Returns a [`Range`] that scans keys in the given range. /// /// Key-value pairs in the range are scanned in ascending order, and key-value pairs that have /// existed since the invocation of the method are guaranteed to be visited if they are not /// removed. However, it is possible to visit removed key-value pairs momentarily. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// use sdd::Guard; /// /// let treeindex: TreeIndex = TreeIndex::new(); /// /// let guard = Guard::new(); /// assert_eq!(treeindex.range(4..=8, &guard).count(), 0); /// ``` #[inline] pub fn range<'t, 'g, Q, R: RangeBounds>( &'t self, range: R, guard: &'g Guard, ) -> Range<'t, 'g, K, V, Q, R> where Q: Comparable + ?Sized, { Range::new(&self.root, range, guard) } } impl Clone for TreeIndex where K: 'static + Clone + Ord, V: 'static + Clone, { #[inline] fn clone(&self) -> Self { let self_clone = Self::default(); for (k, v) in self.iter(&Guard::new()) { let _result: Result<(), (K, V)> = self_clone.insert_sync(k.clone(), v.clone()); } self_clone } } impl Debug for TreeIndex where K: 'static + Clone + Debug + Ord, V: 'static + Clone + Debug, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let guard = Guard::new(); f.debug_map().entries(self.iter(&guard)).finish() } } impl Default for TreeIndex { /// Creates a [`TreeIndex`] with the default parameters. /// /// # Examples /// /// ``` /// use scc::TreeIndex; /// /// let treeindex: TreeIndex = TreeIndex::default(); /// ``` #[inline] fn default() -> Self { Self::new() } } impl Drop for TreeIndex { #[inline] fn drop(&mut self) { self.clear(); } } impl PartialEq for TreeIndex where K: 'static + Clone + Ord, V: 'static + Clone + PartialEq, { #[inline] fn eq(&self, other: &Self) -> bool { // The key order is preserved, therefore comparing iterators suffices. let guard = Guard::new(); Iterator::eq(self.iter(&guard), other.iter(&guard)) } } impl UnwindSafe for TreeIndex {} impl<'t, 'g, K, V> Iter<'t, 'g, K, V> { #[inline] fn new(root: &'t AtomicShared>, guard: &'g Guard) -> Iter<'t, 'g, K, V> { Iter::<'t, 'g, K, V> { root, leaf_scanner: None, guard, } } } impl Debug for Iter<'_, '_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Iter") .field("root", &self.root) .field("leaf_scanner", &self.leaf_scanner) .finish() } } impl<'g, K, V> Iterator for Iter<'_, 'g, K, V> where K: 'static + Clone + Ord, V: 'static + Clone, { type Item = (&'g K, &'g V); #[inline] fn next(&mut self) -> Option { // Starts scanning. if self.leaf_scanner.is_none() { let root_ptr = self.root.load(Acquire, self.guard); if let Some(root_ref) = root_ptr.as_ref() { if let Some(scanner) = root_ref.min(self.guard) { self.leaf_scanner.replace(scanner); } } else { return None; } } // Go to the next entry. if let Some(mut scanner) = self.leaf_scanner.take() { let min_allowed_key = scanner.get().map(|(key, _)| key); if let Some(result) = scanner.next() { self.leaf_scanner.replace(scanner); return Some(result); } // Go to the next leaf node. if let Some(new_scanner) = scanner.jump(min_allowed_key, self.guard) { if let Some(entry) = new_scanner.get() { self.leaf_scanner.replace(new_scanner); return Some(entry); } } } None } } impl FusedIterator for Iter<'_, '_, K, V> where K: 'static + Clone + Ord, V: 'static + Clone, { } impl UnwindSafe for Iter<'_, '_, K, V> {} impl<'t, 'g, K, V, Q: ?Sized, R: RangeBounds> Range<'t, 'g, K, V, Q, R> { #[inline] fn new( root: &'t AtomicShared>, range: R, guard: &'g Guard, ) -> Range<'t, 'g, K, V, Q, R> { Range::<'t, 'g, K, V, Q, R> { root, leaf_scanner: None, bounds: range, check_lower_bound: true, check_upper_bound: false, guard, query: PhantomData, } } } impl<'g, K, V, Q, R> Range<'_, 'g, K, V, Q, R> where K: 'static + Clone + Ord, V: 'static + Clone, Q: Comparable + ?Sized, R: RangeBounds, { #[inline] fn next_unbounded(&mut self) -> Option<(&'g K, &'g V)> { if self.leaf_scanner.is_none() { // Start scanning. let root_ptr = self.root.load(Acquire, self.guard); if let Some(root_ref) = root_ptr.as_ref() { let min_allowed_key = match self.bounds.start_bound() { Excluded(key) | Included(key) => Some(key), Unbounded => { self.check_lower_bound = false; None } }; let mut leaf_scanner = min_allowed_key .and_then(|min_allowed_key| root_ref.max_le_appr(min_allowed_key, self.guard)); if leaf_scanner.is_none() { // No `min_allowed_key` is supplied, or no keys smaller than or equal to // `min_allowed_key` found. if let Some(mut scanner) = root_ref.min(self.guard) { // It's possible that the leaf has just been emptied, so go to the next. scanner.next(); while scanner.get().is_none() { scanner = scanner.jump(None, self.guard)?; } leaf_scanner.replace(scanner); } } if let Some(leaf_scanner) = leaf_scanner { if let Some(result) = leaf_scanner.get() { self.set_check_upper_bound(&leaf_scanner); self.leaf_scanner.replace(leaf_scanner); return Some(result); } } } } // Go to the next entry. if let Some(mut leaf_scanner) = self.leaf_scanner.take() { let min_allowed_key = leaf_scanner.get().map(|(key, _)| key); if let Some(result) = leaf_scanner.next() { self.leaf_scanner.replace(leaf_scanner); return Some(result); } // Go to the next leaf node. if let Some(new_scanner) = leaf_scanner.jump(min_allowed_key, self.guard) { if let Some(entry) = new_scanner.get() { self.set_check_upper_bound(&new_scanner); self.leaf_scanner.replace(new_scanner); return Some(entry); } } } None } #[inline] fn set_check_upper_bound(&mut self, scanner: &Scanner) { self.check_upper_bound = match self.bounds.end_bound() { Excluded(key) => scanner .max_key() .is_some_and(|max_key| key.compare(max_key).is_le()), Included(key) => scanner .max_key() .is_some_and(|max_key| key.compare(max_key).is_lt()), Unbounded => false, }; } } impl> Debug for Range<'_, '_, K, V, Q, R> { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Range") .field("root", &self.root) .field("leaf_scanner", &self.leaf_scanner) .field("check_lower_bound", &self.check_lower_bound) .field("check_upper_bound", &self.check_upper_bound) .finish() } } impl<'g, K, V, Q, R> Iterator for Range<'_, 'g, K, V, Q, R> where K: 'static + Clone + Ord, V: 'static + Clone, Q: Comparable + ?Sized, R: RangeBounds, { type Item = (&'g K, &'g V); #[inline] fn next(&mut self) -> Option { while let Some((k, v)) = self.next_unbounded() { if self.check_lower_bound { match self.bounds.start_bound() { Excluded(key) => { if key.compare(k).is_ge() { continue; } } Included(key) => { if key.compare(k).is_gt() { continue; } } Unbounded => (), } } self.check_lower_bound = false; if self.check_upper_bound { match self.bounds.end_bound() { Excluded(key) => { if key.compare(k).is_gt() { return Some((k, v)); } } Included(key) => { if key.compare(k).is_ge() { return Some((k, v)); } } Unbounded => { return Some((k, v)); } } break; } return Some((k, v)); } None } } impl FusedIterator for Range<'_, '_, K, V, Q, R> where K: 'static + Clone + Ord, V: 'static + Clone, Q: Comparable + ?Sized, R: RangeBounds, { } impl UnwindSafe for Range<'_, '_, K, V, Q, R> where Q: ?Sized, R: RangeBounds + UnwindSafe, { }