papaya-0.2.3/.cargo_vcs_info.json0000644000000001360000000000100123110ustar { "git": { "sha1": "f50ab7f772e2ab18fdf4d16e76a4f7eb9337bcfe" }, "path_in_vcs": "" }papaya-0.2.3/.github/DOCS.md000064400000000000000000000000771046102023000135170ustar 00000000000000Workflows adapted from https://github.com/jonhoo/rust-ci-conf. papaya-0.2.3/.github/workflows/check.yml000064400000000000000000000105071046102023000163010ustar 00000000000000# This workflow runs whenever a PR is opened or updated, or a commit is pushed to main. It runs # several checks: # - fmt: checks that the code is formatted according to rustfmt # - clippy: checks that the code does not contain any clippy warnings # - doc: checks that the code can be documented without errors # - hack: check combinations of feature flags # - msrv: check that the msrv specified in the crate is correct permissions: contents: read # This configuration allows maintainers of this repo to create a branch and pull request based on # the new branch. Restricting the push trigger to the main branch ensures that the PR only gets # built once. on: push: branches: [master] pull_request: # If new code is pushed to a PR branch, then cancel in progress workflows for that PR. Ensures that # we don't waste CI time, and returns results quicker https://github.com/jonhoo/rust-ci-conf/pull/5 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true name: check jobs: fmt: runs-on: ubuntu-latest name: stable / fmt steps: - uses: actions/checkout@v4 with: submodules: true - name: Install stable uses: dtolnay/rust-toolchain@stable with: components: rustfmt - name: cargo fmt --check run: cargo fmt --check clippy: runs-on: ubuntu-latest name: ${{ matrix.toolchain }} / clippy permissions: contents: read checks: write strategy: fail-fast: false matrix: # Get early warning of new lints which are regularly introduced in beta channels. toolchain: [stable, beta] steps: - uses: actions/checkout@v4 with: submodules: true - name: Install ${{ matrix.toolchain }} uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.toolchain }} components: clippy - name: cargo clippy uses: giraffate/clippy-action@v1 with: reporter: 'github-pr-check' github_token: ${{ secrets.GITHUB_TOKEN }} semver: runs-on: ubuntu-latest name: semver steps: - uses: actions/checkout@v4 with: submodules: true - name: Install stable uses: dtolnay/rust-toolchain@stable with: components: rustfmt - name: cargo-semver-checks uses: obi1kenobi/cargo-semver-checks-action@v2 doc: # run docs generation on nightly rather than stable. This enables features like # https://doc.rust-lang.org/beta/unstable-book/language-features/doc-cfg.html which allows an # API be documented as only available in some specific platforms. runs-on: ubuntu-latest name: nightly / doc steps: - uses: actions/checkout@v4 with: submodules: true - name: Install nightly uses: dtolnay/rust-toolchain@nightly - name: cargo doc run: cargo doc --no-deps --all-features env: RUSTDOCFLAGS: --cfg docsrs hack: # cargo-hack checks combinations of feature flags to ensure that features are all additive # which is required for feature unification runs-on: ubuntu-latest name: ubuntu / stable / features steps: - uses: actions/checkout@v4 with: submodules: true - name: Install stable uses: dtolnay/rust-toolchain@stable - name: cargo install cargo-hack uses: taiki-e/install-action@cargo-hack # intentionally no target specifier; see https://github.com/jonhoo/rust-ci-conf/pull/4 # --feature-powerset runs for every combination of features - name: cargo hack run: cargo hack --feature-powerset check msrv: # check that we can build using the minimal rust version that is specified by this crate runs-on: ubuntu-latest # we use a matrix here just because env can't be used in job names # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability strategy: matrix: msrv: ["1.72.0"] name: ubuntu / ${{ matrix.msrv }} steps: - uses: actions/checkout@v4 with: submodules: true - name: Install ${{ matrix.msrv }} uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.msrv }} - name: cargo +${{ matrix.msrv }} check run: cargo check papaya-0.2.3/.github/workflows/safety.yml000064400000000000000000000065531046102023000165250ustar 00000000000000# This workflow runs checks for unsafe code. In crates that don't have any unsafe code, this can be # removed. Runs: # - miri - detects undefined behavior and memory leaks # - address sanitizer - detects memory errors # - leak sanitizer - detects memory leaks # See check.yml for information about how the concurrency cancellation and workflow triggering works permissions: contents: read on: push: branches: [master] pull_request: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true name: safety jobs: sanitizers: runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: submodules: true - name: Install nightly uses: dtolnay/rust-toolchain@nightly - run: | # to get the symbolizer for debug symbol resolution sudo apt install llvm # to fix buggy leak analyzer: # https://github.com/japaric/rust-san#unrealiable-leaksanitizer # ensure there's a profile.dev section if ! grep -qE '^[ \t]*[profile.dev]' Cargo.toml; then echo >> Cargo.toml echo '[profile.dev]' >> Cargo.toml fi # remove pre-existing opt-levels in profile.dev sed -i '/^\s*\[profile.dev\]/,/^\s*\[/ {/^\s*opt-level/d}' Cargo.toml # now set opt-level to 1 sed -i '/^\s*\[profile.dev\]/a opt-level = 1' Cargo.toml cat Cargo.toml name: Enable debug symbols - name: cargo test -Zsanitizer=address # only --lib --tests b/c of https://github.com/rust-lang/rust/issues/53945 run: cargo test --lib --tests --all-features --target x86_64-unknown-linux-gnu env: ASAN_OPTIONS: "detect_odr_violation=0:detect_leaks=0" RUSTFLAGS: "-Z sanitizer=address --cfg papaya_asan" - name: stress tests -Zsanitizer=address run: cargo test --lib --tests --all-features --target x86_64-unknown-linux-gnu -- --ignored --test-threads 1 env: ASAN_OPTIONS: "detect_odr_violation=0:detect_leaks=0" RUSTFLAGS: "-Z sanitizer=address --cfg papaya_asan" - name: cargo test -Zsanitizer=leak if: always() run: cargo test --all-features --target x86_64-unknown-linux-gnu env: LSAN_OPTIONS: "suppressions=lsan-suppressions.txt" RUSTFLAGS: "-Z sanitizer=leak" - name: stress tests -Zsanitizer=leak if: always() run: cargo test --all-features --target x86_64-unknown-linux-gnu -- --ignored --test-threads 1 env: LSAN_OPTIONS: "suppressions=lsan-suppressions.txt" RUSTFLAGS: "-Z sanitizer=leak" miri: runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: submodules: true - run: | echo "NIGHTLY=nightly-$(curl -s https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu/miri)" >> $GITHUB_ENV - name: Install ${{ env.NIGHTLY }} uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.NIGHTLY }} components: miri - name: cargo miri test run: cargo miri test -- --ignored env: # need this until feature(strict_provenance_atomic_ptr) is stabilized MIRIFLAGS: "-Zmiri-permissive-provenance" papaya-0.2.3/.github/workflows/test.yml000064400000000000000000000062611046102023000162050ustar 00000000000000# This is the main CI workflow that runs the test suite on all pushes to main and all pull requests. # It runs the following jobs: # - required: runs the test suite on ubuntu with stable and beta rust toolchains # requirements of this crate, and its dependencies # - os-check: runs the test suite on mac and windows # See check.yml for information about how the concurrency cancellation and workflow triggering works permissions: contents: read on: push: branches: [master] pull_request: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true name: test jobs: required: runs-on: ubuntu-latest timeout-minutes: 15 name: ubuntu / ${{ matrix.toolchain }} strategy: matrix: # run on stable and beta to ensure that tests won't break on the next version of the rust # toolchain toolchain: [stable, beta] steps: - uses: actions/checkout@v4 with: submodules: true - name: Install ${{ matrix.toolchain }} uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.toolchain }} - name: cargo generate-lockfile # enable this ci template to run regardless of whether the lockfile is checked in or not if: hashFiles('Cargo.lock') == '' run: cargo generate-lockfile # https://twitter.com/jonhoo/status/1571290371124260865 - name: cargo test --locked run: cargo test --locked --all-features --all-targets # https://github.com/rust-lang/cargo/issues/6669 - name: cargo test --doc run: cargo test --locked --all-features --doc # run stress tests serially, as they individually spawn many threads to provoke contention - name: stress tests run: cargo test -- --ignored --test-threads 1 os-check: # run cargo test on mac and windows runs-on: ${{ matrix.os }} timeout-minutes: 15 name: ${{ matrix.os }} / stable strategy: fail-fast: false matrix: os: [macos-latest, windows-latest] steps: - uses: actions/checkout@v4 with: submodules: true - name: Install stable uses: dtolnay/rust-toolchain@stable - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' run: cargo generate-lockfile - name: cargo test run: cargo test --locked --all-features --all-targets - name: stress tests run: cargo test -- --ignored --test-threads 1 stress: # run resize stress tests on linux and mac runs-on: ${{ matrix.os }} timeout-minutes: 15 name: ${{ matrix.os }} / stress strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 with: submodules: true - name: Install stable uses: dtolnay/rust-toolchain@stable - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' run: cargo generate-lockfile - name: stress tests run: cargo test --locked -- --ignored --test-threads 1 env: RUSTFLAGS: "--cfg papaya_stress" RUST_MIN_STACK: 8192000 # avoid stack overflows on macOS papaya-0.2.3/.gitignore000064400000000000000000000000241046102023000130650ustar 00000000000000/target /Cargo.lock papaya-0.2.3/BENCHMARKS.md000064400000000000000000000041541046102023000130440ustar 00000000000000# Benchmarks *As always, benchmarks should be taken with a grain of salt. Always measure for your workload.* Below are the benchmark results from the [`conc-map-bench`](https://github.com/xacrimon/conc-map-bench) benchmarking harness under varying workloads. All benchmarks were run on a AMD Ryzen 9 9950X processor, using [`ahash`](https://github.com/tkaitchuck/aHash) and the [`mimalloc`](https://github.com/microsoft/mimalloc) allocator. ### Read Heavy | | | :-------------------------:|:-------------------------: ![](assets/ReadHeavy.ahash.throughput.svg) | ![](assets/ReadHeavy.ahash.latency.svg) ### Exchange | | | :-------------------------:|:-------------------------: ![](assets/Exchange.ahash.throughput.svg) | ![](assets/Exchange.ahash.latency.svg) ### Rapid Grow | | | :-------------------------:|:-------------------------: ![](assets/RapidGrow.ahash.throughput.svg) | ![](assets/RapidGrow.ahash.latency.svg) # Discussion As mentioned in the [performance](../README#performance) section of the guide, `papaya` is optimized read-heavy workloads. As expected, it outperforms all competitors in the read-heavy benchmark. An important guarantee of `papaya` is that reads *never* block under any circumstances. This is crucial for providing consistent read latency regardless of write concurrency. However, it falls short in update-heavy workloads due to allocator pressure and the overhead of memory reclamation, which is necessary for lock-free reads. If your workload is write-heavy and you do not benefit from any of `papaya`'s features, you may wish to consider an alternate hash-table implementation. Additionally, `papaya` does a lot better in terms of latency distribution due to incremental resizing and the lack of bucket locks. Comparing histograms of `insert` latency between `papaya` and `dashmap`, we see that `papaya` manages to keep tail latency lower by a few orders of magnitude. Some latency spikes are unavoidable due to the allocations necessary to maintain a large hash-table, but the distribution is much more consistent (notice the scale of the y-axis). ![](assets/papaya-hist.png) ![](assets/dashmap-hist.png) papaya-0.2.3/Cargo.lock0000644000000565270000000000100103030ustar # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 4 [[package]] name = "addr2line" version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] [[package]] name = "adler" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "anes" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", ] [[package]] name = "base64" version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cast" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "ciborium" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", "serde", ] [[package]] name = "ciborium-io" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", "half", ] [[package]] name = "clap" version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" dependencies = [ "anstyle", "clap_lex", ] [[package]] name = "clap_lex" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "crc32fast" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "criterion" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", "cast", "ciborium", "clap", "criterion-plot", "is-terminal", "itertools", "num-traits", "once_cell", "oorandom", "plotters", "rayon", "regex", "serde", "serde_derive", "serde_json", "tinytemplate", "walkdir", ] [[package]] name = "criterion-plot" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", "itertools", ] [[package]] name = "crossbeam-channel" version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crunchy" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "dashmap" version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", "hashbrown", "lock_api", "once_cell", "parking_lot_core", ] [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "flate2" version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", ] [[package]] name = "getrandom" version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "libc", "wasi", ] [[package]] name = "gimli" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "half" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if", "crunchy", ] [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hdrhistogram" version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ "base64 0.21.7", "byteorder", "crossbeam-channel", "flate2", "nom", "num-traits", ] [[package]] name = "hermit-abi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "is-terminal" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", "libc", "windows-sys", ] [[package]] name = "itertools" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] [[package]] name = "libc" version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "lock_api" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", ] [[package]] name = "log" version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] [[package]] name = "nom" version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", ] [[package]] name = "num-traits" version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ "hermit-abi", "libc", ] [[package]] name = "object" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" dependencies = [ "memchr", ] [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "papaya" version = "0.2.3" dependencies = [ "base64 0.22.1", "criterion", "dashmap", "equivalent", "hdrhistogram", "num_cpus", "rand", "seize", "serde", "serde_json", "tokio", ] [[package]] name = "parking_lot_core" version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", "windows-targets", ] [[package]] name = "pin-project-lite" version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "plotters" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" dependencies = [ "num-traits", "plotters-backend", "plotters-svg", "wasm-bindgen", "web-sys", ] [[package]] name = "plotters-backend" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" [[package]] name = "plotters-svg" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" dependencies = [ "plotters-backend", ] [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] [[package]] name = "quote" version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] [[package]] name = "rand" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", "rand_core", ] [[package]] name = "rand_chacha" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core", ] [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] [[package]] name = "rayon" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", ] [[package]] name = "rayon-core" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", ] [[package]] name = "redox_syscall" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ "bitflags", ] [[package]] name = "regex" version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "ryu" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ "winapi-util", ] [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "seize" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4b8d813387d566f627f3ea1b914c068aac94c40ae27ec43f5f33bde65abefe7" dependencies = [ "libc", "windows-sys", ] [[package]] name = "serde" version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "serde_json" version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", "serde", ] [[package]] name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "syn" version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "tinytemplate" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" dependencies = [ "serde", "serde_json", ] [[package]] name = "tokio" version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "pin-project-lite", ] [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "walkdir" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", ] [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", ] [[package]] name = "winapi-util" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ "windows-sys", ] [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" papaya-0.2.3/Cargo.toml0000644000000043340000000000100103130ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2021" rust-version = "1.72.0" name = "papaya" version = "0.2.3" authors = ["Ibraheem Ahmed "] build = false exclude = ["assets/*"] autolib = false autobins = false autoexamples = false autotests = false autobenches = false description = "A fast and ergonomic concurrent hash-table for read-heavy workloads." readme = "README.md" keywords = [ "concurrent", "hashmap", "atomic", "lock-free", ] categories = [ "algorithms", "concurrency", "data-structures", ] license = "MIT" repository = "https://github.com/ibraheemdev/papaya" [features] default = [] serde = ["dep:serde"] [lib] name = "papaya" path = "src/lib.rs" [[test]] name = "basic" path = "tests/basic.rs" [[test]] name = "basic_set" path = "tests/basic_set.rs" [[test]] name = "common" path = "tests/common.rs" [[test]] name = "cuckoo" path = "tests/cuckoo.rs" [[test]] name = "stress" path = "tests/stress.rs" [[bench]] name = "latency" path = "benches/latency.rs" harness = false [[bench]] name = "single_thread" path = "benches/single_thread.rs" harness = false [dependencies.equivalent] version = "1" [dependencies.seize] version = "0.5" [dependencies.serde] version = "1" optional = true [dev-dependencies.base64] version = "0.22" [dev-dependencies.criterion] version = "0.5" [dev-dependencies.dashmap] version = "5" [dev-dependencies.hdrhistogram] version = "7" [dev-dependencies.num_cpus] version = "1" [dev-dependencies.rand] version = "0.8" [dev-dependencies.serde_json] version = "1" [dev-dependencies.tokio] version = "1" features = [ "fs", "rt", ] [lints.rust.unexpected_cfgs] level = "warn" priority = 0 check-cfg = [ "cfg(papaya_stress)", "cfg(papaya_asan)", ] [profile.test] debug-assertions = true inherits = "release" papaya-0.2.3/Cargo.toml.orig000064400000000000000000000020321046102023000137650ustar 00000000000000[package] name = "papaya" version = "0.2.3" authors = ["Ibraheem Ahmed "] description = "A fast and ergonomic concurrent hash-table for read-heavy workloads." edition = "2021" rust-version = "1.72.0" license = "MIT" readme = "README.md" repository = "https://github.com/ibraheemdev/papaya" categories = ["algorithms", "concurrency", "data-structures"] keywords = ["concurrent", "hashmap", "atomic", "lock-free"] exclude = ["assets/*"] [dependencies] equivalent = "1" seize = "0.5" serde = { version = "1", optional = true } [dev-dependencies] rand = "0.8" base64 = "0.22" hdrhistogram = "7" dashmap = "5" criterion = "0.5" tokio = { version = "1", features = ["fs", "rt"] } num_cpus = "1" serde_json = "1" [features] default = [] serde = ["dep:serde"] [profile.test] inherits = "release" debug-assertions = true [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = [ 'cfg(papaya_stress)', 'cfg(papaya_asan)', ] } [[bench]] name = "single_thread" harness = false [[bench]] name = "latency" harness = false papaya-0.2.3/LICENSE.md000064400000000000000000000020141046102023000125020ustar 00000000000000MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. papaya-0.2.3/README.md000064400000000000000000000034741046102023000123700ustar 00000000000000# `papaya` [crates.io](https://crates.io/crates/papaya) [github](https://github.com/ibraheemdev/papaya) [docs.rs](https://docs.rs/papaya) A fast and ergonomic concurrent hash-table for read-heavy workloads. See [the documentation](https://docs.rs/papaya/latest) to get started. ## Features - An ergonomic lock-free API — no more deadlocks! - Powerful atomic operations. - Seamless usage in async contexts. - Extremely scalable, low-latency reads (see [performance](#performance)). - Predictable latency across all operations. - Efficient memory usage, with garbage collection powered by [`seize`]. ## Performance `papaya` is built with read-heavy workloads in mind. As such, read operations are extremely high throughput and provide consistent performance that scales with concurrency, meaning `papaya` will excel in workloads where reads are more common than writes. In write heavy workloads, `papaya` will still provide competitive performance despite not being it's primary use case. See the [benchmarks] for details. `papaya` aims to provide predictable and consistent latency across all operations. Most operations are lock-free, and those that aren't only block under rare and constrained conditions. `papaya` also features [incremental resizing]. Predictable latency is an important part of performance that doesn't often show up in benchmarks, but has significant implications for real-world usage. [benchmarks]: ./BENCHMARKS.md [`seize`]: https://github.com/ibraheemdev/seize [incremental resizing]: https://docs.rs/papaya/latest/papaya/enum.ResizeMode.html papaya-0.2.3/benches/latency.rs000064400000000000000000000054001046102023000145140ustar 00000000000000use std::fs::File; use std::sync::Barrier; use std::thread; use base64::engine::general_purpose::STANDARD; use base64::write::EncoderWriter; use hdrhistogram::serialization::{Serializer, V2DeflateSerializer}; use hdrhistogram::{Histogram, SyncHistogram}; fn main() { println!("=== papaya (incremental) ==="); p99_insert(papaya::HashMap::new(), |map, i| { map.pin().insert(i, ()); }); p99_concurrent_insert("papaya", papaya::HashMap::new(), |map, i| { map.pin().insert(i, ()); }); println!("=== papaya (blocking) ==="); let map = papaya::HashMap::builder() .resize_mode(papaya::ResizeMode::Blocking) .build(); p99_insert(map.clone(), |map, i| { map.pin().insert(i, ()); }); p99_concurrent_insert("papaya-blocking", map, |map, i| { map.pin().insert(i, ()); }); println!("=== dashmap ==="); p99_insert(dashmap::DashMap::new(), |map, i| { map.insert(i, ()); }); p99_concurrent_insert("dashmap", dashmap::DashMap::new(), |map, i| { map.insert(i, ()); }); } fn p99_insert(map: T, insert: impl Fn(&T, usize)) { const ITEMS: usize = 10_000_000; let mut max = None; for i in 0..ITEMS { let now = std::time::Instant::now(); insert(&map, i); let elapsed = now.elapsed(); if max.map(|max| elapsed > max).unwrap_or(true) { max = Some(elapsed); } } println!("p99 insert: {}ms", max.unwrap().as_millis()); } fn p99_concurrent_insert(name: &str, map: T, insert: impl Fn(&T, usize) + Send + Copy) { const ITEMS: usize = 1_000_000; let barrier = Barrier::new(8); let mut hist = SyncHistogram::::from(Histogram::new(1).unwrap()); thread::scope(|s| { for t in 0..8 { let (barrier, map) = (&barrier, &map); let mut hist = hist.recorder(); s.spawn(move || { barrier.wait(); let mut max = None; for i in 0..ITEMS { let i = (t + 1) * i; let now = std::time::Instant::now(); insert(&map, i); let elapsed = now.elapsed(); if max.map(|max| elapsed > max).unwrap_or(true) { max = Some(elapsed); } hist.record(elapsed.as_micros().try_into().unwrap()) .unwrap(); } println!("p99 concurrent insert: {}ms", max.unwrap().as_millis()); }); } }); hist.refresh(); let mut f = File::create(format!("{name}.hist")).unwrap(); let mut s = V2DeflateSerializer::new(); s.serialize(&hist, &mut EncoderWriter::new(&mut f, &STANDARD)) .unwrap(); } papaya-0.2.3/benches/single_thread.rs000064400000000000000000000035761046102023000157010ustar 00000000000000use std::collections::HashMap; use criterion::{black_box, criterion_group, criterion_main, Criterion}; const SIZE: usize = 10_000; fn compare(c: &mut Criterion) { let mut group = c.benchmark_group("read"); #[derive(Clone, Copy)] struct RandomKeys { state: usize, } impl RandomKeys { fn new() -> Self { RandomKeys { state: 0 } } } impl Iterator for RandomKeys { type Item = usize; fn next(&mut self) -> Option { // Add 1 then multiply by some 32 bit prime. self.state = self.state.wrapping_add(1).wrapping_mul(3_787_392_781); Some(self.state) } } group.bench_function("papaya", |b| { let m = papaya::HashMap::::builder() .collector(seize::Collector::new()) .build(); for i in RandomKeys::new().take(SIZE) { m.pin().insert(i, i); } b.iter(|| { for i in RandomKeys::new().take(SIZE) { black_box(assert_eq!(m.pin().get(&i), Some(&i))); } }); }); group.bench_function("std", |b| { let mut m = HashMap::::default(); for i in RandomKeys::new().take(SIZE) { m.insert(i, i); } b.iter(|| { for i in RandomKeys::new().take(SIZE) { black_box(assert_eq!(m.get(&i), Some(&i))); } }); }); group.bench_function("dashmap", |b| { let m = dashmap::DashMap::::default(); for i in RandomKeys::new().take(SIZE) { m.insert(i, i); } b.iter(|| { for i in RandomKeys::new().take(SIZE) { black_box(assert_eq!(*m.get(&i).unwrap(), i)); } }); }); group.finish(); } criterion_group!(benches, compare); criterion_main!(benches); papaya-0.2.3/src/lib.rs000064400000000000000000000255171046102023000130160ustar 00000000000000//! A fast and ergonomic concurrent hash-table for read-heavy workloads. //! //! # Features //! //! - An ergonomic lock-free API — no more deadlocks! //! - Powerful atomic operations. //! - Seamless usage in async contexts. //! - Extremely scalable, low-latency reads (see [performance](#performance)). //! - Predictable latency across all operations. //! - Efficient memory usage, with garbage collection powered by [`seize`]. //! //! # Overview //! //! The top-level crate documentation is organized as follows: //! //! - [Usage](#usage) shows how to interact with the concurrent `HashMap`. //! - [Consistency](#consistency) describes the guarantees of concurrent operations. //! - [Atomic Operations](#atomic-operations) shows how to perform dynamic operations atomically. //! - [Async Support](#async-support) shows how to use the map in an async context. //! - [Advanced Lifetimes](#advanced-lifetimes) explains how to use guards when working with nested types. //! - [Performance](#performance) provides details of expected performance characteristics. //! //! # Usage //! //! `papaya` aims to provide an ergonomic API without sacrificing performance. [`HashMap`] exposes a lock-free API, enabling it to hand out direct references to objects in the map without the need for wrapper types that are clunky and prone to deadlocks. However, you can't hold on to references forever due to concurrent removals. Because of this, the `HashMap` API is structured around *pinning*. Through a pin you can access the map just like a standard `HashMap`. A pin is similar to a lock guard, so any references that are returned will be tied to the lifetime of the guard. Unlike a lock however, pinning is cheap and can never cause deadlocks. //! //! ```rust //! use papaya::HashMap; //! //! // Create a map. //! let map = HashMap::new(); //! //! // Pin the map. //! let map = map.pin(); //! //! // Use the map as normal. //! map.insert('A', 1); //! assert_eq!(map.get(&'A'), Some(&1)); //! assert_eq!(map.len(), 1); //! ``` //! //! As expected of a concurrent `HashMap`, all operations take a shared reference. This allows the map to be freely pinned and accessed from multiple threads: //! //! ```rust //! use papaya::HashMap; //! //! // Use a map from multiple threads. //! let map = HashMap::new(); //! std::thread::scope(|s| { //! // Insert some values. //! s.spawn(|| { //! let map = map.pin(); //! for i in 'A'..='Z' { //! map.insert(i, 1); //! } //! }); //! //! // Remove the values. //! s.spawn(|| { //! let map = map.pin(); //! for i in 'A'..='Z' { //! map.remove(&i); //! } //! }); //! //! // Read the values. //! s.spawn(|| { //! for (key, value) in map.pin().iter() { //! println!("{key}: {value}"); //! } //! }); //! }); //! ``` //! //! It is important to note that as long as you are holding on to a guard, you are preventing the map from performing garbage collection. Pinning and unpinning the table is relatively cheap but not free, similar to the cost of locking and unlocking an uncontended or lightly contended `Mutex`. Thus guard reuse is encouraged, within reason. See the [`seize`] crate for advanced usage and specifics of the garbage collection algorithm. //! //! # Consistency //! //! Due to the concurrent nature of the map, read and write operations may overlap in time. There is no support for locking the entire table nor individual keys to prevent concurrent access, except through external fine-grained locking. As such, read operations (such as `get`) reflect the results of the *most-recent* write. More formally, a read establishes a *happens-before* relationship with the corresponding write. //! //! Aggregate operations, such as iterators, rely on a weak snapshot of the table and return results reflecting the state of the table at or some point after the creation of the iterator. This means that they may, but are not guaranteed to, reflect concurrent modifications to the table that occur during iteration. Similarly, operations such as `clear` and `clone` rely on iteration and may not produce "perfect" results if the map is being concurrently modified. //! //! Note that to obtain a stable snapshot of the table, aggregate table operations require completing any in-progress resizes. If you rely heavily on iteration or similar operations you should consider configuring [`ResizeMode::Blocking`]. //! //! # Atomic Operations //! //! As mentioned above, `papaya` does not support locking keys to prevent access, which makes performing complex operations more challenging. Instead, `papaya` exposes a number of atomic operations. The most basic of these is [`HashMap::update`], which can be used to update an existing value in the map using a closure: //! //! ```rust //! let map = papaya::HashMap::new(); //! map.pin().insert("poneyland", 42); //! assert_eq!(map.pin().update("poneyland", |e| e + 1), Some(&43)); //! ``` //! //! Note that in the event that the entry is concurrently modified during an `update`, the closure may be called multiple times to retry the operation. For this reason, update operations are intended to be quick and *pure*, as they may be retried or internally memoized. //! //! `papaya` also exposes more powerful atomic operations that serve as a replacement for the [standard entry API](std::collections::hash_map::Entry). These include: //! //! - [`HashMap::update`] //! - [`HashMap::update_or_insert`] //! - [`HashMap::update_or_insert_with`] //! - [`HashMap::get_or_insert`] //! - [`HashMap::get_or_insert_with`] //! - [`HashMap::compute`] //! //! For example, with a standard `HashMap`, `Entry::and_modify` is often paired with `Entry::or_insert`: //! //! ```rust //! use std::collections::HashMap; //! //! let mut map = HashMap::new(); //! // Insert `poneyland` with the value `42` if it doesn't exist, //! // otherwise increment it's value. //! map.entry("poneyland") //! .and_modify(|e| { *e += 1 }) //! .or_insert(42); //! ``` //! //! However, implementing this with a concurrent `HashMap` is tricky as the entry may be modified in-between operations. Instead, you can write the above operation using [`HashMap::update_or_insert`]: //! //! ```rust //! use papaya::HashMap; //! //! let map = HashMap::new(); //! // Insert `poneyland` with the value `42` if it doesn't exist, //! // otherwise increment it's value. //! map.pin().update_or_insert("poneyland", |e| e + 1, 42); //! ``` //! //! Atomic operations are extremely powerful but also easy to misuse. They may be less efficient than update mechanisms tailored for the specific type of data in the map. For example, concurrent counters should avoid using `update` and instead use `AtomicUsize`. Entries that are frequently modified may also benefit from fine-grained locking. //! //! # Async Support //! //! By default, a pinned map guard does not implement `Send` as it is tied to the current thread, similar to a lock guard. This leads to an issue in work-stealing schedulers as guards are not valid across `.await` points. //! //! To overcome this, you can use an *owned* guard. //! //! ```rust //! # use std::sync::Arc; //! use papaya::HashMap; //! //! async fn run(map: Arc>) { //! tokio::spawn(async move { //! // Pin the map with an owned guard. //! let map = map.pin_owned(); //! //! // Hold references across await points. //! let value = map.get(&37); //! tokio::fs::write("db.txt", format!("{value:?}")).await; //! println!("{value:?}"); //! }); //! } //! ``` //! //! Note that owned guards are more expensive to create than regular guards, so they should only be used if necessary. In the above example, you could instead drop the reference and call `get` a second time after the asynchronous call. A more fitting example involves asynchronous iteration: //! //! ```rust //! # use std::sync::Arc; //! use papaya::HashMap; //! //! async fn run(map: Arc>) { //! tokio::spawn(async move { //! for (key, value) in map.pin_owned().iter() { //! tokio::fs::write("db.txt", format!("{key}: {value}\n")).await; //! } //! }); //! } //! ``` //! //! # Advanced Lifetimes //! //! You may run into issues when you try to return a reference to a map contained within an outer type. For example: //! //! ```rust,compile_fail //! pub struct Metrics { //! map: papaya::HashMap> //! } //! //! impl Metrics { //! pub fn get(&self, name: &str) -> Option<&[u64]> { //! // error[E0515]: cannot return value referencing temporary value //! Some(self.map.pin().get(name)?.as_slice()) //! } //! } //! ``` //! //! The solution is to accept a guard in the method directly, tying the lifetime to the caller's stack frame: //! //! ```rust //! use papaya::Guard; //! //! pub struct Metrics { //! map: papaya::HashMap> //! } //! //! impl Metrics { //! pub fn guard(&self) -> impl Guard + '_ { //! self.map.guard() //! } //! //! pub fn get<'guard>(&self, name: &str, guard: &'guard impl Guard) -> Option<&'guard [u64]> { //! Some(self.map.get(name, guard)?.as_slice()) //! } //! } //! ``` //! //! The `Guard` trait supports both local and owned guards. Note the `'guard` lifetime that ties the guard to the returned reference. No wrapper types or guard mapping is necessary. //! //! # Performance //! //! `papaya` is built with read-heavy workloads in mind. As such, read operations are extremely high throughput and provide consistent performance that scales with concurrency, meaning `papaya` will excel in workloads where reads are more common than writes. In write heavy workloads, `papaya` will still provide competitive performance despite not being it's primary use case. See the [benchmarks] for details. //! //! `papaya` aims to provide predictable and consistent latency across all operations. Most operations are lock-free, and those that aren't only block under rare and constrained conditions. `papaya` also features [incremental resizing](ResizeMode). Predictable latency is an important part of performance that doesn't often show up in benchmarks, but has significant implications for real-world usage. //! //! [benchmarks]: https://github.com/ibraheemdev/papaya/blob/master/BENCHMARKS.md #![deny( missing_debug_implementations, missing_docs, dead_code, unsafe_op_in_unsafe_fn )] // Polyfills for unstable APIs related to strict-provenance. #![allow(unstable_name_collisions)] // Stylistic preferences. #![allow(clippy::multiple_bound_locations, clippy::single_match)] // Clippy trips up with pollyfills. #![allow(clippy::incompatible_msrv)] mod map; mod raw; mod set; #[cfg(feature = "serde")] mod serde_impls; pub use equivalent::Equivalent; pub use map::{ Compute, HashMap, HashMapBuilder, HashMapRef, Iter, Keys, OccupiedError, Operation, ResizeMode, Values, }; pub use seize::{Guard, LocalGuard, OwnedGuard}; pub use set::{HashSet, HashSetBuilder, HashSetRef}; papaya-0.2.3/src/map.rs000064400000000000000000001451251046102023000130230ustar 00000000000000use crate::raw::utils::MapGuard; use crate::raw::{self, InsertResult}; use crate::Equivalent; use seize::{Collector, Guard, LocalGuard, OwnedGuard}; use std::collections::hash_map::RandomState; use std::fmt; use std::hash::{BuildHasher, Hash}; use std::marker::PhantomData; /// A concurrent hash table. /// /// Most hash table operations require a [`Guard`](crate::Guard), which can be acquired through /// [`HashMap::guard`] or using the [`HashMap::pin`] API. See the [crate-level documentation](crate#usage) /// for details. pub struct HashMap { raw: raw::HashMap, } // Safety: `HashMap` acts as a single-threaded collection on a single thread. // References to keys and values cannot outlive the map's lifetime on a given // thread. unsafe impl Send for HashMap {} // Safety: We only ever hand out `&{K, V}` through shared references to the map, // and never `&mut {K, V}` except through synchronized memory reclamation. // // However, we require `{K, V}: Send` as keys and values may be removed and dropped // on a different thread than they were created on through shared access to the // `HashMap`. // // Additionally, `HashMap` owns its `seize::Collector` and never exposes it, // so multiple threads cannot be involved in reclamation without sharing the // `HashMap` itself. If this was not true, we would require stricter bounds // on `HashMap` operations themselves. unsafe impl Sync for HashMap {} /// A builder for a [`HashMap`]. /// /// # Examples /// /// ```rust /// use papaya::{HashMap, ResizeMode}; /// use seize::Collector; /// use std::collections::hash_map::RandomState; /// /// let map: HashMap = HashMap::builder() /// // Set the initial capacity. /// .capacity(2048) /// // Set the hasher. /// .hasher(RandomState::new()) /// // Set the resize mode. /// .resize_mode(ResizeMode::Blocking) /// // Set a custom garbage collector. /// .collector(Collector::new().batch_size(128)) /// // Construct the hash map. /// .build(); /// ``` pub struct HashMapBuilder { hasher: S, capacity: usize, collector: Collector, resize_mode: ResizeMode, _kv: PhantomData<(K, V)>, } impl HashMapBuilder { /// Set the hash builder used to hash keys. /// /// Warning: `hash_builder` is normally randomly generated, and is designed /// to allow HashMaps to be resistant to attacks that cause many collisions /// and very poor performance. Setting it manually using this function can /// expose a DoS attack vector. /// /// The `hash_builder` passed should implement the [`BuildHasher`] trait for /// the HashMap to be useful, see its documentation for details. pub fn hasher(self, hasher: S) -> HashMapBuilder { HashMapBuilder { hasher, capacity: self.capacity, collector: self.collector, resize_mode: self.resize_mode, _kv: PhantomData, } } } impl HashMapBuilder { /// Set the initial capacity of the map. /// /// The table should be able to hold at least `capacity` elements before resizing. /// However, the capacity is an estimate, and the table may prematurely resize due /// to poor hash distribution. If `capacity` is 0, the hash map will not allocate. pub fn capacity(self, capacity: usize) -> HashMapBuilder { HashMapBuilder { capacity, hasher: self.hasher, collector: self.collector, resize_mode: self.resize_mode, _kv: PhantomData, } } /// Set the resizing mode of the map. See [`ResizeMode`] for details. pub fn resize_mode(self, resize_mode: ResizeMode) -> Self { HashMapBuilder { resize_mode, hasher: self.hasher, capacity: self.capacity, collector: self.collector, _kv: PhantomData, } } /// Set the [`seize::Collector`] used for garbage collection. /// /// This method may be useful when you want more control over garbage collection. /// /// Note that all `Guard` references used to access the map must be produced by /// the provided `collector`. pub fn collector(self, collector: Collector) -> Self { HashMapBuilder { collector, hasher: self.hasher, capacity: self.capacity, resize_mode: self.resize_mode, _kv: PhantomData, } } /// Construct a [`HashMap`] from the builder, using the configured options. pub fn build(self) -> HashMap { HashMap { raw: raw::HashMap::new(self.capacity, self.hasher, self.collector, self.resize_mode), } } } impl fmt::Debug for HashMapBuilder { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("HashMapBuilder") .field("capacity", &self.capacity) .field("collector", &self.collector) .field("resize_mode", &self.resize_mode) .finish() } } /// Resize behavior for a [`HashMap`]. /// /// Hash maps must resize when the underlying table becomes full, migrating all key and value pairs /// to a new table. This type allows you to configure the resizing behavior when passed to /// [`HashMapBuilder::resize_mode`]. #[derive(Debug)] pub enum ResizeMode { /// Writers copy a constant number of key/value pairs to the new table before making /// progress. /// /// Incremental resizes avoids latency spikes that can occur when insert operations have /// to resize a large table. However, they reduce parallelism during the resize and so can reduce /// overall throughput. Incremental resizing also means reads or write operations during an /// in-progress resize may have to search both the current and new table before succeeding, trading /// off median latency during a resize for tail latency. /// /// This is the default resize mode, with a chunk size of `64`. Incremental(usize), /// All writes to the map must wait till the resize completes before making progress. /// /// Blocking resizes tend to be better in terms of throughput, especially in setups with /// multiple writers that can perform the resize in parallel. However, they can lead to latency /// spikes for insert operations that have to resize large tables. /// /// If insert latency is not a concern, such as if the keys in your map are stable, enabling blocking /// resizes may yield better performance. /// /// Blocking resizing may also be a better option if you rely heavily on iteration or similar /// operations, as they require completing any in-progress resizes for consistency. Blocking, } impl Default for ResizeMode { fn default() -> Self { // Incremental resizing is a good default for most workloads as it avoids // unexpected latency spikes. ResizeMode::Incremental(64) } } impl HashMap { /// Creates an empty `HashMap`. /// /// The hash map is initially created with a capacity of 0, so it will not allocate /// until it is first inserted into. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// let map: HashMap<&str, i32> = HashMap::new(); /// ``` pub fn new() -> HashMap { HashMap::with_capacity_and_hasher(0, RandomState::new()) } /// Creates an empty `HashMap` with the specified capacity. /// /// The table should be able to hold at least `capacity` elements before resizing. /// However, the capacity is an estimate, and the table may prematurely resize due /// to poor hash distribution. If `capacity` is 0, the hash map will not allocate. /// /// Note that the `HashMap` may grow and shrink as elements are inserted or removed, /// but it is guaranteed to never shrink below the initial capacity. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// let map: HashMap<&str, i32> = HashMap::with_capacity(10); /// ``` pub fn with_capacity(capacity: usize) -> HashMap { HashMap::with_capacity_and_hasher(capacity, RandomState::new()) } /// Returns a builder for a `HashMap`. /// /// The builder can be used for more complex configuration, such as using /// a custom [`Collector`], or [`ResizeMode`]. pub fn builder() -> HashMapBuilder { HashMapBuilder { capacity: 0, hasher: RandomState::default(), collector: Collector::new(), resize_mode: ResizeMode::default(), _kv: PhantomData, } } } impl Default for HashMap where S: Default, { fn default() -> Self { HashMap::with_hasher(S::default()) } } impl HashMap { /// Creates an empty `HashMap` which will use the given hash builder to hash /// keys. /// /// Warning: `hash_builder` is normally randomly generated, and is designed /// to allow HashMaps to be resistant to attacks that cause many collisions /// and very poor performance. Setting it manually using this function can /// expose a DoS attack vector. /// /// The `hash_builder` passed should implement the [`BuildHasher`] trait for /// the HashMap to be useful, see its documentation for details. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// use std::hash::RandomState; /// /// let s = RandomState::new(); /// let map = HashMap::with_hasher(s); /// map.pin().insert(1, 2); /// ``` pub fn with_hasher(hash_builder: S) -> HashMap { HashMap::with_capacity_and_hasher(0, hash_builder) } /// Creates an empty `HashMap` with at least the specified capacity, using /// `hash_builder` to hash the keys. /// /// The table should be able to hold at least `capacity` elements before resizing. /// However, the capacity is an estimate, and the table may prematurely resize due /// to poor hash distribution. If `capacity` is 0, the hash map will not allocate. /// /// Warning: `hash_builder` is normally randomly generated, and is designed /// to allow HashMaps to be resistant to attacks that cause many collisions /// and very poor performance. Setting it manually using this function can /// expose a DoS attack vector. /// /// The `hasher` passed should implement the [`BuildHasher`] trait for /// the HashMap to be useful, see its documentation for details. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// use std::hash::RandomState; /// /// let s = RandomState::new(); /// let map = HashMap::with_capacity_and_hasher(10, s); /// map.pin().insert(1, 2); /// ``` pub fn with_capacity_and_hasher(capacity: usize, hash_builder: S) -> HashMap { HashMap { raw: raw::HashMap::new( capacity, hash_builder, Collector::default(), ResizeMode::default(), ), } } /// Returns a pinned reference to the map. /// /// The returned reference manages a guard internally, preventing garbage collection /// for as long as it is held. See the [crate-level documentation](crate#usage) for details. #[inline] pub fn pin(&self) -> HashMapRef<'_, K, V, S, LocalGuard<'_>> { HashMapRef { guard: self.raw.guard(), map: self, } } /// Returns a pinned reference to the map. /// /// Unlike [`HashMap::pin`], the returned reference implements `Send` and `Sync`, /// allowing it to be held across `.await` points in work-stealing schedulers. /// This is especially useful for iterators. /// /// The returned reference manages a guard internally, preventing garbage collection /// for as long as it is held. See the [crate-level documentation](crate#usage) for details. #[inline] pub fn pin_owned(&self) -> HashMapRef<'_, K, V, S, OwnedGuard<'_>> { HashMapRef { guard: self.raw.owned_guard(), map: self, } } /// Returns a guard for use with this map. /// /// Note that holding on to a guard prevents garbage collection. /// See the [crate-level documentation](crate#usage) for details. #[inline] pub fn guard(&self) -> LocalGuard<'_> { self.raw.collector().enter() } /// Returns an owned guard for use with this map. /// /// Owned guards implement `Send` and `Sync`, allowing them to be held across /// `.await` points in work-stealing schedulers. This is especially useful /// for iterators. /// /// Note that holding on to a guard prevents garbage collection. /// See the [crate-level documentation](crate#usage) for details. #[inline] pub fn owned_guard(&self) -> OwnedGuard<'_> { self.raw.collector().enter_owned() } } impl HashMap where K: Hash + Eq, S: BuildHasher, { /// Returns the number of entries in the map. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// /// map.pin().insert(1, "a"); /// map.pin().insert(2, "b"); /// assert!(map.len() == 2); /// ``` #[inline] pub fn len(&self) -> usize { self.raw.len() } /// Returns `true` if the map is empty. Otherwise returns `false`. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// assert!(map.is_empty()); /// map.pin().insert("a", 1); /// assert!(!map.is_empty()); /// ``` #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } /// Returns `true` if the map contains a value for the specified key. /// /// The key may be any borrowed form of the map's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// /// [`Eq`]: std::cmp::Eq /// [`Hash`]: std::hash::Hash /// /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// map.pin().insert(1, "a"); /// assert_eq!(map.pin().contains_key(&1), true); /// assert_eq!(map.pin().contains_key(&2), false); /// ``` #[inline] pub fn contains_key(&self, key: &Q, guard: &impl Guard) -> bool where Q: Equivalent + Hash + ?Sized, { self.get(key, self.raw.verify(guard)).is_some() } /// Returns a reference to the value corresponding to the key. /// /// The key may be any borrowed form of the map's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// /// [`Eq`]: std::cmp::Eq /// [`Hash`]: std::hash::Hash /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// map.pin().insert(1, "a"); /// assert_eq!(map.pin().get(&1), Some(&"a")); /// assert_eq!(map.pin().get(&2), None); /// ``` #[inline] pub fn get<'g, Q>(&self, key: &Q, guard: &'g impl Guard) -> Option<&'g V> where K: 'g, Q: Equivalent + Hash + ?Sized, { match self.raw.get(key, self.raw.verify(guard)) { Some((_, v)) => Some(v), None => None, } } /// Returns the key-value pair corresponding to the supplied key. /// /// The supplied key may be any borrowed form of the map's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// /// [`Eq`]: std::cmp::Eq /// [`Hash`]: std::hash::Hash /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// map.pin().insert(1, "a"); /// assert_eq!(map.pin().get_key_value(&1), Some((&1, &"a"))); /// assert_eq!(map.pin().get_key_value(&2), None); /// ``` #[inline] pub fn get_key_value<'g, Q>(&self, key: &Q, guard: &'g impl Guard) -> Option<(&'g K, &'g V)> where Q: Equivalent + Hash + ?Sized, { self.raw.get(key, self.raw.verify(guard)) } /// Inserts a key-value pair into the map. /// /// If the map did not have this key present, [`None`] is returned. /// /// If the map did have this key present, the value is updated, and the old /// value is returned. The key is not updated, though; this matters for /// types that can be `==` without being identical. See the [standard library /// documentation] for details. /// /// [standard library documentation]: https://doc.rust-lang.org/std/collections/index.html#insert-and-complex-keys /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// assert_eq!(map.pin().insert(37, "a"), None); /// assert_eq!(map.pin().is_empty(), false); /// /// map.pin().insert(37, "b"); /// assert_eq!(map.pin().insert(37, "c"), Some(&"b")); /// assert_eq!(map.pin().get(&37), Some(&"c")); /// ``` #[inline] pub fn insert<'g>(&self, key: K, value: V, guard: &'g impl Guard) -> Option<&'g V> { match self.raw.insert(key, value, true, self.raw.verify(guard)) { InsertResult::Inserted(_) => None, InsertResult::Replaced(value) => Some(value), InsertResult::Error { .. } => unreachable!(), } } /// Tries to insert a key-value pair into the map, and returns /// a reference to the value that was inserted. /// /// If the map already had this key present, nothing is updated, and /// an error containing the existing value is returned. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// let map = map.pin(); /// /// assert_eq!(map.try_insert(37, "a").unwrap(), &"a"); /// /// let err = map.try_insert(37, "b").unwrap_err(); /// assert_eq!(err.current, &"a"); /// assert_eq!(err.not_inserted, "b"); /// ``` #[inline] pub fn try_insert<'g>( &self, key: K, value: V, guard: &'g impl Guard, ) -> Result<&'g V, OccupiedError<'g, V>> { // Safety: Checked the guard above. match self.raw.insert(key, value, false, self.raw.verify(guard)) { InsertResult::Inserted(value) => Ok(value), InsertResult::Error { current, not_inserted, } => Err(OccupiedError { current, not_inserted, }), InsertResult::Replaced(_) => unreachable!(), } } /// Tries to insert a key and value computed from a closure into the map, /// and returns a reference to the value that was inserted. /// /// If the map already had this key present, nothing is updated, and /// the existing value is returned. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// let map = map.pin(); /// /// assert_eq!(map.try_insert_with(37, || "a").unwrap(), &"a"); /// /// let current = map.try_insert_with(37, || "b").unwrap_err(); /// assert_eq!(current, &"a"); /// ``` #[inline] pub fn try_insert_with<'g, F>( &self, key: K, f: F, guard: &'g impl Guard, ) -> Result<&'g V, &'g V> where F: FnOnce() -> V, K: 'g, { self.raw.try_insert_with(key, f, self.raw.verify(guard)) } /// Returns a reference to the value corresponding to the key, or inserts a default value. /// /// If the given key is present, the corresponding value is returned. If it is not present, /// the provided `value` is inserted, and a reference to the newly inserted value is returned. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// assert_eq!(map.pin().get_or_insert("a", 3), &3); /// assert_eq!(map.pin().get_or_insert("a", 6), &3); /// ``` #[inline] pub fn get_or_insert<'g>(&self, key: K, value: V, guard: &'g impl Guard) -> &'g V { // Note that we use `try_insert` instead of `compute` or `get_or_insert_with` here, as it // allows us to avoid the closure indirection. match self.try_insert(key, value, guard) { Ok(inserted) => inserted, Err(OccupiedError { current, .. }) => current, } } /// Returns a reference to the value corresponding to the key, or inserts a default value /// computed from a closure. /// /// If the given key is present, the corresponding value is returned. If it is not present, /// the value computed from `f` is inserted, and a reference to the newly inserted value is /// returned. /// /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// assert_eq!(map.pin().get_or_insert_with("a", || 3), &3); /// assert_eq!(map.pin().get_or_insert_with("a", || 6), &3); /// ``` #[inline] pub fn get_or_insert_with<'g, F>(&self, key: K, f: F, guard: &'g impl Guard) -> &'g V where F: FnOnce() -> V, K: 'g, { self.raw.get_or_insert_with(key, f, self.raw.verify(guard)) } /// Updates an existing entry atomically. /// /// If the value for the specified `key` is present, the new value is computed and stored the /// using the provided update function, and the new value is returned. Otherwise, `None` /// is returned. /// /// /// The update function is given the current value associated with the given key and returns the /// new value to be stored. The operation is applied atomically only if the state of the entry remains /// the same, meaning that it is not concurrently modified in any way. If the entry is /// modified, the operation is retried with the new entry, similar to a traditional [compare-and-swap](https://en.wikipedia.org/wiki/Compare-and-swap) /// operation. /// /// Note that the `update` function should be pure as it may be called multiple times, and the output /// for a given entry may be memoized across retries. /// /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// map.pin().insert("a", 1); /// assert_eq!(map.pin().get(&"a"), Some(&1)); /// /// map.pin().update("a", |v| v + 1); /// assert_eq!(map.pin().get(&"a"), Some(&2)); /// ``` #[inline] pub fn update<'g, F>(&self, key: K, update: F, guard: &'g impl Guard) -> Option<&'g V> where F: Fn(&V) -> V, K: 'g, { self.raw.update(key, update, self.raw.verify(guard)) } /// Updates an existing entry or inserts a default value. /// /// If the value for the specified `key` is present, the new value is computed and stored the /// using the provided update function, and the new value is returned. Otherwise, the provided /// `value` is inserted into the map, and a reference to the newly inserted value is returned. /// /// See [`HashMap::update`] for details about how atomic updates are performed. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// assert_eq!(*map.pin().update_or_insert("a", |i| i + 1, 0), 0); /// assert_eq!(*map.pin().update_or_insert("a", |i| i + 1, 0), 1); /// ``` #[inline] pub fn update_or_insert<'g, F>( &self, key: K, update: F, value: V, guard: &'g impl Guard, ) -> &'g V where F: Fn(&V) -> V, K: 'g, { self.update_or_insert_with(key, update, || value, guard) } /// Updates an existing entry or inserts a default value computed from a closure. /// /// If the value for the specified `key` is present, the new value is computed and stored the /// using the provided update function, and the new value is returned. Otherwise, the value /// computed by `f` is inserted into the map, and a reference to the newly inserted value is /// returned. /// /// See [`HashMap::update`] for details about how atomic updates are performed. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// assert_eq!(*map.pin().update_or_insert_with("a", |i| i + 1, || 0), 0); /// assert_eq!(*map.pin().update_or_insert_with("a", |i| i + 1, || 0), 1); /// ``` #[inline] pub fn update_or_insert_with<'g, U, F>( &self, key: K, update: U, f: F, guard: &'g impl Guard, ) -> &'g V where F: FnOnce() -> V, U: Fn(&V) -> V, K: 'g, { self.raw .update_or_insert_with(key, update, f, self.raw.verify(guard)) } /// Updates an entry with a compare-and-swap (CAS) function. /// /// This method allows you to perform complex operations on the map atomically. The `compute` /// closure is given the current state of the entry and returns the operation that should be /// performed. The operation is applied atomically only if the state of the entry remains the same, /// meaning it is not concurrently modified in any way. /// /// Note that the `compute` function should be pure as it may be called multiple times, and /// the output for a given entry may be memoized across retries. /// /// In most cases you can avoid this method and instead use a higher-level atomic operation. /// See the [crate-level documentation](crate#atomic-operations) for details. /// /// # Examples /// /// ```rust /// use papaya::{HashMap, Operation, Compute}; /// /// let map = HashMap::new(); /// let map = map.pin(); /// /// let compute = |entry| match entry { /// // Remove the value if it is even. /// Some((_key, value)) if value % 2 == 0 => { /// Operation::Remove /// } /// /// // Increment the value if it is odd. /// Some((_key, value)) => { /// Operation::Insert(value + 1) /// } /// /// // Do nothing if the key does not exist /// None => Operation::Abort(()), /// }; /// /// assert_eq!(map.compute('A', compute), Compute::Aborted(())); /// /// map.insert('A', 1); /// assert_eq!(map.compute('A', compute), Compute::Updated { /// old: (&'A', &1), /// new: (&'A', &2), /// }); /// assert_eq!(map.compute('A', compute), Compute::Removed(&'A', &2)); /// ``` #[inline] pub fn compute<'g, F, T>( &self, key: K, compute: F, guard: &'g impl Guard, ) -> Compute<'g, K, V, T> where F: FnMut(Option<(&'g K, &'g V)>) -> Operation, { self.raw.compute(key, compute, self.raw.verify(guard)) } /// Removes a key from the map, returning the value at the key if the key /// was previously in the map. /// /// The key may be any borrowed form of the map's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// map.pin().insert(1, "a"); /// assert_eq!(map.pin().remove(&1), Some(&"a")); /// assert_eq!(map.pin().remove(&1), None); /// ``` #[inline] pub fn remove<'g, Q>(&self, key: &Q, guard: &'g impl Guard) -> Option<&'g V> where K: 'g, Q: Equivalent + Hash + ?Sized, { match self.raw.remove(key, self.raw.verify(guard)) { Some((_, value)) => Some(value), None => None, } } /// Removes a key from the map, returning the stored key and value if the /// key was previously in the map. /// /// The key may be any borrowed form of the map's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// map.pin().insert(1, "a"); /// assert_eq!(map.pin().get(&1), Some(&"a")); /// assert_eq!(map.pin().remove_entry(&1), Some((&1, &"a"))); /// assert_eq!(map.pin().remove(&1), None); /// ``` #[inline] pub fn remove_entry<'g, Q>(&self, key: &Q, guard: &'g impl Guard) -> Option<(&'g K, &'g V)> where K: 'g, Q: Equivalent + Hash + ?Sized, { self.raw.remove(key, self.raw.verify(guard)) } /// Conditionally removes a key from the map based on the provided closure. /// /// If the key is found in the map and the closure returns `true` given the key and its value, /// the key and value are returned successfully. Note that the returned entry is guaranteed to /// be the same entry that the closure returned `true` for. However, the closure returning `true` /// does not guarantee that the entry is removed in the presence of concurrent modifications. /// /// If the key is not found in the map, `Ok(None)` is returned. /// /// If the closure returns `false`, an error is returned containing the entry provided to the closure. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// map.pin().insert(1, "a"); /// /// assert_eq!(map.pin().remove_if(&1, |k, v| *v == "b"), Err((&1, &"a"))); /// assert_eq!(map.pin().get(&1), Some(&"a")); /// assert_eq!(map.pin().remove_if(&1, |k, v| *v == "a"), Ok(Some((&1, &"a")))); /// assert_eq!(map.pin().remove_if(&1, |_, _| true), Ok(None)); /// ``` #[inline] pub fn remove_if<'g, Q, F>( &self, key: &Q, should_remove: F, guard: &'g impl Guard, ) -> Result, (&'g K, &'g V)> where Q: Equivalent + Hash + ?Sized, F: FnMut(&K, &V) -> bool, { self.raw .remove_if(key, should_remove, self.raw.verify(guard)) } /// Tries to reserve capacity for `additional` more elements to be inserted /// in the `HashMap`. /// /// After calling this method, the table should be able to hold at least `capacity` elements /// before resizing. However, the capacity is an estimate, and the table may prematurely resize /// due to poor hash distribution. The collection may also reserve more space to avoid frequent /// reallocations. /// /// # Panics /// /// Panics if the new allocation size overflows `usize`. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map: HashMap<&str, i32> = HashMap::new(); /// map.pin().reserve(10); /// ``` #[inline] pub fn reserve(&self, additional: usize, guard: &impl Guard) { self.raw.reserve(additional, self.raw.verify(guard)) } /// Clears the map, removing all key-value pairs. /// /// Note that this method will block until any in-progress resizes are /// completed before proceeding. See the [consistency](crate#consistency) /// section for details. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::new(); /// /// map.pin().insert(1, "a"); /// map.pin().clear(); /// assert!(map.pin().is_empty()); /// ``` #[inline] pub fn clear(&self, guard: &impl Guard) { self.raw.clear(self.raw.verify(guard)) } /// Retains only the elements specified by the predicate. /// /// In other words, remove all pairs `(k, v)` for which `f(&k, &v)` returns `false`. /// The elements are visited in unsorted (and unspecified) order. /// /// Note the function may be called more than once for a given key if its value is /// concurrently modified during removal. /// /// Additionally, this method will block until any in-progress resizes are /// completed before proceeding. See the [consistency](crate#consistency) /// section for details. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let mut map: HashMap = (0..8).map(|x| (x, x * 10)).collect(); /// map.pin().retain(|&k, _| k % 2 == 0); /// assert_eq!(map.len(), 4); /// ``` #[inline] pub fn retain(&self, f: F, guard: &impl Guard) where F: FnMut(&K, &V) -> bool, { self.raw.retain(f, self.raw.verify(guard)) } /// An iterator visiting all key-value pairs in arbitrary order. /// The iterator element type is `(&K, &V)`. /// /// Note that this method will block until any in-progress resizes are /// completed before proceeding. See the [consistency](crate#consistency) /// section for details. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::from([ /// ("a", 1), /// ("b", 2), /// ("c", 3), /// ]); /// /// for (key, val) in map.pin().iter() { /// println!("key: {key} val: {val}"); /// } #[inline] pub fn iter<'g, G>(&self, guard: &'g G) -> Iter<'g, K, V, G> where G: Guard, { Iter { raw: self.raw.iter(self.raw.verify(guard)), } } /// An iterator visiting all keys in arbitrary order. /// The iterator element type is `&K`. /// /// Note that this method will block until any in-progress resizes are /// completed before proceeding. See the [consistency](crate#consistency) /// section for details. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::from([ /// ("a", 1), /// ("b", 2), /// ("c", 3), /// ]); /// /// for key in map.pin().keys() { /// println!("{key}"); /// } /// ``` #[inline] pub fn keys<'g, G>(&self, guard: &'g G) -> Keys<'g, K, V, G> where G: Guard, { Keys { iter: self.iter(guard), } } /// An iterator visiting all values in arbitrary order. /// The iterator element type is `&V`. /// /// Note that this method will block until any in-progress resizes are /// completed before proceeding. See the [consistency](crate#consistency) /// section for details. /// /// # Examples /// /// ``` /// use papaya::HashMap; /// /// let map = HashMap::from([ /// ("a", 1), /// ("b", 2), /// ("c", 3), /// ]); /// /// for value in map.pin().values() { /// println!("{value}"); /// } /// ``` #[inline] pub fn values<'g, G>(&self, guard: &'g G) -> Values<'g, K, V, G> where G: Guard, { Values { iter: self.iter(guard), } } } /// An operation to perform on given entry in a [`HashMap`]. /// /// See [`HashMap::compute`] for details. #[derive(Debug, PartialEq, Eq)] pub enum Operation { /// Insert the given value. Insert(V), /// Remove the entry from the map. Remove, /// Abort the operation with the given value. Abort(T), } /// The result of a [`compute`](HashMap::compute) operation. /// /// Contains information about the [`Operation`] that was performed. #[derive(Debug, PartialEq, Eq)] pub enum Compute<'g, K, V, T> { /// The given entry was inserted. Inserted(&'g K, &'g V), /// The entry was updated. Updated { /// The entry that was replaced. old: (&'g K, &'g V), /// The entry that was inserted. new: (&'g K, &'g V), }, /// The given entry was removed. Removed(&'g K, &'g V), /// The operation was aborted with the given value. Aborted(T), } /// An error returned by [`try_insert`](HashMap::try_insert) when the key already exists. /// /// Contains the existing value, and the value that was not inserted. #[derive(Debug, PartialEq, Eq)] pub struct OccupiedError<'a, V: 'a> { /// The value in the map that was already present. pub current: &'a V, /// The value which was not inserted, because the entry was already occupied. pub not_inserted: V, } impl PartialEq for HashMap where K: Hash + Eq, V: PartialEq, S: BuildHasher, { fn eq(&self, other: &Self) -> bool { if self.len() != other.len() { return false; } let (guard1, guard2) = (&self.guard(), &other.guard()); let mut iter = self.iter(guard1); iter.all(|(key, value)| other.get(key, guard2).is_some_and(|v| *value == *v)) } } impl Eq for HashMap where K: Hash + Eq, V: Eq, S: BuildHasher, { } impl fmt::Debug for HashMap where K: Hash + Eq + fmt::Debug, V: fmt::Debug, S: BuildHasher, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let guard = self.guard(); f.debug_map().entries(self.iter(&guard)).finish() } } impl Extend<(K, V)> for &HashMap where K: Hash + Eq, S: BuildHasher, { fn extend>(&mut self, iter: T) { // from `hashbrown::HashMap::extend`: // Keys may be already present or show multiple times in the iterator. // Reserve the entire hint lower bound if the map is empty. // Otherwise reserve half the hint (rounded up), so the map // will only resize twice in the worst case. let iter = iter.into_iter(); let reserve = if self.is_empty() { iter.size_hint().0 } else { (iter.size_hint().0 + 1) / 2 }; let guard = self.guard(); self.reserve(reserve, &guard); for (key, value) in iter { self.insert(key, value, &guard); } } } impl<'a, K, V, S> Extend<(&'a K, &'a V)> for &HashMap where K: Copy + Hash + Eq, V: Copy, S: BuildHasher, { fn extend>(&mut self, iter: T) { self.extend(iter.into_iter().map(|(&key, &value)| (key, value))); } } impl From<[(K, V); N]> for HashMap where K: Hash + Eq, { fn from(arr: [(K, V); N]) -> Self { HashMap::from_iter(arr) } } impl FromIterator<(K, V)> for HashMap where K: Hash + Eq, S: BuildHasher + Default, { fn from_iter>(iter: T) -> Self { let mut iter = iter.into_iter(); if let Some((key, value)) = iter.next() { let (lower, _) = iter.size_hint(); let map = HashMap::with_capacity_and_hasher(lower.saturating_add(1), S::default()); // Ideally we could use an unprotected guard here. However, `insert` // returns references to values that were replaced and retired, so // we need a "real" guard. A `raw_insert` method that strictly returns // pointers would fix this. { let map = map.pin(); map.insert(key, value); for (key, value) in iter { map.insert(key, value); } } map } else { Self::default() } } } impl Clone for HashMap where K: Clone + Hash + Eq, V: Clone, S: BuildHasher + Clone, { fn clone(&self) -> HashMap { let other = HashMap::builder() .capacity(self.len()) .hasher(self.raw.hasher.clone()) .collector(seize::Collector::new()) .build(); { let (guard1, guard2) = (&self.guard(), &other.guard()); for (key, value) in self.iter(guard1) { other.insert(key.clone(), value.clone(), guard2); } } other } } /// A pinned reference to a [`HashMap`]. /// /// This type is created with [`HashMap::pin`] and can be used to easily access a [`HashMap`] /// without explicitly managing a guard. See the [crate-level documentation](crate#usage) for details. pub struct HashMapRef<'map, K, V, S, G> { guard: MapGuard, map: &'map HashMap, } impl<'map, K, V, S, G> HashMapRef<'map, K, V, S, G> where K: Hash + Eq, S: BuildHasher, G: Guard, { /// Returns a reference to the inner [`HashMap`]. #[inline] pub fn map(&self) -> &'map HashMap { self.map } /// Returns the number of entries in the map. /// /// See [`HashMap::len`] for details. #[inline] pub fn len(&self) -> usize { self.map.raw.len() } /// Returns `true` if the map is empty. Otherwise returns `false`. /// /// See [`HashMap::is_empty`] for details. #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } /// Returns `true` if the map contains a value for the specified key. /// /// See [`HashMap::contains_key`] for details. #[inline] pub fn contains_key(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.get(key).is_some() } /// Returns a reference to the value corresponding to the key. /// /// See [`HashMap::get`] for details. #[inline] pub fn get(&self, key: &Q) -> Option<&V> where Q: Equivalent + Hash + ?Sized, { match self.map.raw.get(key, &self.guard) { Some((_, v)) => Some(v), None => None, } } /// Returns the key-value pair corresponding to the supplied key. /// /// See [`HashMap::get_key_value`] for details. #[inline] pub fn get_key_value(&self, key: &Q) -> Option<(&K, &V)> where Q: Equivalent + Hash + ?Sized, { self.map.raw.get(key, &self.guard) } /// Inserts a key-value pair into the map. /// /// See [`HashMap::insert`] for details. #[inline] pub fn insert(&self, key: K, value: V) -> Option<&V> { match self.map.raw.insert(key, value, true, &self.guard) { InsertResult::Inserted(_) => None, InsertResult::Replaced(value) => Some(value), InsertResult::Error { .. } => unreachable!(), } } /// Tries to insert a key-value pair into the map, and returns /// a reference to the value that was inserted. /// /// See [`HashMap::try_insert`] for details. #[inline] pub fn try_insert(&self, key: K, value: V) -> Result<&V, OccupiedError<'_, V>> { match self.map.raw.insert(key, value, false, &self.guard) { InsertResult::Inserted(value) => Ok(value), InsertResult::Error { current, not_inserted, } => Err(OccupiedError { current, not_inserted, }), InsertResult::Replaced(_) => unreachable!(), } } /// Tries to insert a key and value computed from a closure into the map, /// and returns a reference to the value that was inserted. /// /// See [`HashMap::try_insert_with`] for details. #[inline] pub fn try_insert_with(&self, key: K, f: F) -> Result<&V, &V> where F: FnOnce() -> V, { self.map.raw.try_insert_with(key, f, &self.guard) } /// Returns a reference to the value corresponding to the key, or inserts a default value. /// /// See [`HashMap::get_or_insert`] for details. #[inline] pub fn get_or_insert(&self, key: K, value: V) -> &V { // Note that we use `try_insert` instead of `compute` or `get_or_insert_with` here, as it // allows us to avoid the closure indirection. match self.try_insert(key, value) { Ok(inserted) => inserted, Err(OccupiedError { current, .. }) => current, } } /// Returns a reference to the value corresponding to the key, or inserts a default value /// computed from a closure. /// /// See [`HashMap::get_or_insert_with`] for details. #[inline] pub fn get_or_insert_with(&self, key: K, f: F) -> &V where F: FnOnce() -> V, { self.map.raw.get_or_insert_with(key, f, &self.guard) } /// Updates an existing entry atomically. /// /// See [`HashMap::update`] for details. #[inline] pub fn update(&self, key: K, update: F) -> Option<&V> where F: Fn(&V) -> V, { self.map.raw.update(key, update, &self.guard) } /// Updates an existing entry or inserts a default value. /// /// See [`HashMap::update_or_insert`] for details. #[inline] pub fn update_or_insert(&self, key: K, update: F, value: V) -> &V where F: Fn(&V) -> V, { self.update_or_insert_with(key, update, || value) } /// Updates an existing entry or inserts a default value computed from a closure. /// /// See [`HashMap::update_or_insert_with`] for details. #[inline] pub fn update_or_insert_with(&self, key: K, update: U, f: F) -> &V where F: FnOnce() -> V, U: Fn(&V) -> V, { self.map .raw .update_or_insert_with(key, update, f, &self.guard) } // Updates an entry with a compare-and-swap (CAS) function. // /// See [`HashMap::compute`] for details. #[inline] pub fn compute<'g, F, T>(&'g self, key: K, compute: F) -> Compute<'g, K, V, T> where F: FnMut(Option<(&'g K, &'g V)>) -> Operation, { self.map.raw.compute(key, compute, &self.guard) } /// Removes a key from the map, returning the value at the key if the key /// was previously in the map. /// /// See [`HashMap::remove`] for details. #[inline] pub fn remove(&self, key: &Q) -> Option<&V> where Q: Equivalent + Hash + ?Sized, { match self.map.raw.remove(key, &self.guard) { Some((_, value)) => Some(value), None => None, } } /// Removes a key from the map, returning the stored key and value if the /// key was previously in the map. /// /// See [`HashMap::remove_entry`] for details. #[inline] pub fn remove_entry(&self, key: &Q) -> Option<(&K, &V)> where Q: Equivalent + Hash + ?Sized, { self.map.raw.remove(key, &self.guard) } /// Conditionally removes a key from the map based on the provided closure. /// /// See [`HashMap::remove_if`] for details. #[inline] pub fn remove_if(&self, key: &Q, should_remove: F) -> Result, (&K, &V)> where Q: Equivalent + Hash + ?Sized, F: FnMut(&K, &V) -> bool, { self.map.raw.remove_if(key, should_remove, &self.guard) } /// Clears the map, removing all key-value pairs. /// /// See [`HashMap::clear`] for details. #[inline] pub fn clear(&self) { self.map.raw.clear(&self.guard) } /// Retains only the elements specified by the predicate. /// /// See [`HashMap::retain`] for details. #[inline] pub fn retain(&mut self, f: F) where F: FnMut(&K, &V) -> bool, { self.map.raw.retain(f, &self.guard) } /// Tries to reserve capacity for `additional` more elements to be inserted /// in the map. /// /// See [`HashMap::reserve`] for details. #[inline] pub fn reserve(&self, additional: usize) { self.map.raw.reserve(additional, &self.guard) } /// An iterator visiting all key-value pairs in arbitrary order. /// The iterator element type is `(&K, &V)`. /// /// See [`HashMap::iter`] for details. #[inline] pub fn iter(&self) -> Iter<'_, K, V, G> { Iter { raw: self.map.raw.iter(&self.guard), } } /// An iterator visiting all keys in arbitrary order. /// The iterator element type is `&K`. /// /// See [`HashMap::keys`] for details. #[inline] pub fn keys(&self) -> Keys<'_, K, V, G> { Keys { iter: self.iter() } } /// An iterator visiting all values in arbitrary order. /// The iterator element type is `&V`. /// /// See [`HashMap::values`] for details. #[inline] pub fn values(&self) -> Values<'_, K, V, G> { Values { iter: self.iter() } } } impl fmt::Debug for HashMapRef<'_, K, V, S, G> where K: Hash + Eq + fmt::Debug, V: fmt::Debug, S: BuildHasher, G: Guard, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_map().entries(self.iter()).finish() } } impl<'a, K, V, S, G> IntoIterator for &'a HashMapRef<'_, K, V, S, G> where K: Hash + Eq, S: BuildHasher, G: Guard, { type Item = (&'a K, &'a V); type IntoIter = Iter<'a, K, V, G>; fn into_iter(self) -> Self::IntoIter { self.iter() } } /// An iterator over a map's entries. /// /// This struct is created by the [`iter`](HashMap::iter) method on [`HashMap`]. See its documentation for details. pub struct Iter<'g, K, V, G> { raw: raw::Iter<'g, K, V, MapGuard>, } impl<'g, K: 'g, V: 'g, G> Iterator for Iter<'g, K, V, G> where G: Guard, { type Item = (&'g K, &'g V); #[inline] fn next(&mut self) -> Option { self.raw.next() } } impl fmt::Debug for Iter<'_, K, V, G> where K: fmt::Debug, V: fmt::Debug, G: Guard, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list() .entries(Iter { raw: self.raw.clone(), }) .finish() } } /// An iterator over a map's keys. /// /// This struct is created by the [`keys`](HashMap::keys) method on [`HashMap`]. See its documentation for details. pub struct Keys<'g, K, V, G> { iter: Iter<'g, K, V, G>, } impl<'g, K: 'g, V: 'g, G> Iterator for Keys<'g, K, V, G> where G: Guard, { type Item = &'g K; #[inline] fn next(&mut self) -> Option { let (key, _) = self.iter.next()?; Some(key) } } impl fmt::Debug for Keys<'_, K, V, G> where K: fmt::Debug, V: fmt::Debug, G: Guard, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("Keys").field(&self.iter).finish() } } /// An iterator over a map's values. /// /// This struct is created by the [`values`](HashMap::values) method on [`HashMap`]. See its documentation for details. pub struct Values<'g, K, V, G> { iter: Iter<'g, K, V, G>, } impl<'g, K: 'g, V: 'g, G> Iterator for Values<'g, K, V, G> where G: Guard, { type Item = &'g V; #[inline] fn next(&mut self) -> Option { let (_, value) = self.iter.next()?; Some(value) } } impl fmt::Debug for Values<'_, K, V, G> where K: fmt::Debug, V: fmt::Debug, G: Guard, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("Values").field(&self.iter).finish() } } papaya-0.2.3/src/raw/alloc.rs000064400000000000000000000153041046102023000141240ustar 00000000000000use std::alloc::Layout; use std::marker::PhantomData; use std::sync::atomic::{AtomicPtr, AtomicU8, Ordering}; use std::{alloc, mem, ptr}; use super::{probe, State}; // A hash-table laid out in a single allocation. // // Note that the `PhantomData` ensures that the hash-table is invariant // with respect to `T`, as this struct is stored behind an `AtomicPtr`. #[repr(transparent)] pub struct RawTable(u8, PhantomData); // The layout of the table allocation. #[repr(C)] struct TableLayout { /// A mask to get an index into the table from a hash. mask: usize, /// The maximum probe limit for this table. limit: usize, /// State for the table resize. state: State, /// An array of metadata for each entry. meta: [AtomicU8; 0], /// An array of entries. entries: [AtomicPtr; 0], } // Manages a table allocation. #[repr(C)] pub struct Table { /// A mask to get an index into the table from a hash. pub mask: usize, /// The maximum probe limit for this table. pub limit: usize, // The raw table allocation. // // Invariant: This pointer is initialized and valid for reads and writes. pub raw: *mut RawTable, } impl Copy for Table {} impl Clone for Table { fn clone(&self) -> Self { *self } } impl Table { // Allocate a table with the provided length and collector. pub fn alloc(len: usize) -> Table { assert!(len.is_power_of_two()); // Pad the meta table to fulfill the alignment requirement of an entry. let len = len.max(mem::align_of::>()); let mask = len - 1; let limit = probe::limit(len); let layout = Table::::layout(len); // Allocate the table, zeroing the entries. // // Safety: The layout for is guaranteed to be non-zero. let ptr = unsafe { alloc::alloc_zeroed(layout) }; if ptr.is_null() { alloc::handle_alloc_error(layout); } // Safety: We just allocated the pointer and ensured it is non-null above. unsafe { // Write the table state. ptr.cast::>().write(TableLayout { mask, limit, meta: [], entries: [], state: State::default(), }); // Initialize the meta table. ptr.add(mem::size_of::>()) .cast::() .write_bytes(super::meta::EMPTY, len); } Table { mask, limit, // Invariant: We allocated and initialized the allocation above. raw: ptr.cast::>(), } } // Creates a `Table` from a raw pointer. // // # Safety // // The pointer must either be null, or a valid pointer created with `Table::alloc`. #[inline] pub unsafe fn from_raw(raw: *mut RawTable) -> Table { if raw.is_null() { return Table { raw, mask: 0, limit: 0, }; } // Safety: The caller guarantees that the pointer is valid. let layout = unsafe { &*raw.cast::>() }; Table { raw, mask: layout.mask, limit: layout.limit, } } // Returns the metadata entry at the given index. // // # Safety // // The index must be in-bounds for the length of the table. #[inline] pub unsafe fn meta(&self, i: usize) -> &AtomicU8 { debug_assert!(i < self.len()); // Safety: The caller guarantees the index is in-bounds. unsafe { let meta = self.raw.add(mem::size_of::>()); &*meta.cast::().add(i) } } // Returns the entry at the given index. // // # Safety // // The index must be in-bounds for the length of the table. #[inline] pub unsafe fn entry(&self, i: usize) -> &AtomicPtr { debug_assert!(i < self.len()); // Safety: The caller guarantees the index is in-bounds. unsafe { let meta = self.raw.add(mem::size_of::>()); let entries = meta.add(self.len()).cast::>(); &*entries.add(i) } } /// Returns the length of the table. #[inline] pub fn len(&self) -> usize { self.mask + 1 } // Returns a reference to the table state. #[inline] pub fn state(&self) -> &State { // Safety: The raw table pointer is always valid for reads and writes. unsafe { &(*self.raw.cast::>()).state } } // Returns a mutable reference to the table state. #[inline] pub fn state_mut(&mut self) -> &mut State { // Safety: The raw table pointer is always valid for reads and writes. unsafe { &mut (*self.raw.cast::>()).state } } // Returns a pointer to the next table, if it has already been created. #[inline] pub fn next_table(&self) -> Option { let next = self.state().next.load(Ordering::Acquire); if !next.is_null() { // Safety: We verified that the pointer is non-null, and the // next pointer is otherwise a valid pointer to a table allocation. return unsafe { Some(Table::from_raw(next)) }; } None } // Deallocate the table. // // # Safety // // The table may not be accessed in any way after this method is // called. pub unsafe fn dealloc(table: Table) { let layout = Self::layout(table.len()); // Safety: The raw table pointer is valid and allocated with `alloc::alloc_zeroed`. // Additionally, the caller guarantees that the allocation will not be accessed after // this point. unsafe { ptr::drop_in_place(table.raw.cast::>()); alloc::dealloc(table.raw.cast::(), layout); }; } // Returns the non-zero layout for a table allocation. fn layout(len: usize) -> Layout { let size = mem::size_of::>() + (mem::size_of::() * len) // Metadata table. + (mem::size_of::>() * len); // Entry pointers. // Layout::from_size_align(size, mem::align_of::>()).unwrap() } } #[test] fn layout() { unsafe { let table: Table = Table::alloc(4); let table: Table = Table::from_raw(table.raw); // The capacity is padded for pointer alignment. assert_eq!(table.mask, 7); assert_eq!(table.len(), 8); Table::dealloc(table); } } papaya-0.2.3/src/raw/mod.rs000064400000000000000000003106421046102023000136140ustar 00000000000000mod alloc; mod probe; pub(crate) mod utils; use std::hash::{BuildHasher, Hash}; use std::mem::MaybeUninit; use std::sync::atomic::{AtomicPtr, AtomicU8, AtomicUsize, Ordering}; use std::sync::Mutex; use std::{hint, panic, ptr}; use self::alloc::{RawTable, Table}; use self::probe::Probe; use self::utils::{untagged, AtomicPtrFetchOps, Counter, Parker, StrictProvenance, Tagged}; use crate::map::{Compute, Operation, ResizeMode}; use crate::Equivalent; use seize::{Collector, LocalGuard, OwnedGuard}; use utils::{MapGuard, Stack, VerifiedGuard}; /// A lock-free hash-table. pub struct HashMap { /// A pointer to the root table. table: AtomicPtr>>, /// Collector for memory reclamation. collector: Collector, /// The resize mode, either blocking or incremental. resize: ResizeMode, /// An atomic counter of the number of keys in the table. count: Counter, /// The initial capacity provided to `HashMap::new`. /// /// The table is guaranteed to never shrink below this capacity. initial_capacity: usize, /// Hasher for keys. pub hasher: S, } /// Resize state for the hash-table. pub struct State { /// The next table used for resizing. pub next: AtomicPtr>, /// A lock acquired to allocate the next table. pub allocating: Mutex<()>, /// The number of entries that have been copied to the next table. pub copied: AtomicUsize, /// The number of entries that have been claimed by copiers, /// but not necessarily copied. pub claim: AtomicUsize, /// The status of the resize. pub status: AtomicU8, /// A thread parker for blocking on copy operations. pub parker: Parker, /// Entries whose retirement has been deferred by later tables. pub deferred: Stack<*mut T>, } impl Default for State { fn default() -> State { State { next: AtomicPtr::new(ptr::null_mut()), allocating: Mutex::new(()), copied: AtomicUsize::new(0), claim: AtomicUsize::new(0), status: AtomicU8::new(State::PENDING), parker: Parker::default(), deferred: Stack::new(), } } } impl State<()> { /// A resize is in-progress. pub const PENDING: u8 = 0; /// The resize has been aborted, continue to the next table. pub const ABORTED: u8 = 1; /// The resize was complete and the table was promoted. pub const PROMOTED: u8 = 2; } // The result of an insert operation. pub enum InsertResult<'g, V> { /// Inserted the given value. Inserted(&'g V), /// Replaced the given value. Replaced(&'g V), /// Error returned by `try_insert`. Error { current: &'g V, not_inserted: V }, } // The raw result of an insert operation. pub enum RawInsertResult<'g, K, V> { /// Inserted the given value. Inserted(&'g V), /// Replaced the given value. Replaced(&'g V), /// Error returned by `try_insert`. Error { current: &'g V, not_inserted: *mut Entry, }, } // An entry in the hash-table. #[repr(C, align(8))] // Reserve the lower 3 bits for pointer tagging. pub struct Entry { /// The key for this entry. pub key: K, /// The value for this entry. pub value: V, } impl Entry<(), ()> { /// The entry is being copied to the new table, no updates are allowed on the old table. /// /// This bit is put down to initiate a copy, forcing all writers to complete the resize /// before making progress. const COPYING: usize = 0b001; /// The entry has been copied to the new table. /// /// This bit is put down after a copy completes. Both readers and writers must go to /// the new table to see the new state of the entry. /// /// In blocking mode this is unused. const COPIED: usize = 0b010; /// The entry was copied from a previous table. /// /// This bit indicates that an entry may still be accessible from previous tables /// because the resize is still in progress, and so it is unsafe to reclaim. /// /// In blocking mode this is unused. const BORROWED: usize = 0b100; } impl utils::Unpack for Entry { /// Mask for an entry pointer, ignoring any tag bits. const MASK: usize = !(Entry::COPYING | Entry::COPIED | Entry::BORROWED); } impl Entry { /// A sentinel pointer for a deleted entry. /// /// Null pointers are never copied to the new table, so this state is safe to use. /// Note that tombstone entries may still be marked as `COPYING`, so this state /// cannot be used for direct equality. const TOMBSTONE: *mut Entry = Entry::COPIED as _; } /// The status of an entry. enum EntryStatus { /// The entry is a tombstone or null (potentially a null copy). Null, /// The entry is being copied. Copied(Tagged>), /// A valid entry. Value(Tagged>), } impl From>> for EntryStatus { /// Returns the status for this entry. #[inline] fn from(entry: Tagged>) -> Self { if entry.ptr.is_null() { EntryStatus::Null } else if entry.tag() & Entry::COPYING != 0 { EntryStatus::Copied(entry) } else { EntryStatus::Value(entry) } } } /// The state of an entry we attempted to update. enum UpdateStatus { /// Successfully replaced the given key and value. Replaced(Tagged>), /// A new entry was written before we could update. Found(EntryStatus), } /// The state of an entry we attempted to insert into. enum InsertStatus { /// Successfully inserted the value. Inserted, /// A new entry was written before we could update. Found(EntryStatus), } impl HashMap { /// Creates new hash-table with the given options. #[inline] pub fn new( capacity: usize, hasher: S, collector: Collector, resize: ResizeMode, ) -> HashMap { // The table is lazily allocated. if capacity == 0 { return HashMap { collector, resize, hasher, initial_capacity: 1, table: AtomicPtr::new(ptr::null_mut()), count: Counter::default(), }; } // Initialize the table and mark it as the root. let mut table = Table::alloc(probe::entries_for(capacity)); *table.state_mut().status.get_mut() = State::PROMOTED; HashMap { hasher, resize, collector, initial_capacity: capacity, table: AtomicPtr::new(table.raw), count: Counter::default(), } } /// Returns a guard for this collector pub fn guard(&self) -> MapGuard> { // Safety: Created the guard from our collector. unsafe { MapGuard::new(self.collector().enter()) } } /// Returns an owned guard for this collector pub fn owned_guard(&self) -> MapGuard> { // Safety: Created the guard from our collector. unsafe { MapGuard::new(self.collector().enter_owned()) } } /// Verify a guard is valid to use with this map. #[inline] pub fn verify<'g, G>(&self, guard: &'g G) -> &'g MapGuard where G: seize::Guard, { assert_eq!( *guard.collector(), self.collector, "Attempted to access map with incorrect guard" ); // Safety: Verified the guard above. unsafe { MapGuard::from_ref(guard) } } /// Returns a reference to the root hash-table. #[inline] fn root(&self, guard: &impl VerifiedGuard) -> Table> { // Load the root table. let raw = guard.protect(&self.table, Ordering::Acquire); // Safety: The root table is either null or a valid table allocation. unsafe { Table::from_raw(raw) } } /// Returns a reference to the collector. #[inline] pub fn collector(&self) -> &Collector { &self.collector } /// Returns the number of entries in the table. #[inline] pub fn len(&self) -> usize { self.count.sum() } /// Returns true if incremental resizing is enabled. #[inline] fn is_incremental(&self) -> bool { matches!(self.resize, ResizeMode::Incremental(_)) } } impl HashMap where K: Hash + Eq, S: BuildHasher, { /// Returns a reference to the entry corresponding to the key. #[inline] pub fn get<'g, Q>(&self, key: &Q, guard: &'g impl VerifiedGuard) -> Option<(&'g K, &'g V)> where Q: Equivalent + Hash + ?Sized, { // Load the root table. let mut table = self.root(guard); // The table has not been initialized yet. if table.raw.is_null() { return None; } let (h1, h2) = self.hash(key); loop { // Initialize the probe state. let mut probe = Probe::start(h1, table.mask); // Probe until we reach the limit. 'probe: while probe.len <= table.limit { // Load the entry metadata first for cheap searches. // // Safety: `probe.i` is always in-bounds for the table length. let meta = unsafe { table.meta(probe.i) }.load(Ordering::Acquire); if meta == h2 { // Load the full entry. // // Safety: `probe.i` is always in-bounds for the table length. let entry = guard .protect(unsafe { table.entry(probe.i) }, Ordering::Acquire) .unpack(); // The entry was deleted, keep probing. if entry.ptr.is_null() { probe.next(table.mask); continue 'probe; } // Safety: We performed a protected load of the pointer using a verified guard with // `Acquire` and ensured that it is non-null, meaning it is valid for reads as long // as we hold the guard. let entry_ref = unsafe { &(*entry.ptr) }; // Check for a full match. if key.equivalent(&entry_ref.key) { // The entry was copied to the new table. // // In blocking resize mode we do not need to perform self check as all writes block // until any resizes are complete, making the root table the source of truth for readers. if entry.tag() & Entry::COPIED != 0 { break 'probe; } // Found the correct entry, return the key and value. return Some((&entry_ref.key, &entry_ref.value)); } } // The key is not in the table. // // It also cannot be in the next table because we have not went over the probe limit. if meta == meta::EMPTY { return None; } probe.next(table.mask); } // In incremental resize mode, we have to check the next table if we found // a copied entry or went over the probe limit. if self.is_incremental() { if let Some(next) = table.next_table() { table = next; continue; } } // Otherwise, the key is not in the table. return None; } } /// Inserts a key-value pair into the table. #[inline] pub fn insert<'g>( &self, key: K, value: V, replace: bool, guard: &'g impl VerifiedGuard, ) -> InsertResult<'g, V> { // Perform the insert. let raw_result = self.insert_inner(key, value, replace, guard); let result = match raw_result { // Updated an entry. RawInsertResult::Replaced(value) => InsertResult::Replaced(value), // Inserted a new entry. RawInsertResult::Inserted(value) => { // Increment the table length. self.count.get(guard).fetch_add(1, Ordering::Relaxed); InsertResult::Inserted(value) } // Failed to insert the entry. RawInsertResult::Error { current, not_inserted, } => { // Safety: We allocated this box above and it was not inserted into the table. let not_inserted = unsafe { Box::from_raw(not_inserted) }; InsertResult::Error { current, not_inserted: not_inserted.value, } } }; result } /// Inserts an entry into the map. #[inline] fn insert_inner<'g>( &self, key: K, value: V, should_replace: bool, guard: &'g impl VerifiedGuard, ) -> RawInsertResult<'g, K, V> { // Allocate the entry to be inserted. let new_entry = untagged(Box::into_raw(Box::new(Entry { key, value }))); // Safety: We just allocated the entry above. let new_ref = unsafe { &(*new_entry.ptr) }; // Load the root table. let mut table = self.root(guard); // Allocate the table if it has not been initialized yet. if table.raw.is_null() { table = self.init(None); } let (h1, h2) = self.hash(&new_ref.key); let mut help_copy = true; loop { // Initialize the probe state. let mut probe = Probe::start(h1, table.mask); // Probe until we reach the limit. let copying = 'probe: loop { if probe.len > table.limit { break None; } // Load the entry metadata first for cheap searches. // // Safety: `probe.i` is always in-bounds for the table length. let meta = unsafe { table.meta(probe.i) }.load(Ordering::Acquire); // The entry is empty, try to insert. let entry = if meta == meta::EMPTY { // Perform the insertion. // // Safety: `probe.i` is always in-bounds for the table length. Additionally, // `new_entry` was allocated above and never shared. match unsafe { self.insert_at(probe.i, h2, new_entry.raw, table, guard) } { // Successfully inserted. InsertStatus::Inserted => return RawInsertResult::Inserted(&new_ref.value), // Lost to a concurrent insert. // // If the key matches, we might be able to update the value. InsertStatus::Found(EntryStatus::Value(found)) | InsertStatus::Found(EntryStatus::Copied(found)) => found, // Otherwise, continue probing. InsertStatus::Found(EntryStatus::Null) => { probe.next(table.mask); continue 'probe; } } } // Found a potential match. else if meta == h2 { // Load the full entry. // // Safety: `probe.i` is always in-bounds for the table length. let entry = guard .protect(unsafe { table.entry(probe.i) }, Ordering::Acquire) .unpack(); // The entry was deleted, keep probing. if entry.ptr.is_null() { probe.next(table.mask); continue 'probe; } // If the key matches, we might be able to update the value. entry } // Otherwise, continue probing. else { probe.next(table.mask); continue 'probe; }; // Safety: We performed a protected load of the pointer using a verified guard with // `Acquire` and ensured that it is non-null, meaning it is valid for reads as long // as we hold the guard. let entry_ref = unsafe { &(*entry.ptr) }; // Check for a full match. if entry_ref.key != new_ref.key { probe.next(table.mask); continue 'probe; } // The entry is being copied to the new table. if entry.tag() & Entry::COPYING != 0 { break 'probe Some(probe.i); } // Return an error for calls to `try_insert`. if !should_replace { return RawInsertResult::Error { current: &entry_ref.value, not_inserted: new_entry.ptr, }; } // Try to update the value. // // Safety: // - `probe.i` is always in-bounds for the table length // - `entry` is a valid non-null entry that was inserted into the map. match unsafe { self.insert_slow(probe.i, entry, new_entry.raw, table, guard) } { // Successfully performed the update. UpdateStatus::Replaced(entry) => { // Safety: `entry` is a valid non-null entry that we found in the map // before replacing it. let value = unsafe { &(*entry.ptr).value }; return RawInsertResult::Replaced(value); } // The entry is being copied. UpdateStatus::Found(EntryStatus::Copied(_)) => break 'probe Some(probe.i), // The entry was deleted before we could update it, continue probing. UpdateStatus::Found(EntryStatus::Null) => { probe.next(table.mask); continue 'probe; } UpdateStatus::Found(EntryStatus::Value(_)) => {} } }; // Prepare to retry in the next table. table = self.prepare_retry_insert(copying, &mut help_copy, table, guard); } } /// The slow-path for `insert`, updating the value. /// /// The returned pointer is guaranteed to be non-null and valid for reads. /// /// # Safety /// /// The safety requirements of `HashMap::update_at` apply. #[cold] #[inline(never)] unsafe fn insert_slow( &self, i: usize, mut entry: Tagged>, new_entry: *mut Entry, table: Table>, guard: &impl VerifiedGuard, ) -> UpdateStatus { loop { // Try to update the value. // // Safety: Guaranteed by caller. match unsafe { self.update_at(i, entry, new_entry, table, guard) } { // Someone else beat us to the update, retry. // // Note that the pointer we find here is a non-null entry that was inserted // into the map. UpdateStatus::Found(EntryStatus::Value(found)) => entry = found, status => return status, } } } /// Prepare to retry an insert operation in the next table. #[cold] #[inline(never)] fn prepare_retry_insert( &self, copying: Option, help_copy: &mut bool, table: Table>, guard: &impl VerifiedGuard, ) -> Table> { // If went over the probe limit or found a copied entry, trigger a resize. let mut next_table = self.get_or_alloc_next(None, table); let next_table = match self.resize { // In blocking mode we must complete the resize before proceeding. ResizeMode::Blocking => self.help_copy(true, &table, guard), // In incremental mode we can perform more granular blocking. ResizeMode::Incremental(_) => { // Help out with the copy. if *help_copy { next_table = self.help_copy(false, &table, guard); } // The entry we want to update is being copied. if let Some(i) = copying { // Wait for the entry to be copied. // // We could race with the copy to insert into the table. However, // this entire code path is very rare and likely to complete quickly, // so blocking allows us to make copies faster. self.wait_copied(i, &table); } next_table } }; // Limit incremental copying to once per operation, for more consistent latency. *help_copy = false; // Continue in the new table. next_table } /// Removes a key from the map, returning the entry for the key if the key was previously in the map. #[inline] pub fn remove<'g, Q>(&self, key: &Q, guard: &'g impl VerifiedGuard) -> Option<(&'g K, &'g V)> where Q: Equivalent + Hash + ?Sized, { #[inline(always)] fn should_remove(_key: &K, _value: &V) -> bool { true } // Safety: `should_remove` unconditionally returns `true`. unsafe { self.remove_if(key, should_remove, guard).unwrap_unchecked() } } /// Removes a key from the map, returning the entry for the key if the key was previously in the map /// and the provided closure returns `true` #[inline] pub fn remove_if<'g, Q, F>( &self, key: &Q, mut should_remove: F, guard: &'g impl VerifiedGuard, ) -> Result, (&'g K, &'g V)> where Q: Equivalent + Hash + ?Sized, F: FnMut(&K, &V) -> bool, { // Load the root table. let mut table = self.root(guard); // The table has not been initialized yet. if table.raw.is_null() { return Ok(None); } let (h1, h2) = self.hash(key); let mut help_copy = true; loop { // Initialize the probe state. let mut probe = Probe::start(h1, table.mask); // Probe until we reach the limit. let copying = 'probe: loop { if probe.len > table.limit { break None; } // Load the entry metadata first for cheap searches. // // Safety: `probe.i` is always in-bounds for the table length. let meta = unsafe { table.meta(probe.i).load(Ordering::Acquire) }; // The key is not in the table. // It also cannot be in the next table because we have not went over the probe limit. if meta == meta::EMPTY { return Ok(None); } // Check for a potential match. if meta != h2 { probe.next(table.mask); continue 'probe; } // Load the full entry. // // Safety: `probe.i` is always in-bounds for the table length. let mut entry = guard .protect(unsafe { table.entry(probe.i) }, Ordering::Acquire) .unpack(); // The entry was deleted, keep probing. if entry.ptr.is_null() { probe.next(table.mask); continue 'probe; } // Check for a full match. // // Safety: We performed a protected load of the pointer using a verified guard with // `Acquire` and ensured that it is non-null, meaning it is valid for reads as long // as we hold the guard. if !key.equivalent(unsafe { &(*entry.ptr).key }) { probe.next(table.mask); continue 'probe; } // The entry is being copied to the new table, we have to complete the copy before // we can remove it. if entry.tag() & Entry::COPYING != 0 { break 'probe Some(probe.i); } loop { // Safety: `entry` is a valid, non-null, protected entry that we found in the map. let entry_ref = unsafe { &(*entry.ptr) }; // Ensure that the entry should be removed. if !should_remove(&entry_ref.key, &entry_ref.value) { return Err((&entry_ref.key, &entry_ref.value)); } // Safety: // - `probe.i` is always in-bounds for the table length // - `entry` is a valid non-null entry that we found in the map. let status = unsafe { self.update_at(probe.i, entry, Entry::TOMBSTONE, table, guard) }; match status { // Successfully removed the entry. UpdateStatus::Replaced(_entry) => { // Mark the entry as a tombstone. // // Note that this might end up being overwritten by the metadata hash // if the initial insertion is lagging behind, but we avoid the RMW // and sacrifice reads in the extremely rare case. // // Safety: `probe.i` is always in-bounds for the table length. unsafe { table .meta(probe.i) .store(meta::TOMBSTONE, Ordering::Release) }; // Decrement the table length. self.count.get(guard).fetch_sub(1, Ordering::Relaxed); // Note that `entry_ref` here is the entry that we just replaced. return Ok(Some((&entry_ref.key, &entry_ref.value))); } // The entry is being copied to the new table, we have to complete the copy // before we can remove. UpdateStatus::Found(EntryStatus::Copied(_)) => break 'probe Some(probe.i), // The entry was deleted. // // We know that at some point during our execution the key was not in the map. UpdateStatus::Found(EntryStatus::Null) => return Ok(None), // Lost to a concurrent update, retry. UpdateStatus::Found(EntryStatus::Value(found)) => entry = found, } } }; // Prepare to retry in the next table. table = match self.prepare_retry(copying, &mut help_copy, table, guard) { Some(table) => table, // The search was exhausted. None => return Ok(None), } } } /// Prepare to retry an operation on an existing key in the next table. /// /// Returns `None` if the recursive search has been exhausted. #[cold] fn prepare_retry( &self, copying: Option, help_copy: &mut bool, table: Table>, guard: &impl VerifiedGuard, ) -> Option>> { let next_table = match self.resize { ResizeMode::Blocking => match copying { // The entry we want to perform the operation on is being copied. // // In blocking mode we must complete the resize before proceeding. Some(_) => self.help_copy(true, &table, guard), // If we went over the probe limit, the key is not in the map. None => return None, }, ResizeMode::Incremental(_) => { // In incremental resize mode, we always have to check the next table. let next_table = table.next_table()?; // Help out with the copy. if *help_copy { self.help_copy(false, &table, guard); } if let Some(i) = copying { // Wait for the entry to be copied. // // We could race with the copy to insert into the table. However, // this entire code path is very rare and likely to complete quickly, // so blocking allows us to make copies faster. self.wait_copied(i, &table); } next_table } }; // Limit incremental copying to once per operation, for more consistent latency. *help_copy = false; // Continue in the new table. Some(next_table) } /// Attempts to insert an entry at the given index. /// /// In the case of an error, the returned pointer is guaranteed to be /// protected and valid for reads as long as the guard is held. /// /// # Safety /// /// The index must be in-bounds for the table. Additionally, `new_entry` must be a /// valid owned pointer to insert into the map. #[inline] unsafe fn insert_at( &self, i: usize, meta: u8, new_entry: *mut Entry, table: Table>, guard: &impl VerifiedGuard, ) -> InsertStatus { // Safety: The caller guarantees that `i` is in-bounds. let entry = unsafe { table.entry(i) }; let meta_entry = unsafe { table.meta(i) }; // Try to claim the empty entry. let found = match guard.compare_exchange( entry, ptr::null_mut(), new_entry, Ordering::Release, Ordering::Acquire, ) { // Successfully claimed the entry. Ok(_) => { // Update the metadata table. meta_entry.store(meta, Ordering::Release); // Return the value we inserted. return InsertStatus::Inserted; } // Lost to a concurrent update. Err(found) => found.unpack(), }; let (meta, status) = match EntryStatus::from(found) { EntryStatus::Value(_) | EntryStatus::Copied(_) => { // Safety: We performed a protected load of the pointer using a verified guard // with `Acquire` and ensured that it is non-null, meaning it is valid for reads // as long as we hold the guard. let key = unsafe { &(*found.ptr).key }; // An entry was inserted, we have to hash it to get the metadata. // // The logic is the same for copied entries here as we have to // check if the key matches and continue the update in the new table. let hash = self.hasher.hash_one(key); (meta::h2(hash), EntryStatus::Value(found)) } // The entry was deleted or null copied. EntryStatus::Null => (meta::TOMBSTONE, EntryStatus::Null), }; // Ensure the meta table is updated to keep the probe chain alive for readers. if meta_entry.load(Ordering::Relaxed) == meta::EMPTY { meta_entry.store(meta, Ordering::Release); } InsertStatus::Found(status) } /// Attempts to replace the value of an existing entry at the given index. /// /// In the case of an error, the returned pointer is guaranteed to be /// protected and valid for reads. /// /// # Safety /// /// - The index must be in-bounds for the table. /// - `current` must be a valid non-null entry that was inserted into the map. /// - `new_entry` must be a valid sentinel or owned pointer to insert into the map. #[inline] unsafe fn update_at( &self, i: usize, current: Tagged>, new_entry: *mut Entry, table: Table>, guard: &impl VerifiedGuard, ) -> UpdateStatus { // Safety: The caller guarantees that `i` is in-bounds. let entry = unsafe { table.entry(i) }; // Try to perform the update. let found = match guard.compare_exchange_weak( entry, current.raw, new_entry, Ordering::Release, Ordering::Acquire, ) { // Successfully updated. Ok(_) => unsafe { // Safety: The caller guarantees that `current` is a valid non-null entry that was // inserted into the map. Additionally, it is now unreachable from this table due // to the CAS above. self.defer_retire(current, &table, guard); return UpdateStatus::Replaced(current); }, // Lost to a concurrent update. Err(found) => found.unpack(), }; UpdateStatus::Found(EntryStatus::from(found)) } /// Reserve capacity for `additional` more elements. #[inline] pub fn reserve(&self, additional: usize, guard: &impl VerifiedGuard) { let mut table = self.root(guard); // The table has not yet been allocated, initialize it. if table.raw.is_null() { table = self.init(Some(probe::entries_for(additional))); } loop { let capacity = probe::entries_for(self.count.sum().checked_add(additional).unwrap()); // We have enough capacity. if table.len() >= capacity { return; } // Race to allocate the new table. self.get_or_alloc_next(Some(capacity), table); // Force the copy to complete. // // Note that this is not strictly necessary for a `reserve` operation. table = self.help_copy(true, &table, guard); } } /// Remove all entries from this table. #[inline] pub fn clear(&self, guard: &impl VerifiedGuard) { // Load the root table. let mut table = self.root(guard); // The table has not been initialized yet. if table.raw.is_null() { return; } loop { // Get a clean copy of the table to delete from. table = self.linearize(table, guard); // Note that this method is not implemented in terms of `retain(|_, _| true)` to avoid // loading entry metadata, as there is no need to provide consistency with `get`. let mut copying = false; 'probe: for i in 0..table.len() { // Load the entry to delete. // // Safety: `i` is in bounds for the table length. let mut entry = guard .protect(unsafe { table.entry(i) }, Ordering::Acquire) .unpack(); loop { // The entry is empty or already deleted. if entry.ptr.is_null() { continue 'probe; } // Found a non-empty entry being copied. if entry.tag() & Entry::COPYING != 0 { // Clear every entry in this table that we can, then deal with the copy. copying = true; continue 'probe; } // Try to delete the entry. // // Safety: `i` is in bounds for the table length. let result = unsafe { table.entry(i).compare_exchange( entry.raw, Entry::TOMBSTONE, Ordering::Release, Ordering::Acquire, ) }; match result { // Successfully deleted the entry. Ok(_) => { // Update the metadata table. // // Safety: `i` is in bounds for the table length. unsafe { table.meta(i).store(meta::TOMBSTONE, Ordering::Release) }; // Decrement the table length. self.count.get(guard).fetch_sub(1, Ordering::Relaxed); // Safety: The caller guarantees that `current` is a valid non-null entry that was // inserted into the map. Additionally, it is now unreachable from this table due // to the CAS above. unsafe { self.defer_retire(entry, &table, guard) }; continue 'probe; } // Lost to a concurrent update, retry. Err(found) => entry = found.unpack(), } } } // We cleared every entry in this table. if !copying { break; } // A resize prevented us from deleting all the entries in this table. // // Complete the resize and retry in the new table. table = self.help_copy(true, &table, guard); } } /// Retains only the elements specified by the predicate. #[inline] pub fn retain(&self, mut f: F, guard: &impl VerifiedGuard) where F: FnMut(&K, &V) -> bool, { // Load the root table. let mut table = self.root(guard); // The table has not been initialized yet. if table.raw.is_null() { return; } loop { // Get a clean copy of the table to delete from. table = self.linearize(table, guard); let mut copying = false; 'probe: for i in 0..table.len() { // Load the entry metadata first to ensure consistency with calls to `get` // for entries that are retained. // // Safety: `i` is in bounds for the table length. let meta = unsafe { table.meta(i) }.load(Ordering::Acquire); // The entry is empty or deleted. if matches!(meta, meta::EMPTY | meta::TOMBSTONE) { continue 'probe; } // Load the entry to delete. // // Safety: `i` is in bounds for the table length. let mut entry = guard .protect(unsafe { table.entry(i) }, Ordering::Acquire) .unpack(); loop { // The entry is empty or already deleted. if entry.ptr.is_null() { continue 'probe; } // Found a non-empty entry being copied. if entry.tag() & Entry::COPYING != 0 { // Clear every entry in this table that we can, then deal with the copy. copying = true; continue 'probe; } // Safety: We performed a protected load of the pointer using a verified guard with // `Acquire` and ensured that it is non-null, meaning it is valid for reads as long // as we hold the guard. let entry_ref = unsafe { &*entry.ptr }; // Should we retain this entry? if f(&entry_ref.key, &entry_ref.value) { continue 'probe; } // Try to delete the entry. // // Safety: `i` is in bounds for the table length. let result = unsafe { table.entry(i).compare_exchange( entry.raw, Entry::TOMBSTONE, Ordering::Release, Ordering::Acquire, ) }; match result { // Successfully deleted the entry. Ok(_) => { // Update the metadata table. // // Safety: `i` is in bounds for the table length. unsafe { table.meta(i).store(meta::TOMBSTONE, Ordering::Release) }; // Decrement the table length. self.count.get(guard).fetch_sub(1, Ordering::Relaxed); // Safety: The caller guarantees that `current` is a valid non-null entry that was // inserted into the map. Additionally, it is now unreachable from this table due // to the CAS above. unsafe { self.defer_retire(entry, &table, guard) }; continue 'probe; } // Lost to a concurrent update, retry. Err(found) => entry = found.unpack(), } } } // We cleared every entry in this table. if !copying { break; } // A resize prevented us from deleting all the entries in this table. // // Complete the resize and retry in the new table. table = self.help_copy(true, &table, guard); } } /// Returns an iterator over the keys and values of this table. #[inline] pub fn iter<'g, G>(&self, guard: &'g G) -> Iter<'g, K, V, G> where G: VerifiedGuard, { // Load the root table. let root = self.root(guard); // The table has not been initialized yet, return a dummy iterator. if root.raw.is_null() { return Iter { i: 0, guard, table: root, }; } // Get a clean copy of the table to iterate over. let table = self.linearize(root, guard); Iter { i: 0, guard, table } } /// Returns the h1 and h2 hash for the given key. #[inline] fn hash(&self, key: &Q) -> (usize, u8) where Q: Hash + ?Sized, { let hash = self.hasher.hash_one(key); (meta::h1(hash), meta::h2(hash)) } } /// A wrapper around a CAS function that manages the computed state. struct ComputeState { /// The CAS function. compute: F, /// A cached insert transition. insert: Option, /// A cached update transition. update: Option>, } /// A cached update transition. struct CachedUpdate { /// The entry that the CAS function was called with. input: *mut Entry, /// The cached result. output: Operation, } impl<'g, F, K, V, T> ComputeState where F: FnMut(Option<(&'g K, &'g V)>) -> Operation, K: 'g, V: 'g, { /// Create a new `ComputeState` for the given function. #[inline] fn new(compute: F) -> ComputeState { ComputeState { compute, insert: None, update: None, } } /// Performs a state transition. /// /// # Safety /// /// The entry pointer must be valid for reads if provided. #[inline] unsafe fn next(&mut self, entry: Option<*mut Entry>) -> Operation { let Some(entry) = entry else { // If there is no current entry, perform a transition for the insert. return match self.insert.take() { // Use the cached insert. Some(value) => Operation::Insert(value), // Otherwise, compute the value to insert. None => (self.compute)(None), }; }; // Otherwise, perform an update transition. match self.update.take() { // Used the cached update if the entry has not changed. Some(CachedUpdate { input, output }) if input == entry => output, // Otherwise, compute the value to update. _ => { // Safety: The caller guarantees that `entry` is valid for reads. let entry_ref = unsafe { &*entry }; (self.compute)(Some((&entry_ref.key, &entry_ref.value))) } } } /// Restores the state if an operation fails. /// /// This allows the result of the compute closure with a given input to be cached. /// This is useful at it avoids calling the closure multiple times if an update needs /// to be retried in a new table. /// /// Additionally, update and insert operations are cached separately, although this /// is not guaranteed in the public API. This means that internal methods can rely on /// `compute(None)` being called at most once. #[inline] fn restore(&mut self, input: Option<*mut Entry>, output: Operation) { match input { Some(input) => self.update = Some(CachedUpdate { input, output }), None => match output { Operation::Insert(value) => self.insert = Some(value), _ => unreachable!(), }, } } } /// A lazy initialized `Entry` allocation. enum LazyEntry { /// An uninitialized entry, containing just the owned key. Uninit(K), /// An allocated entry. Init(*mut Entry>), } impl LazyEntry { /// Returns a reference to the entry's key. #[inline] fn key(&self) -> &K { match self { LazyEntry::Uninit(key) => key, LazyEntry::Init(entry) => unsafe { &(**entry).key }, } } /// Initializes the entry if it has not already been initialized, returning the pointer /// to the entry allocation. #[inline] fn init(&mut self) -> *mut Entry> { match self { LazyEntry::Init(entry) => *entry, LazyEntry::Uninit(key) => { // Safety: we read the current key with `ptr::read` and overwrite the // state with `ptr::write`. We also make sure to abort if the allocator // panics, ensuring the current value is not dropped twice. unsafe { let key = ptr::read(key); let entry = panic::catch_unwind(panic::AssertUnwindSafe(|| { Box::into_raw(Box::new(Entry { value: MaybeUninit::uninit(), key, })) })) .unwrap_or_else(|_| std::process::abort()); ptr::write(self, LazyEntry::Init(entry)); entry } } } } } /// RMW operations. impl HashMap where K: Hash + Eq, S: BuildHasher, { /// Tries to insert a key and value computed from a closure into the map, /// and returns a reference to the value that was inserted. #[inline] pub fn try_insert_with<'g, F>( &self, key: K, f: F, guard: &'g impl VerifiedGuard, ) -> Result<&'g V, &'g V> where F: FnOnce() -> V, K: 'g, { let mut f = Some(f); let compute = |entry| match entry { // There is already an existing value. Some((_, current)) => Operation::Abort(current), // Insert the initial value. // // Note that this case is guaranteed to be executed at most // once as insert values are cached, so this can never panic. None => Operation::Insert((f.take().unwrap())()), }; match self.compute(key, compute, guard) { // Failed to insert, return the existing value. Compute::Aborted(current) => Err(current), // Successfully inserted. Compute::Inserted(_, value) => Ok(value), _ => unreachable!(), } } /// Returns a reference to the value corresponding to the key, or inserts a default value /// computed from a closure. #[inline] pub fn get_or_insert_with<'g, F>(&self, key: K, f: F, guard: &'g impl VerifiedGuard) -> &'g V where F: FnOnce() -> V, K: 'g, { match self.try_insert_with(key, f, guard) { Ok(value) => value, Err(value) => value, } } /// Updates an existing entry atomically, returning the value that was inserted. #[inline] pub fn update<'g, F>( &self, key: K, mut update: F, guard: &'g impl VerifiedGuard, ) -> Option<&'g V> where F: FnMut(&V) -> V, K: 'g, { let compute = |entry| match entry { // There is nothing to update. None => Operation::Abort(()), // Perform the update. Some((_, value)) => Operation::Insert(update(value)), }; match self.compute(key, compute, guard) { // Return the updated value. Compute::Updated { new: (_, value), .. } => Some(value), // There was nothing to update. Compute::Aborted(_) => None, _ => unreachable!(), } } /// Updates an existing entry or inserts a default value computed from a closure. #[inline] pub fn update_or_insert_with<'g, U, F>( &self, key: K, update: U, f: F, guard: &'g impl VerifiedGuard, ) -> &'g V where F: FnOnce() -> V, U: Fn(&V) -> V, K: 'g, { let mut f = Some(f); let compute = |entry| match entry { // Perform the update. Some((_, value)) => Operation::Insert::<_, ()>(update(value)), // Insert the initial value. // // Note that this case is guaranteed to be executed at most // once as insert values are cached, so this can never panic. None => Operation::Insert((f.take().unwrap())()), }; match self.compute(key, compute, guard) { // Return the updated value. Compute::Updated { new: (_, value), .. } => value, // Return the value we inserted. Compute::Inserted(_, value) => value, _ => unreachable!(), } } /// Update an entry with a CAS function. /// /// Note that `compute` closure is guaranteed to be called for a `None` input only once, allowing the /// insertion of values that cannot be cloned or reconstructed. #[inline] pub fn compute<'g, F, T>( &self, key: K, compute: F, guard: &'g impl VerifiedGuard, ) -> Compute<'g, K, V, T> where F: FnMut(Option<(&'g K, &'g V)>) -> Operation, { // Lazy initialize the entry allocation. let mut entry = LazyEntry::Uninit(key); // Perform the update. // // Safety: We just allocated the entry above. let result = unsafe { self.compute_with(&mut entry, ComputeState::new(compute), guard) }; // Deallocate the entry if it was not inserted. if matches!(result, Compute::Removed(..) | Compute::Aborted(_)) { if let LazyEntry::Init(entry) = entry { // Safety: The entry was allocated but not inserted into the map. let _ = unsafe { Box::from_raw(entry) }; } } result } /// Update an entry with a CAS function. /// /// # Safety /// /// The new entry must be a valid owned pointer to insert into the map. #[inline] unsafe fn compute_with<'g, F, T>( &self, new_entry: &mut LazyEntry, mut state: ComputeState, guard: &'g impl VerifiedGuard, ) -> Compute<'g, K, V, T> where F: FnMut(Option<(&'g K, &'g V)>) -> Operation, { // Load the root table. let mut table = self.root(guard); // The table has not yet been allocated. if table.raw.is_null() { // Compute the value to insert. // // Safety: Insert transitions are always sound. match unsafe { state.next(None) } { op @ Operation::Insert(_) => state.restore(None, op), Operation::Remove => panic!("Cannot remove `None` entry."), Operation::Abort(value) => return Compute::Aborted(value), } // Initialize the table. table = self.init(None); } let (h1, h2) = self.hash(new_entry.key()); let mut help_copy = false; loop { // Initialize the probe state. let mut probe = Probe::start(h1, table.mask); // Probe until we reach the limit. let copying = 'probe: loop { if probe.len > table.limit { break 'probe None; } // Load the entry metadata first for cheap searches. // // Safety: `probe.i` is always in-bounds for the table length. let meta = unsafe { table.meta(probe.i) }.load(Ordering::Acquire); // The entry is empty. let mut entry = if meta == meta::EMPTY { // Compute the value to insert. // // Safety: Insert transitions are always sound. let value = match unsafe { state.next(None) } { Operation::Insert(value) => value, Operation::Remove => panic!("Cannot remove `None` entry."), Operation::Abort(value) => return Compute::Aborted(value), }; let new_entry = new_entry.init(); // Safety: `new_entry` was just allocated above and is valid for writes. unsafe { (*new_entry).value = MaybeUninit::new(value) } // Attempt to insert. // // Safety: `probe.i` is always in-bounds for the table length.Additionally, // `new_entry` was allocated above and never shared. match unsafe { self.insert_at(probe.i, h2, new_entry.cast(), table, guard) } { // Successfully inserted. InsertStatus::Inserted => { // Increment the table length. self.count.get(guard).fetch_add(1, Ordering::Relaxed); // Safety: `new_entry` was initialized above. let new_ref = unsafe { &*new_entry.cast::>() }; return Compute::Inserted(&new_ref.key, &new_ref.value); } // Lost to a concurrent insert. // // If the key matches, we might be able to update the value. InsertStatus::Found(EntryStatus::Value(found)) | InsertStatus::Found(EntryStatus::Copied(found)) => { // Cache the previous value // // Safety: `new_entry` was initialized above and was not inserted // into the map. let value = unsafe { (*new_entry).value.assume_init_read() }; state.restore(None, Operation::Insert(value)); found } // The entry was removed or invalidated. InsertStatus::Found(EntryStatus::Null) => { // Cache the previous value. // // Safety: `new_entry` was initialized above and was not inserted // into the map. let value = unsafe { (*new_entry).value.assume_init_read() }; state.restore(None, Operation::Insert(value)); // Continue probing. probe.next(table.mask); continue 'probe; } } } // Found a potential match. else if meta == h2 { // Load the full entry. // // Safety: `probe.i` is always in-bounds for the table length. let found = guard .protect(unsafe { table.entry(probe.i) }, Ordering::Acquire) .unpack(); // The entry was deleted, keep probing. if found.ptr.is_null() { probe.next(table.mask); continue 'probe; } // If the key matches, we might be able to update the value. found } // Otherwise, continue probing. else { probe.next(table.mask); continue 'probe; }; // Check for a full match. // // Safety: We performed a protected load of the pointer using a verified guard with // `Acquire` and ensured that it is non-null, meaning it is valid for reads as long // as we hold the guard. if unsafe { (*entry.ptr).key != *new_entry.key() } { probe.next(table.mask); continue 'probe; } // The entry is being copied to the new table. if entry.tag() & Entry::COPYING != 0 { break 'probe Some(probe.i); } loop { // Compute the value to insert. // // Safety: `entry` is valid for reads. let failure = match unsafe { state.next(Some(entry.ptr)) } { // The operation was aborted. Operation::Abort(value) => return Compute::Aborted(value), // Update the value. Operation::Insert(value) => { let new_entry = new_entry.init(); // Safety: `new_entry` was just allocated above and is valid for writes. unsafe { (*new_entry).value = MaybeUninit::new(value) } // Try to perform the update. // // Safety: // - `probe.i` is always in-bounds for the table length // - `entry` is a valid non-null entry that we found in the map. // - `new_entry` was initialized above and never shared. let status = unsafe { self.update_at(probe.i, entry, new_entry.cast(), table, guard) }; match status { // Successfully updated. UpdateStatus::Replaced(entry) => { // Safety: `entry` is a valid non-null entry that we found in the map // before replacing it. let entry_ref = unsafe { &(*entry.ptr) }; // Safety: `new_entry` was initialized above. let new_ref = unsafe { &*new_entry.cast::>() }; return Compute::Updated { old: (&entry_ref.key, &entry_ref.value), new: (&new_ref.key, &new_ref.value), }; } // The update failed. failure => { // Save the previous value. // // Safety: `new_entry` was initialized above and was not inserted // into the map. let value = unsafe { (*new_entry).value.assume_init_read() }; state.restore(Some(entry.ptr), Operation::Insert(value)); failure } } } // Remove the key from the map. Operation::Remove => { // Try to perform the removal. // // Safety: // - `probe.i` is always in-bounds for the table length // - `entry` is a valid non-null entry that we found in the map. let status = unsafe { self.update_at(probe.i, entry, Entry::TOMBSTONE, table, guard) }; match status { // Successfully removed the entry. UpdateStatus::Replaced(entry) => { // Mark the entry as a tombstone. // // Note that this might end up being overwritten by the metadata hash // if the initial insertion is lagging behind, but we avoid the RMW // and sacrifice reads in the extremely rare case. unsafe { table .meta(probe.i) .store(meta::TOMBSTONE, Ordering::Release) }; // Decrement the table length. self.count.get(guard).fetch_sub(1, Ordering::Relaxed); // Safety: `entry` is a valid non-null entry that we found in the map // before replacing it. let entry_ref = unsafe { &(*entry.ptr) }; return Compute::Removed(&entry_ref.key, &entry_ref.value); } // The remove failed. failure => { // Save the removal operation. state.restore(Some(entry.ptr), Operation::Remove); failure } } } }; match failure { // The entry is being copied to the new table. UpdateStatus::Found(EntryStatus::Copied(_)) => break 'probe Some(probe.i), // The entry was deleted before we could update it. // // We know that at some point during our execution the key was not in the map. UpdateStatus::Found(EntryStatus::Null) => { // Compute the next operation. // // Safety: Insert transitions are always sound. match unsafe { state.next(None) } { Operation::Insert(value) => { // Save the computed value. state.restore(None, Operation::Insert(value)); // Continue probing to find an empty slot. probe.next(table.mask); continue 'probe; } Operation::Remove => panic!("Cannot remove `None` entry."), Operation::Abort(value) => return Compute::Aborted(value), } } // Someone else beat us to the update, retry. UpdateStatus::Found(EntryStatus::Value(found)) => entry = found, _ => unreachable!(), } } }; // Prepare to retry in the next table. if let Some(next_table) = self.prepare_retry(copying, &mut help_copy, table, guard) { table = next_table; continue; } // Otherwise, the key is not in the map. // // Safety: Insert transitions are always sound. match unsafe { state.next(None) } { // Need to insert into the new table. op @ Operation::Insert(_) => { table = self.prepare_retry_insert(None, &mut help_copy, table, guard); state.restore(None, op); } // The operation was aborted. Operation::Abort(value) => return Compute::Aborted(value), Operation::Remove => panic!("Cannot remove `None` entry."), } } } } /// Resize operations. impl HashMap where K: Hash + Eq, S: BuildHasher, { /// Allocate the initial table. #[cold] #[inline(never)] fn init(&self, capacity: Option) -> Table> { const CAPACITY: usize = 32; // Allocate the table and mark it as the root. let mut new = Table::alloc(capacity.unwrap_or(CAPACITY)); *new.state_mut().status.get_mut() = State::PROMOTED; // Race to write the initial table. match self.table.compare_exchange( ptr::null_mut(), new.raw, Ordering::Release, Ordering::Acquire, ) { // Successfully initialized the table. Ok(_) => new, // Someone beat us, deallocate our table and use the table that was written. Err(found) => { // Safety: We allocated the table above and never shared it. unsafe { Table::dealloc(new) } // Safety: The table was just initialized. unsafe { Table::from_raw(found) } } } } /// Returns the next table, allocating it has not already been created. #[cold] #[inline(never)] fn get_or_alloc_next( &self, capacity: Option, table: Table>, ) -> Table> { // Avoid spinning in tests, which can hide race conditions. const SPIN_ALLOC: usize = if cfg!(any(test, debug_assertions)) { 1 } else { 7 }; // The next table is already allocated. if let Some(next) = table.next_table() { return next; } let state = table.state(); // Otherwise, try to acquire the allocation lock. // // Unlike in `init`, we do not race here to prevent unnecessary allocator pressure. let _allocating = match state.allocating.try_lock() { Ok(lock) => lock, // Someone else is currently allocating. Err(_) => { let mut spun = 0; // Spin for a bit, waiting for the table to be initialized. while spun <= SPIN_ALLOC { for _ in 0..(spun * spun) { hint::spin_loop(); } // The table was initialized. if let Some(next) = table.next_table() { return next; } spun += 1; } // Otherwise, we have to block. state.allocating.lock().unwrap() } }; // The table was allocated while we were waiting for the lock. if let Some(next) = table.next_table() { return next; } let current_capacity = table.len(); // Loading the length here is quite expensive, we may want to consider // a probabilistic counter to detect high-deletion workloads. let active_entries = self.len(); let next_capacity = match cfg!(papaya_stress) { // Never grow the table to stress the incremental resizing algorithm. true => current_capacity, // Double the table capacity if we are at least 50% full. false if active_entries >= (current_capacity >> 1) => current_capacity << 1, // Halve the table if we are at most 12.5% full. // // This heuristic is intentionally pessimistic as unnecessarily shrinking // is an expensive operation, but it may change in the future. We also respect // the initial capacity to give the user a way to retain a strict minimum table // size. false if active_entries <= (current_capacity >> 3) => { self.initial_capacity.max(current_capacity >> 1) } // Otherwise keep the capacity the same. // // This can occur due to poor hash distribution or frequent cycling of // insertions and deletions, in which case we want to avoid continuously // growing the table. false => current_capacity, }; let next_capacity = capacity.unwrap_or(next_capacity); assert!( next_capacity <= isize::MAX as usize, "`HashMap` exceeded maximum capacity" ); // Allocate the new table while holding the lock. let next = Table::alloc(next_capacity); state.next.store(next.raw, Ordering::Release); drop(_allocating); next } /// Help along with an existing resize operation, returning the new root table. /// /// If `copy_all` is `false` in incremental resize mode, this returns the current reference's next /// table, not necessarily the new root. #[cold] #[inline(never)] fn help_copy( &self, copy_all: bool, table: &Table>, guard: &impl VerifiedGuard, ) -> Table> { match self.resize { ResizeMode::Blocking => self.help_copy_blocking(table, guard), ResizeMode::Incremental(chunk) => { let copied_to = self.help_copy_incremental(chunk, copy_all, guard); if !copy_all { // If we weren't trying to linearize, we have to write to the next table // even if the copy hasn't completed yet. return table.next_table().unwrap(); } copied_to } } } /// Help along the resize operation until it completes and the next table is promoted. /// /// Should only be called on the root table. fn help_copy_blocking( &self, table: &Table>, guard: &impl VerifiedGuard, ) -> Table> { // Load the next table. let mut next = table.next_table().unwrap(); 'copy: loop { // Make sure we are copying to the correct table. while next.state().status.load(Ordering::Relaxed) == State::ABORTED { next = self.get_or_alloc_next(None, next); } // The copy already completed if self.try_promote(table, &next, 0, guard) { return next; } let copy_chunk = table.len().min(4096); loop { // Every entry has already been claimed. if next.state().claim.load(Ordering::Relaxed) >= table.len() { break; } // Claim a chunk to copy. let copy_start = next.state().claim.fetch_add(copy_chunk, Ordering::Relaxed); // Copy our chunk of entries. let mut copied = 0; for i in 0..copy_chunk { let i = copy_start + i; if i >= table.len() { break; } // Copy the entry. // // Safety: We verified that `i` is in-bounds above. if unsafe { !self.copy_at_blocking(i, table, &next, guard) } { // This table doesn't have space for the next entry. // // Abort the current resize. // // Note that the `SeqCst` is necessary to make the store visible // to threads that are unparked. next.state().status.store(State::ABORTED, Ordering::SeqCst); // Allocate the next table. let allocated = self.get_or_alloc_next(None, next); // Wake anyone waiting for us to finish. let state = table.state(); state.parker.unpark(&state.status); // Retry in a new table. next = allocated; continue 'copy; } copied += 1; } // Are we done? if self.try_promote(table, &next, copied, guard) { return next; } // If the resize was aborted while we were copying, continue in the new table. if next.state().status.load(Ordering::Relaxed) == State::ABORTED { continue 'copy; } } let state = next.state(); // We copied all that we can, wait for the table to be promoted. for spun in 0.. { // Avoid spinning in tests, which can hide race conditions. const SPIN_WAIT: usize = if cfg!(any(test, debug_assertions)) { 1 } else { 7 }; // Note that `Acquire` is necessary here to ensure we see the // relevant modifications to the root table if see the updated // state before parking. // // Otherwise, `Parker::park` will ensure the necessary synchronization // when we are unparked. let status = state.status.load(Ordering::Acquire); // If this copy was aborted, we have to retry in the new table. if status == State::ABORTED { continue 'copy; } // The copy has completed. if status == State::PROMOTED { return next; } // Copy chunks are relatively small and we expect to finish quickly, // so spin for a bit before resorting to parking. if spun <= SPIN_WAIT { for _ in 0..(spun * spun) { hint::spin_loop(); } continue; } // Park until the table is promoted. state .parker .park(&state.status, |status| status == State::PENDING); } } } /// Copy the entry at the given index to the new table. /// /// Returns `true` if the entry was copied into the table or `false` if the table was full. /// /// # Safety /// /// The index must be in-bounds for the table. unsafe fn copy_at_blocking( &self, i: usize, table: &Table>, next_table: &Table>, guard: &impl VerifiedGuard, ) -> bool { // Mark the entry as copying. // // Safety: The caller guarantees that the index is in-bounds. // // Note that we don't need to protect the returned entry here, because // no one is allowed to retire the entry once we put the `COPYING` bit // down until it is inserted into the new table. let entry = unsafe { table.entry(i) } .fetch_or(Entry::COPYING, Ordering::AcqRel) .unpack(); // The entry is a tombstone. if entry.raw == Entry::TOMBSTONE { return true; } // There is nothing to copy, we're done. if entry.ptr.is_null() { // Mark as a tombstone so readers avoid having to load the entry. // // Safety: The caller guarantees that the index is in-bounds. unsafe { table.meta(i) }.store(meta::TOMBSTONE, Ordering::Release); return true; } // Copy the value to the new table. // // Safety: We marked the entry as `COPYING`, ensuring that any updates // or removals wait until we complete the copy, and allowing us to get // away without a protected load. Additionally, we verified that the // entry is non-null, meaning that it is valid for reads. unsafe { self.insert_copy(entry.ptr.unpack(), false, next_table, guard) .is_some() } } /// Help along an in-progress resize incrementally by copying a chunk of entries. /// /// Returns the table that was copied to. fn help_copy_incremental( &self, chunk: usize, block: bool, guard: &impl VerifiedGuard, ) -> Table> { // Always help the highest priority root resize. let table = self.root(guard); // Load the next table. let Some(next) = table.next_table() else { // The copy we tried to help was already promoted. return table; }; loop { // The copy already completed. if self.try_promote(&table, &next, 0, guard) { return next; } loop { // Every entry has already been claimed. if next.state().claim.load(Ordering::Relaxed) >= table.len() { break; } // Claim a chunk to copy. let copy_start = next.state().claim.fetch_add(chunk, Ordering::Relaxed); // Copy our chunk of entries. let mut copied = 0; for i in 0..chunk { let i = copy_start + i; if i >= table.len() { break; } // Copy the entry. // // Safety: We verified that `i` is in-bounds above. unsafe { self.copy_at_incremental(i, &table, &next, guard) }; copied += 1; } // Update the copy state, and try to promote the table. // // Only copy a single chunk if promotion fails, unless we are forced // to complete the resize. if self.try_promote(&table, &next, copied, guard) || !block { return next; } } // There are no entries that we can copy, block if necessary. if !block { return next; } let state = next.state(); for spun in 0.. { // Avoid spinning in tests, which can hide race conditions. const SPIN_WAIT: usize = if cfg!(any(test, debug_assertions)) { 1 } else { 7 }; // The copy has completed. // // Note that `Acquire` is necessary here to ensure we see the // relevant modifications to the root table if see the updated // state before parking. // // Otherwise, `Parker::park` will ensure the necessary synchronization // when we are unparked. let status = state.status.load(Ordering::Acquire); if status == State::PROMOTED { return next; } // Copy chunks are relatively small and we expect to finish quickly, // so spin for a bit before resorting to parking. if spun <= SPIN_WAIT { for _ in 0..(spun * spun) { hint::spin_loop(); } continue; } // Park until the table is promoted. state .parker .park(&state.status, |status| status == State::PENDING); } } } /// Copy the entry at the given index to the new table. /// /// # Safety /// /// The index must be in-bounds for the table. unsafe fn copy_at_incremental( &self, i: usize, table: &Table>, next_table: &Table>, guard: &impl VerifiedGuard, ) { // Safety: The caller guarantees that the index is in-bounds. let entry = unsafe { table.entry(i) }; // Mark the entry as copying. let found = entry.fetch_or(Entry::COPYING, Ordering::AcqRel).unpack(); // The entry is a tombstone. if found.raw == Entry::TOMBSTONE { return; } // There is nothing to copy, we're done. if found.ptr.is_null() { // Mark as a tombstone so readers avoid having to load the entry. // // Safety: The caller guarantees that the index is in-bounds. unsafe { table.meta(i) }.store(meta::TOMBSTONE, Ordering::Release); return; } // Mark the entry as borrowed so writers in the new table know it was copied. let new_entry = found.map_tag(|addr| addr | Entry::BORROWED); // Copy the value to the new table. // // Safety: We marked the entry as `COPYING`, ensuring that any updates // or removals wait until we complete the copy, and allowing us to get // away without a protected load. Additionally, we verified that the // entry is non-null, meaning that it is valid for reads. unsafe { self.insert_copy(new_entry, true, next_table, guard) .unwrap(); } // Mark the entry as copied. let copied = found .raw .map_addr(|addr| addr | Entry::COPYING | Entry::COPIED); // Note that we already wrote the COPYING bit, so no one is writing to the old // entry except us. // // Note that the `SeqCst` is necessary to make the store visible to threads // that are unparked. entry.store(copied, Ordering::SeqCst); // Notify any writers that the copy has completed. table.state().parker.unpark(entry); } // Copy an entry into the table, returning the index it was inserted into. // // This is an optimized version of `insert_entry` where the caller is the only writer // inserting the given key into the new table, as it has already been marked as copying. // // # Safety // // The new entry must be valid for reads. unsafe fn insert_copy( &self, new_entry: Tagged>, resize: bool, table: &Table>, guard: &impl VerifiedGuard, ) -> Option<(Table>, usize)> { // Safety: The new entry is guaranteed to be valid for reads. let key = unsafe { &(*new_entry.ptr).key }; let mut table = *table; let (h1, h2) = self.hash(key); loop { // Initialize the probe state. let mut probe = Probe::start(h1, table.mask); // Probe until we reach the limit. while probe.len <= table.limit { // Safety: `probe.i` is always in-bounds for the table length. let meta_entry = unsafe { table.meta(probe.i) }; // Load the entry metadata first for cheap searches. let meta = meta_entry.load(Ordering::Acquire); // The entry is empty, try to insert. if meta == meta::EMPTY { // Safety: `probe.i` is always in-bounds for the table length. let entry = unsafe { table.entry(probe.i) }; // Try to claim the entry. match guard.compare_exchange( entry, ptr::null_mut(), new_entry.raw, Ordering::Release, Ordering::Acquire, ) { // Successfully inserted. Ok(_) => { // Update the metadata table. meta_entry.store(h2, Ordering::Release); return Some((table, probe.i)); } Err(found) => { let found = found.unpack(); // The entry was deleted or copied. let meta = if found.ptr.is_null() { meta::TOMBSTONE } else { // Safety: We performed a protected load of the pointer using a verified guard with // `Acquire` and ensured that it is non-null, meaning it is valid for reads as long // as we hold the guard. let found_ref = unsafe { &(*found.ptr) }; // Ensure the meta table is updated to avoid breaking the probe chain. let hash = self.hasher.hash_one(&found_ref.key); meta::h2(hash) }; if meta_entry.load(Ordering::Relaxed) == meta::EMPTY { meta_entry.store(meta, Ordering::Release); } } } } // Continue probing. probe.next(table.mask); } if !resize { return None; } // Insert into the next table. table = self.get_or_alloc_next(None, table); } } // Update the copy state and attempt to promote a table to the root. // // Returns `true` if the table was promoted. fn try_promote( &self, table: &Table>, next: &Table>, copied: usize, guard: &impl VerifiedGuard, ) -> bool { let state = next.state(); // Update the copy count. let copied = if copied > 0 { state.copied.fetch_add(copied, Ordering::AcqRel) + copied } else { state.copied.load(Ordering::Acquire) }; // If we copied all the entries in the table, we can try to promote. if copied == table.len() { let root = self.table.load(Ordering::Relaxed); // Only promote root copies. // // We can't promote a nested copy before it's parent has finished, as // it may not contain all the entries in the table. if table.raw == root { // Try to update the root. if self .table .compare_exchange(table.raw, next.raw, Ordering::Release, Ordering::Acquire) .is_ok() { // Successfully promoted the table. // // Note that the `SeqCst` is necessary to make the store visible to threads // that are unparked. state.status.store(State::PROMOTED, Ordering::SeqCst); // Retire the old table. // // Safety: `table.raw` is a valid pointer to the table we just copied from. // Additionally, the CAS above made the previous table unreachable from the // root pointer, allowing it to be safely retired. unsafe { guard.defer_retire(table.raw, |table, collector| { // Note that we do not drop entries because they have been copied to // the new root. drop_table(Table::from_raw(table), collector); }); } } // Wake up any writers waiting for the resize to complete. state.parker.unpark(&state.status); return true; } } // Not ready to promote yet. false } // Completes all pending copies in incremental mode to get a clean copy of the table. // // This is necessary for operations like `iter` or `clear`, where entries in multiple tables // can cause lead to incomplete results. #[inline] fn linearize( &self, mut table: Table>, guard: &impl VerifiedGuard, ) -> Table> { if self.is_incremental() { // If we're in incremental resize mode, we need to complete any in-progress resizes to // ensure we don't miss any entries in the next table. We can't iterate over both because // we risk returning the same entry twice. while table.next_table().is_some() { table = self.help_copy(true, &table, guard); } } table } // Wait for an incremental copy of a given entry to complete. #[cold] #[inline(never)] fn wait_copied(&self, i: usize, table: &Table>) { // Avoid spinning in tests, which can hide race conditions. const SPIN_WAIT: usize = if cfg!(any(test, debug_assertions)) { 1 } else { 5 }; let entry = unsafe { table.entry(i) }; // Spin for a short while, waiting for the entry to be copied. for spun in 0..SPIN_WAIT { // The entry was copied. let entry = entry.load(Ordering::Acquire).unpack(); if entry.tag() & Entry::COPIED != 0 { return; } for _ in 0..(spun * spun) { hint::spin_loop(); } } // Park until the copy completes. let parker = &table.state().parker; parker.park(entry, |entry| entry.addr() & Entry::COPIED == 0); } /// Retire an entry that was removed from the current table, but may still be reachable from /// previous tables. /// /// # Safety /// /// The entry must be a valid pointer that is unreachable from the current table. Additionally, /// it is *undefined behavior* to call this method multiple times for the same entry. #[inline] unsafe fn defer_retire( &self, entry: Tagged>, table: &Table>, guard: &impl VerifiedGuard, ) { match self.resize { // Safety: In blocking resize mode, we only ever write to the root table, so the entry // is inaccessible from all tables. ResizeMode::Blocking => unsafe { guard.defer_retire(entry.ptr, seize::reclaim::boxed); }, // In incremental resize mode, the entry may be accessible in previous tables. ResizeMode::Incremental(_) => { if entry.tag() & Entry::BORROWED == 0 { // Safety: If the entry is not borrowed, meaning it is not in any previous tables, // it is inaccessible even if the current table is not root. Thus we can safely retire. unsafe { guard.defer_retire(entry.ptr, seize::reclaim::boxed) }; return; } let root = self.root(guard); // Check if our table, or any subsequent table, is the root. let mut next = Some(*table); while let Some(table) = next { if table.raw == root.raw { // Safety: The root table is our table or a table that succeeds ours. // Thus any previous tables are unreachable from the root, so we can safely retire. unsafe { guard.defer_retire(entry.ptr, seize::reclaim::boxed) }; return; } next = table.next_table(); } // Otherwise, we have to wait for the table we are copying from to be reclaimed. // // Find the table we are copying from, searching from the root. let mut prev = root; loop { let next = prev.next_table().unwrap(); // Defer the entry to be retired by the table we are copying from. if next.raw == table.raw { prev.state().deferred.push(entry.ptr); return; } prev = next; } } } } } // An iterator over the keys and values of this table. pub struct Iter<'g, K, V, G> { i: usize, table: Table>, guard: &'g G, } impl<'g, K: 'g, V: 'g, G> Iterator for Iter<'g, K, V, G> where G: VerifiedGuard, { type Item = (&'g K, &'g V); #[inline] fn next(&mut self) -> Option { // The table has not yet been allocated. if self.table.raw.is_null() { return None; } loop { // Iterated over every entry in the table, we're done. if self.i >= self.table.len() { return None; } // Load the entry metadata first to ensure consistency with calls to `get`. // // Safety: We verified that `self.i` is in-bounds above. let meta = unsafe { self.table.meta(self.i) }.load(Ordering::Acquire); // The entry is empty or deleted. if matches!(meta, meta::EMPTY | meta::TOMBSTONE) { self.i += 1; continue; } // Load the entry. // // Safety: We verified that `self.i` is in-bounds above. let entry = self .guard .protect(unsafe { self.table.entry(self.i) }, Ordering::Acquire) .unpack(); // The entry was deleted. if entry.ptr.is_null() { self.i += 1; continue; } // Safety: We performed a protected load of the pointer using a verified guard with // `Acquire` and ensured that it is non-null, meaning it is valid for reads as long // as we hold the guard. let entry_ref = unsafe { &(*entry.ptr) }; self.i += 1; return Some((&entry_ref.key, &entry_ref.value)); } } } // Safety: An iterator holds a shared reference to the HashMap // and Guard, and outputs shared references to keys and values. // Thus everything must be `Sync` for the iterator to be `Send` // or `Sync`. // // It is not possible to obtain an owned key, value, or guard // from an iterator, so `Send` is not a required bound. unsafe impl Send for Iter<'_, K, V, G> where K: Sync, V: Sync, G: Sync, { } unsafe impl Sync for Iter<'_, K, V, G> where K: Sync, V: Sync, G: Sync, { } impl Clone for Iter<'_, K, V, G> { #[inline] fn clone(&self) -> Self { Iter { i: self.i, table: self.table, guard: self.guard, } } } impl Drop for HashMap { fn drop(&mut self) { let mut raw = *self.table.get_mut(); // Make sure all objects are reclaimed before the collector is dropped. // // Dropping a table depends on accessing the collector for deferred retirement, // using the shared collector pointer that is invalidated by drop. // // Safety: We have a unique reference to the collector. unsafe { self.collector.reclaim_all() }; // Drop all nested tables and entries. while !raw.is_null() { // Safety: The root and next tables are always valid pointers to a // table allocation, or null. let mut table = unsafe { Table::from_raw(raw) }; // Read the next table pointer before dropping the current one. let next = *table.state_mut().next.get_mut(); // Safety: We have unique access to the table and do // not access the entries after this call. unsafe { drop_entries(table) }; // Safety: We have unique access to the table and do // not access it after this call. unsafe { drop_table(table, &self.collector) }; // Continue for all nested tables. raw = next; } } } // Drop all entries in this table. // // # Safety // // The table entries must not be accessed after this call. unsafe fn drop_entries(table: Table>) { for i in 0..table.len() { // Safety: `i` is in-bounds and we have unique access to the table. let entry = unsafe { (*table.entry(i).as_ptr()).unpack() }; // The entry was copied, or there is nothing to deallocate. if entry.ptr.is_null() || entry.tag() & Entry::COPYING != 0 { continue; } // Drop the entry. // // Safety: We verified that the table is non-null and will // not be accessed after this call. Additionally, we ensured // that the entry is not copied to avoid double freeing entries // that may exist in multiple tables. unsafe { drop(Box::from_raw(entry.ptr)) } } } // Drop the table allocation. // // # Safety // // The table must not be accessed after this call. unsafe fn drop_table(mut table: Table>, collector: &Collector) { // Drop any entries that were deferred during an incremental resize. // // Safety: Entries are deferred after they are made unreachable from the // next table during a resize from this table. This table must have been accessible // from the root for any entry to have been deferred. Thus it is being retired now, // *after* the entry was made inaccessible from the next table. Additionally, for // this table to have been retired, it also must no longer be accessible from the root, // meaning that the entry has been totally removed from the map, and can be safely // retired. table .state_mut() .deferred .drain(|entry| unsafe { collector.retire(entry, seize::reclaim::boxed) }); // Deallocate the table. // // Safety: The caller guarantees that the table will not be accessed after this call. unsafe { Table::dealloc(table) }; } // Entry metadata, inspired by `hashbrown`. mod meta { use std::mem; // Indicates an empty entry. pub const EMPTY: u8 = 0x80; // Indicates an entry that has been deleted. pub const TOMBSTONE: u8 = u8::MAX; // Returns the primary hash for an entry. #[inline] pub fn h1(hash: u64) -> usize { hash as usize } /// Return a byte of hash metadata, used for cheap searches. #[inline] pub fn h2(hash: u64) -> u8 { const MIN_HASH_LEN: usize = if mem::size_of::() < mem::size_of::() { mem::size_of::() } else { mem::size_of::() }; // Grab the top 7 bits of the hash. // // While the hash is normally a full 64-bit value, some hash functions // (such as fxhash) produce a usize result instead, which means that the // top 32 bits are 0 on 32-bit platforms. let top7 = hash >> (MIN_HASH_LEN * 8 - 7); (top7 & 0x7f) as u8 } } papaya-0.2.3/src/raw/probe.rs000064400000000000000000000024021046102023000141340ustar 00000000000000// A quadratic probe sequence. #[derive(Default)] pub struct Probe { // The current index in the probe sequence. pub i: usize, // The current length of the probe sequence. pub len: usize, } impl Probe { // Initialize the probe sequence. #[inline] pub fn start(hash: usize, mask: usize) -> Probe { Probe { i: hash & mask, len: 0, } } // Increment the probe sequence. #[inline] pub fn next(&mut self, mask: usize) { self.len += 1; self.i = (self.i + self.len) & mask; } } // The maximum probe length for table operations. // // Estimating a load factor for the hash-table based on probe lengths allows // the hash-table to avoid loading the length every insert, which is a source // of contention. pub fn limit(capacity: usize) -> usize { // 5 * log2(capacity): Testing shows this gives us a ~85% load factor. 5 * ((usize::BITS as usize) - (capacity.leading_zeros() as usize) - 1) } // Returns an estimate of the number of entries needed to hold `capacity` elements. pub fn entries_for(capacity: usize) -> usize { // We should rarely resize before 75%. let capacity = capacity.checked_mul(8).expect("capacity overflow") / 6; capacity.next_power_of_two() } papaya-0.2.3/src/raw/utils/counter.rs000064400000000000000000000035241046102023000156520ustar 00000000000000use std::sync::{ atomic::{AtomicIsize, Ordering}, OnceLock, }; use super::CachePadded; // A sharded atomic counter. // // Sharding the length counter of `HashMap` is extremely important, // as a single point of contention for insertions/deletions significantly // degrades concurrent performance. // // We can take advantage of the fact that `seize::Guard` pub struct Counter(Box<[CachePadded]>); impl Default for Counter { /// Create a new `Counter`. fn default() -> Counter { // available_parallelism is quite slow (microseconds). static CPUS: OnceLock = OnceLock::new(); let num_cpus = *CPUS.get_or_init(|| { std::thread::available_parallelism() .map(Into::into) .unwrap_or(1) }); // Round up to the next power-of-two for fast modulo. let shards = (0..num_cpus.next_power_of_two()) .map(|_| Default::default()) .collect(); Counter(shards) } } impl Counter { // Return the shard for the given thread ID. #[inline] pub fn get(&self, guard: &impl seize::Guard) -> &AtomicIsize { // Guard thread IDs are essentially perfectly sharded due to // the internal thread ID allocator, which makes contention // very unlikely even with the exact number of shards as CPUs. let shard = guard.thread_id() & (self.0.len() - 1); &self.0[shard].value } // Returns the sum of all counter shards. #[inline] pub fn sum(&self) -> usize { self.0 .iter() .map(|x| x.value.load(Ordering::Relaxed)) .sum::() .try_into() // Depending on the order of deletion/insertions this might be negative, // in which case we assume the map is empty. .unwrap_or(0) } } papaya-0.2.3/src/raw/utils/mod.rs000064400000000000000000000055131046102023000147520ustar 00000000000000mod counter; mod parker; mod stack; mod tagged; pub use counter::Counter; pub use parker::Parker; pub use stack::Stack; pub use tagged::{untagged, AtomicPtrFetchOps, StrictProvenance, Tagged, Unpack}; /// A `seize::Guard` that has been verified to belong to a given map. pub trait VerifiedGuard: seize::Guard {} #[repr(transparent)] pub struct MapGuard(G); impl MapGuard { /// Create a new `MapGuard`. /// /// # Safety /// /// The guard must be valid to use with the given map. pub unsafe fn new(guard: G) -> MapGuard { MapGuard(guard) } /// Create a new `MapGuard` from a reference. /// /// # Safety /// /// The guard must be valid to use with the given map. pub unsafe fn from_ref(guard: &G) -> &MapGuard { // Safety: `VerifiedGuard` is `repr(transparent)` over `G`. unsafe { &*(guard as *const G as *const MapGuard) } } } impl VerifiedGuard for MapGuard where G: seize::Guard {} impl seize::Guard for MapGuard where G: seize::Guard, { #[inline] fn refresh(&mut self) { self.0.refresh(); } #[inline] fn flush(&self) { self.0.flush(); } #[inline] fn collector(&self) -> &seize::Collector { self.0.collector() } #[inline] fn thread_id(&self) -> usize { self.0.thread_id() } #[inline] unsafe fn defer_retire(&self, ptr: *mut T, reclaim: unsafe fn(*mut T, &seize::Collector)) { unsafe { self.0.defer_retire(ptr, reclaim) }; } } /// Pads and aligns a value to the length of a cache line. /// // Source: https://github.com/crossbeam-rs/crossbeam/blob/0f81a6957588ddca9973e32e92e7e94abdad801e/crossbeam-utils/src/cache_padded.rs#L63. #[derive(Clone, Copy, Default, Hash, PartialEq, Eq)] #[cfg_attr( any( target_arch = "x86_64", target_arch = "aarch64", target_arch = "arm64ec", target_arch = "powerpc64", ), repr(align(128)) )] #[cfg_attr( any( target_arch = "arm", target_arch = "mips", target_arch = "mips32r6", target_arch = "mips64", target_arch = "mips64r6", target_arch = "sparc", target_arch = "hexagon", ), repr(align(32)) )] #[cfg_attr(target_arch = "m68k", repr(align(16)))] #[cfg_attr(target_arch = "s390x", repr(align(256)))] #[cfg_attr( not(any( target_arch = "x86_64", target_arch = "aarch64", target_arch = "arm64ec", target_arch = "powerpc64", target_arch = "arm", target_arch = "mips", target_arch = "mips32r6", target_arch = "mips64", target_arch = "mips64r6", target_arch = "sparc", target_arch = "hexagon", target_arch = "m68k", target_arch = "s390x", )), repr(align(64)) )] pub struct CachePadded { value: T, } papaya-0.2.3/src/raw/utils/parker.rs000064400000000000000000000114351046102023000154570ustar 00000000000000use std::collections::HashMap; use std::sync::atomic::{AtomicPtr, AtomicU8, AtomicUsize, Ordering}; use std::sync::Mutex; use std::thread::{self, Thread}; // A simple thread parker. // // This parker is rarely used and relatively naive. Ideally this would just use a futex // but the hashmap needs to park on tagged pointer state so we would either need mixed-sized // atomic accesses (https://github.com/rust-lang/unsafe-code-guidelines/issues/345) which are // questionable, or 64-bit futexes, which are not available on most platforms. // // The parker implementation may be sharded and use intrusive lists if it is found to be // a bottleneck. #[derive(Default)] pub struct Parker { pending: AtomicUsize, state: Mutex, } #[derive(Default)] struct State { count: u64, threads: HashMap>, } impl Parker { // Block the current thread until the park condition is false. // // This method is guaranteed to establish happens-before with the unpark condition // before it returns. pub fn park(&self, atomic: &impl Atomic, should_park: impl Fn(T) -> bool) { let key = atomic as *const _ as usize; loop { // Announce our thread. // // This must be done before inserting our thread to prevent // incorrect decrements if we are unparked in-between inserting // the thread and incrementing the counter. // // Note that the `SeqCst` store here establishes a total order // with the `SeqCst` store that establishes the unpark condition. self.pending.fetch_add(1, Ordering::SeqCst); // Insert our thread into the parker. let id = { let state = &mut *self.state.lock().unwrap(); state.count += 1; let threads = state.threads.entry(key).or_default(); threads.insert(state.count, thread::current()); state.count }; // Check the park condition. // // Note that `SeqCst` is necessary here to participate in the // total order established above. if !should_park(atomic.load(Ordering::SeqCst)) { // Don't need to park, remove our thread if it wasn't already unparked. let thread = { let mut state = self.state.lock().unwrap(); state .threads .get_mut(&key) .and_then(|threads| threads.remove(&id)) }; if thread.is_some() { self.pending.fetch_sub(1, Ordering::Relaxed); } return; } // Park until we are unparked. loop { thread::park(); let mut state = self.state.lock().unwrap(); if !state .threads .get_mut(&key) .is_some_and(|threads| threads.contains_key(&id)) { break; } } // Ensure we were unparked for the correct reason. // // Establish happens-before with the unpark condition. if !should_park(atomic.load(Ordering::Acquire)) { return; } } } // Unpark all threads waiting on the given atomic. // // Note that any modifications must be `SeqCst` to be visible to unparked threads. pub fn unpark(&self, atomic: &impl Atomic) { let key = atomic as *const _ as usize; // Fast-path, no one waiting to be unparked. // // Note that `SeqCst` is necessary here to participate in the // total order established between the increment of `self.pending` // in `park` and the `SeqCst` store of the unpark condition by // the caller. if self.pending.load(Ordering::SeqCst) == 0 { return; } // Remove and unpark any threads waiting on the atomic. let threads = { let mut state = self.state.lock().unwrap(); state.threads.remove(&key) }; if let Some(threads) = threads { self.pending.fetch_sub(threads.len(), Ordering::Relaxed); for (_, thread) in threads { thread.unpark(); } } } } /// A generic atomic variable. pub trait Atomic { /// Load the value using the given ordering. fn load(&self, ordering: Ordering) -> T; } impl Atomic<*mut T> for AtomicPtr { fn load(&self, ordering: Ordering) -> *mut T { self.load(ordering) } } impl Atomic for AtomicU8 { fn load(&self, ordering: Ordering) -> u8 { self.load(ordering) } } papaya-0.2.3/src/raw/utils/stack.rs000064400000000000000000000040201046102023000152700ustar 00000000000000use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; /// A simple lock-free, append-only, stack of pointers. /// /// This stack is used to defer the reclamation of borrowed entries during /// an incremental resize, which is relatively rare. /// /// Alternative, the deletion algorithm could traverse and delete the entry /// from previous tables to ensure it is unreachable from the root. However, /// it's not clear whether this is better than an allocation or even lock. pub struct Stack { head: AtomicPtr>, } /// A node in the stack. struct Node { value: T, next: *mut Node, } impl Stack { /// Create a new `Stack`. pub fn new() -> Self { Self { head: AtomicPtr::new(ptr::null_mut()), } } /// Add an entry to the stack. pub fn push(&self, value: T) { let node = Box::into_raw(Box::new(Node { value, next: ptr::null_mut(), })); loop { // Load the head node. // // `Relaxed` is sufficient here as all reads are through `&mut self`. let head = self.head.load(Ordering::Relaxed); // Link the node to the stack. unsafe { (*node).next = head } // Attempt to push the node. // // `Relaxed` is similarly sufficient here. if self .head .compare_exchange(head, node, Ordering::Relaxed, Ordering::Relaxed) .is_ok() { break; } } } /// Drain all elements from the stack. pub fn drain(&mut self, mut f: impl FnMut(T)) { let mut head = *self.head.get_mut(); while !head.is_null() { // Safety: We have `&mut self` and the node is non-null. let owned_head = unsafe { Box::from_raw(head) }; // Drain the element. f(owned_head.value); // Continue iterating over the stack. head = owned_head.next; } } } papaya-0.2.3/src/raw/utils/tagged.rs000064400000000000000000000066601046102023000154320ustar 00000000000000use std::mem::align_of; use std::sync::atomic::{AtomicPtr, Ordering}; // Polyfill for the unstable strict-provenance APIs. #[allow(clippy::missing_safety_doc)] #[allow(dead_code)] // `strict_provenance` has stabilized on nightly. pub unsafe trait StrictProvenance: Sized { fn addr(self) -> usize; fn map_addr(self, f: impl FnOnce(usize) -> usize) -> Self; fn unpack(self) -> Tagged where T: Unpack; } // Unpack a tagged pointer. pub trait Unpack: Sized { // A mask for the pointer tag bits. const MASK: usize; // This constant, if used, will fail to compile if T doesn't have an alignment // that guarantees all valid pointers have zero in the bits excluded by T::MASK. const ASSERT_ALIGNMENT: () = assert!(align_of::() > !Self::MASK); } unsafe impl StrictProvenance for *mut T { #[inline(always)] fn addr(self) -> usize { self as usize } #[inline(always)] fn map_addr(self, f: impl FnOnce(usize) -> usize) -> Self { f(self.addr()) as Self } #[inline(always)] fn unpack(self) -> Tagged where T: Unpack, { let () = T::ASSERT_ALIGNMENT; Tagged { raw: self, ptr: self.map_addr(|addr| addr & T::MASK), } } } // An unpacked tagged pointer. pub struct Tagged { // The raw tagged pointer. pub raw: *mut T, // The untagged pointer. pub ptr: *mut T, } // Creates a `Tagged` from an untagged pointer. #[inline] pub fn untagged(value: *mut T) -> Tagged { Tagged { raw: value, ptr: value, } } impl Tagged where T: Unpack, { // Returns the tag portion of this pointer. #[inline] pub fn tag(self) -> usize { self.raw.addr() & !T::MASK } // Maps the tag of this pointer. #[inline] pub fn map_tag(self, f: impl FnOnce(usize) -> usize) -> Self { Tagged { raw: self.raw.map_addr(f), ptr: self.ptr, } } } impl Copy for Tagged {} impl Clone for Tagged { fn clone(&self) -> Self { *self } } // Polyfill for the unstable `atomic_ptr_strict_provenance` APIs. pub trait AtomicPtrFetchOps { fn fetch_or(&self, value: usize, ordering: Ordering) -> *mut T; } impl AtomicPtrFetchOps for AtomicPtr { #[inline] fn fetch_or(&self, value: usize, ordering: Ordering) -> *mut T { #[cfg(not(miri))] { use std::sync::atomic::AtomicUsize; // Safety: `AtomicPtr` and `AtomicUsize` are identical in terms // of memory layout. This operation is technically invalid in that // it loses provenance, but there is no stable alternative. unsafe { &*(self as *const AtomicPtr as *const AtomicUsize) } .fetch_or(value, ordering) as *mut T } // Avoid ptr2int under Miri. #[cfg(miri)] { // Returns the ordering for the read in an RMW operation. const fn read_ordering(ordering: Ordering) -> Ordering { match ordering { Ordering::SeqCst => Ordering::SeqCst, Ordering::AcqRel => Ordering::Acquire, _ => Ordering::Relaxed, } } self.fetch_update(ordering, read_ordering(ordering), |ptr| { Some(ptr.map_addr(|addr| addr | value)) }) .unwrap() } } } papaya-0.2.3/src/serde_impls.rs000064400000000000000000000116131046102023000145460ustar 00000000000000use serde::de::{MapAccess, SeqAccess, Visitor}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::fmt::{self, Formatter}; use std::hash::{BuildHasher, Hash}; use std::marker::PhantomData; use crate::{Guard, HashMap, HashMapRef, HashSet, HashSetRef}; struct MapVisitor { _marker: PhantomData>, } impl Serialize for HashMapRef<'_, K, V, S, G> where K: Serialize + Hash + Eq, V: Serialize, G: Guard, S: BuildHasher, { fn serialize(&self, serializer: Sr) -> Result where Sr: Serializer, { serializer.collect_map(self) } } impl Serialize for HashMap where K: Serialize + Hash + Eq, V: Serialize, S: BuildHasher, { fn serialize(&self, serializer: Sr) -> Result where Sr: Serializer, { self.pin().serialize(serializer) } } impl<'de, K, V, S> Deserialize<'de> for HashMap where K: Deserialize<'de> + Hash + Eq, V: Deserialize<'de>, S: Default + BuildHasher, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(MapVisitor::new()) } } impl MapVisitor { pub(crate) fn new() -> Self { Self { _marker: PhantomData, } } } impl<'de, K, V, S> Visitor<'de> for MapVisitor where K: Deserialize<'de> + Hash + Eq, V: Deserialize<'de>, S: Default + BuildHasher, { type Value = HashMap; fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "a map") } fn visit_map(self, mut access: M) -> Result where M: MapAccess<'de>, { let values = match access.size_hint() { Some(size) => HashMap::with_capacity_and_hasher(size, S::default()), None => HashMap::default(), }; { let values = values.pin(); while let Some((key, value)) = access.next_entry()? { values.insert(key, value); } } Ok(values) } } struct SetVisitor { _marker: PhantomData>, } impl Serialize for HashSetRef<'_, K, S, G> where K: Serialize + Hash + Eq, G: Guard, S: BuildHasher, { fn serialize(&self, serializer: Sr) -> Result where Sr: Serializer, { serializer.collect_seq(self) } } impl Serialize for HashSet where K: Serialize + Hash + Eq, S: BuildHasher, { fn serialize(&self, serializer: Sr) -> Result where Sr: Serializer, { self.pin().serialize(serializer) } } impl<'de, K, S> Deserialize<'de> for HashSet where K: Deserialize<'de> + Hash + Eq, S: Default + BuildHasher, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_seq(SetVisitor::new()) } } impl SetVisitor { pub(crate) fn new() -> Self { Self { _marker: PhantomData, } } } impl<'de, K, S> Visitor<'de> for SetVisitor where K: Deserialize<'de> + Hash + Eq, S: Default + BuildHasher, { type Value = HashSet; fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "a set") } fn visit_seq(self, mut access: M) -> Result where M: SeqAccess<'de>, { let values = match access.size_hint() { Some(size) => HashSet::with_capacity_and_hasher(size, S::default()), None => HashSet::default(), }; { let values = values.pin(); while let Some(key) = access.next_element()? { values.insert(key); } } Ok(values) } } #[cfg(test)] mod test { use crate::HashMap; use crate::HashSet; #[test] fn test_map() { let map: HashMap = HashMap::new(); let guard = map.guard(); map.insert(0, 4, &guard); map.insert(1, 3, &guard); map.insert(2, 2, &guard); map.insert(3, 1, &guard); map.insert(4, 0, &guard); let serialized = serde_json::to_string(&map).unwrap(); let deserialized = serde_json::from_str(&serialized).unwrap(); assert_eq!(map, deserialized); } #[test] fn test_set() { let map: HashSet = HashSet::new(); let guard = map.guard(); map.insert(0, &guard); map.insert(1, &guard); map.insert(2, &guard); map.insert(3, &guard); map.insert(4, &guard); let serialized = serde_json::to_string(&map).unwrap(); let deserialized = serde_json::from_str(&serialized).unwrap(); assert_eq!(map, deserialized); } } papaya-0.2.3/src/set.rs000064400000000000000000000630701046102023000130370ustar 00000000000000use crate::raw::utils::MapGuard; use crate::raw::{self, InsertResult}; use crate::Equivalent; use seize::{Collector, Guard, LocalGuard, OwnedGuard}; use crate::map::ResizeMode; use std::collections::hash_map::RandomState; use std::fmt; use std::hash::{BuildHasher, Hash}; use std::marker::PhantomData; /// A concurrent hash set. /// /// Most hash set operations require a [`Guard`](crate::Guard), which can be acquired through /// [`HashSet::guard`] or using the [`HashSet::pin`] API. See the [crate-level documentation](crate#usage) /// for details. pub struct HashSet { raw: raw::HashMap, } // Safety: We only ever hand out &K through shared references to the map, // so normal Send/Sync rules apply. We never expose owned or mutable references // to keys or values. unsafe impl Send for HashSet {} unsafe impl Sync for HashSet {} /// A builder for a [`HashSet`]. /// /// # Examples /// /// ```rust /// use papaya::{HashSet, ResizeMode}; /// use seize::Collector; /// use std::collections::hash_map::RandomState; /// /// let set: HashSet = HashSet::builder() /// // Set the initial capacity. /// .capacity(2048) /// // Set the hasher. /// .hasher(RandomState::new()) /// // Set the resize mode. /// .resize_mode(ResizeMode::Blocking) /// // Set a custom garbage collector. /// .collector(Collector::new().batch_size(128)) /// // Construct the hash set. /// .build(); /// ``` pub struct HashSetBuilder { hasher: S, capacity: usize, collector: Collector, resize_mode: ResizeMode, _kv: PhantomData, } impl HashSetBuilder { /// Set the hash builder used to hash keys. /// /// Warning: `hash_builder` is normally randomly generated, and is designed /// to allow HashSets to be resistant to attacks that cause many collisions /// and very poor performance. Setting it manually using this function can /// expose a DoS attack vector. /// /// The `hash_builder` passed should implement the [`BuildHasher`] trait for /// the HashSet to be useful, see its documentation for details. pub fn hasher(self, hasher: S) -> HashSetBuilder { HashSetBuilder { hasher, capacity: self.capacity, collector: self.collector, resize_mode: self.resize_mode, _kv: PhantomData, } } } impl HashSetBuilder { /// Set the initial capacity of the set. /// /// The set should be able to hold at least `capacity` elements before resizing. /// However, the capacity is an estimate, and the set may prematurely resize due /// to poor hash distribution. If `capacity` is 0, the hash set will not allocate. pub fn capacity(self, capacity: usize) -> HashSetBuilder { HashSetBuilder { capacity, hasher: self.hasher, collector: self.collector, resize_mode: self.resize_mode, _kv: PhantomData, } } /// Set the resizing mode of the set. See [`ResizeMode`] for details. pub fn resize_mode(self, resize_mode: ResizeMode) -> Self { HashSetBuilder { resize_mode, hasher: self.hasher, capacity: self.capacity, collector: self.collector, _kv: PhantomData, } } /// Set the [`seize::Collector`] used for garbage collection. /// /// This method may be useful when you want more control over garbage collection. /// /// Note that all `Guard` references used to access the set must be produced by /// the provided `collector`. pub fn collector(self, collector: Collector) -> Self { HashSetBuilder { collector, hasher: self.hasher, capacity: self.capacity, resize_mode: self.resize_mode, _kv: PhantomData, } } /// Construct a [`HashSet`] from the builder, using the configured options. pub fn build(self) -> HashSet { HashSet { raw: raw::HashMap::new(self.capacity, self.hasher, self.collector, self.resize_mode), } } } impl fmt::Debug for HashSetBuilder { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("HashSetBuilder") .field("capacity", &self.capacity) .field("collector", &self.collector) .field("resize_mode", &self.resize_mode) .finish() } } impl HashSet { /// Creates an empty `HashSet`. /// /// The hash map is initially created with a capacity of 0, so it will not allocate /// until it is first inserted into. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// let map: HashSet<&str> = HashSet::new(); /// ``` pub fn new() -> HashSet { HashSet::with_capacity_and_hasher(0, RandomState::new()) } /// Creates an empty `HashSet` with the specified capacity. /// /// The set should be able to hold at least `capacity` elements before resizing. /// However, the capacity is an estimate, and the set may prematurely resize due /// to poor hash distribution. If `capacity` is 0, the hash set will not allocate. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// let set: HashSet<&str> = HashSet::with_capacity(10); /// ``` pub fn with_capacity(capacity: usize) -> HashSet { HashSet::with_capacity_and_hasher(capacity, RandomState::new()) } /// Returns a builder for a `HashSet`. /// /// The builder can be used for more complex configuration, such as using /// a custom [`Collector`], or [`ResizeMode`]. pub fn builder() -> HashSetBuilder { HashSetBuilder { capacity: 0, hasher: RandomState::default(), collector: Collector::new(), resize_mode: ResizeMode::default(), _kv: PhantomData, } } } impl Default for HashSet where S: Default, { fn default() -> Self { HashSet::with_hasher(S::default()) } } impl HashSet { /// Creates an empty `HashSet` which will use the given hash builder to hash /// keys. /// /// Warning: `hash_builder` is normally randomly generated, and is designed /// to allow HashSets to be resistant to attacks that cause many collisions /// and very poor performance. Setting it manually using this function can /// expose a DoS attack vector. /// /// The `hash_builder` passed should implement the [`BuildHasher`] trait for /// the HashSet to be useful, see its documentation for details. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// use std::hash::RandomState; /// /// let s = RandomState::new(); /// let set = HashSet::with_hasher(s); /// set.pin().insert(1); /// ``` pub fn with_hasher(hash_builder: S) -> HashSet { HashSet::with_capacity_and_hasher(0, hash_builder) } /// Creates an empty `HashSet` with at least the specified capacity, using /// `hash_builder` to hash the keys. /// /// The set should be able to hold at least `capacity` elements before resizing. /// However, the capacity is an estimate, and the set may prematurely resize due /// to poor hash distribution. If `capacity` is 0, the hash set will not allocate. /// /// Warning: `hash_builder` is normally randomly generated, and is designed /// to allow HashSets to be resistant to attacks that cause many collisions /// and very poor performance. Setting it manually using this function can /// expose a DoS attack vector. /// /// The `hasher` passed should implement the [`BuildHasher`] trait for /// the HashSet to be useful, see its documentation for details. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// use std::hash::RandomState; /// /// let s = RandomState::new(); /// let set = HashSet::with_capacity_and_hasher(10, s); /// set.pin().insert(1); /// ``` pub fn with_capacity_and_hasher(capacity: usize, hash_builder: S) -> HashSet { HashSet { raw: raw::HashMap::new( capacity, hash_builder, Collector::default(), ResizeMode::default(), ), } } /// Returns a pinned reference to the set. /// /// The returned reference manages a guard internally, preventing garbage collection /// for as long as it is held. See the [crate-level documentation](crate#usage) for details. #[inline] pub fn pin(&self) -> HashSetRef<'_, K, S, LocalGuard<'_>> { HashSetRef { guard: self.raw.guard(), set: self, } } /// Returns a pinned reference to the set. /// /// Unlike [`HashSet::pin`], the returned reference implements `Send` and `Sync`, /// allowing it to be held across `.await` points in work-stealing schedulers. /// This is especially useful for iterators. /// /// The returned reference manages a guard internally, preventing garbage collection /// for as long as it is held. See the [crate-level documentation](crate#usage) for details. #[inline] pub fn pin_owned(&self) -> HashSetRef<'_, K, S, OwnedGuard<'_>> { HashSetRef { guard: self.raw.owned_guard(), set: self, } } /// Returns a guard for use with this set. /// /// Note that holding on to a guard prevents garbage collection. /// See the [crate-level documentation](crate#usage) for details. #[inline] pub fn guard(&self) -> LocalGuard<'_> { self.raw.collector().enter() } /// Returns an owned guard for use with this set. /// /// Owned guards implement `Send` and `Sync`, allowing them to be held across /// `.await` points in work-stealing schedulers. This is especially useful /// for iterators. /// /// Note that holding on to a guard prevents garbage collection. /// See the [crate-level documentation](crate#usage) for details. #[inline] pub fn owned_guard(&self) -> OwnedGuard<'_> { self.raw.collector().enter_owned() } } impl HashSet where K: Hash + Eq, S: BuildHasher, { /// Returns the number of entries in the set. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set = HashSet::new(); /// /// set.pin().insert(1); /// set.pin().insert(2); /// assert!(set.len() == 2); /// ``` #[inline] pub fn len(&self) -> usize { self.raw.len() } /// Returns `true` if the set is empty. Otherwise returns `false`. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set = HashSet::new(); /// assert!(set.is_empty()); /// set.pin().insert("a"); /// assert!(!set.is_empty()); /// ``` #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } /// Returns `true` if the set contains a value for the specified key. /// /// The key may be any borrowed form of the set's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// /// [`Eq`]: std::cmp::Eq /// [`Hash`]: std::hash::Hash /// /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set = HashSet::new(); /// set.pin().insert(1); /// assert_eq!(set.pin().contains(&1), true); /// assert_eq!(set.pin().contains(&2), false); /// ``` #[inline] pub fn contains(&self, key: &Q, guard: &impl Guard) -> bool where Q: Equivalent + Hash + ?Sized, { self.get(key, self.raw.verify(guard)).is_some() } /// Returns a reference to the value corresponding to the key. /// /// The key may be any borrowed form of the set's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// /// [`Eq`]: std::cmp::Eq /// [`Hash`]: std::hash::Hash /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set = HashSet::new(); /// set.pin().insert(1); /// assert_eq!(set.pin().get(&1), Some(&1)); /// assert_eq!(set.pin().get(&2), None); /// ``` #[inline] pub fn get<'g, Q>(&self, key: &Q, guard: &'g impl Guard) -> Option<&'g K> where Q: Equivalent + Hash + ?Sized, { match self.raw.get(key, self.raw.verify(guard)) { Some((key, _)) => Some(key), None => None, } } /// Inserts a value into the set. /// /// If the set did not have this key present, `true` is returned. /// /// If the set did have this key present, `false` is returned and the old /// value is not updated. This matters for types that can be `==` without /// being identical. See the [standard library documentation] for details. /// /// [standard library documentation]: https://doc.rust-lang.org/std/collections/index.html#insert-and-complex-keys /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set = HashSet::new(); /// assert_eq!(set.pin().insert(37), true); /// assert_eq!(set.pin().is_empty(), false); /// /// set.pin().insert(37); /// assert_eq!(set.pin().insert(37), false); /// assert_eq!(set.pin().get(&37), Some(&37)); /// ``` #[inline] pub fn insert(&self, key: K, guard: &impl Guard) -> bool { match self.raw.insert(key, (), true, self.raw.verify(guard)) { InsertResult::Inserted(_) => true, InsertResult::Replaced(_) => false, InsertResult::Error { .. } => unreachable!(), } } /// Removes a key from the set, returning the value at the key if the key /// was previously in the set. /// /// The key may be any borrowed form of the set's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set = HashSet::new(); /// set.pin().insert(1); /// assert_eq!(set.pin().remove(&1), true); /// assert_eq!(set.pin().remove(&1), false); /// ``` #[inline] pub fn remove(&self, key: &Q, guard: &impl Guard) -> bool where Q: Equivalent + Hash + ?Sized, { match self.raw.remove(key, self.raw.verify(guard)) { Some((_, _)) => true, None => false, } } /// Tries to reserve capacity for `additional` more elements to be inserted /// in the `HashSet`. /// /// After calling this method, the set should be able to hold at least `capacity` elements /// before resizing. However, the capacity is an estimate, and the set may prematurely resize /// due to poor hash distribution. The collection may also reserve more space to avoid frequent /// reallocations. /// /// # Panics /// /// Panics if the new allocation size overflows `usize`. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set: HashSet<&str> = HashSet::new(); /// set.pin().reserve(10); /// ``` #[inline] pub fn reserve(&self, additional: usize, guard: &impl Guard) { self.raw.reserve(additional, self.raw.verify(guard)) } /// Clears the set, removing all values. /// /// Note that this method will block until any in-progress resizes are /// completed before proceeding. See the [consistency](crate#consistency) /// section for details. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set = HashSet::new(); /// /// set.pin().insert(1); /// set.pin().clear(); /// assert!(set.pin().is_empty()); /// ``` #[inline] pub fn clear(&self, guard: &impl Guard) { self.raw.clear(self.raw.verify(guard)) } /// Retains only the elements specified by the predicate. /// /// In other words, remove all values `v` for which `f(&v)` returns `false`. /// The elements are visited in unsorted (and unspecified) order. /// /// Note the function may be called more than once for a given key if its value is /// concurrently modified during removal. /// /// Additionally, this method will block until any in-progress resizes are /// completed before proceeding. See the [consistency](crate#consistency) /// section for details. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let mut set: HashSet = (0..8).collect(); /// set.pin().retain(|&v| v % 2 == 0); /// assert_eq!(set.len(), 4); /// assert_eq!(set.pin().contains(&1), false); /// assert_eq!(set.pin().contains(&2), true); /// ``` #[inline] pub fn retain(&mut self, mut f: F, guard: &impl Guard) where F: FnMut(&K) -> bool, { self.raw.retain(|k, _| f(k), self.raw.verify(guard)) } /// An iterator visiting all values in arbitrary order. /// /// Note that this method will block until any in-progress resizes are /// completed before proceeding. See the [consistency](crate#consistency) /// section for details. /// /// # Examples /// /// ``` /// use papaya::HashSet; /// /// let set = HashSet::from([ /// "a", /// "b", /// "c" /// ]); /// /// for val in set.pin().iter() { /// println!("val: {val}"); /// } #[inline] pub fn iter<'g, G>(&self, guard: &'g G) -> Iter<'g, K, G> where G: Guard, { Iter { raw: self.raw.iter(self.raw.verify(guard)), } } } impl PartialEq for HashSet where K: Hash + Eq, S: BuildHasher, { fn eq(&self, other: &Self) -> bool { if self.len() != other.len() { return false; } let (guard1, guard2) = (&self.guard(), &other.guard()); let mut iter = self.iter(guard1); iter.all(|key| other.get(key, guard2).is_some()) } } impl Eq for HashSet where K: Hash + Eq, S: BuildHasher, { } impl fmt::Debug for HashSet where K: Hash + Eq + fmt::Debug, S: BuildHasher, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let guard = self.guard(); f.debug_set().entries(self.iter(&guard)).finish() } } impl Extend for &HashSet where K: Hash + Eq, S: BuildHasher, { fn extend>(&mut self, iter: T) { // from `hashbrown::HashSet::extend`: // Keys may be already present or show multiple times in the iterator. // Reserve the entire hint lower bound if the set is empty. // Otherwise reserve half the hint (rounded up), so the set // will only resize twice in the worst case. let iter = iter.into_iter(); let reserve = if self.is_empty() { iter.size_hint().0 } else { (iter.size_hint().0 + 1) / 2 }; let guard = self.guard(); self.reserve(reserve, &guard); for key in iter { self.insert(key, &guard); } } } impl<'a, K, S> Extend<&'a K> for &HashSet where K: Copy + Hash + Eq + 'a, S: BuildHasher, { fn extend>(&mut self, iter: T) { self.extend(iter.into_iter().copied()); } } impl From<[K; N]> for HashSet where K: Hash + Eq, { fn from(arr: [K; N]) -> Self { HashSet::from_iter(arr) } } impl FromIterator for HashSet where K: Hash + Eq, S: BuildHasher + Default, { fn from_iter>(iter: T) -> Self { let mut iter = iter.into_iter(); if let Some(key) = iter.next() { let (lower, _) = iter.size_hint(); let set = HashSet::with_capacity_and_hasher(lower.saturating_add(1), S::default()); // Ideally we could use an unprotected guard here. However, `insert` // returns references to values that were replaced and retired, so // we need a "real" guard. A `raw_insert` method that strictly returns // pointers would fix this. { let set = set.pin(); set.insert(key); for key in iter { set.insert(key); } } set } else { Self::default() } } } impl Clone for HashSet where K: Clone + Hash + Eq, S: BuildHasher + Clone, { fn clone(&self) -> HashSet { let other = HashSet::builder() .capacity(self.len()) .hasher(self.raw.hasher.clone()) .collector(seize::Collector::new()) .build(); { let (guard1, guard2) = (&self.guard(), &other.guard()); for key in self.iter(guard1) { other.insert(key.clone(), guard2); } } other } } /// A pinned reference to a [`HashSet`]. /// /// This type is created with [`HashSet::pin`] and can be used to easily access a [`HashSet`] /// without explicitly managing a guard. See the [crate-level documentation](crate#usage) for details. pub struct HashSetRef<'set, K, S, G> { guard: MapGuard, set: &'set HashSet, } impl<'set, K, S, G> HashSetRef<'set, K, S, G> where K: Hash + Eq, S: BuildHasher, G: Guard, { /// Returns a reference to the inner [`HashSet`]. #[inline] pub fn set(&self) -> &'set HashSet { self.set } /// Returns the number of entries in the set. /// /// See [`HashSet::len`] for details. #[inline] pub fn len(&self) -> usize { self.set.raw.len() } /// Returns `true` if the set is empty. Otherwise returns `false`. /// /// See [`HashSet::is_empty`] for details. #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } /// Returns `true` if the set contains a value for the specified key. /// /// See [`HashSet::contains`] for details. #[inline] pub fn contains(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { self.get(key).is_some() } /// Returns a reference to the value corresponding to the key. /// /// See [`HashSet::get`] for details. #[inline] pub fn get(&self, key: &Q) -> Option<&K> where Q: Equivalent + Hash + ?Sized, { match self.set.raw.get(key, &self.guard) { Some((k, _)) => Some(k), None => None, } } /// Inserts a key-value pair into the set. /// /// See [`HashSet::insert`] for details. #[inline] pub fn insert(&self, key: K) -> bool { match self.set.raw.insert(key, (), true, &self.guard) { InsertResult::Inserted(_) => true, InsertResult::Replaced(_) => false, InsertResult::Error { .. } => unreachable!(), } } /// Removes a key from the set, returning the value at the key if the key /// was previously in the set. /// /// See [`HashSet::remove`] for details. #[inline] pub fn remove(&self, key: &Q) -> bool where Q: Equivalent + Hash + ?Sized, { match self.set.raw.remove(key, &self.guard) { Some((_, _)) => true, None => false, } } /// Clears the set, removing all values. /// /// See [`HashSet::clear`] for details. #[inline] pub fn clear(&self) { self.set.raw.clear(&self.guard) } /// Retains only the elements specified by the predicate. /// /// See [`HashSet::retain`] for details. #[inline] pub fn retain(&mut self, mut f: F) where F: FnMut(&K) -> bool, { self.set.raw.retain(|k, _| f(k), &self.guard) } /// Tries to reserve capacity for `additional` more elements to be inserted /// in the set. /// /// See [`HashSet::reserve`] for details. #[inline] pub fn reserve(&self, additional: usize) { self.set.raw.reserve(additional, &self.guard) } /// An iterator visiting all values in arbitrary order. /// The iterator element type is `(&K, &V)`. /// /// See [`HashSet::iter`] for details. #[inline] pub fn iter(&self) -> Iter<'_, K, G> { Iter { raw: self.set.raw.iter(&self.guard), } } } impl fmt::Debug for HashSetRef<'_, K, S, G> where K: Hash + Eq + fmt::Debug, S: BuildHasher, G: Guard, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_set().entries(self.iter()).finish() } } impl<'a, K, S, G> IntoIterator for &'a HashSetRef<'_, K, S, G> where K: Hash + Eq, S: BuildHasher, G: Guard, { type Item = &'a K; type IntoIter = Iter<'a, K, G>; fn into_iter(self) -> Self::IntoIter { self.iter() } } /// An iterator over a set's entries. /// /// This struct is created by the [`iter`](HashSet::iter) method on [`HashSet`]. See its documentation for details. pub struct Iter<'g, K, G> { raw: raw::Iter<'g, K, (), MapGuard>, } impl<'g, K: 'g, G> Iterator for Iter<'g, K, G> where G: Guard, { type Item = &'g K; #[inline] fn next(&mut self) -> Option { self.raw.next().map(|(k, _)| k) } } impl fmt::Debug for Iter<'_, K, G> where K: fmt::Debug, G: Guard, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list() .entries(Iter { raw: self.raw.clone(), }) .finish() } } papaya-0.2.3/tests/basic.rs000064400000000000000000000571751046102023000137110ustar 00000000000000// Adapted from: https://github.com/jonhoo/flurry/blob/main/tests/basic.rs use papaya::{Compute, HashMap, OccupiedError, Operation}; use std::hash::{BuildHasher, BuildHasherDefault, Hasher}; use std::sync::Arc; mod common; use common::with_map; #[test] fn new() { with_map::(|map| drop(map())); } #[test] fn clear() { with_map::(|map| { let map = map(); let guard = map.guard(); { map.insert(0, 1, &guard); map.insert(1, 1, &guard); map.insert(2, 1, &guard); map.insert(3, 1, &guard); map.insert(4, 1, &guard); } map.clear(&guard); assert!(map.is_empty()); }); } #[test] fn insert() { with_map::(|map| { let map = map(); let guard = map.guard(); let old = map.insert(42, 0, &guard); assert!(old.is_none()); }); } #[test] fn get_empty() { with_map::(|map| { let map = map(); let guard = map.guard(); let e = map.get(&42, &guard); assert!(e.is_none()); }); } #[test] fn get_key_value_empty() { with_map::(|map| { let map = map(); let guard = map.guard(); let e = map.get_key_value(&42, &guard); assert!(e.is_none()); }); } #[test] fn remove_empty() { with_map::(|map| { let map = map(); let guard = map.guard(); let old = map.remove(&42, &guard); assert!(old.is_none()); }); } #[test] fn remove_if() { with_map::(|map| { let map = map(); assert_eq!(map.pin().remove_if(&1, |_k, _v| true), Ok(None)); map.pin().insert(1, 0); assert_eq!(map.pin().remove_if(&0, |_k, _v| true), Ok(None)); assert_eq!(map.pin().remove_if(&1, |_k, _v| false), Err((&1, &0))); assert_eq!(map.pin().remove_if(&1, |_k, v| *v == 1), Err((&1, &0))); assert_eq!(map.pin().get(&1), Some(&0)); assert_eq!(map.pin().remove_if(&1, |_k, v| *v == 0), Ok(Some((&1, &0)))); assert_eq!(map.pin().remove_if(&1, |_, _| true), Ok(None)); }); } #[test] fn insert_and_remove() { with_map::(|map| { let map = map(); let guard = map.guard(); map.insert(42, 0, &guard); let old = map.remove(&42, &guard).unwrap(); assert_eq!(old, &0); assert!(map.get(&42, &guard).is_none()); }); } #[test] fn insert_and_get() { with_map::(|map| { let map = map(); map.insert(42, 0, &map.guard()); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &0); } }); } #[test] fn insert_and_get_key_value() { with_map::(|map| { let map = map(); map.insert(42, 0, &map.guard()); { let guard = map.guard(); let e = map.get_key_value(&42, &guard).unwrap(); assert_eq!(e, (&42, &0)); } }); } #[test] fn reinsert() { with_map::(|map| { let map = map(); let guard = map.guard(); map.insert(42, 0, &guard); let old = map.insert(42, 1, &guard); assert_eq!(old, Some(&0)); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &1); } }); } #[test] fn update() { with_map::(|map| { let map = map(); let guard = map.guard(); map.insert(42, 0, &guard); assert_eq!(map.len(), 1); let new = map.update(42, |v| v + 1, &guard); assert_eq!(map.len(), 1); assert_eq!(new, Some(&1)); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &1); } }); } #[test] fn update_empty() { with_map::(|map| { let map = map(); let guard = map.guard(); let new = map.update(42, |v| v + 1, &guard); assert!(new.is_none()); { let guard = map.guard(); assert!(map.get(&42, &guard).is_none()); } }); } #[test] fn update_or_insert() { with_map::(|map| { let map = map(); let guard = map.guard(); let result = map.update_or_insert(42, |v| v + 1, 0, &guard); assert_eq!(result, &0); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &0); } let result = map.update_or_insert(42, |v| v + 1, 0, &guard); assert_eq!(result, &1); assert_eq!(map.len(), 1); let result = map.update_or_insert(42, |v| v + 1, 0, &guard); assert_eq!(result, &2); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &2); } }); } #[test] fn update_or_insert_with() { with_map::(|map| { let map = map(); let guard = map.guard(); let result = map.update_or_insert_with(42, |v| v + 1, || 0, &guard); assert_eq!(result, &0); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &0); } let result = map.update_or_insert_with(42, |v| v + 1, || 0, &guard); assert_eq!(result, &1); assert_eq!(map.len(), 1); let result = map.update_or_insert_with(42, |v| v + 1, || 0, &guard); assert_eq!(result, &2); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &2); } }); } #[test] fn get_or_insert() { with_map::(|map| { let map = map(); let guard = map.guard(); let result = map.get_or_insert(42, 0, &guard); assert_eq!(result, &0); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &0); } let result = map.get_or_insert(42, 1, &guard); assert_eq!(result, &0); assert_eq!(map.len(), 1); }); } #[test] fn try_insert() { with_map::(|map| { let map = map(); let guard = map.guard(); assert_eq!(map.try_insert(42, 1, &guard), Ok(&1)); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &1); } assert_eq!( map.try_insert(42, 2, &guard), Err(OccupiedError { current: &1, not_inserted: 2 }) ); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &1); } assert_eq!(map.try_insert(43, 2, &guard), Ok(&2)); }); } #[test] fn try_insert_with() { with_map::(|map| { let map = map(); let guard = map.guard(); map.try_insert_with(42, || 1, &guard).unwrap(); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &1); } let mut called = false; let insert = || { called = true; 2 }; assert_eq!(map.try_insert_with(42, insert, &guard), Err(&1)); assert_eq!(map.len(), 1); assert!(!called); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &1); } assert_eq!(map.try_insert_with(43, || 2, &guard), Ok(&2)); }); } #[test] fn get_or_insert_with() { with_map::(|map| { let map = map(); let guard = map.guard(); let result = map.get_or_insert_with(42, || 0, &guard); assert_eq!(result, &0); assert_eq!(map.len(), 1); { let guard = map.guard(); let e = map.get(&42, &guard).unwrap(); assert_eq!(e, &0); } let result = map.get_or_insert_with(42, || 1, &guard); assert_eq!(result, &0); assert_eq!(map.len(), 1); }); } #[test] fn compute() { with_map::(|map| { let map = map(); let map = map.pin(); for i in 0..100 { let compute = |entry| match entry { Some((_, value)) if value % 2 == 0 => Operation::Remove, Some((_, value)) => Operation::Insert(value + 1), None => Operation::Abort(()), }; assert_eq!(map.len(), 0); assert_eq!(map.compute(i, compute), Compute::Aborted(())); assert_eq!(map.get(&i), None); map.compute(i, |_| Operation::Insert::<_, ()>(1)); assert_eq!(map.len(), 1); assert_eq!( map.compute(i, compute), Compute::Updated { old: (&i, &1), new: (&i, &2), } ); assert_eq!(map.compute(i, compute), Compute::Removed(&i, &2)); assert_eq!(map.len(), 0); } }); } #[test] fn concurrent_insert() { with_map::(|map| { let map = map(); let map = Arc::new(map); let map1 = map.clone(); let t1 = std::thread::spawn(move || { for i in 0..64 { map1.insert(i, 0, &map1.guard()); } }); let map2 = map.clone(); let t2 = std::thread::spawn(move || { for i in 0..64 { map2.insert(i, 1, &map2.guard()); } }); t1.join().unwrap(); t2.join().unwrap(); let guard = map.guard(); for i in 0..64 { let v = map.get(&i, &guard).unwrap(); assert!(v == &0 || v == &1); let kv = map.get_key_value(&i, &guard).unwrap(); assert!(kv == (&i, &0) || kv == (&i, &1)); } }); } #[test] fn concurrent_remove() { with_map::(|map| { let map = map(); let map = Arc::new(map); { let guard = map.guard(); for i in 0..64 { map.insert(i, i, &guard); } } let map1 = map.clone(); let t1 = std::thread::spawn(move || { let guard = map1.guard(); for i in 0..64 { if let Some(v) = map1.remove(&i, &guard) { assert_eq!(v, &i); } } }); let map2 = map.clone(); let t2 = std::thread::spawn(move || { let guard = map2.guard(); for i in 0..64 { if let Some(v) = map2.remove(&i, &guard) { assert_eq!(v, &i); } } }); t1.join().unwrap(); t2.join().unwrap(); // after joining the threads, the map should be empty let guard = map.guard(); for i in 0..64 { assert!(map.get(&i, &guard).is_none()); } }); } #[test] fn concurrent_update() { with_map::(|map| { let map = map(); let map = Arc::new(map); { let guard = map.guard(); for i in 0..64 { map.insert(i, i, &guard); } } let map1 = map.clone(); let t1 = std::thread::spawn(move || { let guard = map1.guard(); for i in 0..64 { let new = *map1.update(i, |v| v + 1, &guard).unwrap(); assert!(new == i + 1 || new == i + 2); } }); let map2 = map.clone(); let t2 = std::thread::spawn(move || { let guard = map2.guard(); for i in 0..64 { let new = *map2.update(i, |v| v + 1, &guard).unwrap(); assert!(new == i + 1 || new == i + 2); } }); t1.join().unwrap(); t2.join().unwrap(); // after joining the threads, the map should be empty let guard = map.guard(); for i in 0..64 { assert_eq!(map.get(&i, &guard), Some(&(i + 2))); } }); } #[test] #[cfg(not(miri))] fn concurrent_resize_and_get() { if cfg!(papaya_stress) { return; } with_map::(|map| { let map = map(); let map = Arc::new(map); { let guard = map.guard(); for i in 0..1024 { map.insert(i, i, &guard); } } let map1 = map.clone(); // t1 is using reserve to trigger a bunch of resizes let t1 = std::thread::spawn(move || { let guard = map1.guard(); // there should be 2 ** 10 capacity already, so trigger additional resizes for power in 11..16 { map1.reserve(1 << power, &guard); } }); let map2 = map.clone(); // t2 is retrieving existing keys a lot, attempting to encounter a BinEntry::Moved let t2 = std::thread::spawn(move || { let guard = map2.guard(); for _ in 0..32 { for i in 0..1024 { let v = map2.get(&i, &guard).unwrap(); assert_eq!(v, &i); } } }); t1.join().unwrap(); t2.join().unwrap(); // make sure all the entries still exist after all the resizes { let guard = map.guard(); for i in 0..1024 { let v = map.get(&i, &guard).unwrap(); assert_eq!(v, &i); } } }); } #[test] fn current_kv_dropped() { let dropped1 = Arc::new(0); let dropped2 = Arc::new(0); with_map::, Arc>(|map| { let map = map(); map.insert(dropped1.clone(), dropped2.clone(), &map.guard()); assert_eq!(Arc::strong_count(&dropped1), 2); assert_eq!(Arc::strong_count(&dropped2), 2); drop(map); // dropping the map should immediately drop (not deferred) all keys and values assert_eq!(Arc::strong_count(&dropped1), 1); assert_eq!(Arc::strong_count(&dropped2), 1); }); } #[test] fn empty_maps_equal() { with_map::(|map1| { let map1 = map1(); with_map::(|map2| { let map2 = map2(); assert_eq!(map1, map2); assert_eq!(map2, map1); }); }); } #[test] fn different_size_maps_not_equal() { with_map::(|map1| { let map1 = map1(); with_map::(|map2| { let map2 = map2(); { let guard1 = map1.guard(); let guard2 = map2.guard(); map1.insert(1, 0, &guard1); map1.insert(2, 0, &guard1); map1.insert(3, 0, &guard1); map2.insert(1, 0, &guard2); map2.insert(2, 0, &guard2); } assert_ne!(map1, map2); assert_ne!(map2, map1); }); }); } #[test] fn same_values_equal() { with_map::(|map1| { let map1 = map1(); with_map::(|map2| { let map2 = map2(); { map1.pin().insert(1, 0); map2.pin().insert(1, 0); } assert_eq!(map1, map2); assert_eq!(map2, map1); }); }); } #[test] fn different_values_not_equal() { with_map::(|map1| { let map1 = map1(); with_map::(|map2| { let map2 = map2(); { map1.pin().insert(1, 0); map2.pin().insert(1, 1); } assert_ne!(map1, map2); assert_ne!(map2, map1); }); }); } #[test] fn clone_map_empty() { with_map::<&'static str, u32>(|map| { let map = map(); let cloned_map = map.clone(); assert_eq!(map.len(), cloned_map.len()); assert_eq!(&map, &cloned_map); assert_eq!(cloned_map.len(), 0); }); } #[test] // Test that same values exists in both maps (original and cloned) fn clone_map_filled() { with_map::<&'static str, u32>(|map| { let map = map(); map.insert("FooKey", 0, &map.guard()); map.insert("BarKey", 10, &map.guard()); let cloned_map = map.clone(); assert_eq!(map.len(), cloned_map.len()); assert_eq!(&map, &cloned_map); // test that we are not mapping the same tables map.insert("NewItem", 100, &map.guard()); assert_ne!(&map, &cloned_map); }); } #[test] fn default() { with_map::(|map| { let map = map(); let guard = map.guard(); map.insert(42, 0, &guard); assert_eq!(map.get(&42, &guard), Some(&0)); }); } #[test] fn debug() { with_map::(|map| { let map = map(); let guard = map.guard(); map.insert(42, 0, &guard); map.insert(16, 8, &guard); let formatted = format!("{:?}", map); assert!(formatted == "{42: 0, 16: 8}" || formatted == "{16: 8, 42: 0}"); }); } #[test] fn extend() { if cfg!(papaya_stress) { return; } with_map::(|map| { let map = map(); let guard = map.guard(); let mut entries: Vec<(usize, usize)> = vec![(42, 0), (16, 6), (38, 42)]; entries.sort_unstable(); (&map).extend(entries.clone().into_iter()); let mut collected: Vec<(usize, usize)> = map .iter(&guard) .map(|(key, value)| (*key, *value)) .collect(); collected.sort_unstable(); assert_eq!(entries, collected); }); } #[test] fn extend_ref() { if cfg!(papaya_stress) { return; } with_map::(|map| { let map = map(); let mut entries: Vec<(&usize, &usize)> = vec![(&42, &0), (&16, &6), (&38, &42)]; entries.sort(); (&map).extend(entries.clone().into_iter()); let guard = map.guard(); let mut collected: Vec<(&usize, &usize)> = map.iter(&guard).collect(); collected.sort(); assert_eq!(entries, collected); }); } #[test] fn from_iter_empty() { use std::iter::FromIterator; let entries: Vec<(usize, usize)> = Vec::new(); let map: HashMap = HashMap::from_iter(entries.into_iter()); assert_eq!(map.len(), 0) } #[test] fn from_iter_repeated() { use std::iter::FromIterator; let entries = vec![(0, 1), (0, 2), (0, 3)]; let map: HashMap<_, _> = HashMap::from_iter(entries.into_iter()); let map = map.pin(); assert_eq!(map.len(), 1); assert_eq!(map.iter().collect::>(), vec![(&0, &3)]) } #[test] fn len() { with_map::(|map| { let map = map(); let len = if cfg!(miri) { 100 } else { 10_000 }; for i in 0..len { map.pin().insert(i, i + 1); } assert_eq!(map.len(), len); }); } #[test] fn iter() { if cfg!(papaya_stress) { return; } with_map::(|map| { let map = map(); let len = if cfg!(miri) { 100 } else { 10_000 }; for i in 0..len { assert_eq!(map.pin().insert(i, i + 1), None); } let v: Vec<_> = (0..len).map(|i| (i, i + 1)).collect(); let mut got: Vec<_> = map.pin().iter().map(|(&k, &v)| (k, v)).collect(); got.sort(); assert_eq!(v, got); }); } #[test] fn retain_empty() { with_map::(|map| { let map = map(); map.pin().retain(|_, _| false); assert_eq!(map.len(), 0); }); } #[test] fn retain_all_false() { with_map::(|map| { let map = map(); for i in 0..100 { map.pin().insert(i, i); } map.pin().retain(|_, _| false); assert_eq!(map.len(), 0); }); } #[test] fn retain_all_true() { with_map::(|map| { let map = map(); for i in 0..100 { map.pin().insert(i, i); } map.pin().retain(|_, _| true); assert_eq!(map.len(), 100); }); } #[test] fn retain_some() { with_map::(|map| { let map = map(); for i in 0..100 { map.pin().insert(i, i); } map.pin().retain(|_, v| *v < 5); assert_eq!(map.len(), 5); let mut got: Vec<_> = map.pin().values().copied().collect(); got.sort(); assert_eq!(got, [0, 1, 2, 3, 4]); }); } #[test] fn mixed() { const LEN: usize = if cfg!(miri) { 48 } else { 1024 }; with_map::(|map| { let map = map(); assert!(map.pin().get(&100).is_none()); map.pin().insert(100, 101); assert_eq!(map.pin().get(&100), Some(&101)); map.pin().update(100, |x| x + 2); assert_eq!(map.pin().get(&100), Some(&103)); assert!(map.pin().get(&200).is_none()); map.pin().insert(200, 202); assert_eq!(map.pin().get(&200), Some(&202)); assert!(map.pin().get(&300).is_none()); assert_eq!(map.pin().remove(&100), Some(&103)); assert_eq!(map.pin().remove(&200), Some(&202)); assert!(map.pin().remove(&300).is_none()); assert!(map.pin().get(&100).is_none()); assert!(map.pin().get(&200).is_none()); assert!(map.pin().get(&300).is_none()); for i in 0..LEN { assert_eq!(map.pin().insert(i, i + 1), None); } for i in 0..LEN { assert_eq!(map.pin().get(&i), Some(&(i + 1))); } for i in 0..LEN { assert_eq!(map.pin().update(i, |i| i - 1), Some(&i)); } for i in 0..LEN { assert_eq!(map.pin().get(&i), Some(&i)); } for i in 0..LEN { assert_eq!(map.pin().remove(&i), Some(&i)); } for i in 0..LEN { assert_eq!(map.pin().get(&i), None); } for i in 0..(LEN * 2) { assert_eq!(map.pin().insert(i, i + 1), None); } for i in 0..(LEN * 2) { assert_eq!(map.pin().get(&i), Some(&(i + 1))); } }); } // run tests with hashers that create unrealistically long probe sequences mod hasher { use super::*; fn check() { let range = if cfg!(miri) { 0..16 } else { 0..100 }; with_map::(|map| { let map = map(); let guard = map.guard(); for i in range.clone() { map.insert(i, i, &guard); } assert!(!map.contains_key(&i32::min_value(), &guard)); assert!(!map.contains_key(&(range.start - 1), &guard)); for i in range.clone() { assert!(map.contains_key(&i, &guard)); } assert!(!map.contains_key(&range.end, &guard)); assert!(!map.contains_key(&i32::max_value(), &guard)); }); } #[test] fn test_zero_hasher() { #[derive(Default)] pub struct ZeroHasher; impl Hasher for ZeroHasher { fn finish(&self) -> u64 { 0 } fn write(&mut self, _: &[u8]) {} } check::>(); } #[test] fn test_max_hasher() { #[derive(Default)] struct MaxHasher; impl Hasher for MaxHasher { fn finish(&self) -> u64 { u64::max_value() } fn write(&mut self, _: &[u8]) {} } check::>(); } } papaya-0.2.3/tests/basic_set.rs000064400000000000000000000340341046102023000145510ustar 00000000000000// Adapted from: https://github.com/jonhoo/flurry/blob/main/tests/basic.rs use papaya::HashSet; use std::hash::{BuildHasher, BuildHasherDefault, Hasher}; use std::sync::Arc; mod common; use common::with_set; #[test] fn new() { with_set::(|set| drop(set())); } #[test] fn clear() { with_set::(|set| { let set = set(); let guard = set.guard(); { set.insert(0, &guard); set.insert(1, &guard); set.insert(2, &guard); set.insert(3, &guard); set.insert(4, &guard); } set.clear(&guard); assert!(set.is_empty()); }); } #[test] fn insert() { with_set::(|set| { let set = set(); let guard = set.guard(); assert_eq!(set.insert(42, &guard), true); assert_eq!(set.insert(42, &guard), false); assert_eq!(set.len(), 1); }); } #[test] fn get_empty() { with_set::(|set| { let set = set(); let guard = set.guard(); let e = set.get(&42, &guard); assert!(e.is_none()); }); } #[test] fn remove_empty() { with_set::(|set| { let set = set(); let guard = set.guard(); assert_eq!(set.remove(&42, &guard), false); }); } #[test] fn insert_and_remove() { with_set::(|set| { let set = set(); let guard = set.guard(); assert!(set.insert(42, &guard)); assert!(set.remove(&42, &guard)); assert!(set.get(&42, &guard).is_none()); }); } #[test] fn insert_and_get() { with_set::(|set| { let set = set(); set.insert(42, &set.guard()); { let guard = set.guard(); let e = set.get(&42, &guard).unwrap(); assert_eq!(e, &42); } }); } #[test] fn reinsert() { with_set::(|set| { let set = set(); let guard = set.guard(); assert!(set.insert(42, &guard)); assert!(!set.insert(42, &guard)); { let guard = set.guard(); let e = set.get(&42, &guard).unwrap(); assert_eq!(e, &42); } }); } #[test] fn concurrent_insert() { with_set::(|set| { let set = set(); let set = Arc::new(set); let set1 = set.clone(); let t1 = std::thread::spawn(move || { for i in 0..64 { set1.insert(i, &set1.guard()); } }); let set2 = set.clone(); let t2 = std::thread::spawn(move || { for i in 0..64 { set2.insert(i, &set2.guard()); } }); t1.join().unwrap(); t2.join().unwrap(); let guard = set.guard(); for i in 0..64 { let v = set.get(&i, &guard).unwrap(); assert!(v == &i); } }); } #[test] fn concurrent_remove() { with_set::(|set| { let set = set(); let set = Arc::new(set); { let guard = set.guard(); for i in 0..64 { set.insert(i, &guard); } } let set1 = set.clone(); let t1 = std::thread::spawn(move || { let guard = set1.guard(); for i in 0..64 { set1.remove(&i, &guard); } }); let set2 = set.clone(); let t2 = std::thread::spawn(move || { let guard = set2.guard(); for i in 0..64 { set2.remove(&i, &guard); } }); t1.join().unwrap(); t2.join().unwrap(); // after joining the threads, the set should be empty let guard = set.guard(); for i in 0..64 { assert!(set.get(&i, &guard).is_none()); } }); } #[test] #[cfg(not(miri))] fn concurrent_resize_and_get() { if cfg!(papaya_stress) { return; } with_set::(|set| { let set = set(); let set = Arc::new(set); { let guard = set.guard(); for i in 0..1024 { set.insert(i, &guard); } } let set1 = set.clone(); // t1 is using reserve to trigger a bunch of resizes let t1 = std::thread::spawn(move || { let guard = set1.guard(); // there should be 2 ** 10 capacity already, so trigger additional resizes for power in 11..16 { set1.reserve(1 << power, &guard); } }); let set2 = set.clone(); // t2 is retrieving existing keys a lot, attempting to encounter a BinEntry::Moved let t2 = std::thread::spawn(move || { let guard = set2.guard(); for _ in 0..32 { for i in 0..1024 { let v = set2.get(&i, &guard).unwrap(); assert_eq!(v, &i); } } }); t1.join().unwrap(); t2.join().unwrap(); // make sure all the entries still exist after all the resizes { let guard = set.guard(); for i in 0..1024 { let v = set.get(&i, &guard).unwrap(); assert_eq!(v, &i); } } }); } #[test] fn current_kv_dropped() { let dropped1 = Arc::new(0); with_set::>(|set| { let set = set(); set.insert(dropped1.clone(), &set.guard()); assert_eq!(Arc::strong_count(&dropped1), 2); drop(set); // dropping the set should immediately drop (not deferred) all keys and values assert_eq!(Arc::strong_count(&dropped1), 1); }); } #[test] fn empty_sets_equal() { with_set::(|set1| { let set1 = set1(); with_set::(|set2| { let set2 = set2(); assert_eq!(set1, set2); assert_eq!(set2, set1); }); }); } #[test] fn different_size_sets_not_equal() { with_set::(|set1| { let set1 = set1(); with_set::(|set2| { let set2 = set2(); { let guard1 = set1.guard(); let guard2 = set2.guard(); set1.insert(1, &guard1); set1.insert(2, &guard1); set1.insert(3, &guard1); set2.insert(1, &guard2); set2.insert(2, &guard2); } assert_ne!(set1, set2); assert_ne!(set2, set1); }); }); } #[test] fn same_values_equal() { with_set::(|set1| { let set1 = set1(); with_set::(|set2| { let set2 = set2(); { set1.pin().insert(1); set2.pin().insert(1); } assert_eq!(set1, set2); assert_eq!(set2, set1); }); }); } #[test] fn different_values_not_equal() { with_set::(|set1| { let set1 = set1(); with_set::(|set2| { let set2 = set2(); { set1.pin().insert(1); set2.pin().insert(2); } assert_ne!(set1, set2); assert_ne!(set2, set1); }); }); } #[test] fn clone_set_empty() { with_set::<&'static str>(|set| { let set = set(); let cloned_set = set.clone(); assert_eq!(set.len(), cloned_set.len()); assert_eq!(&set, &cloned_set); assert_eq!(cloned_set.len(), 0); }); } #[test] // Test that same values exists in both sets (original and cloned) fn clone_set_filled() { with_set::<&'static str>(|set| { let set = set(); set.insert("FooKey", &set.guard()); set.insert("BarKey", &set.guard()); let cloned_set = set.clone(); assert_eq!(set.len(), cloned_set.len()); assert_eq!(&set, &cloned_set); // test that we are not setting the same tables set.insert("NewItem", &set.guard()); assert_ne!(&set, &cloned_set); }); } #[test] fn default() { with_set::(|set| { let set = set(); let guard = set.guard(); set.insert(42, &guard); assert_eq!(set.get(&42, &guard), Some(&42)); }); } #[test] fn debug() { with_set::(|set| { let set = set(); let guard = set.guard(); set.insert(42, &guard); set.insert(16, &guard); let formatted = format!("{:?}", set); assert!(formatted == "{42, 16}" || formatted == "{16, 42}"); }); } #[test] fn extend() { if cfg!(papaya_stress) { return; } with_set::(|set| { let set = set(); let guard = set.guard(); let mut entries: Vec = vec![42, 16, 38]; entries.sort_unstable(); (&set).extend(entries.clone().into_iter()); let mut collected: Vec = set.iter(&guard).map(|key| *key).collect(); collected.sort_unstable(); assert_eq!(entries, collected); }); } #[test] fn extend_ref() { if cfg!(papaya_stress) { return; } with_set::(|set| { let set = set(); let mut entries: Vec<&usize> = vec![&42, &36, &18]; entries.sort(); (&set).extend(entries.clone().into_iter()); let guard = set.guard(); let mut collected: Vec<&usize> = set.iter(&guard).collect(); collected.sort(); assert_eq!(entries, collected); }); } #[test] fn from_iter_empty() { use std::iter::FromIterator; let entries: Vec = Vec::new(); let set: HashSet = HashSet::from_iter(entries.into_iter()); assert_eq!(set.len(), 0) } #[test] fn from_iter_repeated() { use std::iter::FromIterator; let entries = vec![0, 0, 0]; let set: HashSet<_> = HashSet::from_iter(entries.into_iter()); let set = set.pin(); assert_eq!(set.len(), 1); assert_eq!(set.iter().collect::>(), vec![&0]) } #[test] fn len() { with_set::(|set| { let set = set(); let len = if cfg!(miri) { 100 } else { 10_000 }; for i in 0..len { set.pin().insert(i); } assert_eq!(set.len(), len); }); } #[test] fn iter() { if cfg!(papaya_stress) { return; } with_set::(|set| { let set = set(); let len = if cfg!(miri) { 100 } else { 10_000 }; for i in 0..len { assert_eq!(set.pin().insert(i), true); } let v: Vec<_> = (0..len).collect(); let mut got: Vec<_> = set.pin().iter().map(|&k| k).collect(); got.sort(); assert_eq!(v, got); }); } #[test] fn retain_empty() { with_set::(|set| { let set = set(); set.pin().retain(|_| false); assert_eq!(set.len(), 0); }); } #[test] fn retain_all_false() { with_set::(|set| { let set = set(); for i in 0..10 { set.pin().insert(i); } set.pin().retain(|_| false); assert_eq!(set.len(), 0); }); } #[test] fn retain_all_true() { with_set::(|set| { let set = set(); for i in 0..10 { set.pin().insert(i); } set.pin().retain(|_| true); assert_eq!(set.len(), 10); }); } #[test] fn retain_some() { with_set::(|set| { let set = set(); for i in 0..10 { set.pin().insert(i); } set.pin().retain(|&k| k >= 5); assert_eq!(set.len(), 5); let mut got: Vec<_> = set.pin().iter().copied().collect(); got.sort(); assert_eq!(got, [5, 6, 7, 8, 9]); }); } #[test] fn mixed() { const LEN: usize = if cfg!(miri) { 48 } else { 1024 }; with_set::(|set| { let set = set(); assert!(set.pin().get(&100).is_none()); set.pin().insert(100); assert_eq!(set.pin().get(&100), Some(&100)); assert!(set.pin().get(&200).is_none()); set.pin().insert(200); assert_eq!(set.pin().get(&200), Some(&200)); assert!(set.pin().get(&300).is_none()); assert_eq!(set.pin().remove(&100), true); assert_eq!(set.pin().remove(&200), true); assert_eq!(set.pin().remove(&300), false); assert!(set.pin().get(&100).is_none()); assert!(set.pin().get(&200).is_none()); assert!(set.pin().get(&300).is_none()); for i in 0..LEN { assert_eq!(set.pin().insert(i), true); } for i in 0..LEN { assert_eq!(set.pin().get(&i), Some(&i)); } for i in 0..LEN { assert_eq!(set.pin().remove(&i), true); } for i in 0..LEN { assert_eq!(set.pin().get(&i), None); } for i in 0..(LEN * 2) { assert_eq!(set.pin().insert(i), true); } for i in 0..(LEN * 2) { assert_eq!(set.pin().get(&i), Some(&i)); } }); } // run tests with hashers that create unrealistically long probe sequences mod hasher { use super::*; fn check() { let range = if cfg!(miri) { 0..16 } else { 0..100 }; with_set::(|set| { let set = set(); let guard = set.guard(); for i in range.clone() { set.insert(i, &guard); } assert!(!set.contains(&i32::min_value(), &guard)); assert!(!set.contains(&(range.start - 1), &guard)); for i in range.clone() { assert!(set.contains(&i, &guard)); } assert!(!set.contains(&range.end, &guard)); assert!(!set.contains(&i32::max_value(), &guard)); }); } #[test] fn test_zero_hasher() { #[derive(Default)] pub struct ZeroHasher; impl Hasher for ZeroHasher { fn finish(&self) -> u64 { 0 } fn write(&mut self, _: &[u8]) {} } check::>(); } #[test] fn test_max_hasher() { #[derive(Default)] struct MaxHasher; impl Hasher for MaxHasher { fn finish(&self) -> u64 { u64::max_value() } fn write(&mut self, _: &[u8]) {} } check::>(); } } papaya-0.2.3/tests/common.rs000064400000000000000000000045271046102023000141110ustar 00000000000000#![allow(dead_code)] use papaya::{HashMap, HashSet, ResizeMode}; use seize::Collector; // Run the test on different configurations of a `HashMap`. pub fn with_map(mut test: impl FnMut(&dyn Fn() -> HashMap)) { let collector = || Collector::new().batch_size(128); // Blocking resize mode. if !cfg!(papaya_stress) { test( &(|| { HashMap::builder() .collector(collector()) .resize_mode(ResizeMode::Blocking) .build() }), ); } // Incremental resize mode with a small chunk to stress operations on nested tables. test( &(|| { HashMap::builder() .collector(collector()) .resize_mode(ResizeMode::Incremental(1)) .build() }), ); // Incremental resize mode with a medium-sized chunk to promote interference with incremental // resizing. test( &(|| { HashMap::builder() .collector(collector()) .resize_mode(ResizeMode::Incremental(128)) .build() }), ); } // Run the test on different configurations of a `HashSet`. pub fn with_set(mut test: impl FnMut(&dyn Fn() -> HashSet)) { // Blocking resize mode. if !cfg!(papaya_stress) { test(&(|| HashSet::builder().resize_mode(ResizeMode::Blocking).build())); } // Incremental resize mode with a small chunk to stress operations on nested tables. test( &(|| { HashSet::builder() .resize_mode(ResizeMode::Incremental(1)) .build() }), ); // Incremental resize mode with a medium-sized chunk to promote interference with incremental // resizing. test( &(|| { HashSet::builder() .resize_mode(ResizeMode::Incremental(128)) .build() }), ); } // Prints a log message if `RUST_LOG=debug` is set. #[macro_export] macro_rules! debug { ($($x:tt)*) => { if std::env::var("RUST_LOG").as_deref() == Ok("debug") { println!($($x)*); } }; } // Returns the number of threads to use for stress testing. pub fn threads() -> usize { if cfg!(miri) { 2 } else { num_cpus::get_physical().next_power_of_two() } } papaya-0.2.3/tests/cuckoo.rs000064400000000000000000000174671046102023000141130ustar 00000000000000// Adapted from: https://github.com/jonhoo/flurry/blob/main/tests/cuckoo/stress.rs use papaya::{HashMap, ResizeMode}; use rand::distributions::{Distribution, Uniform}; use std::sync::atomic::Ordering; use std::sync::Mutex; use std::sync::{atomic::AtomicBool, Arc}; use std::thread; #[cfg(not(miri))] mod cfg { /// Number of keys and values to work with. pub const NUM_KEYS: usize = 1 << 14; /// Number of threads that should be started. pub const NUM_THREADS: usize = 4; /// How long the stress test will run (in milliseconds). pub const TEST_LEN: u64 = 10_000; } #[cfg(miri)] mod cfg { /// Number of keys and values to work with. pub const NUM_KEYS: usize = 1 << 10; /// Number of threads that should be started. pub const NUM_THREADS: usize = 4; /// How long the stress test will run (in milliseconds). pub const TEST_LEN: u64 = 5000; } type Key = usize; type Value = usize; struct Environment { table1: HashMap, table2: HashMap, keys: Vec, vals1: Mutex>, vals2: Mutex>, ind_dist: Uniform, val_dist1: Uniform, val_dist2: Uniform, in_table: Mutex>, in_use: Mutex>, finished: AtomicBool, } impl Environment { pub fn new() -> Self { let mut keys = Vec::with_capacity(cfg::NUM_KEYS); let mut in_use = Vec::with_capacity(cfg::NUM_KEYS); for i in 0..cfg::NUM_KEYS { keys.push(i); in_use.push(AtomicBool::new(false)); } Self { table1: HashMap::new(), table2: HashMap::new(), keys, vals1: Mutex::new(vec![0usize; cfg::NUM_KEYS]), vals2: Mutex::new(vec![0usize; cfg::NUM_KEYS]), ind_dist: Uniform::from(0..cfg::NUM_KEYS - 1), val_dist1: Uniform::from(Value::min_value()..Value::max_value()), val_dist2: Uniform::from(Value::min_value()..Value::max_value()), in_table: Mutex::new(vec![false; cfg::NUM_KEYS]), in_use: Mutex::new(in_use), finished: AtomicBool::new(false), } } } fn stress_insert_thread(env: Arc) { let mut rng = rand::thread_rng(); let guard1 = env.table1.guard(); let guard2 = env.table2.guard(); while !env.finished.load(Ordering::SeqCst) { let idx = env.ind_dist.sample(&mut rng); let in_use = env.in_use.lock().unwrap(); if (*in_use)[idx] .compare_exchange(false, true, Ordering::SeqCst, Ordering::Relaxed) .is_ok() { let key = env.keys[idx]; let val1 = env.val_dist1.sample(&mut rng); let val2 = env.val_dist2.sample(&mut rng); let res1 = if !env.table1.contains_key(&key, &guard1) { env.table1 .insert(key, val1, &guard1) .map_or(true, |_| false) } else { false }; let res2 = if !env.table2.contains_key(&key, &guard2) { env.table2 .insert(key, val2, &guard2) .map_or(true, |_| false) } else { false }; let mut in_table = env.in_table.lock().unwrap(); assert_ne!(res1, (*in_table)[idx]); assert_ne!(res2, (*in_table)[idx]); if res1 { assert_eq!(Some(&val1), env.table1.get(&key, &guard1)); assert_eq!(Some(&val2), env.table2.get(&key, &guard2)); let mut vals1 = env.vals1.lock().unwrap(); let mut vals2 = env.vals2.lock().unwrap(); (*vals1)[idx] = val1; (*vals2)[idx] = val2; (*in_table)[idx] = true; } (*in_use)[idx].swap(false, Ordering::SeqCst); } } } fn stress_delete_thread(env: Arc) { let mut rng = rand::thread_rng(); let guard1 = env.table1.guard(); let guard2 = env.table2.guard(); while !env.finished.load(Ordering::SeqCst) { let idx = env.ind_dist.sample(&mut rng); let in_use = env.in_use.lock().unwrap(); if (*in_use)[idx] .compare_exchange(false, true, Ordering::SeqCst, Ordering::Relaxed) .is_ok() { let key = env.keys[idx]; let res1 = env.table1.remove(&key, &guard1).map_or(false, |_| true); let res2 = env.table2.remove(&key, &guard2).map_or(false, |_| true); let mut in_table = env.in_table.lock().unwrap(); assert_eq!(res1, (*in_table)[idx]); assert_eq!(res2, (*in_table)[idx]); if res1 { assert!(env.table1.get(&key, &guard1).is_none()); assert!(env.table2.get(&key, &guard2).is_none()); (*in_table)[idx] = false; } (*in_use)[idx].swap(false, Ordering::SeqCst); } } } fn stress_find_thread(env: Arc) { let mut rng = rand::thread_rng(); let guard1 = env.table1.guard(); let guard2 = env.table2.guard(); while !env.finished.load(Ordering::SeqCst) { let idx = env.ind_dist.sample(&mut rng); let in_use = env.in_use.lock().unwrap(); if (*in_use)[idx] .compare_exchange(false, true, Ordering::SeqCst, Ordering::Relaxed) .is_ok() { let key = env.keys[idx]; let in_table = env.in_table.lock().unwrap(); let val1 = (*env.vals1.lock().unwrap())[idx]; let val2 = (*env.vals2.lock().unwrap())[idx]; let value = env.table1.get(&key, &guard1); if value.is_some() { assert_eq!(&val1, value.unwrap()); assert!((*in_table)[idx]); } let value = env.table2.get(&key, &guard2); if value.is_some() { assert_eq!(&val2, value.unwrap()); assert!((*in_table)[idx]); } (*in_use)[idx].swap(false, Ordering::SeqCst); } } } #[test] #[ignore] #[cfg(not(papaya_stress))] fn stress_test_blocking() { let mut root = Environment::new(); root.table1 = HashMap::builder().resize_mode(ResizeMode::Blocking).build(); root.table2 = HashMap::builder().resize_mode(ResizeMode::Blocking).build(); run(Arc::new(root)); } #[test] #[ignore] fn stress_test_incremental() { let mut root = Environment::new(); root.table1 = HashMap::builder() .resize_mode(ResizeMode::Incremental(1024)) .build(); root.table2 = HashMap::builder() .resize_mode(ResizeMode::Incremental(1024)) .build(); run(Arc::new(root)); } #[test] #[ignore] fn stress_test_incremental_slow() { let mut root = Environment::new(); root.table1 = HashMap::builder() .resize_mode(ResizeMode::Incremental(1)) .build(); root.table2 = HashMap::builder() .resize_mode(ResizeMode::Incremental(1)) .build(); run(Arc::new(root)); } fn run(root: Arc) { let mut threads = Vec::new(); for _ in 0..cfg::NUM_THREADS { let env = Arc::clone(&root); threads.push(thread::spawn(move || stress_insert_thread(env))); let env = Arc::clone(&root); threads.push(thread::spawn(move || stress_delete_thread(env))); let env = Arc::clone(&root); threads.push(thread::spawn(move || stress_find_thread(env))); } thread::sleep(std::time::Duration::from_millis(cfg::TEST_LEN)); root.finished.swap(true, Ordering::SeqCst); for t in threads { t.join().expect("failed to join thread"); } if !cfg!(papaya_stress) { let in_table = &*root.in_table.lock().unwrap(); let num_filled = in_table.iter().filter(|b| **b).count(); assert_eq!(num_filled, root.table1.pin().len()); assert_eq!(num_filled, root.table2.pin().len()); } } papaya-0.2.3/tests/stress.rs000064400000000000000000001146261046102023000141460ustar 00000000000000use papaya::{Compute, HashMap, Operation}; use rand::prelude::*; use std::hash::Hash; use std::ops::Range; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::sync::Barrier; use std::thread; mod common; use common::{threads, with_map}; // Call `contains_key` in parallel for a shared set of keys. #[test] #[ignore] fn contains_key_stress() { const ENTRIES: usize = match () { _ if cfg!(miri) => 64, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 12, _ => 1 << 19, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let mut content = vec![0; ENTRIES]; { let guard = map.guard(); for k in 0..ENTRIES { map.insert(k, k, &guard); content[k] = k; } } let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { for _ in 0..threads { s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in 0..ENTRIES { let key = content[i % content.len()]; assert!(map.contains_key(&key, &guard)); } }); } }); } }); } // Call `insert` in parallel with each thread inserting a distinct set of keys. #[test] #[ignore] fn insert_stress() { const ENTRIES: usize = match () { _ if cfg!(miri) => 64, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 12, _ => 1 << 17, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug)] struct Random(usize); fn random() -> Random { Random(rand::thread_rng().gen()) } with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { for _ in 0..threads { s.spawn(|| { barrier.wait(); for _ in 0..ENTRIES { let key = random(); map.insert(key, key, &map.guard()); assert!(map.contains_key(&key, &map.guard())); } }); } }); assert_eq!(map.len(), ENTRIES * threads); } }); } // Call `insert` in parallel on a small shared set of keys. #[test] #[ignore] fn insert_overwrite_stress() { const ENTRIES: usize = if cfg!(miri) { 64 } else { 256 }; const OPERATIONS: usize = match () { _ if cfg!(miri) => 1, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 9, _ => 1 << 10, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; let entries = || { let mut entries = (0..(OPERATIONS)) .flat_map(|_| (0..ENTRIES)) .collect::>(); let mut rng = rand::thread_rng(); entries.shuffle(&mut rng); entries }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let counters = (0..ENTRIES) .map(|_| AtomicUsize::new(0)) .collect::>(); let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { let mut handles = Vec::with_capacity(threads); for _ in 0..threads { let h = s.spawn(|| { let mut seen = (0..ENTRIES) .map(|_| Vec::with_capacity(OPERATIONS)) .collect::>(); let entries = entries(); barrier.wait(); for i in entries { let value = counters[i].fetch_add(1, Ordering::Relaxed); if let Some(&prev) = map.insert(i, value, &map.guard()) { // Keep track of values we overwrite. seen[i].push(prev); } assert!(map.contains_key(&i, &map.guard())); } seen }); handles.push(h); } let mut seen = (0..ENTRIES) .map(|_| Vec::::with_capacity(threads * OPERATIONS)) .collect::>(); for h in handles { let values = h.join().unwrap(); for (i, values) in values.iter().enumerate() { seen[i].extend(values); } } for i in 0..seen.len() { seen[i].push(*map.pin().get(&i).unwrap()); } // Ensure every insert operation was consistent. let operations = (0..(threads * OPERATIONS)).collect::>(); for mut values in seen { values.sort(); assert_eq!(values, operations); } }); assert_eq!(map.len(), ENTRIES); } }); } // Call `update` in parallel for a small shared set of keys. #[test] #[ignore] fn update_stress() { const ENTRIES: usize = if cfg!(miri) { 64 } else { 256 }; const OPERATIONS: usize = match () { _ if cfg!(miri) => 1, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 9, _ => 1 << 10, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 48 }; let entries = || { let mut entries = (0..(OPERATIONS)) .flat_map(|_| (0..ENTRIES)) .collect::>(); let mut rng = rand::thread_rng(); entries.shuffle(&mut rng); entries }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); { let guard = map.guard(); for i in 0..ENTRIES { map.insert(i, 0, &guard); } } let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { for _ in 0..threads { s.spawn(|| { let entries = entries(); barrier.wait(); for i in entries { let guard = map.guard(); let new = *map.update(i, |v| v + 1, &guard).unwrap(); assert!((0..=(threads * OPERATIONS)).contains(&new)); } }); } }); let guard = map.guard(); for i in 0..ENTRIES { assert_eq!(*map.get(&i, &guard).unwrap(), threads * OPERATIONS); } } }); } // Call `update` in parallel for a shared set of keys, with a single thread dedicated // to inserting unrelated keys. This is likely to cause interference with incremental resizing. #[test] #[ignore] fn update_insert_stress() { const ENTRIES: usize = match () { _ if cfg!(miri) => 64, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 12, _ => 1 << 18, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 48 }; with_map(|map| { let map = map(); { let guard = map.guard(); for i in 0..ENTRIES { map.insert(i, 0, &guard); } } for t in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let threads = threads(); let barrier = Barrier::new(threads); let threads = &threads; thread::scope(|s| { for _ in 0..(threads - 1) { s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in 0..ENTRIES { let new = *map.update(i, |v| v + 1, &guard).unwrap(); assert!((0..=(threads * (t + 1))).contains(&new)); } }); } s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in ENTRIES..(ENTRIES * 2) { map.insert(i, usize::MAX, &guard); } }); }); let guard = map.guard(); for i in 0..ENTRIES { assert_eq!(*map.get(&i, &guard).unwrap(), (threads - 1) * (t + 1)); } for i in ENTRIES..(ENTRIES * 2) { assert_eq!(*map.get(&i, &guard).unwrap(), usize::MAX); } } }); } // Call `update_or_insert` in parallel for a small shared set of keys. // Stresses the `insert` -> `update` transition in `compute`. #[test] #[ignore] fn update_or_insert_stress() { const ENTRIES: usize = if cfg!(miri) { 64 } else { 256 }; const OPERATIONS: usize = match () { _ if cfg!(miri) => 1, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 9, _ => 1 << 10, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 48 }; let threads = threads(); let entries = (0..(threads * OPERATIONS)) .flat_map(|_| (0..ENTRIES)) .collect::>(); let chunk = ENTRIES * OPERATIONS; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let mut entries = entries.clone(); let mut rng = rand::thread_rng(); entries.shuffle(&mut rng); let barrier = Barrier::new(threads); thread::scope(|s| { for t in 0..threads { let range = (chunk * t)..(chunk * (t + 1)); s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in &entries[range] { map.update_or_insert(*i, |v| v + 1, 1, &guard); } }); } }); let guard = map.guard(); for i in 0..ENTRIES { assert_eq!(*map.get(&i, &guard).unwrap(), threads * OPERATIONS); } assert_eq!(map.len(), ENTRIES); } }); } // Call `update_or_insert` in parallel for a small shared set of keys, with some // threads removing and reinserting values. // // Stresses the `update` <-> `insert` transition in `compute`. #[test] #[ignore] fn remove_update_or_insert_stress() { const ENTRIES: usize = if cfg!(miri) { 64 } else { 256 }; const OPERATIONS: usize = match () { _ if cfg!(miri) => 1, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 4, _ => 1 << 9, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; let threads = threads(); let entries = || { let mut entries = (0..(OPERATIONS)) .flat_map(|_| (0..ENTRIES)) .collect::>(); let mut rng = rand::thread_rng(); entries.shuffle(&mut rng); entries }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let group = threads.checked_div(2).unwrap(); let barrier = Barrier::new(threads + 1); thread::scope(|s| { for _ in 0..group { s.spawn(|| { let entries = entries(); barrier.wait(); let guard = map.guard(); for i in entries { map.update_or_insert(i, |v| v + 1, 1, &guard); } }); } for _ in 0..group { s.spawn(|| { let entries = entries(); barrier.wait(); let guard = map.guard(); for i in entries { if let Some(&value) = map.remove(&i, &guard) { map.update_or_insert(i, |v| v + value, value, &guard); } } }); } s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in ENTRIES..(ENTRIES * OPERATIONS) { map.insert(i, usize::MAX, &guard); } }); }); let guard = map.guard(); assert_eq!(map.len(), ENTRIES * OPERATIONS); for i in 0..ENTRIES { assert_eq!(*map.get(&i, &guard).unwrap(), group * OPERATIONS); } for i in ENTRIES..(ENTRIES * OPERATIONS) { assert_eq!(*map.get(&i, &guard).unwrap(), usize::MAX); } } }); } // Call `update_or_insert` in parallel for a small shared set of keys, with some // threads conditionally removing and reinserting values. // // Stresses the `remove` <-> `update` transition in `compute`. #[test] #[ignore] fn conditional_remove_update_or_insert_stress() { const ENTRIES: usize = if cfg!(miri) { 64 } else { 256 }; const OPERATIONS: usize = match () { _ if cfg!(miri) => 1, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 5, _ => 1 << 9, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; let threads = threads(); let entries = || { let mut entries = (0..(OPERATIONS)) .flat_map(|_| (0..ENTRIES)) .collect::>(); let mut rng = rand::thread_rng(); entries.shuffle(&mut rng); entries }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let group = threads.checked_div(2).unwrap(); let barrier = Barrier::new(threads + 1); thread::scope(|s| { for _ in 0..group { s.spawn(|| { let entries = entries(); barrier.wait(); let guard = map.guard(); for i in entries { map.update_or_insert(i, |v| v + 1, 1, &guard); } }); } for _ in 0..group { s.spawn(|| { let entries = entries(); barrier.wait(); let guard = map.guard(); for i in entries { let compute = |entry| match entry { Some((_, value)) if value % 2 == 0 => Operation::Remove, _ => Operation::Abort(()), }; if let Compute::Removed(_, &value) = map.compute(i, compute, &guard) { map.update_or_insert(i, |v| v + value, value, &guard); } } }); } s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in ENTRIES..(ENTRIES * OPERATIONS) { map.insert(i, usize::MAX, &guard); } }); }); let guard = map.guard(); assert_eq!(map.len(), ENTRIES * OPERATIONS); for i in 0..ENTRIES { assert_eq!(*map.get(&i, &guard).unwrap(), group * OPERATIONS); } for i in ENTRIES..(ENTRIES * OPERATIONS) { assert_eq!(*map.get(&i, &guard).unwrap(), usize::MAX); } } }); } // Similar to `conditional_remove_update_or_insert_stress` except uses `HashMap::remove_if`. #[test] #[ignore] fn remove_if_update_or_insert_stress() { const ENTRIES: usize = if cfg!(miri) { 64 } else { 256 }; const OPERATIONS: usize = match () { _ if cfg!(miri) => 1, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 5, _ => 1 << 9, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; let threads = threads(); let entries = || { let mut entries = (0..(OPERATIONS)) .flat_map(|_| (0..ENTRIES)) .collect::>(); let mut rng = rand::thread_rng(); entries.shuffle(&mut rng); entries }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let group = threads.checked_div(2).unwrap(); let barrier = Barrier::new(threads + 1); thread::scope(|s| { for _ in 0..group { s.spawn(|| { let entries = entries(); barrier.wait(); let guard = map.guard(); for i in entries { map.update_or_insert(i, |v| v + 1, 1, &guard); } }); } for _ in 0..group { s.spawn(|| { let entries = entries(); barrier.wait(); let guard = map.guard(); for i in entries { if let Ok(Some((_k, &value))) = map.remove_if(&i, |_k, v| v % 2 == 0, &guard) { map.update_or_insert(i, |v| v + value, value, &guard); } } }); } s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in ENTRIES..(ENTRIES * OPERATIONS) { map.insert(i, usize::MAX, &guard); } }); }); let guard = map.guard(); assert_eq!(map.len(), ENTRIES * OPERATIONS); for i in 0..ENTRIES { assert_eq!(*map.get(&i, &guard).unwrap(), group * OPERATIONS); } for i in ENTRIES..(ENTRIES * OPERATIONS) { assert_eq!(*map.get(&i, &guard).unwrap(), usize::MAX); } } }); } // Call `remove` and `insert` in parallel for a shared set of keys. #[test] #[ignore] fn insert_remove_stress() { const ENTRIES: usize = if cfg!(miri) { 64 } else { 256 }; const OPERATIONS: usize = match () { _ if cfg!(miri) => 1, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 8, _ => 1 << 9, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 48 }; let entries = || { let mut entries = (0..(OPERATIONS)) .flat_map(|_| (0..ENTRIES)) .collect::>(); let mut rng = rand::thread_rng(); entries.shuffle(&mut rng); entries }; let threads = threads(); with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let group = threads.checked_div(2).unwrap(); let barrier = Barrier::new(threads); thread::scope(|s| { for _ in 0..group { s.spawn(|| { let entries = entries(); barrier.wait(); let guard = map.guard(); for i in entries { map.insert(i, i, &guard); } }); } for _ in 0..group { s.spawn(|| { let entries = entries(); barrier.wait(); let guard = map.guard(); for i in entries { if map.remove(&i, &guard).is_some() { map.insert(i, i, &guard); } } }); } }); let guard = map.guard(); assert_eq!(map.len(), ENTRIES); for i in 0..ENTRIES { assert_eq!(map.get(&i, &guard), Some(&i)); } } }); } // Call `remove` in parallel for a shared set of keys with other threads calling `update`, // and a dedicated thread for inserting unrelated keys. This is likely to cause interference // with incremental resizing. #[test] #[ignore] fn remove_mixed_stress() { const ENTRIES: usize = match () { _ if cfg!(miri) => 64, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 11, _ => 1 << 17, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 64 }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); { let guard = map.guard(); for i in 0..ENTRIES { map.insert(i, 0, &guard); } } let threads = threads().max(3); let barrier = Barrier::new(threads); thread::scope(|s| { for _ in 0..(threads - 2) { s.spawn(|| { let mut entries = (0..ENTRIES).collect::>(); let mut rng = rand::thread_rng(); entries.shuffle(&mut rng); barrier.wait(); let guard = map.guard(); loop { let mut empty = true; for &i in entries.iter() { if map.update(i, |v| v + 1, &guard).is_some() { empty = false; } } if empty { break; } } }); } s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in 0..ENTRIES { map.remove(&i, &guard); } for i in 0..ENTRIES { assert_eq!(map.get(&i, &guard), None); } }); s.spawn(|| { barrier.wait(); let guard = map.guard(); for i in ENTRIES..(ENTRIES * 2) { map.insert(i, usize::MAX, &guard); } }); }); let guard = map.guard(); for i in 0..ENTRIES { assert_eq!(map.get(&i, &guard), None); } for i in ENTRIES..(ENTRIES * 2) { assert_eq!(*map.get(&i, &guard).unwrap(), usize::MAX); } assert_eq!(map.len(), ENTRIES); } }); } // Performs insert and remove operations with each thread operating on a distinct set of keys. #[test] #[ignore] fn insert_remove_chunk_stress() { const ENTRIES: usize = match () { _ if cfg!(miri) => 48, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 11, _ => 1 << 17, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 48 }; let run = |barrier: &Barrier, chunk: Range, map: &HashMap, threads: usize| { barrier.wait(); for i in chunk.clone() { assert_eq!(map.pin().insert(i, i), None); } for i in chunk.clone() { assert_eq!(map.pin().get(&i), Some(&i)); } for i in chunk.clone() { assert_eq!(map.pin().remove(&i), Some(&i)); } for i in chunk.clone() { assert_eq!(map.pin().get(&i), None); } if !cfg!(papaya_stress) { for (&k, &v) in map.pin().iter() { assert!(k < ENTRIES * threads); assert!(v == k); } } }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { for i in 0..threads { let map = ↦ let barrier = &barrier; let chunk = (ENTRIES * i)..(ENTRIES * (i + 1)); s.spawn(move || run(barrier, chunk, map, threads)); } }); if !cfg!(papaya_stress) { let got: Vec<_> = map.pin().into_iter().map(|(&k, &v)| (k, v)).collect(); assert_eq!(got, []); } assert_eq!(map.len(), 0); } }); } // Performs a mix of operations with each thread operating on a distinct set of keys. #[test] #[ignore] fn mixed_chunk_stress() { const ENTRIES: usize = match () { _ if cfg!(miri) => 48, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 11, _ => 1 << 16, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; let run = |barrier: &Barrier, chunk: Range, map: &HashMap, threads: usize| { barrier.wait(); for i in chunk.clone() { assert_eq!(map.pin().insert(i, i + 1), None); } for i in chunk.clone() { assert_eq!(map.pin().get(&i), Some(&(i + 1))); } for i in chunk.clone() { assert_eq!(map.pin().update(i, |i| i - 1), Some(&i)); } for i in chunk.clone() { assert_eq!(map.pin().remove(&i), Some(&i)); } for i in chunk.clone() { assert_eq!(map.pin().get(&i), None); } for i in chunk.clone() { assert_eq!(map.pin().insert(i, i + 1), None); } for i in chunk.clone() { assert_eq!(map.pin().get(&i), Some(&(i + 1))); } if !cfg!(papaya_stress) { for (&k, &v) in map.pin().iter() { assert!(k < ENTRIES * threads); assert!(v == k || v == k + 1); } } }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { for i in 0..threads { let map = ↦ let barrier = &barrier; let chunk = (ENTRIES * i)..(ENTRIES * (i + 1)); s.spawn(move || run(barrier, chunk, map, threads)); } }); if !cfg!(papaya_stress) { let v: Vec<_> = (0..ENTRIES * threads).map(|i| (i, i + 1)).collect(); let mut got: Vec<_> = map.pin().iter().map(|(&k, &v)| (k, v)).collect(); got.sort(); assert_eq!(v, got); } assert_eq!(map.len(), ENTRIES * threads); } }); } // Performs a mix of operations with each thread operating on a specific entry within // a distinct set of keys. This is more likely to cause interference with incremental resizing. #[test] #[ignore] fn mixed_entry_stress() { const ENTRIES: usize = match () { _ if cfg!(miri) => 100, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 11, _ => 1 << 11, }; const OPERATIONS: usize = if cfg!(miri) { 1 } else { 72 }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; let run = |barrier: &Barrier, chunk: Range, map: &HashMap, threads: usize| { barrier.wait(); for i in chunk.clone() { for _ in 0..OPERATIONS { assert_eq!(map.pin().insert(i, i + 1), None); assert_eq!(map.pin().get(&i), Some(&(i + 1))); assert_eq!(map.pin().update(i, |i| i + 1), Some(&(i + 2))); assert_eq!(map.pin().remove(&i), Some(&(i + 2))); assert_eq!(map.pin().get(&i), None); assert_eq!(map.pin().update(i, |i| i + 1), None); } } for i in chunk.clone() { assert_eq!(map.pin().get(&i), None); } if !cfg!(papaya_stress) { for (&k, &v) in map.pin().iter() { assert!(k < ENTRIES * threads); assert!(v == k + 1 || v == k + 2); } } }; with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { for i in 0..threads { let map = ↦ let barrier = &barrier; let chunk = (ENTRIES * i)..(ENTRIES * (i + 1)); s.spawn(move || run(barrier, chunk, map, threads)); } }); if !cfg!(papaya_stress) { let got: Vec<_> = map.pin().iter().map(|(&k, &v)| (k, v)).collect(); assert_eq!(got, []); } assert_eq!(map.len(), 0); } }); } // Clears the map during concurrent insertion. // // This test is relatively vague as there are few guarantees observable from concurrent calls // to `clear` but still useful for Miri. #[test] #[ignore] fn clear_stress() { if cfg!(papaya_stress) { return; } const ENTRIES: usize = match () { _ if cfg!(miri) => 64, _ if cfg!(papaya_asan) => 1 << 12, _ => 1 << 17, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug)] struct Random(usize); fn random() -> Random { Random(rand::thread_rng().gen()) } with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { for _ in 0..(threads - 1) { s.spawn(|| { barrier.wait(); for _ in 0..ENTRIES { let key = random(); map.pin().insert(key, key); } }); } s.spawn(|| { barrier.wait(); for _ in 0..(threads * 20) { map.pin().clear(); } }); }); map.pin().clear(); assert_eq!(map.len(), 0); assert_eq!(map.pin().iter().count(), 0); } }); } // Retains the map during concurrent insertion. #[test] #[ignore] fn retain_stress() { if cfg!(papaya_stress) { return; } const ENTRIES: usize = match () { _ if cfg!(miri) => 64, _ if cfg!(papaya_asan) => 1 << 12, _ => 1 << 17, }; const ITERATIONS: usize = if cfg!(miri) { 1 } else { 32 }; #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug)] struct Random(usize); fn random() -> Random { Random(rand::thread_rng().gen()) } with_map(|map| { for _ in (0..ITERATIONS).inspect(|e| debug!("{e}/{ITERATIONS}")) { let map = map(); let threads = threads(); let barrier = Barrier::new(threads); thread::scope(|s| { for _ in 0..(threads - 1) { s.spawn(|| { barrier.wait(); for _ in 0..ENTRIES { let key = random(); map.insert(key, key, &map.guard()); assert!(map.contains_key(&key, &map.guard())); } }); } s.spawn(|| { barrier.wait(); for _ in 0..(threads * 20) { map.pin().retain(|k, v| { assert_eq!(k, v); true }); } }); }); assert_eq!(map.len(), ENTRIES * (threads - 1)); assert_eq!(map.pin().iter().count(), ENTRIES * (threads - 1)); } }); } // Adapted from: https://github.com/jonhoo/flurry/tree/main/tests/jdk #[test] #[ignore] fn everything() { const SIZE: usize = match () { _ if cfg!(miri) => 1 << 5, _ if cfg!(papaya_stress) || cfg!(papaya_asan) => 1 << 13, _ => 1 << 20, }; // There must be more things absent than present! const ABSENT_SIZE: usize = SIZE << 1; const ABSENT_MASK: usize = ABSENT_SIZE - 1; let mut rng = rand::thread_rng(); with_map(|map| { let map = map(); let mut keys: Vec<_> = (0..ABSENT_SIZE + SIZE).collect(); keys.shuffle(&mut rng); let absent_keys = &keys[0..ABSENT_SIZE]; let keys = &keys[ABSENT_SIZE..]; // put (absent) t3(&map, keys, SIZE); // put (present) t3(&map, keys, 0); // contains_key (present & absent) t7(&map, keys, absent_keys); // contains_key (present) t4(&map, keys, SIZE); // contains_key (absent) t4(&map, absent_keys, 0); // get t6(&map, keys, absent_keys, SIZE, ABSENT_MASK); // get (present) t1(&map, keys, SIZE); // get (absent) t1(&map, absent_keys, 0); // remove (absent) t2(&map, absent_keys, 0); // remove (present) t5(&map, keys, SIZE / 2); // put (half present) t3(&map, keys, SIZE / 2); // iter, keys, values (present) if !cfg!(papaya_stress) { ittest1(&map, SIZE); ittest2(&map, SIZE); ittest3(&map, SIZE); } }); fn t1(map: &HashMap, keys: &[K], expect: usize) where K: Sync + Send + Clone + Hash + Ord, V: Sync + Send, { let mut sum = 0; let iters = 4; let guard = map.guard(); for _ in 0..iters { for key in keys { if map.get(key, &guard).is_some() { sum += 1; } } } assert_eq!(sum, expect * iters); } fn t2(map: &HashMap, keys: &[K], expect: usize) where K: Sync + Send + Copy + Hash + Ord + std::fmt::Display, { let mut sum = 0; let guard = map.guard(); for key in keys { if map.remove(key, &guard).is_some() { sum += 1; } } assert_eq!(sum, expect); } fn t3(map: &HashMap, keys: &[K], expect: usize) where K: Sync + Send + Copy + Hash + Ord, { let mut sum = 0; let guard = map.guard(); for i in 0..keys.len() { if map.insert(keys[i], 0, &guard).is_none() { sum += 1; } } assert_eq!(sum, expect); } fn t4(map: &HashMap, keys: &[K], expect: usize) where K: Sync + Send + Copy + Hash + Ord, { let mut sum = 0; let guard = map.guard(); for i in 0..keys.len() { if map.contains_key(&keys[i], &guard) { sum += 1; } } assert_eq!(sum, expect); } fn t5(map: &HashMap, keys: &[K], expect: usize) where K: Sync + Send + Copy + Hash + Ord, { let mut sum = 0; let guard = map.guard(); let mut i = keys.len() as isize - 2; while i >= 0 { if map.remove(&keys[i as usize], &guard).is_some() { sum += 1; } i -= 2; } assert_eq!(sum, expect); } fn t6(map: &HashMap, keys1: &[K], keys2: &[K], expect: usize, mask: usize) where K: Sync + Send + Clone + Hash + Ord, V: Sync + Send, { let mut sum = 0; let guard = map.guard(); for i in 0..expect { if map.get(&keys1[i], &guard).is_some() { sum += 1; } if map.get(&keys2[i & mask], &guard).is_some() { sum += 1; } } assert_eq!(sum, expect); } fn t7(map: &HashMap, k1: &[K], k2: &[K]) where K: Sync + Send + Copy + Hash + Ord, { let mut sum = 0; let guard = map.guard(); for i in 0..k1.len() { if map.contains_key(&k1[i], &guard) { sum += 1; } if map.contains_key(&k2[i], &guard) { sum += 1; } } assert_eq!(sum, k1.len()); } fn ittest1(map: &HashMap, expect: usize) where K: Sync + Send + Copy + Hash + Eq, { let mut sum = 0; let guard = map.guard(); for _ in map.keys(&guard) { sum += 1; } assert_eq!(sum, expect); } fn ittest2(map: &HashMap, expect: usize) where K: Sync + Send + Copy + Hash + Eq, { let mut sum = 0; let guard = map.guard(); for _ in map.values(&guard) { sum += 1; } assert_eq!(sum, expect); } fn ittest3(map: &HashMap, expect: usize) where K: Sync + Send + Copy + Hash + Eq, { let mut sum = 0; let guard = map.guard(); for _ in map.iter(&guard) { sum += 1; } assert_eq!(sum, expect); } }