pax_global_header00006660000000000000000000000064147553214410014520gustar00rootroot0000000000000052 comment=7a2dff5bee7a98420a56c9cc320a88d68756e8c8 prometheus-client-mmap-1.2.9/000077500000000000000000000000001475532144100161305ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/.cargo/000077500000000000000000000000001475532144100173015ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/.cargo/config.toml000066400000000000000000000012441475532144100214440ustar00rootroot00000000000000[target.aarch64-apple-darwin] # Without this flag, when linking static libruby, the linker removes symbols # (such as `_rb_ext_ractor_safe`) which it thinks are dead code... but they are # not, and they need to be included for the `embed` feature to work with static # Ruby. rustflags = [ "-C", "link-dead-code=on", "-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup", ] [target.x86_64-apple-darwin] rustflags = [ "-C", "link-dead-code=on", "-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup", ] [target.aarch64-unknown-linux-gnu] rustflags = [ "-C", "link-dead-code=on" ] [target.x86_64-unknown-linux-gnu] rustflags = [ "-C", "link-dead-code=on" ] prometheus-client-mmap-1.2.9/.clang-format000066400000000000000000000000651475532144100205040ustar00rootroot00000000000000BasedOnStyle: Google IndentWidth: 4 ColumnLimit: 120 prometheus-client-mmap-1.2.9/.coveralls.yml000066400000000000000000000000301475532144100207140ustar00rootroot00000000000000service_name: travis-ci prometheus-client-mmap-1.2.9/.gitignore000066400000000000000000000001331475532144100201150ustar00rootroot00000000000000Gemfile.lock coverage/ lib/*.bundle lib/*.so pkg/ prometheus-client-mmap*.gem target/ tmp/ prometheus-client-mmap-1.2.9/.gitlab-ci.yml000066400000000000000000000147771475532144100206040ustar00rootroot00000000000000variables: FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: "true" include: - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' - component: gitlab.com/gitlab-org/components/gem-release/gem-release@1dbde0475a4c2a51d2fb7e7b0ee0eab875a43802 inputs: build_native_gems: "true" smoke_test_before_script: "gem install pkg/*-x86_64-linux-gnu.gem" smoke_test_script: "ruby -r 'prometheus/client' -e \"exit 0\"" target_platforms: - aarch64-linux-gnu - aarch64-linux-musl - arm64-darwin - x86_64-darwin - x86_64-linux-gnu - x86_64-linux-musl stages: - test - smoke_test - deploy .install-rust: &install-rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --quiet --default-toolchain=1.73.0 --profile=minimal - source "$HOME/.cargo/env" .install-ruby: &install-ruby - ruby -v - bundle --version - bundle config --local path vendor - bundle install -j $(nproc) .install-ruby-and-compile: &install-ruby-and-compile - *install-ruby - bundle exec rake compile .test-ruby: &test-ruby - bundle exec rake spec cache: key: ${CI_JOB_IMAGE} paths: - vendor/ruby before_script: - apt-get update - apt-get install -y curl ruby ruby-dev build-essential llvm-dev libclang-dev clang - *install-rust - *install-ruby-and-compile .test-job: &test-job image: ruby:${RUBY_VERSION} stage: test variables: prometheus_multiproc_dir: tmp/ BUILDER_IMAGE_REVISION: "5.22.0" RUBY_VERSION: "3.2" SKIP_C_EXTENSION_TESTING: "1" script: - ls -al lib/prometheus/client - '[ $(stat -c %a "lib/prometheus/client.rb") -eq 644 ] || exit 1' - bundle exec rake spec - cd ext/fast_mmaped_file_rs && cargo nextest run artifacts: paths: - coverage/ ruby: <<: *test-job before_script: - apt-get update - apt-get install -y llvm-dev libclang-dev clang - *install-rust - *install-ruby-and-compile - curl -LsSf https://get.nexte.st/0.9/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin parallel: matrix: - RUBY_VERSION: ["3.1", "3.2", "3.3", "3.4"] ruby-head: <<: *test-job image: rubylang/ruby:master-debug-dev before_script: - apt-get update - apt-get install -y llvm-dev libclang-dev clang - *install-rust - *install-ruby-and-compile - curl -LsSf https://get.nexte.st/0.9/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin builder:centos_7: <<: *test-job before_script: - source /opt/rh/llvm-toolset-7/enable - *install-ruby-and-compile script: - *test-ruby image: registry.gitlab.com/gitlab-org/gitlab-omnibus-builder/centos_7:${BUILDER_IMAGE_REVISION} builder:almalinux_8: <<: *test-job before_script: - *install-ruby-and-compile script: - *test-ruby image: registry.gitlab.com/gitlab-org/gitlab-omnibus-builder/almalinux_8:${BUILDER_IMAGE_REVISION} i386/debian:bookworm: <<: *test-job image: i386/debian:bookworm before_script: - apt-get update - apt-get install -y curl ruby ruby-dev ruby-bundler build-essential llvm-dev libclang-dev clang git - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --quiet --default-toolchain=1.73.0 --profile=minimal --default-host=i686-unknown-linux-gnu - source "$HOME/.cargo/env" - *install-ruby-and-compile - curl -LsSf https://get.nexte.st/0.9/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - export LANG=C.UTF-8 script: - *test-ruby archlinux:ruby:3.0: <<: *test-job image: archlinux/archlinux:base before_script: - pacman -Sy --noconfirm git gcc clang make ruby ruby-bundler ruby-rdoc ruby-rake which grep gawk procps-ng glibc - *install-rust - *install-ruby-and-compile - curl -LsSf https://get.nexte.st/0.9/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin script: - *test-ruby fuzzbert: image: ruby:3.2 variables: prometheus_multiproc_dir: tmp/ script: - bundle exec fuzzbert fuzz/**/fuzz_*.rb --handler PrintAndExitHandler --limit 10000 rustfmt check: image: debian:bookworm-slim before_script: - apt-get update - apt-get install -y curl - *install-rust - rustup component add rustfmt script: - cargo fmt --manifest-path ext/fast_mmaped_file_rs/Cargo.toml -- --check clippy check: image: debian:bookworm-slim before_script: - apt-get update - apt-get install -y curl ruby ruby-dev build-essential llvm-dev libclang-dev clang git - *install-rust - rustup component add clippy script: - cargo clippy --manifest-path ext/fast_mmaped_file_rs/Cargo.toml rspec Address-Sanitizer: image: ruby:3.1 before_script: - apt-get update - apt-get install -y git build-essential curl llvm-dev libclang-dev clang - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --quiet --default-toolchain=nightly-2023-10-05 --profile=minimal - source "$HOME/.cargo/env" - rustup component add rust-src - *install-ruby allow_failure: true script: - RB_SYS_EXTRA_CARGO_ARGS='-Zbuild-std' CARGO_BUILD_TARGET=x86_64-unknown-linux-gnu bundle exec rake compile -- --enable-address-sanitizer - export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 - export ASAN_OPTIONS=atexit=true:verbosity=1 - export LSAN_OPTIONS=suppressions=known-leaks-suppression.txt - prometheus_rust_mmaped_file=false bundle exec rspec > /dev/null - bundle exec rspec > /dev/null parsing Address-Sanitizer: image: ruby:3.1 before_script: - apt-get update - apt-get install -y git build-essential curl llvm-dev libclang-dev clang - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --quiet --default-toolchain=nightly-2023-10-05 --profile=minimal - source "$HOME/.cargo/env" - rustup component add rust-src - *install-ruby-and-compile allow_failure: true script: - bundle exec rspec - RB_SYS_EXTRA_CARGO_ARGS='-Zbuild-std' CARGO_BUILD_TARGET=x86_64-unknown-linux-gnu bundle exec rake compile -- --enable-address-sanitizer - DB_FILE=$(basename $(find tmp -name '*.db' | head -n 1)) - test -n $DB_FILE - for ((i=1;i<=100;i++)); do cp tmp/$DB_FILE tmp/$i$DB_FILE; done - export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 - export ASAN_OPTIONS=atexit=true:verbosity=1 - export LSAN_OPTIONS=suppressions=known-leaks-suppression.txt - bundle exec bin/parse -t tmp/*.db > /dev/null pages: image: ruby:3.1 stage: deploy dependencies: - "ruby: [3.1]" script: - mv coverage/ public/ artifacts: paths: - public expire_in: 30 days only: - master prometheus-client-mmap-1.2.9/.rubocop.yml000066400000000000000000000004751475532144100204100ustar00rootroot00000000000000AllCops: TargetRubyVersion: 2.2 Exclude: - lib/prometheus/client/version.rb - tmp AlignHash: EnforcedHashRocketStyle: table Style/TrailingCommaInArguments: EnforcedStyleForMultiline: comma Style/TrailingCommaInLiteral: EnforcedStyleForMultiline: comma Metrics/AbcSize: Max: 18 prometheus-client-mmap-1.2.9/.tool-versions000066400000000000000000000000141475532144100207470ustar00rootroot00000000000000rust 1.73.0 prometheus-client-mmap-1.2.9/.travis.yml000066400000000000000000000001201475532144100202320ustar00rootroot00000000000000sudo: false language: ruby rvm: - 1.9.3 - 2.2.5 - 2.3.1 - jruby-9.0.5.0 prometheus-client-mmap-1.2.9/AUTHORS.md000066400000000000000000000012041475532144100175740ustar00rootroot00000000000000The Prometheus project was started by Matt T. Proud (emeritus) and Julius Volz in 2012. Maintainers of this repository: * Tobias Schmidt The following individuals have contributed code to this repository (listed in alphabetical order): * Andrea Bernardo Ciddio * Bjoern Rabenstein * Dmitry Krasnoukhov * Julius Volz * Kirk Russell * Patrick Ellis * Roland Rifandi Utama * Sergio Gil * Tobias Schmidt prometheus-client-mmap-1.2.9/CHANGELOG.md000066400000000000000000000117621475532144100177500ustar00rootroot00000000000000## v1.2.9 - Include top-level Cargo.lock and Cargo.toml in gem !203 - cargo: Update Rust crate tempfile to 3.17 !201 - cargo: Update Rust crate smallvec to 1.14 !200 ## v1.2.8 - Fix seg faults due to the use of transmute !196 ## v1.2.7 - Make a copy of WeakMap values to update in attempt to solve memory issues in staging !194 - Update rb-sys and other dependencies !193 ## v1.2.6 - Use each_pair instead of each_value to iterate through WeakMap !183 ## v1.2.5 - Ignore nil WeakMap values !180 ## v1.2.4 - No code changes. Retagging due to smoke test faliure. ## v1.2.3 - Fix GNU gems not building !178 ## v1.2.2 - Ship a -linux-gnu gem instead of -linux gem !177 ## v1.2.1 - No code changes. Retagging due to version update mistake. ## v1.2.0 - Update to Ruby 3.4 and produce musl builds !174 ## v1.1.1 - Cargo: Update dependencies for shlex security fix !149 - Revert merge request causing high committed RAM !151 ## v1.1.0 - Add support for Ruby 3.3 in precompiled gems !144 ## v1.0.3 - Don't publish Ruby gem with world-writeable files !146 ## v1.0.2 - mmap: Use 'with_errno' helper to construct error !141 ## v1.0.1 - file_info: Don't error out if file is expanded !139 ## v1.0.0 - Make Rust the default extension and remove C extension !121 - Update to Rust 1.73.0 !135 - mmap: Detect unexpected file size changes !137 ## v0.28.1 - Avoid file truncation in munmap !130 - ci: Disable testing of C extension !131 ## v0.28.0 - Add helper to provide Puma worker PIDs !128 ## v0.27.0 - Allow use of `instance` and `job` labels !125 - Fix header formatting issue !126 - cargo: Update to magnus 0.6.0 !127 ## v0.26.1 - Use Kernel#warn instead of Prometheus::Client.logger.info when Rust extension not available !124 ## v0.26.0 - Enable Rust extension by default !122 - ci: Fix address sanitizer jobs !123 ## v0.25.0 - fix: make C and Rust extensions compatible with Ruby 3.3 RString changes !120 ## v0.24.5 - file_entry: Use serde_json to parse entries !119 ## v0.24.4 - ci: Run pages and ASAN on Ruby 3.1 !117 - parser: Don't assume values don't contain commas !118 ## v0.24.3 - mmap: Use C types when interfacing with Ruby !116 ## v0.24.2 - Start tracking shared child strings !114 - Convert 'type_' to non-static Symbol !115 ## v0.24.1 - ci: Fix smoke test !113 ## v0.24.0 - Expose Rust extension for use in marshaling metrics and read/write values !111 - Fix i386/debian CI job and refactor cache key handling !110 ## v0.23.1 - Use c_long for Ruby string length !109 ## v0.23.0 - Drop musl precompiled gem and relax RubyGems dependency !106 ## v0.22.0 - Re-implement write path in Rust !103 ## v0.21.0 - Remove 'rustc' check from 'Rakefile' !97 - Add support for precompiled gems !99 - Refactor 'bin/setup' to remove uninitialized vars !100 - ci: create precompiled gems and push to Rubygems automatically !101 - Require RubyGems >= v3.3.22 !101 ## v0.20.3 - Check for 'rustc' in 'extconf.rb' !95 ## v0.20.2 - Allocate EntryMap keys only when needed !92 - Don't auto-install Rust toolchain on 'gem install' !93 ## v0.20.1 - Install Rust extension to 'lib' !90 ## v0.20.0 - Use system page size !84 - Implement 'to_metrics' in Rust !85 ## v0.19.1 - No changes; v0.19.0 gem pulled in some unnecessary files. ## v0.19.0 - Fix seg fault after memory is unmapped !80 ## v0.18.0 - pushgateway: add grouping_key feature !76 ## v0.17.0 - Fix crash when trying to inspect all strings !74 ## v0.16.2 - No code changes. Retagging due to extraneous file included in package. ## v0.16.1 - Improve LabelSetValidator debug messages !69 - Properly rescue Oj exceptions !70 ## v0.16.0 - Make sure on reset we release file descriptors for open files !63 ## v0.15.0 - Make labels order independent !60 ## v0.14.0 - Remove deprecated taint mechanism logic !59 ## v0.13.0 - Gauge: add decrement method to gauge metric type !57 - Update push.rb to use newest client_ruby code !56 ## v0.12.0 - Remove deprecated rb_safe_level() and rb_secure() calls !53 ## v0.11.0 - Include filename in IOError exception !47 - Fix clang-format violations !49 - CI: use libasan5 !50 - Truncate MmappedDict#inspect output !51 ## v0.10.0 - Eliminate SIGBUS errors in parsing metrics !43 - Make it easier to diagnose clang-format failures !44 - Add Ruby 2.6 and 2.7 to CI builds !45 ## v0.9.10 - Extend `Prometheus::Client.reinitialize_on_pid_change` method to receive `:force` param !40 With `force: true` it reinitializes all metrics files. With `force: false` (default) it reinitializes only on changed PID (as it was before). In any case, it keeps the registry (as it was before). Thus, the change is backward compatible. ## v0.9.9 - Do not allow label values that will corrupt the metrics !38 ## v0.9.7 - Restore Prometheus logger !36 ## v0.9.7 - Disable warning if prometheus_multiproc_dir is not set !35 ## v0.9.6 - Add missing `pid=` label for metrics without labels !31 ## v0.9.5 - Set multiprocess_files_dir config to temp directory by default https://gitlab.com/gitlab-org/prometheus-client-mmap/merge_requests/28 prometheus-client-mmap-1.2.9/CONTRIBUTING.md000066400000000000000000000010451475532144100203610ustar00rootroot00000000000000# Contributing Prometheus uses GitHub to manage reviews of pull requests. * If you have a trivial fix or improvement, go ahead and create a pull request, addressing (with `@...`) one or more of the maintainers (see [AUTHORS.md](AUTHORS.md)) in the description of the pull request. * If you plan to do something more involved, first discuss your ideas on our [mailing list](https://groups.google.com/forum/?fromgroups#!forum/prometheus-developers). This will avoid unnecessary work and surely give you and us a good deal of inspiration. prometheus-client-mmap-1.2.9/Cargo.lock000066400000000000000000000476121475532144100200470ustar00rootroot00000000000000# This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 3 [[package]] name = "ahash" version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "once_cell", "version_check", "zerocopy", ] [[package]] name = "aho-corasick" version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "allocator-api2" version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bindgen" version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.8.0", "cexpr", "clang-sys", "itertools", "lazy_static", "lazycell", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", "syn", ] [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "bstr" version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "regex-automata", "serde", ] [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cexpr" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ "nom", ] [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clang-sys" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", "libloading", ] [[package]] name = "cpufeatures" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", ] [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", ] [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "errno" version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", "windows-sys", ] [[package]] name = "fast_mmaped_file_rs" version = "0.1.0" dependencies = [ "bstr", "hashbrown", "indoc", "libc", "magnus", "memmap2", "nix", "rand", "rb-sys", "rb-sys-env 0.2.2", "serde", "serde_json", "sha2", "smallvec", "tempfile", "thiserror", ] [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "generic-array" version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", ] [[package]] name = "getrandom" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] name = "getrandom" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", "libc", "wasi 0.13.3+wasi-0.2.2", "windows-targets", ] [[package]] name = "glob" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", ] [[package]] name = "indoc" version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "itertools" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libloading" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets", ] [[package]] name = "linux-raw-sys" version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "magnus" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d87ae53030f3a22e83879e666cb94e58a7bdf31706878a0ba48752994146dab" dependencies = [ "magnus-macros", "rb-sys", "rb-sys-env 0.1.2", "seq-macro", ] [[package]] name = "magnus-macros" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" dependencies = [ "libc", ] [[package]] name = "memoffset" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg", ] [[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "nix" version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags 1.3.2", "cfg-if", "libc", "memoffset", "pin-utils", ] [[package]] name = "nom" version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", ] [[package]] name = "once_cell" version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "pin-utils" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "ppv-lite86" version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ "zerocopy", ] [[package]] name = "proc-macro2" version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] [[package]] name = "quote" version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] [[package]] name = "rand" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", "rand_core", ] [[package]] name = "rand_chacha" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core", ] [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom 0.2.15", ] [[package]] name = "rb-sys" version = "0.9.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cfdfe3935ea21d9d3b0251f3258ff8773d9525bc422e27847ca4c567214f0b0" dependencies = [ "rb-sys-build", ] [[package]] name = "rb-sys-build" version = "0.9.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dd2fa891aed8ff298d6ee0301e4c704a5bf469fbc9e9a46d7c1af26f020ad1" dependencies = [ "bindgen", "lazy_static", "proc-macro2", "quote", "regex", "shell-words", "syn", ] [[package]] name = "rb-sys-env" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb" [[package]] name = "rb-sys-env" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08f8d2924cf136a1315e2b4c7460a39f62ef11ee5d522df9b2750fab55b868b6" [[package]] name = "regex" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-hash" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", "windows-sys", ] [[package]] name = "ryu" version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "seq-macro" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "serde_json" version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", "ryu", "serde", ] [[package]] name = "sha2" version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", "digest", ] [[package]] name = "shell-words" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "smallvec" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" dependencies = [ "serde", ] [[package]] name = "syn" version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "tempfile" version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a40f762a77d2afa88c2d919489e390a12bdd261ed568e60cfa7e48d4e20f0d33" dependencies = [ "cfg-if", "fastrand", "getrandom 0.3.1", "once_cell", "rustix", "windows-sys", ] [[package]] name = "thiserror" version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "typenum" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" version = "0.13.3+wasi-0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" dependencies = [ "wit-bindgen-rt", ] [[package]] name = "windows-sys" version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "wit-bindgen-rt" version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" dependencies = [ "bitflags 2.8.0", ] [[package]] name = "zerocopy" version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", "syn", ] prometheus-client-mmap-1.2.9/Cargo.toml000066400000000000000000000003741475532144100200640ustar00rootroot00000000000000# This Cargo.toml is here to let externals tools (IDEs, etc.) know that this is # a Rust project. Your extensions dependencies should be added to the Cargo.toml # in the ext/ directory. [workspace] members = ["ext/fast_mmaped_file_rs"] resolver = "2" prometheus-client-mmap-1.2.9/Gemfile000066400000000000000000000007201475532144100174220ustar00rootroot00000000000000source 'https://rubygems.org' gemspec def ruby_version?(constraint) Gem::Dependency.new('', constraint).match?('', RUBY_VERSION) end group :test do gem 'oj', '> 3' gem 'json', '< 2.0' if ruby_version?('< 2.0') gem 'simplecov' gem 'rack', '< 2.0' if ruby_version?('< 2.2.2') gem 'rack-test' gem "rake", "> 12.3.2" gem 'rb_sys', '~> 0.9' gem 'rubocop', ruby_version?('< 2.0') ? '< 0.42' : nil gem 'tins', '< 1.7' if ruby_version?('< 2.0') end prometheus-client-mmap-1.2.9/LICENSE000066400000000000000000000261351475532144100171440ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. prometheus-client-mmap-1.2.9/NOTICE000066400000000000000000000002731475532144100170360ustar00rootroot00000000000000Prometheus instrumentation library for Ruby applications Copyright 2013-2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). prometheus-client-mmap-1.2.9/README.md000066400000000000000000000217321475532144100174140ustar00rootroot00000000000000# Prometheus Ruby Mmap Client This Prometheus library is fork of [Prometheus Ruby Client](https://github.com/prometheus/client_ruby) that uses mmap'ed files to share metrics from multiple processes. This allows efficient metrics processing for Ruby web apps running in multiprocess setups like Unicorn. A suite of instrumentation metric primitives for Ruby that can be exposed through a HTTP interface. Intended to be used together with a [Prometheus server][1]. [![Gem Version][4]](http://badge.fury.io/rb/prometheus-client-mmap) [![Build Status][3]](https://gitlab.com/gitlab-org/prometheus-client-mmap/commits/master) ## Installation prometheus-client-mmap ships with [precompiled native gems](https://rubygems.org/gems/prometheus-client-mmap). Install the gem via: ```shell gem install prometheus-client-mmap ``` ### Requirements for building from source Building from source requires a number of dependencies: * Rust v1.65+ * clang 5.0 or higher for [bindgen](https://rust-lang.github.io/rust-bindgen/requirements.html) * `make` * A C compiler (for legacy extension, which will be removed soon) ## Usage ### Overview ```ruby require 'prometheus/client' # returns a default registry prometheus = Prometheus::Client.registry # create a new counter metric http_requests = Prometheus::Client::Counter.new(:http_requests, 'A counter of HTTP requests made') # register the metric prometheus.register(http_requests) # equivalent helper function http_requests = prometheus.counter(:http_requests, 'A counter of HTTP requests made') # start using the counter http_requests.increment ``` ## Rust extension This gem now uses a rewritten Rust extension instead of C. implementation that reads the metric files and outputs the multiprocess metrics to text. This implementation is significantly faster than the C extension. ### Rack middleware There are two [Rack][2] middlewares available, one to expose a metrics HTTP endpoint to be scraped by a prometheus server ([Exporter][9]) and one to trace all HTTP requests ([Collector][10]). It's highly recommended to enable gzip compression for the metrics endpoint, for example by including the `Rack::Deflater` middleware. ```ruby # config.ru require 'rack' require 'prometheus/client/rack/collector' require 'prometheus/client/rack/exporter' use Rack::Deflater, if: ->(env, status, headers, body) { body.any? && body[0].length > 512 } use Prometheus::Client::Rack::Collector use Prometheus::Client::Rack::Exporter run ->(env) { [200, {'Content-Type' => 'text/html'}, ['OK']] } ``` Start the server and have a look at the metrics endpoint: [http://localhost:5000/metrics](http://localhost:5000/metrics). For further instructions and other scripts to get started, have a look at the integrated [example application](examples/rack/README.md). ### Pushgateway The Ruby client can also be used to push its collected metrics to a [Pushgateway][8]. This comes in handy with batch jobs or in other scenarios where it's not possible or feasible to let a Prometheus server scrape a Ruby process. TLS and HTTP basic authentication are supported. ```ruby require 'prometheus/client' require 'prometheus/client/push' registry = Prometheus::Client.registry # ... register some metrics, set/increment/observe/etc. their values # push the registry state to the default gateway Prometheus::Client::Push.new(job: 'my-batch-job').add(registry) # optional: specify a grouping key that uniquely identifies a job instance, and gateway. # # Note: the labels you use in the grouping key must not conflict with labels set on the # metrics being pushed. If they do, an error will be raised. Prometheus::Client::Push.new( job: 'my-batch-job', gateway: 'https://example.domain:1234', grouping_key: { instance: 'some-instance', extra_key: 'foobar' } ).add(registry) # If you want to replace any previously pushed metrics for a given grouping key, # use the #replace method. # # Unlike #add, this will completely replace the metrics under the specified grouping key # (i.e. anything currently present in the pushgateway for the specified grouping key, but # not present in the registry for that grouping key will be removed). # # See https://github.com/prometheus/pushgateway#put-method for a full explanation. Prometheus::Client::Push.new(job: 'my-batch-job').replace(registry) # If you want to delete all previously pushed metrics for a given grouping key, # use the #delete method. Prometheus::Client::Push.new(job: 'my-batch-job').delete ``` ## Metrics The following metric types are currently supported. ### Counter Counter is a metric that exposes merely a sum or tally of things. ```ruby counter = Prometheus::Client::Counter.new(:service_requests_total, '...') # increment the counter for a given label set counter.increment({ service: 'foo' }) # increment by a given value counter.increment({ service: 'bar' }, 5) # get current value for a given label set counter.get({ service: 'bar' }) # => 5 ``` ### Gauge Gauge is a metric that exposes merely an instantaneous value or some snapshot thereof. ```ruby gauge = Prometheus::Client::Gauge.new(:room_temperature_celsius, '...') # set a value gauge.set({ room: 'kitchen' }, 21.534) # retrieve the current value for a given label set gauge.get({ room: 'kitchen' }) # => 21.534 ``` ### Histogram A histogram samples observations (usually things like request durations or response sizes) and counts them in configurable buckets. It also provides a sum of all observed values. ```ruby histogram = Prometheus::Client::Histogram.new(:service_latency_seconds, '...') # record a value histogram.observe({ service: 'users' }, Benchmark.realtime { service.call(arg) }) # retrieve the current bucket values histogram.get({ service: 'users' }) # => { 0.005 => 3, 0.01 => 15, 0.025 => 18, ..., 2.5 => 42, 5 => 42, 10 = >42 } ``` ### Summary Summary, similar to histograms, is an accumulator for samples. It captures Numeric data and provides an efficient percentile calculation mechanism. ```ruby summary = Prometheus::Client::Summary.new(:service_latency_seconds, '...') # record a value summary.observe({ service: 'database' }, Benchmark.realtime { service.call() }) # retrieve the current quantile values summary.get({ service: 'database' }) # => { 0.5 => 0.1233122, 0.9 => 3.4323, 0.99 => 5.3428231 } ``` ## Configuration ### Memory mapped files storage location Set `prometheus_multiproc_dir` environment variable to the path where you want metric files to be stored. Example: ``` prometheus_multiproc_dir=/tmp ``` ## Pitfalls ### PID cardinality In multiprocess setup e.g. running under Unicorn or Puma, having worker process restart often can lead to performance problems when proccesing metric files. By default each process using Prometheus metrics will create a set of files based on that process PID. With high worker churn this will lead to creation of thousands of files and in turn will cause very noticable slowdown when displaying metrics To reduce this problem, a surrogate process id can be used. Set of all such IDs needs have low cardinality, and each process id must be unique among all running process. For Unicorn and Puma a worker id/number can be used to greatly speedup the metrics rendering. If you are using Unicorn, add this line to your `configure` block: ```ruby config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider) ``` If you are using Puma, add this line to your `configure` block: ```ruby config.pid_provider = Prometheus::Client::Support::Puma.method(:worker_pid_provider) ``` ## Tools ### `bin/parse` This command can be used to parse metric files located on the filesystem just like a metric exporter would. It outputs either `json` formatted raw data or digested data in prometheus `text` format. #### Usage: ```bash $ ./bin/parse -h Usage: parse [options] files... -t, --to-prometheus-text format output using Prometheus text formatter -p, --profile enable profiling -h, --help Show this message ``` ## Development After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. To install this gem onto your local machine, run `bundle exec rake install`. ### Releasing a new version To release a new version: 1. Update `lib/prometheus/client/version.rb` with the version number. 1. Update `CHANGELOG.md` with the changes in the release. 1. Create a merge request and merge it to `master`. 1. Push a new tag to the repository. The new version with precompiled, native gems will automatically be published to [RubyGems](https://rubygems.org/gems/prometheus-client-mmap) when the pipeline for the tag completes. [1]: https://github.com/prometheus/prometheus [2]: http://rack.github.io/ [3]: https://gitlab.com/gitlab-org/prometheus-client-mmap/badges/master/pipeline.svg [4]: https://badge.fury.io/rb/prometheus-client.svg [8]: https://github.com/prometheus/pushgateway [9]: lib/prometheus/client/rack/exporter.rb [10]: lib/prometheus/client/rack/collector.rb prometheus-client-mmap-1.2.9/Rakefile000066400000000000000000000034301475532144100175750ustar00rootroot00000000000000require 'bundler' require 'rake/clean' require "rake_compiler_dock" require 'rspec/core/rake_task' require 'rubocop/rake_task' require 'rake/extensiontask' require 'gem_publisher' require 'rb_sys' ruby_cc_version = RakeCompilerDock.ruby_cc_version(">= 3.1") cross_platforms = %w[ aarch64-linux-gnu aarch64-linux-musl arm64-darwin x86_64-darwin x86_64-linux-gnu x86_64-linux-musl ] CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'], FileList['**/extconf.h'], FileList['**/Makefile'], FileList['pkg/'] desc 'Default: run specs' task default: [:spec] # test alias task test: :spec desc 'Run specs' RSpec::Core::RakeTask.new do |t| t.rspec_opts = '--require ./spec/spec_helper.rb' end desc 'Lint code' RuboCop::RakeTask.new Bundler::GemHelper.install_tasks desc 'Publish gem to RubyGems.org' task :publish_gem do |_t| gem = GemPublisher.publish_if_updated('prometheus-client-mmap.gemspec', :rubygems) puts "Published #{gem}" if gem end task :console do exec 'irb -r prometheus -I ./lib' end gemspec = Gem::Specification.load(File.expand_path('../prometheus-client-mmap.gemspec', __FILE__)) Gem::PackageTask.new(gemspec) Rake::ExtensionTask.new('fast_mmaped_file_rs', gemspec) do |ext| ext.source_pattern = "*.{rs,toml}" ext.cross_compile = true ext.cross_platform = cross_platforms end namespace "gem" do task "prepare" do sh "bundle" end cross_platforms.each do |plat| desc "Build the native gem for #{plat}" task plat => "prepare" do ENV["RCD_IMAGE"] = "rbsys/#{plat}:#{RbSys::VERSION}" RakeCompilerDock.sh <<~SH, platform: plat bundle && \ RUBY_CC_VERSION="#{ruby_cc_version}" \ bundle exec rake native:#{plat} pkg/#{gemspec.full_name}-#{plat}.gem SH end end end prometheus-client-mmap-1.2.9/bin/000077500000000000000000000000001475532144100167005ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/bin/console000077500000000000000000000001331475532144100202650ustar00rootroot00000000000000#!/usr/bin/env ruby require 'bundler/setup' require 'prometheus' require 'pry' Pry.start prometheus-client-mmap-1.2.9/bin/parse000077500000000000000000000046701475532144100177470ustar00rootroot00000000000000#!/usr/bin/env ruby $LOAD_PATH << File.expand_path('../../lib', __FILE__) require 'prometheus/client' require 'prometheus/client/helper/plain_file' require 'prometheus/client/helper/metrics_processing' require 'prometheus/client/helper/metrics_representation' require 'json' require 'optparse' require 'fast_mmaped_file_rs' options = {} OptionParser.new do |opts| opts.banner = 'Usage: parse [options] files...' opts.on('-t', '--to-prometheus-text', 'format output using Prometheus text formatter') do |v| options[:prom_text] = v end opts.on('-s', '--to-prometheus-text-slow', 'format output using Prometheus Ruby based text formatter') do |v| options[:prom_text_slow] = v end opts.on('-p', '--profile', 'enable profiling') do |v| options[:profile] = v end opts.on_tail('-h', '--help', 'Show this message') do puts opts exit end end.parse! class Object def transform(&block) yield self end end # Output closely resembling binary file contents # best used with `jq` def to_json(files) files.map { |f| Prometheus::Client::Helper::PlainFile.new(f) } .map { |f| { filepath: f.filepath, entries: entries_to_json(f.parsed_entries(true)) } } .transform { |s| JSON.dump(s); nil } end def to_prom_text(files) files.map {|f| Prometheus::Client::Helper::PlainFile.new(f) } .map { |f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] } .transform { |files| FastMmapedFileRs.to_metrics(files.to_a) } end def to_prom_text_slow(files) files.map { |f| Prometheus::Client::Helper::PlainFile.new(f) } .each_with_object({}) { |f, metrics| f.to_metrics(metrics, true) } .transform(&Prometheus::Client::Helper::MetricsProcessing.method(:merge_metrics)) .transform(&Prometheus::Client::Helper::MetricsRepresentation.method(:to_text)) end def entries_to_json(entries) entries.map { |e, v| entry_to_json(*JSON.load(e)).merge(value: v) } end def entry_to_json(metric=nil, name=nil, labels=[], label_values=[]) { metric: metric, name: name, labels: labels.zip(label_values).to_h } end def run(profile = false) if profile require 'ruby-prof' RubyProf.start yield result = RubyProf.stop printer = RubyProf::FlatPrinter.new(result) printer.print(STDERR) else yield end end run(options[:profile]) do if options[:prom_text] puts to_prom_text(ARGV) elsif options[:prom_text_slow] puts to_prom_text_slow(ARGV) else puts to_json(ARGV) end end prometheus-client-mmap-1.2.9/bin/setup000077500000000000000000000022261475532144100177700ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail IFS=$'\n\t' set -vx bundle install bundle exec rake compile if cargo nextest --version > /dev/null 2>&1; then exit; fi # Check if rust is managed by 'asdf' if command -v cargo | grep '.asdf/shims'; then # This will fail if no rust version has been specified in asdf rust_path="$(asdf where rust)/bin" # Check for $CARGO_HOME that may not be in $HOME # We use '/dev/null' as a fallback value known to be present and not a directory elif [ -d "${CARGO_HOME:-/dev/null}/bin" ]; then rust_path="${CARGO_HOME}/bin" # Default path for rustup.rs elif [ -d "${HOME}/.cargo/bin" ]; then rust_path="${HOME}/.cargo/bin" else echo "No rust toolchain found, skipping installation of 'cargo nextest'" exit fi if [ "$(uname -s)" = 'Darwin' ]; then host_os='mac' elif [ "$(uname -s)" = 'Linux' ] && [ "$(uname -m)" = 'x86_64' ]; then host_os='linux' else echo "Auto-install for 'cargo nextest' only available on MacOS and x86_64 Linux. Download manually from https://nexte.st/" exit fi echo "Installing 'cargo nextest'..." curl -LsSf "https://get.nexte.st/latest/${host_os}" | tar zxf - -C "${rust_path}" prometheus-client-mmap-1.2.9/example.rb000066400000000000000000000016501475532144100201120ustar00rootroot00000000000000$LOAD_PATH.unshift("./lib") require 'prometheus/client' require 'prometheus/client/formats/text.rb' require 'pp' prometheus = Prometheus::Client.registry counter = Prometheus::Client::Counter.new(:mycounter, 'Example counter') gauge = Prometheus::Client::Gauge.new(:mygauge, 'Example gauge', {}, :livesum) histogram = Prometheus::Client::Histogram.new(:myhistogram, 'Example histogram', {}, [0, 1, 2]) prometheus.register(counter) prometheus.register(gauge) prometheus.register(histogram) counter.increment({'foo': 'bar'}, 2) counter.increment({'foo': 'biz'}, 4) gauge.set({'foo': 'bar'}, 3) gauge.set({'foo': 'biz'}, 3) gauge.decrement({'foo': 'bar'}, 1) histogram.observe({'foo': 'bar'}, 0.5) histogram.observe({'foo': 'biz'}, 0.5) histogram.observe({'foo': 'bar'}, 1.5) histogram.observe({'foo': 'biz'}, 2) #puts Prometheus::Client::Formats::Text.marshal(prometheus) puts Prometheus::Client::Formats::Text.marshal_multiprocess prometheus-client-mmap-1.2.9/examples/000077500000000000000000000000001475532144100177465ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/examples/rack/000077500000000000000000000000001475532144100206665ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/examples/rack/.gitignore000066400000000000000000000000231475532144100226510ustar00rootroot00000000000000data/ Gemfile.lock prometheus-client-mmap-1.2.9/examples/rack/Gemfile000066400000000000000000000001371475532144100221620ustar00rootroot00000000000000source 'https://rubygems.org' gem 'prometheus-client', path: '../..' gem 'rack' gem 'unicorn' prometheus-client-mmap-1.2.9/examples/rack/README.md000066400000000000000000000027411475532144100221510ustar00rootroot00000000000000# Rack example A simple Rack application which shows how to use prometheus' `Rack::Exporter` and `Rack::Collector` rack middlwares. ## Run the example ### Standalone Execute the provided `run` script: ```bash bundle install bundle exec ./run ``` This will start the rack app, run a few requests against it, print the output of `/metrics` and terminate. ### With a Prometheus server Start a Prometheus server with the provided config: ```bash prometheus -config.file ./prometheus.yaml ``` In another terminal, start the application server: ```bash bundle install bundle exec unicorn -c ./unicorn.conf ``` You can now open the [example app](http://localhost:5000/) and its [metrics page](http://localhost:5000/metrics) to inspect the output. The running Prometheus server can be used to [play around with the metrics][rate-query]. [rate-query]: http://localhost:9090/graph#%5B%7B%22range_input%22%3A%221h%22%2C%22expr%22%3A%22rate(http_request_duration_seconds_count%5B1m%5D)%22%2C%22tab%22%3A0%7D%5D ## Collector The example shown in [`config.ru`](config.ru) is a trivial rack application using the default collector and exporter middlewares. In order to use a custom label builder in the collector, change the line to something like this: ```ruby use Prometheus::Client::Rack::Collector do |env| { method: env['REQUEST_METHOD'].downcase, host: env['HTTP_HOST'].to_s, path: env['PATH_INFO'].to_s, http_version: env['HTTP_VERSION'].to_s, } end ``` prometheus-client-mmap-1.2.9/examples/rack/config.ru000077500000000000000000000005041475532144100225050ustar00rootroot00000000000000require 'rack' require 'prometheus/client/rack/collector' require 'prometheus/client/rack/exporter' use Rack::Deflater, if: ->(_, _, _, body) { body.any? && body[0].length > 512 } use Prometheus::Client::Rack::Collector use Prometheus::Client::Rack::Exporter run ->(_) { [200, { 'Content-Type' => 'text/html' }, ['OK']] } prometheus-client-mmap-1.2.9/examples/rack/prometheus.yml000066400000000000000000000003531475532144100236050ustar00rootroot00000000000000global: scrape_interval: "15s" scrape_configs: - job_name: "prometheus" static_configs: - targets: - "localhost:9090" - job_name: "rack-example" static_configs: - targets: - "localhost:5000" prometheus-client-mmap-1.2.9/examples/rack/run000077500000000000000000000013221475532144100214160ustar00rootroot00000000000000#!/bin/bash -e trap 'kill $(jobs -p)' EXIT installed() { which $1 > /dev/null } log() { echo >&2 $1 } fatal() { log $1 exit 1 } if ! installed vegeta; then if ! installed go; then fatal "Could not find go. Either run the examples manually or install" fi go get github.com/tsenart/vegeta go install github.com/tsenart/vegeta fi PORT=5000 URL=http://127.0.0.1:${PORT}/ log "starting example server" bundle install --quiet bundle exec unicorn -p ${PORT} -c unicorn.conf &>> /dev/null & # wait until unicorn is available sleep 1 log "sending requests for 5 seconds" printf "GET ${URL}\nPOST ${URL}\nDELETE ${URL}" | vegeta attack -duration 5s &>> /dev/null log "printing /metrics" curl -s "${URL}metrics" prometheus-client-mmap-1.2.9/examples/rack/unicorn.conf000066400000000000000000000000601475532144100232060ustar00rootroot00000000000000listen 5000 worker_processes 1 preload_app true prometheus-client-mmap-1.2.9/ext/000077500000000000000000000000001475532144100167305ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/000077500000000000000000000000001475532144100227135ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/Cargo.toml000066400000000000000000000020721475532144100246440ustar00rootroot00000000000000[package] name = "fast_mmaped_file_rs" version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] hashbrown = "0.14" libc = "0.2" magnus = { version = "0.7", features = ["rb-sys"] } memmap2 = "0.9" # v0.26 cannot be built on CentOS 7 https://github.com/nix-rust/nix/issues/1972 nix = { version = "0.25", features = ["mman"] } # mman used for MsFlags rb-sys = { version = "0.9", features = ["stable-api-compiled-fallback"] } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["raw_value"] } smallvec = { version = "1.14", features = ["serde"] } thiserror = "2.0" [dev-dependencies] bstr = "1.11" indoc = "2.0" # We need the `embed` feature to run tests, but this triggers failures when building as a Gem. magnus = { version = "0.7", features = ["rb-sys","embed"] } rand = "0.8" sha2 = "0.10" tempfile = "3.17" [build-dependencies] rb-sys-env = "0.2.2" [lib] # Integration tests won't work if crate is only `cdylib`. crate-type = ["cdylib","lib"] prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/README.md000066400000000000000000000030601475532144100241710ustar00rootroot00000000000000# Testing ## Running Tests Use [cargo nextest](https://nexte.st/) to execute the Rust unit tests. ```sh $ cargo nextest run ``` ## Why not use 'cargo test'? We need to embed Ruby into the test binary to access Ruby types. This requires us to run `magnus::embed::init()` no more than once before calling Ruby. See [the magnus docs](https://docs.rs/magnus/latest/magnus/embed/fn.init.html) for more details. If we try to create separate `#[test]` functions that call `init()` these will conflict, as Cargo runs tests in parallel using a single process with separate threads. Running `cargo test` will result in errors like: ``` ---- file_info::test::with_ruby stdout ---- thread 'file_info::test::with_ruby' panicked at 'Ruby already initialized' ``` The simplest workaround for this is to avoid using `cargo test` to run unit tests. [nextest](https://nexte.st/) is an alternate test harness that runs each test as its own process, enabling each test to intitialize Ruby without conflict. ## 'symbol not found' errors when running tests If you see errors like the following when running tests: ``` Caused by: for `fast_mmaped_file_rs`, command `/Users/myuser/prometheus-client-mmap/ext/fast_mmaped_file_rs/target/debug/deps/fast_mmaped_file_rs-c81ccc96a6484e04 --list --format terse` exited with signal 6 (SIGABRT) --- stdout: --- stderr: dyld[17861]: symbol not found in flat namespace '_rb_cArray' ``` Clearing the build cache will resolve the problem. ```sh $ cargo clean ``` This is probably due to separate features being used with `magnus` in development builds. prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/build.rs000066400000000000000000000001511475532144100243550ustar00rootroot00000000000000fn main() -> Result<(), Box> { let _ = rb_sys_env::activate()?; Ok(()) } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/extconf.rb000066400000000000000000000013761475532144100247150ustar00rootroot00000000000000require "mkmf" require "rb_sys/mkmf" if find_executable('rustc') create_rust_makefile("fast_mmaped_file_rs") do |r| r.auto_install_rust_toolchain = false if enable_config('fail-on-warning') r.extra_rustflags = ["-Dwarnings"] end if enable_config('debug') r.profile = :dev end if enable_config('address-sanitizer') r.extra_rustflags = ["-Zsanitizer=address"] end # `rb_sys/mkmf` passes all arguments after `--` directly to `cargo rustc`. # We use this awful hack to keep compatibility with existing flags used by # the C implementation. trimmed_argv = ARGV.take_while { |arg| arg != "--" } ARGV = trimmed_argv end else raise 'rustc not found. prometheus-client-mmap now requires Rust.' end prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/000077500000000000000000000000001475532144100235025ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/error.rs000066400000000000000000000133311475532144100252020ustar00rootroot00000000000000use magnus::{exception, Ruby}; use std::any; use std::fmt::Display; use std::io; use std::path::Path; use thiserror::Error; use crate::util; use crate::PROM_EPARSING_ERROR; /// A lightweight representation of Ruby ExceptionClasses. #[derive(PartialEq, Eq, Clone, Copy, Debug)] pub enum RubyError { Arg, Encoding, Frozen, Index, Io, NoMem, PromParsing, Runtime, Type, } impl From for magnus::ExceptionClass { fn from(err: RubyError) -> magnus::ExceptionClass { match err { RubyError::Arg => exception::arg_error(), RubyError::Encoding => exception::encoding_error(), RubyError::Frozen => exception::frozen_error(), RubyError::Index => exception::index_error(), RubyError::Io => exception::io_error(), RubyError::NoMem => exception::no_mem_error(), RubyError::Runtime => exception::runtime_error(), RubyError::PromParsing => { // UNWRAP: this will panic if called outside of a Ruby thread. let ruby = Ruby::get().unwrap(); ruby.get_inner(&PROM_EPARSING_ERROR) } RubyError::Type => exception::type_error(), } } } /// Errors returned internally within the crate. Methods called directly by Ruby return /// `magnus::error::Error` as do functions that interact heavily with Ruby. This can be /// converted into a `magnus::error::Error` at the boundary between Rust and Ruby. #[derive(PartialEq, Eq, Error, Debug)] pub enum MmapError { /// A read or write was made while another thread had mutable access to the mmap. #[error("read/write operation attempted while mmap was being written to")] ConcurrentAccess, /// An error message used to exactly match the messages returned by the C /// implementation. #[error("{0}")] Legacy(String, RubyError), /// A String had invalid UTF-8 sequences. #[error("{0}")] Encoding(String), /// A failed attempt to cast an integer from one type to another. #[error("failed to cast {object_name} {value} from {from} to {to}")] FailedCast { from: &'static str, to: &'static str, value: String, object_name: String, }, /// The mmap was frozen when a mutable operation was attempted. #[error("mmap")] Frozen, /// An io operation failed. #[error("failed to {operation} path '{path}': {err}")] Io { operation: String, path: String, err: String, }, #[error("string length gt {}", i32::MAX)] KeyLength, /// Failed to allocate memory. #[error("Couldn't allocate for {0} memory")] OutOfMemory(usize), /// A memory operation fell outside of the containers bounds. #[error("offset {index} out of bounds of len {len}")] OutOfBounds { index: String, len: String }, /// A numeric operation overflowed. #[error("overflow when {op} {value} and {added} of type {ty}")] Overflow { value: String, added: String, op: String, ty: &'static str, }, /// A miscellaneous error. #[error("{0}")] Other(String), /// A failure when parsing a `.db` file containing Prometheus metrics. #[error("{0}")] PromParsing(String), /// No mmap open. #[error("unmapped file")] UnmappedFile, /// A custom error message with `strerror(3)` appended. #[error("{0}")] WithErrno(String), } impl MmapError { pub fn legacy>(msg: T, ruby_err: RubyError) -> Self { MmapError::Legacy(msg.into(), ruby_err) } pub fn failed_cast(value: T, object_name: &str) -> Self { MmapError::FailedCast { from: any::type_name::(), to: any::type_name::(), value: value.to_string(), object_name: object_name.to_string(), } } pub fn io(operation: &str, path: &Path, err: io::Error) -> Self { MmapError::Io { operation: operation.to_string(), path: path.display().to_string(), err: err.to_string(), } } pub fn overflowed(value: T, added: T, op: &str) -> Self { MmapError::Overflow { value: value.to_string(), added: added.to_string(), op: op.to_string(), ty: any::type_name::(), } } pub fn out_of_bounds(index: T, len: T) -> Self { MmapError::OutOfBounds { index: index.to_string(), len: len.to_string(), } } pub fn with_errno>(msg: T) -> Self { let strerror = util::strerror(util::errno()); MmapError::WithErrno(format!("{}: ({strerror})", msg.into())) } pub fn ruby_err(&self) -> RubyError { match self { MmapError::ConcurrentAccess => RubyError::Arg, MmapError::Legacy(_, e) => *e, MmapError::Encoding(_) => RubyError::Encoding, MmapError::Io { .. } => RubyError::Io, MmapError::FailedCast { .. } => RubyError::Arg, MmapError::Frozen => RubyError::Frozen, MmapError::KeyLength => RubyError::Arg, MmapError::Overflow { .. } => RubyError::Arg, MmapError::OutOfBounds { .. } => RubyError::Index, MmapError::OutOfMemory { .. } => RubyError::NoMem, MmapError::Other(_) => RubyError::Arg, MmapError::PromParsing(_) => RubyError::PromParsing, MmapError::UnmappedFile => RubyError::Io, MmapError::WithErrno(_) => RubyError::Io, } } } impl From for magnus::error::Error { fn from(err: MmapError) -> magnus::error::Error { magnus::error::Error::new(err.ruby_err().into(), err.to_string()) } } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/file_entry.rs000066400000000000000000000710111475532144100262100ustar00rootroot00000000000000use magnus::Symbol; use serde::Deserialize; use serde_json::value::RawValue; use smallvec::SmallVec; use std::fmt::Write; use std::str; use crate::error::{MmapError, RubyError}; use crate::file_info::FileInfo; use crate::raw_entry::RawEntry; use crate::Result; use crate::{SYM_GAUGE, SYM_LIVESUM, SYM_MAX, SYM_MIN}; /// A metrics entry extracted from a `*.db` file. #[derive(Clone, Debug)] pub struct FileEntry { pub data: EntryData, pub meta: EntryMetadata, } /// String slices pointing to the fields of a borrowed `Entry`'s JSON data. #[derive(Deserialize, Debug)] pub struct MetricText<'a> { pub family_name: &'a str, pub metric_name: &'a str, pub labels: SmallVec<[&'a str; 4]>, #[serde(borrow)] pub values: SmallVec<[&'a RawValue; 4]>, } /// The primary data payload for a `FileEntry`, the JSON string and the /// associated pid, if significant. Used as the key for `EntryMap`. #[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Debug)] pub struct EntryData { pub json: String, pub pid: Option, } impl<'a> PartialEq> for EntryData { fn eq(&self, other: &BorrowedData) -> bool { self.pid.as_deref() == other.pid && self.json == other.json } } impl<'a> TryFrom> for EntryData { type Error = MmapError; fn try_from(borrowed: BorrowedData) -> Result { let mut json = String::new(); if json.try_reserve_exact(borrowed.json.len()).is_err() { return Err(MmapError::OutOfMemory(borrowed.json.len())); } json.push_str(borrowed.json); Ok(Self { json, // Don't bother checking for allocation failure, typically ~10 bytes pid: borrowed.pid.map(|p| p.to_string()), }) } } /// A borrowed copy of the JSON string and pid for a `FileEntry`. We use this /// to check if a given string/pid combination is present in the `EntryMap`, /// copying them to owned values only when needed. #[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Debug)] pub struct BorrowedData<'a> { pub json: &'a str, pub pid: Option<&'a str>, } impl<'a> BorrowedData<'a> { pub fn new( raw_entry: &'a RawEntry, file_info: &'a FileInfo, pid_significant: bool, ) -> Result { let json = str::from_utf8(raw_entry.json()) .map_err(|e| MmapError::Encoding(format!("invalid UTF-8 in entry JSON: {e}")))?; let pid = if pid_significant { Some(file_info.pid.as_str()) } else { None }; Ok(Self { json, pid }) } } /// The metadata associated with a `FileEntry`. The value in `EntryMap`. #[derive(Clone, Debug)] pub struct EntryMetadata { pub multiprocess_mode: Symbol, pub type_: Symbol, pub value: f64, } impl EntryMetadata { /// Construct a new `FileEntry`, copying the JSON string from the `RawEntry` /// into an internal buffer. pub fn new(mmap_entry: &RawEntry, file: &FileInfo) -> Result { let value = mmap_entry.value(); Ok(EntryMetadata { multiprocess_mode: file.multiprocess_mode, type_: file.type_, value, }) } /// Combine values with another `EntryMetadata`. pub fn merge(&mut self, other: &Self) { if self.type_ == SYM_GAUGE { match self.multiprocess_mode { s if s == SYM_MIN => self.value = self.value.min(other.value), s if s == SYM_MAX => self.value = self.value.max(other.value), s if s == SYM_LIVESUM => self.value += other.value, _ => self.value = other.value, } } else { self.value += other.value; } } /// Validate if pid is significant for metric. pub fn is_pid_significant(&self) -> bool { let mp = self.multiprocess_mode; self.type_ == SYM_GAUGE && !(mp == SYM_MIN || mp == SYM_MAX || mp == SYM_LIVESUM) } } impl FileEntry { /// Convert the sorted entries into a String in Prometheus metrics format. pub fn entries_to_string(entries: Vec) -> Result { // We guesstimate that lines are ~100 bytes long, preallocate the string to // roughly that size. let mut out = String::new(); out.try_reserve(entries.len() * 128) .map_err(|_| MmapError::OutOfMemory(entries.len() * 128))?; let mut prev_name: Option = None; let entry_count = entries.len(); let mut processed_count = 0; for entry in entries { let metrics_data = match serde_json::from_str::(&entry.data.json) { Ok(m) => { if m.labels.len() != m.values.len() { continue; } m } // We don't exit the function here so the total number of invalid // entries can be calculated below. Err(_) => continue, }; match prev_name.as_ref() { Some(p) if p == metrics_data.family_name => {} _ => { entry.append_header(metrics_data.family_name, &mut out); prev_name = Some(metrics_data.family_name.to_owned()); } } entry.append_entry(metrics_data, &mut out)?; writeln!(&mut out, " {}", entry.meta.value) .map_err(|e| MmapError::Other(format!("Failed to append to output: {e}")))?; processed_count += 1; } if processed_count != entry_count { return Err(MmapError::legacy( format!("Processed entries {processed_count} != map entries {entry_count}"), RubyError::Runtime, )); } Ok(out) } fn append_header(&self, family_name: &str, out: &mut String) { out.push_str("# HELP "); out.push_str(family_name); out.push_str(" Multiprocess metric\n"); out.push_str("# TYPE "); out.push_str(family_name); out.push(' '); out.push_str(&self.meta.type_.name().expect("name was invalid UTF-8")); out.push('\n'); } fn append_entry(&self, json_data: MetricText, out: &mut String) -> Result<()> { out.push_str(json_data.metric_name); if json_data.labels.is_empty() { if let Some(pid) = self.data.pid.as_ref() { out.push_str("{pid=\""); out.push_str(pid); out.push_str("\"}"); } return Ok(()); } out.push('{'); let it = json_data.labels.iter().zip(json_data.values.iter()); for (i, (&key, val)) in it.enumerate() { out.push_str(key); out.push('='); match val.get() { "null" => out.push_str("\"\""), s if s.starts_with('"') => out.push_str(s), s => { // Quote numeric values. out.push('"'); out.push_str(s); out.push('"'); } } if i < json_data.labels.len() - 1 { out.push(','); } } if let Some(pid) = self.data.pid.as_ref() { out.push_str(",pid=\""); out.push_str(pid); out.push('"'); } out.push('}'); Ok(()) } } #[cfg(test)] mod test { use bstr::BString; use indoc::indoc; use super::*; use crate::file_info::FileInfo; use crate::raw_entry::RawEntry; use crate::testhelper::{TestEntry, TestFile}; #[test] fn test_entries_to_string() { struct TestCase { name: &'static str, multiprocess_mode: &'static str, json: &'static [&'static str], values: &'static [f64], pids: &'static [&'static str], expected_out: Option<&'static str>, expected_err: Option, } let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let tc = vec![ TestCase { name: "one metric, pid significant", multiprocess_mode: "all", json: &[r#"["family","name",["label_a","label_b"],["value_a","value_b"]]"#], values: &[1.0], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name{label_a="value_a",label_b="value_b",pid="worker-1"} 1 "##}), expected_err: None, }, TestCase { name: "one metric, no pid", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],["value_a","value_b"]]"#], values: &[1.0], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name{label_a="value_a",label_b="value_b"} 1 "##}), expected_err: None, }, TestCase { name: "many labels", multiprocess_mode: "min", json: &[ r#"["family","name",["label_a","label_b","label_c","label_d","label_e"],["value_a","value_b","value_c","value_d","value_e"]]"#, ], values: &[1.0], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name{label_a="value_a",label_b="value_b",label_c="value_c",label_d="value_d",label_e="value_e"} 1 "##}), expected_err: None, }, TestCase { name: "floating point shown", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],["value_a","value_b"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name{label_a="value_a",label_b="value_b"} 1.5 "##}), expected_err: None, }, TestCase { name: "numeric value", multiprocess_mode: "min", json: &[ r#"["family","name",["label_a","label_b","label_c"],["value_a",403,-0.2E5]]"#, ], values: &[1.5], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name{label_a="value_a",label_b="403",label_c="-0.2E5"} 1.5 "##}), expected_err: None, }, TestCase { name: "null value", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],["value_a",null]]"#], values: &[1.5], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name{label_a="value_a",label_b=""} 1.5 "##}), expected_err: None, }, TestCase { name: "comma in value", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],["value_a","value,_b"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name{label_a="value_a",label_b="value,_b"} 1.5 "##}), expected_err: None, }, TestCase { name: "no labels, pid significant", multiprocess_mode: "all", json: &[r#"["family","name",[],[]]"#], values: &[1.0], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name{pid="worker-1"} 1 "##}), expected_err: None, }, TestCase { name: "no labels, no pid", multiprocess_mode: "min", json: &[r#"["family","name",[],[]]"#], values: &[1.0], pids: &["worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge name 1 "##}), expected_err: None, }, TestCase { name: "two metrics, same family, pid significant", multiprocess_mode: "all", json: &[ r#"["family","first",["label_a","label_b"],["value_a","value_b"]]"#, r#"["family","second",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0, 2.0], pids: &["worker-1", "worker-1"], expected_out: Some(indoc! {r##"# HELP family Multiprocess metric # TYPE family gauge first{label_a="value_a",label_b="value_b",pid="worker-1"} 1 second{label_a="value_a",label_b="value_b",pid="worker-1"} 2 "##}), expected_err: None, }, TestCase { name: "two metrics, different family, pid significant", multiprocess_mode: "min", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, r#"["second_family","second_name",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0, 2.0], pids: &["worker-1", "worker-1"], expected_out: Some(indoc! {r##"# HELP first_family Multiprocess metric # TYPE first_family gauge first_name{label_a="value_a",label_b="value_b"} 1 # HELP second_family Multiprocess metric # TYPE second_family gauge second_name{label_a="value_a",label_b="value_b"} 2 "##}), expected_err: None, }, TestCase { name: "three metrics, two different families, pid significant", multiprocess_mode: "all", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, r#"["first_family","second_name",["label_a","label_b"],["value_a","value_b"]]"#, r#"["second_family","second_name",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0, 2.0, 3.0], pids: &["worker-1", "worker-1", "worker-1"], expected_out: Some(indoc! {r##"# HELP first_family Multiprocess metric # TYPE first_family gauge first_name{label_a="value_a",label_b="value_b",pid="worker-1"} 1 second_name{label_a="value_a",label_b="value_b",pid="worker-1"} 2 # HELP second_family Multiprocess metric # TYPE second_family gauge second_name{label_a="value_a",label_b="value_b",pid="worker-1"} 3 "##}), expected_err: None, }, TestCase { name: "same metrics, pid significant, separate workers", multiprocess_mode: "all", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0, 2.0], pids: &["worker-1", "worker-2"], expected_out: Some(indoc! {r##"# HELP first_family Multiprocess metric # TYPE first_family gauge first_name{label_a="value_a",label_b="value_b",pid="worker-1"} 1 first_name{label_a="value_a",label_b="value_b",pid="worker-2"} 2 "##}), expected_err: None, }, TestCase { name: "same metrics, pid not significant, separate workers", multiprocess_mode: "max", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0, 2.0], pids: &["worker-1", "worker-2"], expected_out: Some(indoc! {r##"# HELP first_family Multiprocess metric # TYPE first_family gauge first_name{label_a="value_a",label_b="value_b"} 1 first_name{label_a="value_a",label_b="value_b"} 2 "##}), expected_err: None, }, TestCase { name: "entry fails to parse", multiprocess_mode: "min", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, r#"[not valid"#, ], values: &[1.0, 2.0], pids: &["worker-1", "worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 1 != map entries 2".to_owned(), RubyError::Runtime, )), }, TestCase { name: "too many values", multiprocess_mode: "min", json: &[r#"["family","name",["label_a"],["value_a","value,_b"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "no values", multiprocess_mode: "min", json: &[r#"["family","name",["label_a"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "no labels or values", multiprocess_mode: "min", json: &[r#"["family","name","foo"]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "too many leading brackets", multiprocess_mode: "min", json: &[r#"[["family","name",["label_a","label_b"],["value_a","value_b"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "too many trailing brackets", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],["value_a","value_b"]]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "too many leading label brackets", multiprocess_mode: "min", json: &[r#"["family","name",[["label_a","label_b"],["value_a","value_b"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "too many leading label brackets", multiprocess_mode: "min", json: &[r#"["family","name",[["label_a","label_b"],["value_a","value_b"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "too many leading value brackets", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],[["value_a","value_b"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "misplaced bracket", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],]["value_a","value_b"]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "comma in numeric", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],["value_a",403,0]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, TestCase { name: "non-e letter in numeric", multiprocess_mode: "min", json: &[r#"["family","name",["label_a","label_b"],["value_a",-2.0c5]]"#], values: &[1.5], pids: &["worker-1"], expected_out: None, expected_err: Some(MmapError::legacy( "Processed entries 0 != map entries 1".to_owned(), RubyError::Runtime, )), }, ]; for case in tc { let name = case.name; let input_bytes: Vec = case .json .iter() .zip(case.values) .map(|(&s, &value)| TestEntry::new(s, value).as_bstring()) .collect(); let mut file_infos = Vec::new(); for pid in case.pids { let TestFile { file, path, dir: _dir, } = TestFile::new(b"foobar"); let info = FileInfo { file, path, len: case.json.len(), multiprocess_mode: Symbol::new(case.multiprocess_mode), type_: Symbol::new("gauge"), pid: pid.to_string(), }; file_infos.push(info); } let file_entries: Vec = input_bytes .iter() .map(|s| RawEntry::from_slice(s).unwrap()) .zip(file_infos) .map(|(entry, info)| { let meta = EntryMetadata::new(&entry, &info).unwrap(); let borrowed = BorrowedData::new(&entry, &info, meta.is_pid_significant()).unwrap(); let data = EntryData::try_from(borrowed).unwrap(); FileEntry { data, meta } }) .collect(); let output = FileEntry::entries_to_string(file_entries); if let Some(expected_out) = case.expected_out { assert_eq!( expected_out, output.as_ref().unwrap(), "test case: {name} - output" ); } if let Some(expected_err) = case.expected_err { assert_eq!( expected_err, output.unwrap_err(), "test case: {name} - error" ); } } } #[test] fn test_merge() { struct TestCase { name: &'static str, metric_type: &'static str, multiprocess_mode: &'static str, values: &'static [f64], expected_value: f64, } let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let tc = vec![ TestCase { name: "gauge max", metric_type: "gauge", multiprocess_mode: "max", values: &[1.0, 5.0], expected_value: 5.0, }, TestCase { name: "gauge min", metric_type: "gauge", multiprocess_mode: "min", values: &[1.0, 5.0], expected_value: 1.0, }, TestCase { name: "gauge livesum", metric_type: "gauge", multiprocess_mode: "livesum", values: &[1.0, 5.0], expected_value: 6.0, }, TestCase { name: "gauge all", metric_type: "gauge", multiprocess_mode: "all", values: &[1.0, 5.0], expected_value: 5.0, }, TestCase { name: "not a gauge", metric_type: "histogram", multiprocess_mode: "max", values: &[1.0, 5.0], expected_value: 6.0, }, ]; for case in tc { let name = case.name; let json = r#"["family","metric",["label_a","label_b"],["value_a","value_b"]]"#; let TestFile { file, path, dir: _dir, } = TestFile::new(b"foobar"); let info = FileInfo { file, path, len: json.len(), multiprocess_mode: Symbol::new(case.multiprocess_mode), type_: Symbol::new(case.metric_type), pid: "worker-1".to_string(), }; let input_bytes: Vec = case .values .iter() .map(|&value| TestEntry::new(json, value).as_bstring()) .collect(); let entries: Vec = input_bytes .iter() .map(|s| RawEntry::from_slice(s).unwrap()) .map(|entry| { let meta = EntryMetadata::new(&entry, &info).unwrap(); let borrowed = BorrowedData::new(&entry, &info, meta.is_pid_significant()).unwrap(); let data = EntryData::try_from(borrowed).unwrap(); FileEntry { data, meta } }) .collect(); let mut entry_a = entries[0].clone(); let entry_b = entries[1].clone(); entry_a.meta.merge(&entry_b.meta); assert_eq!( case.expected_value, entry_a.meta.value, "test case: {name} - value" ); } } } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/file_info.rs000066400000000000000000000160001475532144100257770ustar00rootroot00000000000000use magnus::exception::*; use magnus::{Error, RString, Symbol, Value}; use std::ffi::OsString; use std::fs::File; use std::io::{self, Read, Seek}; use std::os::unix::ffi::OsStringExt; use std::path::PathBuf; use crate::err; use crate::error::{MmapError, RubyError}; use crate::util; use crate::Result; /// The details of a `*.db` file. #[derive(Debug)] pub struct FileInfo { pub file: File, pub path: PathBuf, pub len: usize, pub multiprocess_mode: Symbol, pub type_: Symbol, pub pid: String, } impl FileInfo { /// Receive the details of a file from Ruby and store as a `FileInfo`. pub fn open_from_params(params: &[Value; 4]) -> magnus::error::Result { if params.len() != 4 { return Err(err!( arg_error(), "wrong number of arguments {} instead of 4", params.len() )); } let filepath = RString::from_value(params[0]) .ok_or_else(|| err!(arg_error(), "can't convert filepath to String"))?; // SAFETY: We immediately copy the string buffer from Ruby, preventing // it from being mutated out from under us. let path_bytes: Vec<_> = unsafe { filepath.as_slice().to_owned() }; let path = PathBuf::from(OsString::from_vec(path_bytes)); let mut file = File::open(&path).map_err(|_| { err!( arg_error(), "Can't open {}, errno: {}", path.display(), util::errno() ) })?; let stat = file .metadata() .map_err(|_| err!(io_error(), "Can't stat file, errno: {}", util::errno()))?; let length = util::cast_chk::<_, usize>(stat.len(), "file size")?; let multiprocess_mode = Symbol::from_value(params[1]) .ok_or_else(|| err!(arg_error(), "expected multiprocess_mode to be a symbol"))?; let type_ = Symbol::from_value(params[2]) .ok_or_else(|| err!(arg_error(), "expected file type to be a symbol"))?; let pid = RString::from_value(params[3]) .ok_or_else(|| err!(arg_error(), "expected pid to be a String"))?; file.rewind() .map_err(|_| err!(io_error(), "Can't fseek 0, errno: {}", util::errno()))?; Ok(Self { file, path, len: length, multiprocess_mode, type_, pid: pid.to_string()?, }) } /// Read the contents of the associated file into the buffer provided by /// the caller. pub fn read_from_file(&mut self, buf: &mut Vec) -> Result<()> { buf.clear(); buf.try_reserve(self.len).map_err(|_| { MmapError::legacy( format!("Can't malloc {}, errno: {}", self.len, util::errno()), RubyError::Io, ) })?; match self.file.read_to_end(buf) { Ok(n) if n == self.len => Ok(()), // A worker may expand the file between our `stat` and `read`, no harm done. Ok(n) if n > self.len => { self.len = n; Ok(()) } Ok(_) => Err(MmapError::io( "read", &self.path, io::Error::from(io::ErrorKind::UnexpectedEof), )), Err(e) => Err(MmapError::io("read", &self.path, e)), } } } #[cfg(test)] mod test { use magnus::{eval, RArray, Symbol}; use rand::{thread_rng, Rng}; use sha2::{Digest, Sha256}; use std::fs; use std::io::Write; use super::*; use crate::testhelper::TestFile; #[test] fn test_open_from_params() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let file_data = b"foobar"; let TestFile { file: _file, path, dir: _dir, } = TestFile::new(file_data); let pid = "worker-1_0"; let args = RArray::from_value( eval(&format!("['{}', :max, :gauge, '{pid}']", path.display())).unwrap(), ) .unwrap(); let arg0 = args.shift().unwrap(); let arg1 = args.shift().unwrap(); let arg2 = args.shift().unwrap(); let arg3 = args.shift().unwrap(); let out = FileInfo::open_from_params(&[arg0, arg1, arg2, arg3]); assert!(out.is_ok()); let out = out.unwrap(); assert_eq!(out.path, path); assert_eq!(out.len, file_data.len()); assert_eq!(out.multiprocess_mode, Symbol::new("max")); assert_eq!(out.type_, Symbol::new("gauge")); assert_eq!(out.pid, pid); } #[test] fn test_read_from_file() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); const BUF_LEN: usize = 1 << 20; // 1MiB // Create a buffer with random data. let mut buf = vec![0u8; BUF_LEN]; thread_rng().fill(buf.as_mut_slice()); let TestFile { file, path, dir: _dir, } = TestFile::new(&buf); let mut info = FileInfo { file, path: path.clone(), len: buf.len(), multiprocess_mode: Symbol::new("puma"), type_: Symbol::new("max"), pid: "worker-0_0".to_string(), }; let mut out_buf = Vec::new(); info.read_from_file(&mut out_buf).unwrap(); assert_eq!(buf.len(), out_buf.len(), "buffer lens"); let mut in_hasher = Sha256::new(); in_hasher.update(&buf); let in_hash = in_hasher.finalize(); let mut out_hasher = Sha256::new(); out_hasher.update(&out_buf); let out_hash = out_hasher.finalize(); assert_eq!(in_hash, out_hash, "content hashes"); } #[test] fn test_read_from_file_resized() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); const BUF_LEN: usize = 1 << 14; // 16KiB // Create a buffer with random data. let mut buf = vec![0u8; BUF_LEN]; thread_rng().fill(buf.as_mut_slice()); let TestFile { file, path, dir: _dir, } = TestFile::new(&buf); let mut info = FileInfo { file, path: path.clone(), len: buf.len(), multiprocess_mode: Symbol::new("puma"), type_: Symbol::new("max"), pid: "worker-0_0".to_string(), }; let mut resized_file = fs::OpenOptions::new() .write(true) .append(true) .open(path) .unwrap(); // Write data to file after it has been `stat`ed in the // constructor. resized_file.write_all(&[1; 1024]).unwrap(); let mut out_buf = Vec::new(); info.read_from_file(&mut out_buf).unwrap(); assert_eq!(BUF_LEN + 1024, info.len, "resized file updated len"); } } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/lib.rs000066400000000000000000000052371475532144100246250ustar00rootroot00000000000000use magnus::exception::*; use magnus::prelude::*; use magnus::value::{Fixnum, Lazy, LazyId}; use magnus::{class, define_class, exception, function, method, Ruby}; use std::mem::size_of; use crate::mmap::MmapedFile; pub mod error; pub mod file_entry; pub mod file_info; mod macros; pub mod map; pub mod mmap; pub mod raw_entry; pub mod util; #[cfg(test)] mod testhelper; type Result = std::result::Result; const MAP_SHARED: i64 = libc::MAP_SHARED as i64; const HEADER_SIZE: usize = 2 * size_of::(); static SYM_GAUGE: LazyId = LazyId::new("gauge"); static SYM_MIN: LazyId = LazyId::new("min"); static SYM_MAX: LazyId = LazyId::new("max"); static SYM_LIVESUM: LazyId = LazyId::new("livesum"); static SYM_PID: LazyId = LazyId::new("pid"); static SYM_SAMPLES: LazyId = LazyId::new("samples"); static PROM_EPARSING_ERROR: Lazy = Lazy::new(|_| { let prom_err = define_class( "PrometheusParsingError", exception::runtime_error().as_r_class(), ) .expect("failed to create class `PrometheusParsingError`"); ExceptionClass::from_value(prom_err.as_value()) .expect("failed to create exception class from `PrometheusParsingError`") }); #[magnus::init] fn init(ruby: &Ruby) -> magnus::error::Result<()> { // Initialize the static symbols LazyId::force(&SYM_GAUGE, ruby); LazyId::force(&SYM_MIN, ruby); LazyId::force(&SYM_MAX, ruby); LazyId::force(&SYM_LIVESUM, ruby); LazyId::force(&SYM_PID, ruby); LazyId::force(&SYM_SAMPLES, ruby); // Initialize `PrometheusParsingError` class. Lazy::force(&PROM_EPARSING_ERROR, ruby); let klass = define_class("FastMmapedFileRs", class::object())?; klass.undef_default_alloc_func(); // UNWRAP: We know `MAP_SHARED` fits in a `Fixnum`. klass.const_set("MAP_SHARED", Fixnum::from_i64(MAP_SHARED).unwrap())?; klass.define_singleton_method("to_metrics", function!(MmapedFile::to_metrics, 1))?; // Required for subclassing to work klass.define_alloc_func::(); klass.define_singleton_method("new", method!(MmapedFile::new, -1))?; klass.define_method("initialize", method!(MmapedFile::initialize, 1))?; klass.define_method("slice", method!(MmapedFile::slice, -1))?; klass.define_method("sync", method!(MmapedFile::sync, -1))?; klass.define_method("munmap", method!(MmapedFile::munmap, 0))?; klass.define_method("used", method!(MmapedFile::load_used, 0))?; klass.define_method("used=", method!(MmapedFile::save_used, 1))?; klass.define_method("fetch_entry", method!(MmapedFile::fetch_entry, 3))?; klass.define_method("upsert_entry", method!(MmapedFile::upsert_entry, 3))?; Ok(()) } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/macros.rs000066400000000000000000000005471475532144100253420ustar00rootroot00000000000000#[macro_export] macro_rules! err { (with_errno: $err_t:expr, $($arg:expr),*) => { { let err = format!($($arg),*); let strerror = strerror(errno()); Error::new($err_t, format!("{err} ({strerror})")) } }; ($err_t:expr, $($arg:expr),*) => { Error::new($err_t, format!($($arg),*)) }; } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/map.rs000066400000000000000000000403561475532144100246350ustar00rootroot00000000000000use hashbrown::hash_map::RawEntryMut; use hashbrown::HashMap; use magnus::{exception::*, Error, RArray}; use std::hash::{BuildHasher, Hash, Hasher}; use std::mem::size_of; use crate::error::MmapError; use crate::file_entry::{BorrowedData, EntryData, EntryMetadata, FileEntry}; use crate::file_info::FileInfo; use crate::raw_entry::RawEntry; use crate::util::read_u32; use crate::Result; use crate::{err, HEADER_SIZE}; /// A HashMap of JSON strings and their associated metadata. /// Used to print metrics in text format. /// /// The map key is the entry's JSON string and an optional pid string. The latter /// allows us to have multiple entries on the map for multiple pids using the /// same string. #[derive(Default, Debug)] pub struct EntryMap(HashMap); impl EntryMap { /// Construct a new EntryMap. pub fn new() -> Self { Self(HashMap::new()) } /// Given a list of files, read each one into memory and parse the metrics it contains. pub fn aggregate_files(&mut self, list_of_files: RArray) -> magnus::error::Result<()> { // Pre-allocate the `HashMap` and validate we don't OOM. The C implementation // ignores allocation failures here. We perform this check to avoid potential // panics. We assume ~1,000 entries per file, so 72 KiB allocated per file. self.0 .try_reserve(list_of_files.len() * 1024) .map_err(|_| { err!( no_mem_error(), "Couldn't allocate for {} memory", size_of::() * list_of_files.len() * 1024 ) })?; // We expect file sizes between 4KiB and 4MiB. Pre-allocate 16KiB to reduce reallocations // a bit. let mut buf = Vec::new(); buf.try_reserve(16_384) .map_err(|_| err!(no_mem_error(), "Couldn't allocate for {} memory", 16_384))?; for item in list_of_files.each() { let params = RArray::from_value(item?).expect("file list was not a Ruby Array"); if params.len() != 4 { return Err(err!( arg_error(), "wrong number of arguments {} instead of 4", params.len() )); } let params = params.to_value_array::<4>()?; let mut file_info = FileInfo::open_from_params(¶ms)?; file_info.read_from_file(&mut buf)?; self.process_buffer(file_info, &buf)?; } Ok(()) } /// Consume the `EntryMap` and convert the key/value into`FileEntry` /// objects, sorting them by their JSON strings. pub fn into_sorted(self) -> Result> { let mut sorted = Vec::new(); // To match the behavior of the C version, pre-allocate the entries // and check for allocation failure. Generally idiomatic Rust would // `collect` the iterator into a new `Vec` in place, but this panics // if it can't allocate and we want to continue execution in that // scenario. if sorted.try_reserve_exact(self.0.len()).is_err() { return Err(MmapError::OutOfMemory( self.0.len() * size_of::(), )); } sorted.extend( self.0 .into_iter() .map(|(data, meta)| FileEntry { data, meta }), ); sorted.sort_unstable_by(|x, y| x.data.cmp(&y.data)); Ok(sorted) } /// Check if the `EntryMap` already contains the JSON string. /// If yes, update the associated value, if not insert the /// entry into the map. pub fn merge_or_store(&mut self, data: BorrowedData, meta: EntryMetadata) -> Result<()> { // Manually hash the `BorrowedData` and perform an equality check on the // key. This allows us to perform the comparison without allocating a // new `EntryData` that may not be needed. let mut state = self.0.hasher().build_hasher(); data.hash(&mut state); let hash = state.finish(); match self.0.raw_entry_mut().from_hash(hash, |k| k == &data) { RawEntryMut::Vacant(entry) => { // Allocate a new `EntryData` as the JSON/pid combination is // not present in the map. let owned = EntryData::try_from(data)?; entry.insert(owned, meta); } RawEntryMut::Occupied(mut entry) => { let existing = entry.get_mut(); existing.merge(&meta); } } Ok(()) } /// Parse metrics data from a `.db` file and store in the `EntryMap`. fn process_buffer(&mut self, file_info: FileInfo, source: &[u8]) -> Result<()> { if source.len() < HEADER_SIZE { // Nothing to read, OK. return Ok(()); } // CAST: no-op on 32-bit, widening on 64-bit. let used = read_u32(source, 0)? as usize; if used > source.len() { return Err(MmapError::PromParsing(format!( "source file {} corrupted, used {used} > file size {}", file_info.path.display(), source.len() ))); } let mut pos = HEADER_SIZE; while pos + size_of::() < used { let raw_entry = RawEntry::from_slice(&source[pos..used])?; if pos + raw_entry.total_len() > used { return Err(MmapError::PromParsing(format!( "source file {} corrupted, used {used} < stored data length {}", file_info.path.display(), pos + raw_entry.total_len() ))); } let meta = EntryMetadata::new(&raw_entry, &file_info)?; let data = BorrowedData::new(&raw_entry, &file_info, meta.is_pid_significant())?; self.merge_or_store(data, meta)?; pos += raw_entry.total_len(); } Ok(()) } } #[cfg(test)] mod test { use magnus::Symbol; use std::mem; use super::*; use crate::file_entry::FileEntry; use crate::testhelper::{self, TestFile}; impl EntryData { /// A helper function for tests to convert owned data to references. fn as_borrowed(&self) -> BorrowedData { BorrowedData { json: &self.json, pid: self.pid.as_deref(), } } } #[test] fn test_into_sorted() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let entries = vec![ FileEntry { data: EntryData { json: "zzzzzz".to_string(), pid: Some("worker-0_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("max"), type_: Symbol::new("gauge"), value: 1.0, }, }, FileEntry { data: EntryData { json: "zzz".to_string(), pid: Some("worker-0_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("max"), type_: Symbol::new("gauge"), value: 1.0, }, }, FileEntry { data: EntryData { json: "zzzaaa".to_string(), pid: Some("worker-0_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("max"), type_: Symbol::new("gauge"), value: 1.0, }, }, FileEntry { data: EntryData { json: "aaa".to_string(), pid: Some("worker-0_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("max"), type_: Symbol::new("gauge"), value: 1.0, }, }, FileEntry { data: EntryData { json: "ooo".to_string(), pid: Some("worker-1_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("all"), type_: Symbol::new("gauge"), value: 1.0, }, }, FileEntry { data: EntryData { json: "ooo".to_string(), pid: Some("worker-0_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("all"), type_: Symbol::new("gauge"), value: 1.0, }, }, ]; let mut map = EntryMap::new(); for entry in entries { map.0.insert(entry.data, entry.meta); } let result = map.into_sorted(); assert!(result.is_ok()); let sorted = result.unwrap(); assert_eq!(sorted.len(), 6); assert_eq!(sorted[0].data.json, "aaa"); assert_eq!(sorted[1].data.json, "ooo"); assert_eq!(sorted[1].data.pid.as_deref(), Some("worker-0_0")); assert_eq!(sorted[2].data.json, "ooo"); assert_eq!(sorted[2].data.pid.as_deref(), Some("worker-1_0")); assert_eq!(sorted[3].data.json, "zzz"); assert_eq!(sorted[4].data.json, "zzzaaa"); assert_eq!(sorted[5].data.json, "zzzzzz"); } #[test] fn test_merge_or_store() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let key = "foobar"; let starting_entry = FileEntry { data: EntryData { json: key.to_string(), pid: Some("worker-0_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("all"), type_: Symbol::new("gauge"), value: 1.0, }, }; let matching_entry = FileEntry { data: EntryData { json: key.to_string(), pid: Some("worker-0_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("all"), type_: Symbol::new("gauge"), value: 5.0, }, }; let same_key_different_worker = FileEntry { data: EntryData { json: key.to_string(), pid: Some("worker-1_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("all"), type_: Symbol::new("gauge"), value: 100.0, }, }; let unmatched_entry = FileEntry { data: EntryData { json: "another key".to_string(), pid: Some("worker-0_0".to_string()), }, meta: EntryMetadata { multiprocess_mode: Symbol::new("all"), type_: Symbol::new("gauge"), value: 1.0, }, }; let mut map = EntryMap::new(); map.0 .insert(starting_entry.data.clone(), starting_entry.meta.clone()); let matching_borrowed = matching_entry.data.as_borrowed(); map.merge_or_store(matching_borrowed, matching_entry.meta) .unwrap(); assert_eq!( 5.0, map.0.get(&starting_entry.data).unwrap().value, "value updated" ); assert_eq!(1, map.0.len(), "no entry added"); let same_key_different_worker_borrowed = same_key_different_worker.data.as_borrowed(); map.merge_or_store( same_key_different_worker_borrowed, same_key_different_worker.meta, ) .unwrap(); assert_eq!( 5.0, map.0.get(&starting_entry.data).unwrap().value, "value unchanged" ); assert_eq!(2, map.0.len(), "additional entry added"); let unmatched_entry_borrowed = unmatched_entry.data.as_borrowed(); map.merge_or_store(unmatched_entry_borrowed, unmatched_entry.meta) .unwrap(); assert_eq!( 5.0, map.0.get(&starting_entry.data).unwrap().value, "value unchanged" ); assert_eq!(3, map.0.len(), "entry added"); } #[test] fn test_process_buffer() { struct TestCase { name: &'static str, json: &'static [&'static str], values: &'static [f64], used: Option, expected_ct: usize, expected_err: Option, } let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let tc = vec![ TestCase { name: "single entry", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0], used: None, expected_ct: 1, expected_err: None, }, TestCase { name: "multiple entries", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, r#"["second_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0, 2.0], used: None, expected_ct: 2, expected_err: None, }, TestCase { name: "empty", json: &[], values: &[], used: None, expected_ct: 0, expected_err: None, }, TestCase { name: "used too long", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0], used: Some(9999), expected_ct: 0, expected_err: Some(MmapError::PromParsing(String::new())), }, TestCase { name: "used too short", json: &[ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#, ], values: &[1.0], used: Some(15), expected_ct: 0, expected_err: Some(MmapError::out_of_bounds(88, 7)), }, ]; for case in tc { let name = case.name; let input_bytes = testhelper::entries_to_db(case.json, case.values, case.used); let TestFile { file, path, dir: _dir, } = TestFile::new(&input_bytes); let info = FileInfo { file, path, len: case.json.len(), multiprocess_mode: Symbol::new("max"), type_: Symbol::new("gauge"), pid: "worker-1".to_string(), }; let mut map = EntryMap::new(); let result = map.process_buffer(info, &input_bytes); assert_eq!(case.expected_ct, map.0.len(), "test case: {name} - count"); if let Some(expected_err) = case.expected_err { // Validate we have the right enum type for the error. Error // messages contain the temp dir path and can't be predicted // exactly. assert_eq!( mem::discriminant(&expected_err), mem::discriminant(&result.unwrap_err()), "test case: {name} - failure" ); } else { assert_eq!(Ok(()), result, "test case: {name} - success"); assert_eq!( case.json.len(), map.0.len(), "test case: {name} - all entries captured" ); } } } } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/mmap.rs000066400000000000000000001010701475532144100250010ustar00rootroot00000000000000use magnus::exception::*; use magnus::prelude::*; use magnus::rb_sys::{AsRawValue, FromRawValue}; use magnus::typed_data::Obj; use magnus::value::Fixnum; use magnus::{eval, scan_args, Error, Integer, RArray, RClass, RHash, RString, Value}; use nix::libc::{c_char, c_long, c_ulong}; use rb_sys::rb_str_new_static; use std::fs::File; use std::io::{prelude::*, SeekFrom}; use std::mem; use std::path::Path; use std::ptr::NonNull; use std::sync::RwLock; use crate::err; use crate::error::MmapError; use crate::file_entry::FileEntry; use crate::map::EntryMap; use crate::raw_entry::RawEntry; use crate::util::{self, CheckedOps}; use crate::Result; use crate::HEADER_SIZE; use inner::InnerMmap; mod inner; #[cfg(ruby_gte_3_4)] /// The Ruby `STR_SHARED` flag, aka `FL_USER0`. /// This was changed from `FL_USER2` in https://github.com/ruby/ruby/commit/6deeec5d459ecff5ec4628523b14ac7379fd942e. const STR_SHARED: c_ulong = 1 << (12); #[cfg(ruby_lte_3_3)] /// The Ruby `STR_SHARED` flag, aka `FL_USER2`. const STR_SHARED: c_ulong = 1 << (14); /// The Ruby `STR_NOEMBED` flag, aka `FL_USER1`. const STR_NOEMBED: c_ulong = 1 << (13); /// A Rust struct wrapped in a Ruby object, providing access to a memory-mapped /// file used to store, update, and read out Prometheus metrics. /// /// - File format: /// - Header: /// - 4 bytes: u32 - total size of metrics in file. /// - 4 bytes: NUL byte padding. /// - Repeating metrics entries: /// - 4 bytes: u32 - entry JSON string size. /// - `N` bytes: UTF-8 encoded JSON string used as entry key. /// - (8 - (4 + `N`) % 8) bytes: 1 to 8 padding space (0x20) bytes to /// reach 8-byte alignment. /// - 8 bytes: f64 - entry value. /// /// All numbers are saved in native-endian format. /// /// Generated via [luismartingarcia/protocol](https://github.com/luismartingarcia/protocol): /// /// /// ``` /// protocol "Used:4,Pad:4,K1 Size:4,K1 Name:4,K1 Value:8,K2 Size:4,K2 Name:4,K2 Value:8" /// /// 0 1 2 3 /// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 /// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ /// | Used | Pad |K1 Size|K1 Name| K1 Value |K2 Size|K2 Name| /// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ /// | K2 Value | /// +-+-+-+-+-+-+-+ /// ``` // // The API imposed by `magnus` requires all methods to use shared borrows. // This means we can't store any mutable state in the top-level struct, // and must store the interior data behind a `RwLock`, which adds run-time // checks that mutable operations have no concurrent read or writes. // // We are further limited by the need to support subclassing in Ruby, which // requires us to define an allocation function for the class, the // `magnus::class::define_alloc_func()` function. This needs a support the // `Default` trait, so a `File` cannot directly help by the object being // constructed. Having the `RwLock` hold an `Option` of the interior object // resolves this. #[derive(Debug, Default)] #[magnus::wrap(class = "FastMmapedFileRs", free_immediately, size)] pub struct MmapedFile(RwLock>); impl MmapedFile { /// call-seq: /// new(file) /// /// create a new Mmap object /// /// * file /// /// /// Creates a mapping that's shared with all other processes /// mapping the same area of the file. pub fn new(klass: RClass, args: &[Value]) -> magnus::error::Result> { let args = scan_args::scan_args::<(RString,), (), (), (), (), ()>(args)?; let path = args.required.0; let lock = MmapedFile(RwLock::new(None)); let obj = Obj::wrap_as(lock, klass); let _: Value = obj.funcall("initialize", (path,))?; Ok(obj) } /// Initialize a new `FastMmapedFileRs` object. This must be defined in /// order for inheritance to work. pub fn initialize(rb_self: Obj, fname: String) -> magnus::error::Result<()> { let file = File::options() .read(true) .write(true) .open(&fname) .map_err(|_| err!(arg_error(), "Can't open {}", fname))?; let inner = InnerMmap::new(fname.into(), file)?; rb_self.insert_inner(inner)?; let weak_klass = RClass::from_value(eval("ObjectSpace::WeakMap")?) .ok_or_else(|| err!(no_method_error(), "unable to create WeakMap"))?; let weak_obj_tracker = weak_klass.new_instance(())?; // We will need to iterate over strings backed by the mmapped file, but // don't want to prevent the GC from reaping them when the Ruby code // has finished with them. `ObjectSpace::WeakMap` allows us to track // them without extending their lifetime. // // https://ruby-doc.org/core-3.0.0/ObjectSpace/WeakMap.html rb_self.ivar_set("@weak_obj_tracker", weak_obj_tracker)?; Ok(()) } /// Read the list of files provided from Ruby and convert them to a Prometheus /// metrics String. pub fn to_metrics(file_list: RArray) -> magnus::error::Result { let mut map = EntryMap::new(); map.aggregate_files(file_list)?; let sorted = map.into_sorted()?; FileEntry::entries_to_string(sorted).map_err(|e| e.into()) } /// Document-method: [] /// Document-method: slice /// /// call-seq: [](args) /// /// Element reference - with the following syntax: /// /// self[nth] /// /// retrieve the nth character /// /// self[start..last] /// /// return a substring from start to last /// /// self[start, length] /// /// return a substring of lenght characters from start pub fn slice(rb_self: Obj, args: &[Value]) -> magnus::error::Result { // The C implementation would trigger a GC cycle via `rb_gc_force_recycle` // if the `MM_PROTECT` flag is set, but in practice this is never used. // We omit this logic, particularly because `rb_gc_force_recycle` is a // no-op as of Ruby 3.1. let rs_self = &*rb_self; let str = rs_self.str(rb_self)?; rs_self._slice(rb_self, str, args) } fn _slice( &self, rb_self: Obj, str: RString, args: &[Value], ) -> magnus::error::Result { let substr: RString = str.funcall("[]", args)?; // Track shared child strings which use the same backing storage. if Self::rb_string_is_shared(substr) { (*rb_self).track_rstring(rb_self, substr)?; } // The C implementation does this, perhaps to validate that the len we // provided is actually being used. (*rb_self).inner_mut(|inner| { inner.set_len(str.len()); Ok(()) })?; Ok(substr) } /// Document-method: msync /// Document-method: sync /// Document-method: flush /// /// call-seq: msync /// /// flush the file pub fn sync(&self, args: &[Value]) -> magnus::error::Result<()> { use nix::sys::mman::MsFlags; let mut ms_async = false; let args = scan_args::scan_args::<(), (Option,), (), (), (), ()>(args)?; if let Some(flag) = args.optional.0 { let flag = MsFlags::from_bits(flag).unwrap_or(MsFlags::empty()); ms_async = flag.contains(MsFlags::MS_ASYNC); } // The `memmap2` crate does not support the `MS_INVALIDATE` flag. We ignore that // flag if passed in, checking only for `MS_ASYNC`. In practice no arguments are ever // passed to this function, but we do this to maintain compatibility with the // C implementation. self.inner_mut(|inner| inner.flush(ms_async)) .map_err(|e| e.into()) } /// Document-method: munmap /// Document-method: unmap /// /// call-seq: munmap /// /// terminate the association pub fn munmap(rb_self: Obj) -> magnus::error::Result<()> { let rs_self = &*rb_self; rs_self.inner_mut(|inner| { // We are about to release the backing mmap for Ruby's String // objects. If Ruby attempts to read from them the program will // segfault. We update the length of all Strings to zero so Ruby // does not attempt to access the now invalid address between now // and when GC eventually reaps the objects. // // See the following for more detail: // https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/issues/39 // https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/issues/41 // https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/merge_requests/80 inner.set_len(0); Ok(()) })?; // Update each String object to be zero-length. let cap = util::cast_chk::<_, c_long>(rs_self.capacity(), "capacity")?; rs_self.update_weak_map(rb_self, rs_self.as_mut_ptr(), cap)?; // Remove the `InnerMmap` from the `RwLock`. This will drop // end of this function, unmapping and closing the file. let _ = rs_self.take_inner()?; Ok(()) } /// Fetch the `used` header from the `.db` file, the length /// in bytes of the data written to the file. pub fn load_used(&self) -> magnus::error::Result { let used = self.inner(|inner| inner.load_used())?; Ok(Integer::from_u64(used as u64)) } /// Update the `used` header for the `.db` file, the length /// in bytes of the data written to the file. pub fn save_used(rb_self: Obj, used: Fixnum) -> magnus::error::Result { let rs_self = &*rb_self; let used_uint = used.to_u32()?; // If the underlying mmap is smaller than the header, then resize to fit. // The file has already been expanded to page size when first opened, so // even if the map is less than HEADER_SIZE, we're not at risk of a // SIGBUS. if rs_self.capacity() < HEADER_SIZE { rs_self.expand_to_fit(rb_self, HEADER_SIZE)?; } rs_self.inner_mut(|inner| inner.save_used(used_uint))?; Ok(used) } /// Fetch the value associated with a key from the mmap. /// If no entry is present, initialize with the default /// value provided. pub fn fetch_entry( rb_self: Obj, positions: RHash, key: RString, default_value: f64, ) -> magnus::error::Result { let rs_self = &*rb_self; let position: Option = positions.lookup(key)?; if let Some(pos) = position { let pos = pos.to_usize()?; return rs_self .inner(|inner| inner.load_value(pos)) .map_err(|e| e.into()); } rs_self.check_expand(rb_self, key.len())?; let value_offset: usize = rs_self.inner_mut(|inner| { // SAFETY: We must not call any Ruby code for the lifetime of this borrow. unsafe { inner.initialize_entry(key.as_slice(), default_value) } })?; // CAST: no-op on 64-bit, widening on 32-bit. positions.aset(key, Integer::from_u64(value_offset as u64))?; rs_self.load_value(value_offset) } /// Update the value of an existing entry, if present. Otherwise create a new entry /// for the key. pub fn upsert_entry( rb_self: Obj, positions: RHash, key: RString, value: f64, ) -> magnus::error::Result { let rs_self = &*rb_self; let position: Option = positions.lookup(key)?; if let Some(pos) = position { let pos = pos.to_usize()?; return rs_self .inner_mut(|inner| { inner.save_value(pos, value)?; // TODO just return `value` here instead of loading it? // This is how the C implementation did it, but I don't // see what the extra load gains us. inner.load_value(pos) }) .map_err(|e| e.into()); } rs_self.check_expand(rb_self, key.len())?; let value_offset: usize = rs_self.inner_mut(|inner| { // SAFETY: We must not call any Ruby code for the lifetime of this borrow. unsafe { inner.initialize_entry(key.as_slice(), value) } })?; // CAST: no-op on 64-bit, widening on 32-bit. positions.aset(key, Integer::from_u64(value_offset as u64))?; rs_self.load_value(value_offset) } /// Creates a Ruby String containing the section of the mmapped file that /// has been written to. fn str(&self, rb_self: Obj) -> magnus::error::Result { let val_id = (*rb_self).inner(|inner| { let ptr = inner.as_ptr(); let len = inner.len(); // SAFETY: This is safe so long as the data provided to Ruby meets its // requirements. When unmapping the file this will no longer be the // case, see the comment on `munmap` for how we handle this. Ok(unsafe { rb_str_new_static(ptr as _, len as _) }) })?; // SAFETY: We know that rb_str_new_static returns a VALUE. let val = unsafe { Value::from_raw(val_id) }; // UNWRAP: We created this value as a string above. let str = RString::from_value(val).unwrap(); // Freeze the root string so it can't be mutated out from under any // substrings created. This object is never exposed to callers. str.freeze(); // Track the RString in our `WeakMap` so we can update its address if // we re-mmap the backing file. (*rb_self).track_rstring(rb_self, str)?; Ok(str) } /// If we reallocate, any live Ruby strings provided by the `str()` method /// will be invalidated. We need to iterate over them using and update their /// heap pointers to the newly allocated memory region. fn update_weak_map( &self, rb_self: Obj, old_ptr: *const c_char, old_cap: c_long, ) -> magnus::error::Result<()> { let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?; let new_len = self.inner(|inner| util::cast_chk::<_, c_long>(inner.len(), "mmap len"))?; // Convert WeakMap to an Array of values to to avoid garbage collection issues let values: Value = tracker.funcall("values", ())?; // Iterate over the values of the `WeakMap`. for val in values.enumeratorize("each", ()) { let rb_string = val?; let str = RString::from_value(rb_string) .ok_or_else(|| err!(arg_error(), "weakmap value was not a string"))?; // SAFETY: We're messing with Ruby's internals here, YOLO. unsafe { // Convert the magnus wrapper type to a raw string exposed by `rb_sys`, // which provides access to its internals. let mut raw_str = Self::rb_string_internal(str); // Shared string have their own `ptr` and `len` values, but `aux` // is the id of the parent string so the GC can track this // dependency. The `ptr` will always be an offset from the base // address of the mmap, and `len` will be the length of the mmap // less the offset from the base. if Self::rb_string_is_shared(str) && new_len > 0 { // Calculate how far into the original mmap the shared string // started and update to the equivalent address in the new // one. let substr_ptr = raw_str.as_ref().as_.heap.ptr; let offset = substr_ptr.offset_from(old_ptr); raw_str.as_mut().as_.heap.ptr = self.as_mut_ptr().offset(offset); let current_len = str.len() as c_long; let new_shared_len = old_cap + current_len; self.update_rstring_len(raw_str, new_shared_len); continue; } // Update the string to point to the new mmapped file. // We're matching the behavior of Ruby's `str_new_static` function. // See https://github.com/ruby/ruby/blob/e51014f9c05aa65cbf203442d37fef7c12390015/string.c#L1030-L1053 // // We deliberately do _NOT_ increment the `capa` field of the // string to match the new `len`. We were initially doing this, // but consistently triggered GCs in the middle of updating the // string pointers, causing a segfault. // // See https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/issues/45 raw_str.as_mut().as_.heap.ptr = self.as_mut_ptr(); self.update_rstring_len(raw_str, new_len); } } Ok(()) } /// Check that the mmap is large enough to contain the value to be added, /// and expand it to fit if necessary. fn check_expand(&self, rb_self: Obj, key_len: usize) -> magnus::error::Result<()> { // CAST: no-op on 32-bit, widening on 64-bit. let used = self.inner(|inner| inner.load_used())? as usize; let entry_len = RawEntry::calc_total_len(key_len)?; // We need the mmapped region to contain at least one byte beyond the // written data to create a NUL- terminated C string. Validate that // new length does not exactly match or exceed the length of the mmap. while self.capacity() <= used.add_chk(entry_len)? { self.expand_to_fit(rb_self, self.capacity().mul_chk(2)?)?; } Ok(()) } /// Expand the underlying file until it is long enough to fit `target_cap`. /// This will remove the existing mmap, expand the file, then update any /// strings held by the `WeakMap` to point to the newly mmapped address. fn expand_to_fit(&self, rb_self: Obj, target_cap: usize) -> magnus::error::Result<()> { if target_cap < self.capacity() { return Err(err!(arg_error(), "Can't reduce the size of mmap")); } let mut new_cap = self.capacity(); while new_cap < target_cap { new_cap = new_cap.mul_chk(2)?; } if new_cap != self.capacity() { let old_ptr = self.as_mut_ptr(); let old_cap = util::cast_chk::<_, c_long>(self.capacity(), "capacity")?; // Drop the old mmap. let (mut file, path) = self.take_inner()?.munmap(); self.expand_file(&mut file, &path, target_cap)?; // Re-mmap the expanded file. let new_inner = InnerMmap::reestablish(path, file, target_cap)?; self.insert_inner(new_inner)?; return self.update_weak_map(rb_self, old_ptr, old_cap); } Ok(()) } /// Use lseek(2) to seek past the end of the file and write a NUL byte. This /// creates a file hole that expands the size of the file without consuming /// disk space until it is actually written to. fn expand_file(&self, file: &mut File, path: &Path, len: usize) -> Result<()> { if len == 0 { return Err(MmapError::overflowed(0, -1, "adding")); } // CAST: no-op on 64-bit, widening on 32-bit. let len = len as u64; match file.seek(SeekFrom::Start(len - 1)) { Ok(_) => {} Err(_) => { return Err(MmapError::with_errno(format!("Can't lseek {}", len - 1))); } } match file.write(&[0x0]) { Ok(1) => {} _ => { return Err(MmapError::with_errno(format!( "Can't extend {}", path.display() ))); } } Ok(()) } fn track_rstring(&self, rb_self: Obj, str: RString) -> magnus::error::Result<()> { let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?; // Use the string's Id as the key in the `WeakMap`. let key = str.as_raw(); let _: Value = tracker.funcall("[]=", (key, str))?; Ok(()) } /// The total capacity of the underlying mmap. #[inline] fn capacity(&self) -> usize { // UNWRAP: This is actually infallible, but we need to // wrap it in a `Result` for use with `inner()`. self.inner(|inner| Ok(inner.capacity())).unwrap() } fn load_value(&self, position: usize) -> magnus::error::Result { self.inner(|inner| inner.load_value(position)) .map_err(|e| e.into()) } fn as_mut_ptr(&self) -> *mut c_char { // UNWRAP: This is actually infallible, but we need to // wrap it in a `Result` for use with `inner()`. self.inner(|inner| Ok(inner.as_mut_ptr() as *mut c_char)) .unwrap() } /// Takes a closure with immutable access to InnerMmap. Will fail if the inner /// object has a mutable borrow or has been dropped. fn inner(&self, func: F) -> Result where F: FnOnce(&InnerMmap) -> Result, { let inner_opt = self.0.try_read().map_err(|_| MmapError::ConcurrentAccess)?; let inner = inner_opt.as_ref().ok_or(MmapError::UnmappedFile)?; func(inner) } /// Takes a closure with mutable access to InnerMmap. Will fail if the inner /// object has an existing mutable borrow, or has been dropped. fn inner_mut(&self, func: F) -> Result where F: FnOnce(&mut InnerMmap) -> Result, { let mut inner_opt = self .0 .try_write() .map_err(|_| MmapError::ConcurrentAccess)?; let inner = inner_opt.as_mut().ok_or(MmapError::UnmappedFile)?; func(inner) } /// Take ownership of the `InnerMmap` from the `RwLock`. /// Will fail if a mutable borrow is already held or the inner /// object has been dropped. fn take_inner(&self) -> Result { let mut inner_opt = self .0 .try_write() .map_err(|_| MmapError::ConcurrentAccess)?; match (*inner_opt).take() { Some(i) => Ok(i), None => Err(MmapError::UnmappedFile), } } /// Move `new_inner` into the `RwLock`. /// Will return an error if a mutable borrow is already held. fn insert_inner(&self, new_inner: InnerMmap) -> Result<()> { let mut inner_opt = self .0 .try_write() .map_err(|_| MmapError::ConcurrentAccess)?; (*inner_opt).replace(new_inner); Ok(()) } /// Check if an RString is shared. Shared string use the same underlying /// storage as their parent, taking an offset from the start. By default /// they must run to the end of the parent string. fn rb_string_is_shared(rb_str: RString) -> bool { // SAFETY: We only hold a reference to the raw object for the duration // of this function, and no Ruby code is called. let flags = unsafe { let raw_str = Self::rb_string_internal(rb_str); raw_str.as_ref().basic.flags }; let shared_flags = STR_SHARED | STR_NOEMBED; flags & shared_flags == shared_flags } /// Convert `magnus::RString` into the raw binding used by `rb_sys::RString`. /// We need this to manually change the pointer and length values for strings /// when moving the mmap to a new file. /// /// SAFETY: Calling Ruby code while the returned object is held may result /// in it being mutated or dropped. unsafe fn rb_string_internal(rb_str: RString) -> NonNull { NonNull::new_unchecked(rb_str.as_raw() as *mut rb_sys::RString) } #[cfg(ruby_lte_3_2)] unsafe fn update_rstring_len(&self, mut raw_str: NonNull, new_len: c_long) { raw_str.as_mut().as_.heap.len = new_len; } #[cfg(ruby_gte_3_3)] unsafe fn update_rstring_len(&self, mut raw_str: NonNull, new_len: c_long) { raw_str.as_mut().len = new_len; } } #[cfg(test)] mod test { use magnus::error::Error; use magnus::eval; use magnus::Range; use nix::unistd::{sysconf, SysconfVar}; use std::mem::size_of; use super::*; use crate::raw_entry::RawEntry; use crate::testhelper::TestFile; /// Create a wrapped MmapedFile object. fn create_obj() -> Obj { let TestFile { file: _file, path, dir: _dir, } = TestFile::new(&[0u8; 8]); let path_str = path.display().to_string(); let rpath = RString::new(&path_str); eval!("FastMmapedFileRs.new(path)", path = rpath).unwrap() } /// Add three entries to the mmap. Expected length is 56, 3x 16-byte /// entries with 8-byte header. fn populate_entries(rb_self: &Obj) -> RHash { let positions = RHash::from_value(eval("{}").unwrap()).unwrap(); MmapedFile::upsert_entry(*rb_self, positions, RString::new("a"), 0.0).unwrap(); MmapedFile::upsert_entry(*rb_self, positions, RString::new("b"), 1.0).unwrap(); MmapedFile::upsert_entry(*rb_self, positions, RString::new("c"), 2.0).unwrap(); positions } #[test] fn test_new() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let TestFile { file, path, dir: _dir, } = TestFile::new(&[0u8; 8]); let path_str = path.display().to_string(); let rpath = RString::new(&path_str); // Object created successfully let result: std::result::Result, Error> = eval!("FastMmapedFileRs.new(path)", path = rpath); assert!(result.is_ok()); // Weak map added let obj = result.unwrap(); let weak_tracker: Value = obj.ivar_get("@weak_obj_tracker").unwrap(); assert_eq!("ObjectSpace::WeakMap", weak_tracker.class().inspect()); // File expanded to page size let page_size = sysconf(SysconfVar::PAGE_SIZE).unwrap().unwrap() as u64; let stat = file.metadata().unwrap(); assert_eq!(page_size, stat.len()); // Used set to header size assert_eq!( HEADER_SIZE as u64, obj.load_used().unwrap().to_u64().unwrap() ); } #[test] fn test_slice() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let obj = create_obj(); let _ = populate_entries(&obj); // Validate header updated with new length let header_range = Range::new(0, HEADER_SIZE, true).unwrap().as_value(); let header_slice = MmapedFile::slice(obj, &[header_range]).unwrap(); assert_eq!([56, 0, 0, 0, 0, 0, 0, 0], unsafe { header_slice.as_slice() }); let value_range = Range::new(HEADER_SIZE, 24, true).unwrap().as_value(); let value_slice = MmapedFile::slice(obj, &[value_range]).unwrap(); // Validate string length assert_eq!(1u32.to_ne_bytes(), unsafe { &value_slice.as_slice()[0..4] }); // Validate string and padding assert_eq!("a ", unsafe { String::from_utf8_lossy(&value_slice.as_slice()[4..8]) }); // Validate value assert_eq!(0.0f64.to_ne_bytes(), unsafe { &value_slice.as_slice()[8..16] }); } #[test] fn test_slice_resize() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); fn assert_internals( obj: Obj, parent_id: c_ulong, child_id: c_ulong, unshared_id: c_ulong, ) { let rs_self = &*obj; let tracker: Value = obj.ivar_get("@weak_obj_tracker").unwrap(); let mmap_ptr = rs_self.as_mut_ptr(); let mmap_len = rs_self.capacity(); let mut parent_checked = false; let mut child_checked = false; for val in tracker.enumeratorize("each_value", ()) { let rb_string = val.unwrap(); let str = RString::from_value(rb_string).unwrap(); unsafe { let raw_str = MmapedFile::rb_string_internal(str); if str.as_raw() == child_id { assert_eq!(parent_id, raw_str.as_ref().as_.heap.aux.shared); let child_offset = mmap_len as isize - str.len() as isize; assert_eq!(mmap_ptr.offset(child_offset), raw_str.as_ref().as_.heap.ptr); child_checked = true; } else if str.as_raw() == parent_id { assert_eq!(parent_id, str.as_raw()); assert_eq!(mmap_ptr, raw_str.as_ref().as_.heap.ptr); assert_eq!(mmap_len as c_long, str.len() as c_long); assert!(raw_str.as_ref().basic.flags & (STR_SHARED | STR_NOEMBED) > 0); assert!(str.is_frozen()); parent_checked = true; } else if str.as_raw() == unshared_id { panic!("tracking unshared string"); } else { panic!("unknown string"); } } } assert!(parent_checked && child_checked); } let obj = create_obj(); let _ = populate_entries(&obj); let rs_self = &*obj; // Create a string containing the full mmap. let parent_str = rs_self.str(obj).unwrap(); let parent_id = parent_str.as_raw(); // Ruby's shared strings are only created when they go to the end of // original string. let len = rs_self.inner(|inner| Ok(inner.len())).unwrap(); let shareable_range = Range::new(1, len - 1, false).unwrap().as_value(); // This string should re-use the parent's buffer with an offset and have // the parent's id in `as.heap.aux.shared` let child_str = rs_self._slice(obj, parent_str, &[shareable_range]).unwrap(); let child_id = child_str.as_raw(); // A range that does not reach the end of the parent will not be shared. assert!(len > 4); let unshareable_range = Range::new(0, 4, false).unwrap().as_value(); // This string should NOT be tracked, it should own its own buffer. let unshared_str = rs_self ._slice(obj, parent_str, &[unshareable_range]) .unwrap(); let unshared_id = unshared_str.as_raw(); assert!(!MmapedFile::rb_string_is_shared(unshared_str)); assert_internals(obj, parent_id, child_id, unshared_id); let orig_ptr = rs_self.as_mut_ptr(); // Expand a bunch to ensure we remap for _ in 0..16 { rs_self.expand_to_fit(obj, rs_self.capacity() * 2).unwrap(); } let new_ptr = rs_self.as_mut_ptr(); assert!(orig_ptr != new_ptr); // If we haven't updated the pointer to the newly remapped file this will segfault. let _: Value = eval!("puts parent", parent = parent_str).unwrap(); let _: Value = eval!("puts child", child = child_str).unwrap(); let _: Value = eval!("puts unshared", unshared = unshared_str).unwrap(); // Confirm that tracked strings are still valid. assert_internals(obj, parent_id, child_id, unshared_id); } #[test] fn test_dont_fill_mmap() { let _cleanup = unsafe { magnus::embed::init() }; let ruby = magnus::Ruby::get().unwrap(); crate::init(&ruby).unwrap(); let obj = create_obj(); let positions = populate_entries(&obj); let rs_self = &*obj; rs_self.expand_to_fit(obj, 1024).unwrap(); let current_used = rs_self.inner(|inner| inner.load_used()).unwrap() as usize; let current_cap = rs_self.inner(|inner| Ok(inner.len())).unwrap(); // Create a new entry that exactly fills the capacity of the mmap. let val_len = current_cap - current_used - HEADER_SIZE - size_of::() - size_of::(); assert_eq!( current_cap, RawEntry::calc_total_len(val_len).unwrap() + current_used ); let str = String::from_utf8(vec![b'A'; val_len]).unwrap(); MmapedFile::upsert_entry(obj, positions, RString::new(&str), 1.0).unwrap(); // Validate that we have expanded the mmap, ensuring a trailing NUL. assert!(rs_self.capacity() > current_cap); } } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/mmap/000077500000000000000000000000001475532144100244345ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/mmap/inner.rs000066400000000000000000000546521475532144100261310ustar00rootroot00000000000000use libc::off_t; use memmap2::{MmapMut, MmapOptions}; use nix::libc::c_long; use std::fs::File; use std::mem::size_of; use std::ops::Range; use std::os::unix::prelude::{AsRawFd, RawFd}; use std::path::PathBuf; use crate::error::{MmapError, RubyError}; use crate::raw_entry::RawEntry; use crate::util::CheckedOps; use crate::util::{self, errno, read_f64, read_u32}; use crate::Result; use crate::HEADER_SIZE; /// A mmapped file and its metadata. Ruby never directly interfaces /// with this struct. #[derive(Debug)] pub(super) struct InnerMmap { /// The handle of the file being mmapped. When resizing the /// file we must drop the `InnerMmap` while keeping this open, /// truncate/extend the file, and establish a new `InnerMmap` to /// re-map it. file: File, /// The path of the file. path: PathBuf, /// The mmap itself. When initializing a new entry the length of /// the mmap is used for bounds checking. map: MmapMut, /// The length of data written to the file, used to validate /// whether a `load/save_value` call is in bounds and the length /// we truncate the file to when unmapping. /// /// Equivalent to `i_mm->t->real` in the C implementation. len: usize, } impl InnerMmap { /// Constructs a new `InnerMmap`, mmapping `path`. /// Use when mmapping a file for the first time. When re-mapping a file /// after expanding it the `reestablish` function should be used. pub fn new(path: PathBuf, file: File) -> Result { let stat = file.metadata().map_err(|e| { MmapError::legacy( format!("Can't stat {}: {e}", path.display()), RubyError::Arg, ) })?; let file_size = util::cast_chk::<_, usize>(stat.len(), "file length")?; // We need to ensure the underlying file descriptor is at least a page size. // Otherwise, we could get a SIGBUS error if mmap() attempts to read or write // past the file. let reserve_size = Self::next_page_boundary(file_size)?; // Cast: no-op. Self::reserve_mmap_file_bytes(file.as_raw_fd(), reserve_size as off_t).map_err(|e| { MmapError::legacy( format!( "Can't reserve {reserve_size} bytes for memory-mapped file in {}: {e}", path.display() ), RubyError::Io, ) })?; // Ensure we always have space for the header. let map_len = file_size.max(HEADER_SIZE); // SAFETY: There is the possibility of UB if the file is modified outside of // this program. let map = unsafe { MmapOptions::new().len(map_len).map_mut(&file) }.map_err(|e| { MmapError::legacy(format!("mmap failed ({}): {e}", errno()), RubyError::Arg) })?; let len = file_size; Ok(Self { file, path, map, len, }) } /// Re-mmap a file that was previously mapped. pub fn reestablish(path: PathBuf, file: File, map_len: usize) -> Result { // SAFETY: There is the possibility of UB if the file is modified outside of // this program. let map = unsafe { MmapOptions::new().len(map_len).map_mut(&file) }.map_err(|e| { MmapError::legacy(format!("mmap failed ({}): {e}", errno()), RubyError::Arg) })?; // TODO should we keep this as the old len? We'd want to be able to truncate // to the old length at this point if closing the file. Matching C implementation // for now. let len = map_len; Ok(Self { file, path, map, len, }) } /// Add a new metrics entry to the end of the mmap. This will fail if the mmap is at /// capacity. Callers must expand the file first. /// /// SAFETY: Must not call any Ruby code for the lifetime of `key`, otherwise we risk /// Ruby mutating the underlying `RString`. pub unsafe fn initialize_entry(&mut self, key: &[u8], value: f64) -> Result { // CAST: no-op on 32-bit, widening on 64-bit. let current_used = self.load_used()? as usize; let entry_length = RawEntry::calc_total_len(key.len())?; let new_used = current_used.add_chk(entry_length)?; // Increasing capacity requires expanding the file and re-mmapping it, we can't // perform this from `InnerMmap`. if self.capacity() < new_used { return Err(MmapError::Other(format!( "mmap capacity {} less than {}", self.capacity(), new_used ))); } let bytes = self.map.as_mut(); let value_offset = RawEntry::save(&mut bytes[current_used..new_used], key, value)?; // Won't overflow as value_offset is less than new_used. let position = current_used + value_offset; let new_used32 = util::cast_chk::<_, u32>(new_used, "used")?; self.save_used(new_used32)?; Ok(position) } /// Save a metrics value to an existing entry in the mmap. pub fn save_value(&mut self, offset: usize, value: f64) -> Result<()> { if self.len.add_chk(size_of::())? <= offset { return Err(MmapError::out_of_bounds( offset + size_of::(), self.len, )); } if offset < HEADER_SIZE { return Err(MmapError::Other(format!( "writing to offset {offset} would overwrite file header" ))); } let value_bytes = value.to_ne_bytes(); let value_range = self.item_range(offset, value_bytes.len())?; let bytes = self.map.as_mut(); bytes[value_range].copy_from_slice(&value_bytes); Ok(()) } /// Load a metrics value from an entry in the mmap. pub fn load_value(&self, offset: usize) -> Result { if self.len.add_chk(size_of::())? <= offset { return Err(MmapError::out_of_bounds( offset + size_of::(), self.len, )); } read_f64(self.map.as_ref(), offset) } /// The length of data written to the file. /// With a new file this is only set when Ruby calls `slice` on /// `FastMmapedFileRs`, so even if data has been written to the /// mmap attempts to read will fail until a String is created. /// When an existing file is read we set this value immediately. /// /// Equivalent to `i_mm->t->real` in the C implementation. #[inline] pub fn len(&self) -> usize { self.len } /// The total length in bytes of the mmapped file. /// /// Equivalent to `i_mm->t->len` in the C implementation. #[inline] pub fn capacity(&self) -> usize { self.map.len() } /// Update the length of the mmap considered to be written. pub fn set_len(&mut self, len: usize) { self.len = len; } /// Returns a raw pointer to the mmap. pub fn as_ptr(&self) -> *const u8 { self.map.as_ptr() } /// Returns a mutable raw pointer to the mmap. /// For use in updating RString internals which requires a mutable pointer. pub fn as_mut_ptr(&self) -> *mut u8 { self.map.as_ptr().cast_mut() } /// Perform an msync(2) on the mmap, flushing all changes written /// to disk. The sync may optionally be performed asynchronously. pub fn flush(&mut self, f_async: bool) -> Result<()> { if f_async { self.map .flush_async() .map_err(|_| MmapError::legacy(format!("msync({})", errno()), RubyError::Arg)) } else { self.map .flush() .map_err(|_| MmapError::legacy(format!("msync({})", errno()), RubyError::Arg)) } } /// Load the `used` header containing the size of the metrics data written. pub fn load_used(&self) -> Result { match read_u32(self.map.as_ref(), 0) { // CAST: we know HEADER_SIZE fits in a u32. Ok(0) => Ok(HEADER_SIZE as u32), u => u, } } /// Update the `used` header to the value provided. /// value provided. pub fn save_used(&mut self, used: u32) -> Result<()> { let bytes = self.map.as_mut(); bytes[..size_of::()].copy_from_slice(&used.to_ne_bytes()); Ok(()) } /// Drop self, which performs an munmap(2) on the mmap, /// returning the open `File` and `PathBuf` so the /// caller can expand the file and re-mmap it. pub fn munmap(self) -> (File, PathBuf) { (self.file, self.path) } // From https://stackoverflow.com/a/22820221: The difference with // ftruncate(2) is that (on file systems supporting it, e.g. Ext4) // disk space is indeed reserved by posix_fallocate but ftruncate // extends the file by adding holes (and without reserving disk // space). #[cfg(target_os = "linux")] fn reserve_mmap_file_bytes(fd: RawFd, len: off_t) -> nix::Result<()> { nix::fcntl::posix_fallocate(fd, 0, len) } // We simplify the reference implementation since we generally // don't need to reserve more than a page size. #[cfg(not(target_os = "linux"))] fn reserve_mmap_file_bytes(fd: RawFd, len: off_t) -> nix::Result<()> { nix::unistd::ftruncate(fd, len) } fn item_range(&self, start: usize, len: usize) -> Result> { let offset_end = start.add_chk(len)?; if offset_end >= self.capacity() { return Err(MmapError::out_of_bounds(offset_end, self.capacity())); } Ok(start..offset_end) } fn next_page_boundary(len: usize) -> Result { use nix::unistd::{self, SysconfVar}; let len = c_long::try_from(len) .map_err(|_| MmapError::failed_cast::<_, c_long>(len, "file len"))?; let mut page_size = match unistd::sysconf(SysconfVar::PAGE_SIZE) { Ok(Some(p)) if p > 0 => p, Ok(Some(p)) => { return Err(MmapError::legacy( format!("Invalid page size {p}"), RubyError::Io, )) } Ok(None) => { return Err(MmapError::legacy( "No system page size found", RubyError::Io, )) } Err(_) => { return Err(MmapError::legacy( "Failed to get system page size: {e}", RubyError::Io, )) } }; while page_size < len { page_size = page_size.mul_chk(2)?; } Ok(page_size) } } #[cfg(test)] mod test { use nix::unistd::{self, SysconfVar}; use super::*; use crate::testhelper::{self, TestEntry, TestFile}; use crate::HEADER_SIZE; #[test] fn test_new() { struct TestCase { name: &'static str, existing: bool, expected_len: usize, } let page_size = unistd::sysconf(SysconfVar::PAGE_SIZE).unwrap().unwrap(); let json = r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#; let value = 1.0; let entry_len = TestEntry::new(json, value).as_bytes().len(); let tc = vec![ TestCase { name: "empty file", existing: false, expected_len: 0, }, TestCase { name: "existing file", existing: true, expected_len: HEADER_SIZE + entry_len, }, ]; for case in tc { let name = case.name; let data = match case.existing { true => testhelper::entries_to_db(&[json], &[1.0], None), false => Vec::new(), }; let TestFile { file: original_file, path, dir: _dir, } = TestFile::new(&data); let original_stat = original_file.metadata().unwrap(); let inner = InnerMmap::new(path.clone(), original_file).unwrap(); let updated_file = File::open(&path).unwrap(); let updated_stat = updated_file.metadata().unwrap(); assert!( updated_stat.len() > original_stat.len(), "test case: {name} - file has been extended" ); assert_eq!( updated_stat.len(), page_size as u64, "test case: {name} - file extended to page size" ); assert_eq!( inner.capacity() as u64, original_stat.len().max(HEADER_SIZE as u64), "test case: {name} - mmap capacity matches original file len, unless smaller than HEADER_SIZE" ); assert_eq!( case.expected_len, inner.len(), "test case: {name} - len set" ); } } #[test] fn test_reestablish() { struct TestCase { name: &'static str, target_len: usize, expected_len: usize, } let json = r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#; let tc = vec![TestCase { name: "ok", target_len: 4096, expected_len: 4096, }]; for case in tc { let name = case.name; let data = testhelper::entries_to_db(&[json], &[1.0], None); let TestFile { file: original_file, path, dir: _dir, } = TestFile::new(&data); let inner = InnerMmap::reestablish(path.clone(), original_file, case.target_len).unwrap(); assert_eq!( case.target_len, inner.capacity(), "test case: {name} - mmap capacity set to target len", ); assert_eq!( case.expected_len, inner.len(), "test case: {name} - len set" ); } } #[test] fn test_initialize_entry() { struct TestCase { name: &'static str, empty: bool, used: Option, expected_used: Option, expected_value_offset: Option, expected_err: Option, } let json = r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#; let value = 1.0; let entry_len = TestEntry::new(json, value).as_bytes().len(); let tc = vec![ TestCase { name: "empty file, not expanded by outer mmap", empty: true, used: None, expected_used: None, expected_value_offset: None, expected_err: Some(MmapError::Other(format!( "mmap capacity {HEADER_SIZE} less than {}", entry_len + HEADER_SIZE, ))), }, TestCase { name: "data in file", empty: false, used: None, expected_used: Some(HEADER_SIZE as u32 + (entry_len * 2) as u32), expected_value_offset: Some(176), expected_err: None, }, TestCase { name: "data in file, invalid used larger than file", empty: false, used: Some(10_000), expected_used: None, expected_value_offset: None, expected_err: Some(MmapError::Other(format!( "mmap capacity 4096 less than {}", 10_000 + entry_len ))), }, ]; for case in tc { let name = case.name; let data = match case.empty { true => Vec::new(), false => testhelper::entries_to_db(&[json], &[1.0], case.used), }; let TestFile { file, path, dir: _dir, } = TestFile::new(&data); if !case.empty { // Ensure the file is large enough to have additional entries added. // Normally the outer mmap handles this. file.set_len(4096).unwrap(); } let mut inner = InnerMmap::new(path, file).unwrap(); let result = unsafe { inner.initialize_entry(json.as_bytes(), value) }; if let Some(expected_used) = case.expected_used { assert_eq!( expected_used, inner.load_used().unwrap(), "test case: {name} - used" ); } if let Some(expected_value_offset) = case.expected_value_offset { assert_eq!( expected_value_offset, *result.as_ref().unwrap(), "test case: {name} - value_offset" ); } if let Some(expected_err) = case.expected_err { assert_eq!( expected_err, result.unwrap_err(), "test case: {name} - error" ); } } } #[test] fn test_save_value() { let json = r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#; let value = 1.0; let upper_bound = TestEntry::new(json, value).as_bytes().len() + HEADER_SIZE; let value_offset = upper_bound - size_of::(); struct TestCase { name: &'static str, empty: bool, len: Option, offset: usize, expected_err: Option, } let tc = vec![ TestCase { name: "existing file, in bounds", empty: false, len: None, offset: upper_bound - size_of::() - 1, expected_err: None, }, TestCase { name: "existing file, out of bounds", empty: false, len: Some(100), offset: upper_bound * 2, expected_err: Some(MmapError::out_of_bounds( upper_bound * 2 + size_of::(), 100, )), }, TestCase { name: "existing file, off by one", empty: false, len: None, offset: value_offset + 1, expected_err: Some(MmapError::out_of_bounds( value_offset + 1 + size_of::(), upper_bound, )), }, TestCase { name: "empty file cannot be saved to", empty: true, len: None, offset: 8, expected_err: Some(MmapError::out_of_bounds(8 + size_of::(), 0)), }, TestCase { name: "overwrite header", empty: false, len: None, offset: 7, expected_err: Some(MmapError::Other( "writing to offset 7 would overwrite file header".to_string(), )), }, ]; for case in tc { let name = case.name; let mut data = match case.empty { true => Vec::new(), false => testhelper::entries_to_db(&[json], &[1.0], None), }; if let Some(len) = case.len { // Pad input to desired length. data.append(&mut vec![0xff; len - upper_bound]); } let TestFile { file, path, dir: _dir, } = TestFile::new(&data); let mut inner = InnerMmap::new(path, file).unwrap(); let result = inner.save_value(case.offset, value); if let Some(expected_err) = case.expected_err { assert_eq!( expected_err, result.unwrap_err(), "test case: {name} - expected err" ); } else { assert!(result.is_ok(), "test case: {name} - success"); assert_eq!( value, util::read_f64(&inner.map, case.offset).unwrap(), "test case: {name} - value saved" ); } } } #[test] fn test_load_value() { let json = r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#; let value = 1.0; let total_len = TestEntry::new(json, value).as_bytes().len() + HEADER_SIZE; let value_offset = total_len - size_of::(); struct TestCase { name: &'static str, offset: usize, expected_err: Option, } let tc = vec![ TestCase { name: "in bounds", offset: value_offset, expected_err: None, }, TestCase { name: "out of bounds", offset: value_offset * 2, expected_err: Some(MmapError::out_of_bounds( value_offset * 2 + size_of::(), total_len, )), }, TestCase { name: "off by one", offset: value_offset + 1, expected_err: Some(MmapError::out_of_bounds( value_offset + 1 + size_of::(), total_len, )), }, ]; for case in tc { let name = case.name; let data = testhelper::entries_to_db(&[json], &[1.0], None); let TestFile { file, path, dir: _dir, } = TestFile::new(&data); let inner = InnerMmap::new(path, file).unwrap(); let result = inner.load_value(case.offset); if let Some(expected_err) = case.expected_err { assert_eq!( expected_err, result.unwrap_err(), "test case: {name} - expected err" ); } else { assert!(result.is_ok(), "test case: {name} - success"); assert_eq!(value, result.unwrap(), "test case: {name} - value loaded"); } } } } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/raw_entry.rs000066400000000000000000000370771475532144100261000ustar00rootroot00000000000000use std::mem::size_of; use crate::error::MmapError; use crate::util; use crate::util::CheckedOps; use crate::Result; /// The logic to save a `MetricsEntry`, or parse one from a byte slice. #[derive(PartialEq, Eq, Clone, Debug)] pub struct RawEntry<'a> { bytes: &'a [u8], encoded_len: usize, } impl<'a> RawEntry<'a> { /// Save an entry to the mmap, returning the value offset in the newly created entry. pub fn save(bytes: &'a mut [u8], key: &[u8], value: f64) -> Result { let total_len = Self::calc_total_len(key.len())?; if total_len > bytes.len() { return Err(MmapError::Other(format!( "entry length {total_len} larger than slice length {}", bytes.len() ))); } // CAST: `calc_len` runs `check_encoded_len`, we know the key len // is less than i32::MAX. No risk of overflows or failed casts. let key_len: u32 = key.len() as u32; // Write the key length to the mmap. bytes[..size_of::()].copy_from_slice(&key_len.to_ne_bytes()); // Advance slice past the size. let bytes = &mut bytes[size_of::()..]; bytes[..key.len()].copy_from_slice(key); // Advance to end of key. let bytes = &mut bytes[key.len()..]; let pad_len = Self::padding_len(key.len()); bytes[..pad_len].fill(b' '); let bytes = &mut bytes[pad_len..]; bytes[..size_of::()].copy_from_slice(&value.to_ne_bytes()); Self::calc_value_offset(key.len()) } /// Parse a byte slice starting into an `MmapEntry`. pub fn from_slice(bytes: &'a [u8]) -> Result { // CAST: no-op on 32-bit, widening on 64-bit. let encoded_len = util::read_u32(bytes, 0)? as usize; let total_len = Self::calc_total_len(encoded_len)?; // Confirm the value is in bounds of the slice provided. if total_len > bytes.len() { return Err(MmapError::out_of_bounds(total_len, bytes.len())); } // Advance slice past length int and cut at end of entry. let bytes = &bytes[size_of::()..total_len]; Ok(Self { bytes, encoded_len }) } /// Read the `f64` value of an entry from memory. #[inline] pub fn value(&self) -> f64 { // We've stripped off the leading u32, don't include that here. let offset = self.encoded_len + Self::padding_len(self.encoded_len); // UNWRAP: We confirm in the constructor that the value offset // is in-range for the slice. util::read_f64(self.bytes, offset).unwrap() } /// The length of the entry key without padding. #[inline] pub fn encoded_len(&self) -> usize { self.encoded_len } /// Returns a slice with the JSON string in the entry, excluding padding. #[inline] pub fn json(&self) -> &[u8] { &self.bytes[..self.encoded_len] } /// Calculate the total length of an `MmapEntry`, including the string length, /// string, padding, and value. #[inline] pub fn total_len(&self) -> usize { // UNWRAP:: We confirmed in the constructor that this doesn't overflow. Self::calc_total_len(self.encoded_len).unwrap() } /// Calculate the total length of an `MmapEntry`, including the string length, /// string, padding, and value. Validates encoding_len is within expected bounds. #[inline] pub fn calc_total_len(encoded_len: usize) -> Result { Self::calc_value_offset(encoded_len)?.add_chk(size_of::()) } /// Calculate the value offset of an `MmapEntry`, including the string length, /// string, padding. Validates encoding_len is within expected bounds. #[inline] pub fn calc_value_offset(encoded_len: usize) -> Result { Self::check_encoded_len(encoded_len)?; Ok(size_of::() + encoded_len + Self::padding_len(encoded_len)) } /// Calculate the number of padding bytes to add to the value key to reach /// 8-byte alignment. Does not validate key length. #[inline] pub fn padding_len(encoded_len: usize) -> usize { 8 - (size_of::() + encoded_len) % 8 } #[inline] fn check_encoded_len(encoded_len: usize) -> Result<()> { if encoded_len as u64 > i32::MAX as u64 { return Err(MmapError::KeyLength); } Ok(()) } } #[cfg(test)] mod test { use bstr::ByteSlice; use super::*; use crate::testhelper::TestEntry; #[test] fn test_from_slice() { #[derive(PartialEq, Default, Debug)] struct TestCase { name: &'static str, input: TestEntry, expected_enc_len: Option, expected_err: Option, } let tc = vec![ TestCase { name: "ok", input: TestEntry { header: 61, json: r#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, padding_len: 7, value: 1.0, }, expected_enc_len: Some(61), ..Default::default() }, TestCase { name: "zero length key", input: TestEntry { header: 0, json: "", padding_len: 4, value: 1.0, }, expected_enc_len: Some(0), ..Default::default() }, TestCase { name: "header value too large", input: TestEntry { header: i32::MAX as u32 + 1, json: "foo", padding_len: 1, value: 0.0, }, expected_err: Some(MmapError::KeyLength), ..Default::default() }, TestCase { name: "header value much longer than json len", input: TestEntry { header: 256, json: "foobar", padding_len: 6, value: 1.0, }, expected_err: Some(MmapError::out_of_bounds(272, 24)), ..Default::default() }, TestCase { // Situations where encoded_len is wrong but padding makes the // value offset the same are not caught. name: "header off by one", input: TestEntry { header: 4, json: "123", padding_len: 1, value: 1.0, }, expected_err: Some(MmapError::out_of_bounds(24, 16)), ..Default::default() }, ]; for case in tc { let name = case.name; let input = case.input.as_bstring(); let resp = RawEntry::from_slice(&input); if case.expected_err.is_none() { let expected_buf = case.input.as_bytes_no_header(); let resp = resp.as_ref().unwrap(); let bytes = resp.bytes; assert_eq!(expected_buf, bytes.as_bstr(), "test case: {name} - bytes",); assert_eq!( resp.json(), case.input.json.as_bytes(), "test case: {name} - json matches" ); assert_eq!( resp.total_len(), case.input.as_bstring().len(), "test case: {name} - total_len matches" ); assert_eq!( resp.encoded_len(), case.input.json.len(), "test case: {name} - encoded_len matches" ); assert!( resp.json().iter().all(|&c| c != b' '), "test case: {name} - no spaces in json" ); let padding_len = RawEntry::padding_len(case.input.json.len()); assert!( bytes[resp.encoded_len..resp.encoded_len + padding_len] .iter() .all(|&c| c == b' '), "test case: {name} - padding is spaces" ); assert_eq!( resp.value(), case.input.value, "test case: {name} - value is correct" ); } if let Some(expected_enc_len) = case.expected_enc_len { assert_eq!( expected_enc_len, resp.as_ref().unwrap().encoded_len, "test case: {name} - encoded len", ); } if let Some(expected_err) = case.expected_err { assert_eq!(expected_err, resp.unwrap_err(), "test case: {name} - error",); } } } #[test] fn test_save() { struct TestCase { name: &'static str, key: &'static [u8], value: f64, buf_len: usize, expected_entry: Option, expected_resp: Result, } // TODO No test case to validate keys with len > i32::MAX, adding a static that large crashes // the test binary. let tc = vec![ TestCase { name: "ok", key: br#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, value: 256.0, buf_len: 256, expected_entry: Some(TestEntry { header: 61, json: r#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, padding_len: 7, value: 256.0, }), expected_resp: Ok(72), }, TestCase { name: "zero length key", key: b"", value: 1.0, buf_len: 256, expected_entry: Some(TestEntry { header: 0, json: "", padding_len: 4, value: 1.0, }), expected_resp: Ok(8), }, TestCase { name: "infinite value", key: br#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, value: f64::INFINITY, buf_len: 256, expected_entry: Some(TestEntry { header: 61, json: r#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, padding_len: 7, value: f64::INFINITY, }), expected_resp: Ok(72), }, TestCase { name: "buf len matches entry len", key: br#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, value: 1.0, buf_len: 80, expected_entry: Some(TestEntry { header: 61, json: r#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, padding_len: 7, value: 1.0, }), expected_resp: Ok(72), }, TestCase { name: "buf much too short", key: br#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, value: 1.0, buf_len: 5, expected_entry: None, expected_resp: Err(MmapError::Other(format!( "entry length {} larger than slice length {}", 80, 5, ))), }, TestCase { name: "buf short by one", key: br#"["metric","name",["label_a","label_b"],["value_a","value_b"]]"#, value: 1.0, buf_len: 79, expected_entry: None, expected_resp: Err(MmapError::Other(format!( "entry length {} larger than slice length {}", 80, 79, ))), }, ]; for case in tc { let mut buf = vec![0; case.buf_len]; let resp = RawEntry::save(&mut buf, case.key, case.value); assert_eq!( case.expected_resp, resp, "test case: {} - response", case.name, ); if let Some(e) = case.expected_entry { let expected_buf = e.as_bstring(); assert_eq!( expected_buf, buf[..expected_buf.len()].as_bstr(), "test case: {} - buffer state", case.name ); let header_len = u32::from_ne_bytes(buf[..size_of::()].try_into().unwrap()); assert_eq!( case.key.len(), header_len as usize, "test case: {} - size header", case.name, ); } } } #[test] fn test_calc_value_offset() { struct TestCase { name: &'static str, encoded_len: usize, expected_value_offset: Option, expected_total_len: Option, expected_err: Option, } let tc = vec![ TestCase { name: "ok", encoded_len: 8, expected_value_offset: Some(16), expected_total_len: Some(24), expected_err: None, }, TestCase { name: "padding length one", encoded_len: 3, expected_value_offset: Some(8), expected_total_len: Some(16), expected_err: None, }, TestCase { name: "padding length eight", encoded_len: 4, expected_value_offset: Some(16), expected_total_len: Some(24), expected_err: None, }, TestCase { name: "encoded len gt i32::MAX", encoded_len: i32::MAX as usize + 1, expected_value_offset: None, expected_total_len: None, expected_err: Some(MmapError::KeyLength), }, ]; for case in tc { let name = case.name; if let Some(expected_value_offset) = case.expected_value_offset { assert_eq!( expected_value_offset, RawEntry::calc_value_offset(case.encoded_len).unwrap(), "test case: {name} - value offset" ); } if let Some(expected_total_len) = case.expected_total_len { assert_eq!( expected_total_len, RawEntry::calc_total_len(case.encoded_len).unwrap(), "test case: {name} - total len" ); } if let Some(expected_err) = case.expected_err { assert_eq!( expected_err, RawEntry::calc_value_offset(case.encoded_len).unwrap_err(), "test case: {name} - err" ); } } } #[test] fn test_padding_len() { for encoded_len in 0..64 { let padding = RawEntry::padding_len(encoded_len); // Validate we're actually aligning to 8 bytes. assert!((size_of::() + encoded_len + padding) % 8 == 0) } } } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/testhelper.rs000066400000000000000000000132661475532144100262370ustar00rootroot00000000000000use bstr::{BString, B}; use std::fs::File; use std::io::{Read, Seek, Write}; use std::path::PathBuf; use tempfile::{tempdir, TempDir}; use crate::raw_entry::RawEntry; use crate::HEADER_SIZE; #[derive(PartialEq, Default, Debug)] pub struct TestEntry { pub header: u32, pub json: &'static str, pub padding_len: usize, pub value: f64, } impl TestEntry { pub fn new(json: &'static str, value: f64) -> Self { TestEntry { header: json.len() as u32, json, padding_len: RawEntry::padding_len(json.len()), value, } } pub fn as_bytes(&self) -> Vec { [ B(&self.header.to_ne_bytes()), self.json.as_bytes(), &vec![b' '; self.padding_len], B(&self.value.to_ne_bytes()), ] .concat() } pub fn as_bstring(&self) -> BString { [ B(&self.header.to_ne_bytes()), self.json.as_bytes(), &vec![b' '; self.padding_len], B(&self.value.to_ne_bytes()), ] .concat() .into() } pub fn as_bytes_no_header(&self) -> BString { [ self.json.as_bytes(), &vec![b' '; self.padding_len], B(&self.value.to_ne_bytes()), ] .concat() .into() } } /// Format the data for a `.db` file. /// Optional header value can be used to set an invalid `used` size. pub fn entries_to_db(entries: &[&'static str], values: &[f64], header: Option) -> Vec { let mut out = Vec::new(); let entry_bytes: Vec<_> = entries .iter() .zip(values) .flat_map(|(e, val)| TestEntry::new(e, *val).as_bytes()) .collect(); let used = match header { Some(u) => u, None => (entry_bytes.len() + HEADER_SIZE) as u32, }; out.extend(used.to_ne_bytes()); out.extend([0x0u8; 4]); // Padding. out.extend(entry_bytes); out } /// A temporary file, path, and dir for use with testing. #[derive(Debug)] pub struct TestFile { pub file: File, pub path: PathBuf, pub dir: TempDir, } impl TestFile { pub fn new(file_data: &[u8]) -> TestFile { let dir = tempdir().unwrap(); let path = dir.path().join("test.db"); let mut file = File::options() .create(true) .read(true) .write(true) .open(&path) .unwrap(); file.write_all(file_data).unwrap(); file.sync_all().unwrap(); file.rewind().unwrap(); // We need to keep `dir` in scope so it doesn't drop before the files it // contains, which may prevent cleanup. TestFile { file, path, dir } } } mod test { use super::*; #[test] fn test_entry_new() { let json = "foobar"; let value = 1.0f64; let expected = TestEntry { header: 6, json, padding_len: 6, value, }; let actual = TestEntry::new(json, value); assert_eq!(expected, actual); } #[test] fn test_entry_bytes() { let json = "foobar"; let value = 1.0f64; let expected = [ &6u32.to_ne_bytes(), B(json), &[b' '; 6], &value.to_ne_bytes(), ] .concat(); let actual = TestEntry::new(json, value).as_bstring(); assert_eq!(expected, actual); } #[test] fn test_entry_bytes_no_header() { let json = "foobar"; let value = 1.0f64; let expected = [B(json), &[b' '; 6], &value.to_ne_bytes()].concat(); let actual = TestEntry::new(json, value).as_bytes_no_header(); assert_eq!(expected, actual); } #[test] fn test_entries_to_db_header_correct() { let json = &["foobar", "qux"]; let values = &[1.0, 2.0]; let out = entries_to_db(json, values, None); assert_eq!(48u32.to_ne_bytes(), out[0..4], "used set correctly"); assert_eq!([0u8; 4], out[4..8], "padding set"); assert_eq!( TestEntry::new(json[0], values[0]).as_bytes(), out[8..32], "first entry matches" ); assert_eq!( TestEntry::new(json[1], values[1]).as_bytes(), out[32..48], "second entry matches" ); } #[test] fn test_entries_to_db_header_wrong() { let json = &["foobar", "qux"]; let values = &[1.0, 2.0]; const WRONG_USED: u32 = 1000; let out = entries_to_db(json, values, Some(WRONG_USED)); assert_eq!( WRONG_USED.to_ne_bytes(), out[0..4], "used set to value requested" ); assert_eq!([0u8; 4], out[4..8], "padding set"); assert_eq!( TestEntry::new(json[0], values[0]).as_bytes(), out[8..32], "first entry matches" ); assert_eq!( TestEntry::new(json[1], values[1]).as_bytes(), out[32..48], "second entry matches" ); } #[test] fn test_file() { let mut test_file = TestFile::new(b"foobar"); let stat = test_file.file.metadata().unwrap(); assert_eq!(6, stat.len(), "file length"); assert_eq!( 0, test_file.file.stream_position().unwrap(), "at start of file" ); let mut out_buf = vec![0u8; 256]; let read_result = test_file.file.read(&mut out_buf); assert!(read_result.is_ok()); assert_eq!(6, read_result.unwrap(), "file is readable"); let write_result = test_file.file.write(b"qux"); assert!(write_result.is_ok()); assert_eq!(3, write_result.unwrap(), "file is writable"); } } prometheus-client-mmap-1.2.9/ext/fast_mmaped_file_rs/src/util.rs000066400000000000000000000067771475532144100250460ustar00rootroot00000000000000use nix::errno::Errno; use nix::libc::c_long; use std::fmt::Display; use std::io; use std::mem::size_of; use crate::error::MmapError; use crate::Result; /// Wrapper around `checked_add()` that converts failures /// to `MmapError::Overflow`. pub trait CheckedOps: Sized { fn add_chk(self, rhs: Self) -> Result; fn mul_chk(self, rhs: Self) -> Result; } impl CheckedOps for usize { fn add_chk(self, rhs: Self) -> Result { self.checked_add(rhs) .ok_or_else(|| MmapError::overflowed(self, rhs, "adding")) } fn mul_chk(self, rhs: Self) -> Result { self.checked_mul(rhs) .ok_or_else(|| MmapError::overflowed(self, rhs, "multiplying")) } } impl CheckedOps for c_long { fn add_chk(self, rhs: Self) -> Result { self.checked_add(rhs) .ok_or_else(|| MmapError::overflowed(self, rhs, "adding")) } fn mul_chk(self, rhs: Self) -> Result { self.checked_mul(rhs) .ok_or_else(|| MmapError::overflowed(self, rhs, "multiplying")) } } /// A wrapper around `TryFrom`, returning `MmapError::FailedCast` on error. pub fn cast_chk(val: T, name: &str) -> Result where T: Copy + Display, U: std::convert::TryFrom, { U::try_from(val).map_err(|_| MmapError::failed_cast::(val, name)) } /// Retrieve errno(3). pub fn errno() -> i32 { // UNWRAP: This will always return `Some` when called from `last_os_error()`. io::Error::last_os_error().raw_os_error().unwrap() } /// Get the error string associated with errno(3). /// Equivalent to strerror(3). pub fn strerror(errno: i32) -> &'static str { Errno::from_i32(errno).desc() } /// Read a `u32` value from a byte slice starting from `offset`. #[inline] pub fn read_u32(buf: &[u8], offset: usize) -> Result { if let Some(slice) = buf.get(offset..offset + size_of::()) { // UNWRAP: We can safely unwrap the conversion from slice to array as we // the source and targets are constructed here with the same length. let out: &[u8; size_of::()] = slice.try_into().unwrap(); return Ok(u32::from_ne_bytes(*out)); } Err(MmapError::out_of_bounds(offset, buf.len())) } /// Read an `f64` value from a byte slice starting from `offset`. #[inline] pub fn read_f64(buf: &[u8], offset: usize) -> Result { if let Some(slice) = buf.get(offset..offset + size_of::()) { // UNWRAP: We can safely unwrap the conversion from slice to array as we // can be sure the target array has same length as the source slice. let out: &[u8; size_of::()] = slice.try_into().unwrap(); return Ok(f64::from_ne_bytes(*out)); } Err(MmapError::out_of_bounds( offset + size_of::(), buf.len(), )) } #[cfg(test)] mod test { use super::*; #[test] fn test_read_u32() { let buf = 1u32.to_ne_bytes(); assert!(matches!(read_u32(&buf, 0), Ok(1)), "index ok"); assert!(read_u32(&buf, 10).is_err(), "index out of range"); assert!( read_u32(&buf, 1).is_err(), "index in range but end out of range" ); } #[test] fn test_read_f64() { let buf = 1.00f64.to_ne_bytes(); let ok = read_f64(&buf, 0); assert!(ok.is_ok()); assert_eq!(ok.unwrap(), 1.00); assert!(read_f64(&buf, 10).is_err(), "index out of range"); assert!( read_f64(&buf, 1).is_err(), "index in range but end out of range" ); } } prometheus-client-mmap-1.2.9/fuzz/000077500000000000000000000000001475532144100171265ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/fuzz/prometheus/000077500000000000000000000000001475532144100213215ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/fuzz/prometheus/client/000077500000000000000000000000001475532144100225775ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/fuzz/prometheus/client/helpers/000077500000000000000000000000001475532144100242415ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/fuzz/prometheus/client/helpers/fuzz_mmaped_file.rb000066400000000000000000000040711475532144100301100ustar00rootroot00000000000000require 'fuzzbert' require 'tempfile' require 'prometheus/client' require 'prometheus/client/helper/mmaped_file' require 'fast_mmaped_file_rs' module MmapedFileHelper def self.assert_equals(a, b) raise "#{a} not equal #{b}" unless a == b || a.is_a?(Float) && a.nan? && b.is_a?(Float) && b.nan? end def self.process(filepath) f = Prometheus::Client::Helper::MmapedFile.open(filepath) metrics = {} f.to_metrics(metrics, true) positions = {} f.entries(true).each do |data, encoded_len, value_offset, pos| encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset)) positions[encoded] = pos assert_equals(f.fetch_entry(positions, encoded, value - 1), value) end f.upsert_entry(positions, 'key', 0.1) assert_equals(f.fetch_entry(positions, 'key', 0), 0.1) f.upsert_entry(positions, 'key2', 0.2) assert_equals(f.fetch_entry(positions, 'key2', 0), 0.2) rescue Prometheus::Client::Helper::EntryParser::ParsingError rescue PrometheusParsingError ensure # fuzzbert wraps the process in a trap context. Ruby doesn't allow # mutexes to be synchronized in that context, so we need to spawn another # thread to close the file. Thread.new { f.close }.join(0.5) end end class PrintAndExitHandler def handle(error_data) puts error_data[:id] p error_data[:data] puts error_data[:pid] puts error_data[:status] Process.exit(1) end end fuzz 'MmapedFile' do deploy do |data| tmpfile = Tempfile.new('mmmaped_file') tmpfile.write(data) tmpfile.close MmapedFileHelper.process(tmpfile.path) tmpfile.unlink end data 'completely random' do FuzzBert::Generators.random end data 'should have 10000 bytes used and first entry' do c = FuzzBert::Container.new c << FuzzBert::Generators.fixed([10000, 0, 11, '[1,1,[],[]] ', 1].pack('LLLA12d')) c << FuzzBert::Generators.random(2) c << FuzzBert::Generators.fixed([0, 0].pack('CC')) c << FuzzBert::Generators.fixed('[1,1,[],[]]') c << FuzzBert::Generators.random c.generator end end prometheus-client-mmap-1.2.9/known-leaks-suppression.txt000066400000000000000000000006211475532144100235110ustar00rootroot00000000000000leak:compile_quantifier_node leak:compile_tree leak:heap_page_allocate leak:objspace_xmalloc0 leak:objspace_xrealloc leak:onig_new leak:onig_compile leak:onig_compile_ruby leak:onig_bbuf_init leak:onig_region_resize leak:onig_st_insert_strend leak:parse_exp leak:ruby_init_setproctitle leak:rb_reg_initialize leak:rb_threadptr_root_fiber_setup leak:ruby_xcalloc leak:/lib/x86_64-linux-gnu/libruby-2.3 prometheus-client-mmap-1.2.9/lib/000077500000000000000000000000001475532144100166765ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/lib/prometheus.rb000066400000000000000000000001371475532144100214170ustar00rootroot00000000000000# Prometheus is a generic time-series collection and computation server. module Prometheus end prometheus-client-mmap-1.2.9/lib/prometheus/000077500000000000000000000000001475532144100210715ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/lib/prometheus/client.rb000066400000000000000000000033251475532144100226770ustar00rootroot00000000000000require 'prometheus/client/registry' require 'prometheus/client/configuration' require 'prometheus/client/mmaped_value' module Prometheus # Client is a ruby implementation for a Prometheus compatible client. module Client class << self attr_writer :configuration def configuration @configuration ||= Configuration.new end def configure yield(configuration) end # Returns a default registry object def registry @registry ||= Registry.new end def logger configuration.logger end def pid configuration.pid_provider.call end # Resets the registry and reinitializes all metrics files. # Use case: clean up everything in specs `before` block, # to prevent leaking the state between specs which are updating metrics. def reset! @registry = nil ::Prometheus::Client::MmapedValue.reset_and_reinitialize end def cleanup! Dir.glob("#{configuration.multiprocess_files_dir}/*.db").each { |f| File.unlink(f) if File.exist?(f) } end # With `force: false`: reinitializes metric files only for processes with the changed PID. # With `force: true`: reinitializes all metrics files. # Always keeps the registry. # Use case (`force: false`): pick up new metric files on each worker start, # without resetting already registered files for the master or previously initialized workers. def reinitialize_on_pid_change(force: false) if force ::Prometheus::Client::MmapedValue.reset_and_reinitialize else ::Prometheus::Client::MmapedValue.reinitialize_on_pid_change end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/000077500000000000000000000000001475532144100223475ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/lib/prometheus/client/configuration.rb000066400000000000000000000013401475532144100255410ustar00rootroot00000000000000require 'prometheus/client/registry' require 'prometheus/client/mmaped_value' require 'prometheus/client/page_size' require 'logger' require 'tmpdir' module Prometheus module Client class Configuration attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider def initialize @value_class = ::Prometheus::Client::MmapedValue @initial_mmap_file_size = ::Prometheus::Client::PageSize.page_size(fallback_page_size: 4096) @logger = Logger.new($stdout) @pid_provider = Process.method(:pid) @multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do Dir.mktmpdir("prometheus-mmap") end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/counter.rb000066400000000000000000000011061475532144100243510ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client/metric' module Prometheus module Client # Counter is a metric that exposes merely a sum or tally of things. class Counter < Metric def type :counter end def increment(labels = {}, by = 1) raise ArgumentError, 'increment must be a non-negative number' if by < 0 label_set = label_set_for(labels) synchronize { @values[label_set].increment(by) } end private def default(labels) value_object(type, @name, @name, labels) end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/formats/000077500000000000000000000000001475532144100240225ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/lib/prometheus/client/formats/text.rb000066400000000000000000000052321475532144100253350ustar00rootroot00000000000000require 'prometheus/client/uses_value_type' require 'prometheus/client/helper/json_parser' require 'prometheus/client/helper/plain_file' require 'prometheus/client/helper/metrics_processing' require 'prometheus/client/helper/metrics_representation' module Prometheus module Client module Formats # Text format is human readable mainly used for manual inspection. module Text MEDIA_TYPE = 'text/plain'.freeze VERSION = '0.0.4'.freeze CONTENT_TYPE = "#{MEDIA_TYPE}; version=#{VERSION}".freeze class << self def marshal(registry) metrics = registry.metrics.map do |metric| samples = metric.values.flat_map do |label_set, value| representation(metric, label_set, value) end [metric.name, { type: metric.type, help: metric.docstring, samples: samples }] end Helper::MetricsRepresentation.to_text(metrics) end def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir) file_list = Dir.glob(File.join(path, '*.db')).sort .map {|f| Helper::PlainFile.new(f) } .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] } FastMmapedFileRs.to_metrics(file_list.to_a) end private def load_metrics(path) metrics = {} Dir.glob(File.join(path, '*.db')).sort.each do |f| Helper::PlainFile.new(f).to_metrics(metrics) end metrics end def representation(metric, label_set, value) labels = metric.base_labels.merge(label_set) if metric.type == :summary summary(metric.name, labels, value) elsif metric.type == :histogram histogram(metric.name, labels, value) else [[metric.name, labels, value.get]] end end def summary(name, set, value) rv = value.get.map do |q, v| [name, set.merge(quantile: q), v] end rv << ["#{name}_sum", set, value.get.sum] rv << ["#{name}_count", set, value.get.total] rv end def histogram(name, set, value) # |metric_name, labels, value| rv = value.get.map do |q, v| [name, set.merge(le: q), v] end rv << [name, set.merge(le: '+Inf'), value.get.total] rv << ["#{name}_sum", set, value.get.sum] rv << ["#{name}_count", set, value.get.total] rv end end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/gauge.rb000066400000000000000000000021171475532144100237650ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client/metric' module Prometheus module Client # A Gauge is a metric that exposes merely an instantaneous value or some # snapshot thereof. class Gauge < Metric def initialize(name, docstring, base_labels = {}, multiprocess_mode=:all) super(name, docstring, base_labels) if value_class.multiprocess and ![:min, :max, :livesum, :liveall, :all].include?(multiprocess_mode) raise ArgumentError, 'Invalid multiprocess mode: ' + multiprocess_mode end @multiprocess_mode = multiprocess_mode end def type :gauge end def default(labels) value_object(type, @name, @name, labels, @multiprocess_mode) end # Sets the value for the given label set def set(labels, value) @values[label_set_for(labels)].set(value) end def increment(labels, value) @values[label_set_for(labels)].increment(value) end def decrement(labels, value) @values[label_set_for(labels)].decrement(value) end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/helper/000077500000000000000000000000001475532144100236265ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/lib/prometheus/client/helper/entry_parser.rb000066400000000000000000000101361475532144100266710ustar00rootroot00000000000000require 'prometheus/client/helper/json_parser' module Prometheus module Client module Helper module EntryParser class ParsingError < RuntimeError; end MINIMUM_SIZE = 8 START_POSITION = 8 VALUE_BYTES = 8 ENCODED_LENGTH_BYTES = 4 def used slice(0..3).unpack('l')[0] end def parts @parts ||= File.basename(filepath, '.db') .split('_') .map { |e| e.gsub(/-\d+$/, '') } # remove trailing -number end def type parts[0].to_sym end def pid (parts[2..-1] || []).join('_') end def multiprocess_mode parts[1] end def empty? size < MINIMUM_SIZE || used.zero? end def entries(ignore_errors = false) return Enumerator.new {} if empty? Enumerator.new do |yielder| used_ = used # cache used to avoid unnecessary unpack operations pos = START_POSITION # used + padding offset while pos < used_ && pos < size && pos > 0 data = slice(pos..-1) unless data raise ParsingError, "data slice is nil at pos #{pos}" unless ignore_errors pos += 8 next end encoded_len, first_encoded_bytes = data.unpack('LL') if encoded_len.nil? || encoded_len.zero? || first_encoded_bytes.nil? || first_encoded_bytes.zero? # do not parse empty data pos += 8 next end entry_len = ENCODED_LENGTH_BYTES + encoded_len padding_len = 8 - entry_len % 8 value_offset = entry_len + padding_len # align to 8 bytes pos += value_offset if value_offset > 0 && (pos + VALUE_BYTES) <= size # if positions are safe yielder.yield data, encoded_len, value_offset, pos else raise ParsingError, "data slice is nil at pos #{pos}" unless ignore_errors end pos += VALUE_BYTES end end end def parsed_entries(ignore_errors = false) result = entries(ignore_errors).map do |data, encoded_len, value_offset, _| begin encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset)) [encoded, value] rescue ArgumentError => e Prometheus::Client.logger.debug("Error processing data: #{bin_to_hex(data[0, 7])} len: #{encoded_len} value_offset: #{value_offset}") raise ParsingError, e unless ignore_errors end end result.reject!(&:nil?) if ignore_errors result end def to_metrics(metrics = {}, ignore_errors = false) parsed_entries(ignore_errors).each do |key, value| begin metric_name, name, labelnames, labelvalues = JsonParser.load(key) labelnames ||= [] labelvalues ||= [] metric = metrics.fetch(metric_name, metric_name: metric_name, help: 'Multiprocess metric', type: type, samples: []) if type == :gauge metric[:multiprocess_mode] = multiprocess_mode metric[:samples] += [[name, labelnames.zip(labelvalues) + [['pid', pid]], value]] else # The duplicates and labels are fixed in the next for. metric[:samples] += [[name, labelnames.zip(labelvalues), value]] end metrics[metric_name] = metric rescue JSON::ParserError => e raise ParsingError(e) unless ignore_errors end end metrics.reject! { |e| e.nil? } if ignore_errors metrics end private def bin_to_hex(s) s.each_byte.map { |b| b.to_s(16) }.join end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/helper/file_locker.rb000066400000000000000000000023611475532144100264330ustar00rootroot00000000000000module Prometheus module Client module Helper class FileLocker class << self LOCK_FILE_MUTEX = Mutex.new def lock_to_process(filepath) LOCK_FILE_MUTEX.synchronize do @file_locks ||= {} return false if @file_locks[filepath] file = File.open(filepath, 'ab') if file.flock(File::LOCK_NB | File::LOCK_EX) @file_locks[filepath] = file return true else return false end end end def unlock(filepath) LOCK_FILE_MUTEX.synchronize do @file_locks ||= {} return false unless @file_locks[filepath] file = @file_locks[filepath] file.flock(File::LOCK_UN) file.close @file_locks.delete(filepath) end end def unlock_all LOCK_FILE_MUTEX.synchronize do @file_locks ||= {} @file_locks.values.each do |file| file.flock(File::LOCK_UN) file.close end @file_locks = {} end end end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/helper/json_parser.rb000066400000000000000000000007011475532144100264760ustar00rootroot00000000000000require 'json' module Prometheus module Client module Helper module JsonParser class << self if defined?(Oj) def load(s) Oj.load(s) rescue Oj::ParseError, EncodingError => e raise JSON::ParserError.new(e.message) end else def load(s) JSON.parse(s) end end end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/helper/metrics_processing.rb000066400000000000000000000030251475532144100300550ustar00rootroot00000000000000module Prometheus module Client module Helper module MetricsProcessing def self.merge_metrics(metrics) metrics.each_value do |metric| metric[:samples] = merge_samples(metric[:samples], metric[:type], metric[:multiprocess_mode]).map do |(name, labels), value| [name, labels.to_h, value] end end end def self.merge_samples(raw_samples, metric_type, multiprocess_mode) samples = {} raw_samples.each do |name, labels, value| without_pid = labels.reject { |l| l[0] == 'pid' } case metric_type when :gauge case multiprocess_mode when 'min' s = samples.fetch([name, without_pid], value) samples[[name, without_pid]] = [s, value].min when 'max' s = samples.fetch([name, without_pid], value) samples[[name, without_pid]] = [s, value].max when 'livesum' s = samples.fetch([name, without_pid], 0.0) samples[[name, without_pid]] = s + value else # all/liveall samples[[name, labels]] = value end else # Counter, Histogram and Summary. s = samples.fetch([name, without_pid], 0.0) samples[[name, without_pid]] = s + value end end samples end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/helper/metrics_representation.rb000066400000000000000000000026521475532144100307500ustar00rootroot00000000000000module Prometheus module Client module Helper module MetricsRepresentation METRIC_LINE = '%s%s %s'.freeze TYPE_LINE = '# TYPE %s %s'.freeze HELP_LINE = '# HELP %s %s'.freeze LABEL = '%s="%s"'.freeze SEPARATOR = ','.freeze DELIMITER = "\n".freeze REGEX = { doc: /[\n\\]/, label: /[\n\\"]/ }.freeze REPLACE = { "\n" => '\n', '\\' => '\\\\', '"' => '\"' }.freeze def self.to_text(metrics) lines = [] metrics.each do |name, metric| lines << format(HELP_LINE, name, escape(metric[:help])) lines << format(TYPE_LINE, name, metric[:type]) metric[:samples].each do |metric_name, labels, value| lines << metric(metric_name, format_labels(labels), value) end end # there must be a trailing delimiter (lines << nil).join(DELIMITER) end def self.metric(name, labels, value) format(METRIC_LINE, name, labels, value) end def self.format_labels(set) return if set.empty? strings = set.each_with_object([]) do |(key, value), memo| memo << format(LABEL, key, escape(value, :label)) end "{#{strings.join(SEPARATOR)}}" end def self.escape(string, format = :doc) string.to_s.gsub(REGEX[format], REPLACE) end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/helper/mmaped_file.rb000066400000000000000000000032351475532144100264200ustar00rootroot00000000000000require 'prometheus/client/helper/entry_parser' require 'prometheus/client/helper/file_locker' # load precompiled extension if available begin ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION) require_relative "../../../#{ruby_version}/fast_mmaped_file_rs" rescue LoadError require 'fast_mmaped_file_rs' end module Prometheus module Client module Helper class MmapedFile < FastMmapedFileRs include EntryParser attr_reader :filepath, :size def initialize(filepath) @filepath = filepath File.open(filepath, 'a+b') do |file| file.truncate(initial_mmap_file_size) if file.size < MINIMUM_SIZE @size = file.size end super(filepath) end def close munmap FileLocker.unlock(filepath) end private def initial_mmap_file_size Prometheus::Client.configuration.initial_mmap_file_size end public class << self def open(filepath) MmapedFile.new(filepath) end def ensure_exclusive_file(file_prefix = 'mmaped_file') (0..Float::INFINITY).lazy .map { |f_num| "#{file_prefix}_#{Prometheus::Client.pid}-#{f_num}.db" } .map { |filename| File.join(Prometheus::Client.configuration.multiprocess_files_dir, filename) } .find { |path| Helper::FileLocker.lock_to_process(path) } end def open_exclusive_file(file_prefix = 'mmaped_file') filename = Helper::MmapedFile.ensure_exclusive_file(file_prefix) open(filename) end end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/helper/plain_file.rb000066400000000000000000000010271475532144100262550ustar00rootroot00000000000000require 'prometheus/client/helper/entry_parser' module Prometheus module Client module Helper # Parses DB files without using mmap class PlainFile include EntryParser attr_reader :filepath def source @data ||= File.read(filepath, mode: 'rb') end def initialize(filepath) @filepath = filepath end def slice(*args) source.slice(*args) end def size source.length end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/histogram.rb000066400000000000000000000046601475532144100246770ustar00rootroot00000000000000require 'prometheus/client/metric' require 'prometheus/client/uses_value_type' module Prometheus module Client # A histogram samples observations (usually things like request durations # or response sizes) and counts them in configurable buckets. It also # provides a sum of all observed values. class Histogram < Metric # Value represents the state of a Histogram at a given point. class Value < Hash include UsesValueType attr_accessor :sum, :total, :total_inf def initialize(type, name, labels, buckets) @sum = value_object(type, name, "#{name}_sum", labels) @total = value_object(type, name, "#{name}_count", labels) @total_inf = value_object(type, name, "#{name}_bucket", labels.merge(le: "+Inf")) buckets.each do |bucket| self[bucket] = value_object(type, name, "#{name}_bucket", labels.merge(le: bucket.to_s)) end end def observe(value) @sum.increment(value) @total.increment() @total_inf.increment() each_key do |bucket| self[bucket].increment() if value <= bucket end end def get() hash = {} each_key do |bucket| hash[bucket] = self[bucket].get() end hash end end # DEFAULT_BUCKETS are the default Histogram buckets. The default buckets # are tailored to broadly measure the response time (in seconds) of a # network service. (From DefBuckets client_golang) DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10].freeze # Offer a way to manually specify buckets def initialize(name, docstring, base_labels = {}, buckets = DEFAULT_BUCKETS) raise ArgumentError, 'Unsorted buckets, typo?' unless sorted? buckets @buckets = buckets super(name, docstring, base_labels) end def type :histogram end def observe(labels, value) label_set = label_set_for(labels) synchronize { @values[label_set].observe(value) } end private def default(labels) # TODO: default function needs to know key of hash info (label names and values) Value.new(type, @name, labels, @buckets) end def sorted?(bucket) bucket.each_cons(2).all? { |i, j| i <= j } end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/label_set_validator.rb000066400000000000000000000045101475532144100266730ustar00rootroot00000000000000# encoding: UTF-8 module Prometheus module Client # LabelSetValidator ensures that all used label sets comply with the # Prometheus specification. class LabelSetValidator RESERVED_LABELS = [].freeze class LabelSetError < StandardError; end class InvalidLabelSetError < LabelSetError; end class InvalidLabelError < LabelSetError; end class ReservedLabelError < LabelSetError; end def initialize(reserved_labels = []) @reserved_labels = (reserved_labels + RESERVED_LABELS).freeze @validated = {} end def valid?(labels) unless labels.is_a?(Hash) raise InvalidLabelSetError, "#{labels} is not a valid label set" end labels.all? do |key, value| validate_symbol(key) validate_name(key) validate_reserved_key(key) validate_value(key, value) end end def validate(labels) return labels if @validated.key?(labels.hash) valid?(labels) unless @validated.empty? || match?(labels, @validated.first.last) raise InvalidLabelSetError, "labels must have the same signature: (#{label_diff(labels, @validated.first.last)})" end @validated[labels.hash] = labels end private def label_diff(a, b) "expected keys: #{b.keys.sort}, got: #{a.keys.sort}" end def match?(a, b) a.keys.sort == b.keys.sort end def validate_symbol(key) return true if key.is_a?(Symbol) raise InvalidLabelError, "label #{key} is not a symbol" end def validate_name(key) return true unless key.to_s.start_with?('__') raise ReservedLabelError, "label #{key} must not start with __" end def validate_reserved_key(key) return true unless @reserved_labels.include?(key) raise ReservedLabelError, "#{key} is reserved" end def validate_value(key, value) return true if value.is_a?(String) || value.is_a?(Numeric) || value.is_a?(Symbol) || value.is_a?(FalseClass) || value.is_a?(TrueClass) || value.nil? raise InvalidLabelError, "#{key} does not contain a valid value (type #{value.class})" end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/metric.rb000066400000000000000000000037651475532144100241720ustar00rootroot00000000000000require 'thread' require 'prometheus/client/label_set_validator' require 'prometheus/client/uses_value_type' module Prometheus module Client class Metric include UsesValueType attr_reader :name, :docstring, :base_labels def initialize(name, docstring, base_labels = {}) @mutex = Mutex.new @validator = case type when :summary LabelSetValidator.new(['quantile']) when :histogram LabelSetValidator.new(['le']) else LabelSetValidator.new end @values = Hash.new { |hash, key| hash[key] = default(key) } validate_name(name) validate_docstring(docstring) @validator.valid?(base_labels) @name = name @docstring = docstring @base_labels = base_labels end # Returns the value for the given label set def get(labels = {}) label_set = label_set_for(labels) @validator.valid?(label_set) @values[label_set].get end # Returns all label sets with their values def values synchronize do @values.each_with_object({}) do |(labels, value), memo| memo[labels] = value end end end private def touch_default_value @values[label_set_for({})] end def default(labels) value_object(type, @name, @name, labels) end def validate_name(name) return true if name.is_a?(Symbol) raise ArgumentError, 'given name must be a symbol' end def validate_docstring(docstring) return true if docstring.respond_to?(:empty?) && !docstring.empty? raise ArgumentError, 'docstring must be given' end def label_set_for(labels) @validator.validate(@base_labels.merge(labels)) end def synchronize(&block) @mutex.synchronize(&block) end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/mmaped_dict.rb000066400000000000000000000041051475532144100251420ustar00rootroot00000000000000require 'prometheus/client/helper/mmaped_file' require 'prometheus/client/helper/plain_file' require 'prometheus/client' module Prometheus module Client class ParsingError < StandardError end # A dict of doubles, backed by an mmapped file. # # The file starts with a 4 byte int, indicating how much of it is used. # Then 4 bytes of padding. # There's then a number of entries, consisting of a 4 byte int which is the # size of the next field, a utf-8 encoded string key, padding to an 8 byte # alignment, and then a 8 byte float which is the value. class MmapedDict attr_reader :m, :used, :positions def self.read_all_values(f) Helper::PlainFile.new(f).entries.map do |data, encoded_len, value_offset, _| encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset)) [encoded, value] end end def initialize(m) @mutex = Mutex.new @m = m # @m.mlock # TODO: Ensure memory is locked to RAM @positions = {} read_all_positions.each do |key, pos| @positions[key] = pos end rescue StandardError => e raise ParsingError, "exception #{e} while processing metrics file #{path}" end def read_value(key) @m.fetch_entry(@positions, key, 0.0) end def write_value(key, value) @m.upsert_entry(@positions, key, value) end def path @m.filepath if @m end def close @m.sync @m.close rescue TypeError => e Prometheus::Client.logger.warn("munmap raised error #{e}") end def inspect "#<#{self.class}:0x#{(object_id << 1).to_s(16)}>" end private def init_value(key) @m.add_entry(@positions, key, 0.0) end # Yield (key, pos). No locking is performed. def read_all_positions @m.entries.map do |data, encoded_len, _, absolute_pos| encoded, = data.unpack(format('@4A%d', encoded_len)) [encoded, absolute_pos] end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/mmaped_value.rb000066400000000000000000000067161475532144100253450ustar00rootroot00000000000000require 'prometheus/client' require 'prometheus/client/mmaped_dict' require 'json' module Prometheus module Client # A float protected by a mutex backed by a per-process mmaped file. class MmapedValue VALUE_LOCK = Mutex.new @@files = {} @@pid = -1 def initialize(type, metric_name, name, labels, multiprocess_mode = '') @file_prefix = type.to_s @metric_name = metric_name @name = name @labels = labels if type == :gauge @file_prefix += '_' + multiprocess_mode.to_s end @pid = -1 @mutex = Mutex.new initialize_file end def increment(amount = 1) @mutex.synchronize do initialize_file if pid_changed? @value += amount write_value(@key, @value) @value end end def decrement(amount = 1) increment(-amount) end def set(value) @mutex.synchronize do initialize_file if pid_changed? @value = value write_value(@key, @value) @value end end def get @mutex.synchronize do initialize_file if pid_changed? return @value end end def pid_changed? @pid != Process.pid end # method needs to be run in VALUE_LOCK mutex def unsafe_reinitialize_file(check_pid = true) unsafe_initialize_file if !check_pid || pid_changed? end def self.reset_and_reinitialize VALUE_LOCK.synchronize do @@pid = Process.pid @@files = {} ObjectSpace.each_object(MmapedValue).each do |v| v.unsafe_reinitialize_file(false) end end end def self.reset_on_pid_change if pid_changed? @@pid = Process.pid @@files = {} end end def self.reinitialize_on_pid_change VALUE_LOCK.synchronize do reset_on_pid_change ObjectSpace.each_object(MmapedValue, &:unsafe_reinitialize_file) end end def self.pid_changed? @@pid != Process.pid end def self.multiprocess true end private def initialize_file VALUE_LOCK.synchronize do unsafe_initialize_file end end def unsafe_initialize_file self.class.reset_on_pid_change @pid = Process.pid unless @@files.has_key?(@file_prefix) unless @file.nil? @file.close end mmaped_file = Helper::MmapedFile.open_exclusive_file(@file_prefix) @@files[@file_prefix] = MmapedDict.new(mmaped_file) end @file = @@files[@file_prefix] @key = rebuild_key @value = read_value(@key) end def rebuild_key keys = @labels.keys.sort values = @labels.values_at(*keys) [@metric_name, @name, keys, values].to_json end def write_value(key, val) @file.write_value(key, val) rescue StandardError => e Prometheus::Client.logger.warn("writing value to #{@file.path} failed with #{e}") Prometheus::Client.logger.debug(e.backtrace.join("\n")) end def read_value(key) @file.read_value(key) rescue StandardError => e Prometheus::Client.logger.warn("reading value from #{@file.path} failed with #{e}") Prometheus::Client.logger.debug(e.backtrace.join("\n")) 0 end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/page_size.rb000066400000000000000000000006051475532144100246430ustar00rootroot00000000000000require 'open3' module Prometheus module Client module PageSize def self.page_size(fallback_page_size: 4096) stdout, status = Open3.capture2('getconf PAGESIZE') return fallback_page_size if status.nil? || !status.success? page_size = stdout.chomp.to_i return fallback_page_size if page_size <= 0 page_size end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/push.rb000066400000000000000000000147331475532144100236630ustar00rootroot00000000000000# encoding: UTF-8 require 'base64' require 'thread' require 'net/http' require 'uri' require 'erb' require 'set' require 'prometheus/client' require 'prometheus/client/formats/text' require 'prometheus/client/label_set_validator' module Prometheus # Client is a ruby implementation for a Prometheus compatible client. module Client # Push implements a simple way to transmit a given registry to a given # Pushgateway. class Push class HttpError < StandardError; end class HttpRedirectError < HttpError; end class HttpClientError < HttpError; end class HttpServerError < HttpError; end DEFAULT_GATEWAY = 'http://localhost:9091'.freeze PATH = '/metrics/job/%s'.freeze SUPPORTED_SCHEMES = %w(http https).freeze attr_reader :job, :gateway, :path def initialize(job:, gateway: DEFAULT_GATEWAY, grouping_key: {}, **kwargs) raise ArgumentError, "job cannot be nil" if job.nil? raise ArgumentError, "job cannot be empty" if job.empty? @validator = LabelSetValidator.new() @validator.validate(grouping_key) @mutex = Mutex.new @job = job @gateway = gateway || DEFAULT_GATEWAY @grouping_key = grouping_key @path = build_path(job, grouping_key) @uri = parse("#{@gateway}#{@path}") validate_no_basic_auth!(@uri) @http = Net::HTTP.new(@uri.host, @uri.port) @http.use_ssl = (@uri.scheme == 'https') @http.open_timeout = kwargs[:open_timeout] if kwargs[:open_timeout] @http.read_timeout = kwargs[:read_timeout] if kwargs[:read_timeout] end def basic_auth(user, password) @user = user @password = password end def add(registry) synchronize do request(Net::HTTP::Post, registry) end end def replace(registry) synchronize do request(Net::HTTP::Put, registry) end end def delete synchronize do request(Net::HTTP::Delete) end end private def parse(url) uri = URI.parse(url) unless SUPPORTED_SCHEMES.include?(uri.scheme) raise ArgumentError, 'only HTTP gateway URLs are supported currently.' end uri rescue URI::InvalidURIError => e raise ArgumentError, "#{url} is not a valid URL: #{e}" end def build_path(job, grouping_key) path = format(PATH, ERB::Util::url_encode(job)) grouping_key.each do |label, value| if value.include?('/') encoded_value = Base64.urlsafe_encode64(value) path += "/#{label}@base64/#{encoded_value}" # While it's valid for the urlsafe_encode64 function to return an # empty string when the input string is empty, it doesn't work for # our specific use case as we're putting the result into a URL path # segment. A double slash (`//`) can be normalised away by HTTP # libraries, proxies, and web servers. # # For empty strings, we use a single padding character (`=`) as the # value. # # See the pushgateway docs for more details: # # https://github.com/prometheus/pushgateway/blob/6393a901f56d4dda62cd0f6ab1f1f07c495b6354/README.md#url elsif value.empty? path += "/#{label}@base64/=" else path += "/#{label}/#{ERB::Util::url_encode(value)}" end end path end def request(req_class, registry = nil) validate_no_label_clashes!(registry) if registry req = req_class.new(@uri) req.content_type = Formats::Text::CONTENT_TYPE req.basic_auth(@user, @password) if @user req.body = Formats::Text.marshal(registry) if registry response = @http.request(req) validate_response!(response) response end def synchronize @mutex.synchronize { yield } end def validate_no_basic_auth!(uri) if uri.user || uri.password raise ArgumentError, <<~EOF Setting Basic Auth credentials in the gateway URL is not supported, please call the `basic_auth` method. Received username `#{uri.user}` in gateway URL. Instead of passing Basic Auth credentials like this: ``` push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://user:password@localhost:9091") ``` please pass them like this: ``` push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://localhost:9091") push.basic_auth("user", "password") ``` While URLs do support passing Basic Auth credentials using the `http://user:password@example.com/` syntax, the username and password in that syntax have to follow the usual rules for URL encoding of characters per RFC 3986 (https://datatracker.ietf.org/doc/html/rfc3986#section-2.1). Rather than place the burden of correctly performing that encoding on users of this gem, we decided to have a separate method for supplying Basic Auth credentials, with no requirement to URL encode the characters in them. EOF end end def validate_no_label_clashes!(registry) # There's nothing to check if we don't have a grouping key return if @grouping_key.empty? # We could be doing a lot of comparisons, so let's do them against a # set rather than an array grouping_key_labels = @grouping_key.keys.to_set registry.metrics.each do |metric| metric.values.keys.first.keys.each do |label| if grouping_key_labels.include?(label) raise LabelSetValidator::InvalidLabelSetError, "label :#{label} from grouping key collides with label of the " \ "same name from metric :#{metric.name} and would overwrite it" end end end end def validate_response!(response) status = Integer(response.code) if status >= 300 message = "status: #{response.code}, message: #{response.message}, body: #{response.body}" if status <= 399 raise HttpRedirectError, message elsif status <= 499 raise HttpClientError, message else raise HttpServerError, message end end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/rack/000077500000000000000000000000001475532144100232675ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/lib/prometheus/client/rack/collector.rb000066400000000000000000000047031475532144100256060ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client' module Prometheus module Client module Rack # Collector is a Rack middleware that provides a sample implementation of # a HTTP tracer. The default label builder can be modified to export a # different set of labels per recorded metric. class Collector attr_reader :app, :registry def initialize(app, options = {}, &label_builder) @app = app @registry = options[:registry] || Client.registry @label_builder = label_builder || DEFAULT_LABEL_BUILDER init_request_metrics init_exception_metrics end def call(env) # :nodoc: trace(env) { @app.call(env) } end protected DEFAULT_LABEL_BUILDER = proc do |env| { method: env['REQUEST_METHOD'].downcase, host: env['HTTP_HOST'].to_s, path: env['PATH_INFO'].to_s, } end def init_request_metrics @requests = @registry.counter( :http_requests_total, 'A counter of the total number of HTTP requests made.', ) @durations = @registry.summary( :http_request_duration_seconds, 'A summary of the response latency.', ) @durations_hist = @registry.histogram( :http_req_duration_seconds, 'A histogram of the response latency.', ) end def init_exception_metrics @exceptions = @registry.counter( :http_exceptions_total, 'A counter of the total number of exceptions raised.', ) end def trace(env) start = Time.now yield.tap do |response| duration = (Time.now - start).to_f record(labels(env, response), duration) end rescue => exception @exceptions.increment(exception: exception.class.name) raise end def labels(env, response) @label_builder.call(env).tap do |labels| labels[:code] = response.first.to_s end end def record(labels, duration) @requests.increment(labels) @durations.observe(labels, duration) @durations_hist.observe(labels, duration) rescue => exception @exceptions.increment(exception: exception.class.name) raise nil end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/rack/exporter.rb000066400000000000000000000050551475532144100254710ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client' require 'prometheus/client/formats/text' module Prometheus module Client module Rack # Exporter is a Rack middleware that provides a sample implementation of # a Prometheus HTTP client API. class Exporter attr_reader :app, :registry, :path FORMATS = [Formats::Text].freeze FALLBACK = Formats::Text def initialize(app, options = {}) @app = app @registry = options[:registry] || Client.registry @path = options[:path] || '/metrics' @acceptable = build_dictionary(FORMATS, FALLBACK) end def call(env) if env['PATH_INFO'] == @path format = negotiate(env['HTTP_ACCEPT'], @acceptable) format ? respond_with(format) : not_acceptable(FORMATS) else @app.call(env) end end private def negotiate(accept, formats) accept = '*/*' if accept.to_s.empty? parse(accept).each do |content_type, _| return formats[content_type] if formats.key?(content_type) end nil end def parse(header) header.to_s.split(/\s*,\s*/).map do |type| attributes = type.split(/\s*;\s*/) quality = extract_quality(attributes) [attributes.join('; '), quality] end.sort_by(&:last).reverse end def extract_quality(attributes, default = 1.0) quality = default attributes.delete_if do |attr| quality = attr.split('q=').last.to_f if attr.start_with?('q=') end quality end def respond_with(format) response = if Prometheus::Client.configuration.value_class.multiprocess format.marshal_multiprocess else format.marshal end [ 200, { 'Content-Type' => format::CONTENT_TYPE }, [response], ] end def not_acceptable(formats) types = formats.map { |format| format::MEDIA_TYPE } [ 406, { 'Content-Type' => 'text/plain' }, ["Supported media types: #{types.join(', ')}"], ] end def build_dictionary(formats, fallback) formats.each_with_object('*/*' => fallback) do |format, memo| memo[format::CONTENT_TYPE] = format memo[format::MEDIA_TYPE] = format end end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/registry.rb000066400000000000000000000027321475532144100245500ustar00rootroot00000000000000# encoding: UTF-8 require 'thread' require 'prometheus/client/counter' require 'prometheus/client/summary' require 'prometheus/client/gauge' require 'prometheus/client/histogram' module Prometheus module Client # Registry class Registry class AlreadyRegisteredError < StandardError; end def initialize @metrics = {} @mutex = Mutex.new end def register(metric) name = metric.name @mutex.synchronize do if exist?(name.to_sym) raise AlreadyRegisteredError, "#{name} has already been registered" else @metrics[name.to_sym] = metric end end metric end def counter(name, docstring, base_labels = {}) register(Counter.new(name, docstring, base_labels)) end def summary(name, docstring, base_labels = {}) register(Summary.new(name, docstring, base_labels)) end def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) register(Gauge.new(name, docstring, base_labels, multiprocess_mode)) end def histogram(name, docstring, base_labels = {}, buckets = Histogram::DEFAULT_BUCKETS) register(Histogram.new(name, docstring, base_labels, buckets)) end def exist?(name) @metrics.key?(name) end def get(name) @metrics[name.to_sym] end def metrics @metrics.values end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/simple_value.rb000066400000000000000000000007071475532144100253650ustar00rootroot00000000000000require 'json' module Prometheus module Client class SimpleValue def initialize(_type, _metric_name, _name, _labels, *_args) @value = 0.0 end def set(value) @value = value end def increment(by = 1) @value += by end def decrement(by = 1) @value -= by end def get @value end def self.multiprocess false end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/summary.rb000066400000000000000000000032241475532144100243720ustar00rootroot00000000000000require 'prometheus/client/metric' require 'prometheus/client/uses_value_type' module Prometheus module Client # Summary is an accumulator for samples. It captures Numeric data and # provides an efficient quantile calculation mechanism. class Summary < Metric extend Gem::Deprecate # Value represents the state of a Summary at a given point. class Value < Hash include UsesValueType attr_accessor :sum, :total def initialize(type, name, labels) @sum = value_object(type, name, "#{name}_sum", labels) @total = value_object(type, name, "#{name}_count", labels) end def observe(value) @sum.increment(value) @total.increment end end def initialize(name, docstring, base_labels = {}) super(name, docstring, base_labels) end def type :summary end # Records a given value. def observe(labels, value) label_set = label_set_for(labels) synchronize { @values[label_set].observe(value) } end alias add observe deprecate :add, :observe, 2016, 10 # Returns the value for the given label set def get(labels = {}) @validator.valid?(labels) synchronize do @values[labels].sum.get end end # Returns all label sets with their values def values synchronize do @values.each_with_object({}) do |(labels, value), memo| memo[labels] = value.sum end end end private def default(labels) Value.new(type, @name, labels) end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/support/000077500000000000000000000000001475532144100240635ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/lib/prometheus/client/support/puma.rb000066400000000000000000000017741475532144100253630ustar00rootroot00000000000000module Prometheus module Client module Support module Puma extend self def worker_pid_provider wid = worker_id if wid = worker_id wid else "process_id_#{Process.pid}" end end private def object_based_worker_id return unless defined?(::Puma::Cluster::Worker) workers = ObjectSpace.each_object(::Puma::Cluster::Worker) return if workers.nil? workers_first = workers.first workers_first.index unless workers_first.nil? end def program_name $PROGRAM_NAME end def worker_id if matchdata = program_name.match(/puma.*cluster worker ([0-9]+):/) "puma_#{matchdata[1]}" elsif object_worker_id = object_based_worker_id "puma_#{object_worker_id}" elsif program_name.include?('puma') 'puma_master' end end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/support/unicorn.rb000066400000000000000000000014241475532144100260660ustar00rootroot00000000000000module Prometheus module Client module Support module Unicorn def self.worker_pid_provider wid = worker_id if wid.nil? "process_id_#{Process.pid}" else "worker_id_#{wid}" end end def self.worker_id match = $0.match(/worker\[([^\]]+)\]/) if match match[1] else object_based_worker_id end end def self.object_based_worker_id return unless defined?(::Unicorn::Worker) workers = ObjectSpace.each_object(::Unicorn::Worker) return if workers.nil? workers_first = workers.first workers_first.nr unless workers_first.nil? end end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/uses_value_type.rb000066400000000000000000000013331475532144100261100ustar00rootroot00000000000000require 'prometheus/client/simple_value' module Prometheus module Client # Module providing convenience methods for creating value_object module UsesValueType def value_class Prometheus::Client.configuration.value_class end def value_object(type, metric_name, name, labels, *args) value_class.new(type, metric_name, name, labels, *args) rescue StandardError => e Prometheus::Client.logger.info("error #{e} while creating instance of #{value_class} defaulting to SimpleValue") Prometheus::Client.logger.debug("error #{e} backtrace #{e.backtrace.join("\n")}") Prometheus::Client::SimpleValue.new(type, metric_name, name, labels) end end end end prometheus-client-mmap-1.2.9/lib/prometheus/client/version.rb000066400000000000000000000001111475532144100243520ustar00rootroot00000000000000module Prometheus module Client VERSION = '1.2.9'.freeze end end prometheus-client-mmap-1.2.9/prometheus-client-mmap.gemspec000066400000000000000000000027621475532144100241030ustar00rootroot00000000000000#encoding: utf-8 $LOAD_PATH.push File.expand_path('../lib', __FILE__) require 'prometheus/client/version' Gem::Specification.new do |s| s.name = 'prometheus-client-mmap' s.version = Prometheus::Client::VERSION s.summary = 'A suite of instrumentation metric primitives ' \ 'that can be exposed through a web services interface.' s.authors = ['Tobias Schmidt', 'PaweÅ‚ Chojnacki', 'Stan Hu', 'Will Chandler'] s.email = ['ts@soundcloud.com', 'pawel@gitlab.com', 'stanhu@gmail.com', 'wchandler@gitlab.com'] s.homepage = 'https://gitlab.com/gitlab-org/prometheus-client-mmap' s.license = 'Apache-2.0' s.files = `git ls-files Cargo.lock Cargo.toml README.md .tool-versions lib ext vendor`.split("\n") s.require_paths = ['lib'] s.extensions = Dir.glob('{ext/**/extconf.rb}') # This C extension uses ObjectSpace::WeakRef with Integer keys (https://bugs.ruby-lang.org/issues/16035) s.required_ruby_version = '>= 3.1' s.add_dependency "base64" s.add_dependency "bigdecimal" s.add_dependency "logger" s.add_dependency "rb_sys", "~> 0.9.109" s.add_development_dependency 'fuzzbert', '~> 1.0', '>= 1.0.4' s.add_development_dependency 'gem_publisher', '~> 1' s.add_development_dependency 'pry', '~> 0.12.2' s.add_development_dependency "rake-compiler", "~> 1.2.7" s.add_development_dependency "rspec", "~> 3.2" s.add_development_dependency 'ruby-prof', '~> 1.7' end prometheus-client-mmap-1.2.9/spec/000077500000000000000000000000001475532144100170625ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/spec/examples/000077500000000000000000000000001475532144100207005ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/spec/examples/metric_example.rb000066400000000000000000000022771475532144100242330ustar00rootroot00000000000000# encoding: UTF-8 shared_examples_for Prometheus::Client::Metric do subject { described_class.new(:foo, 'foo description') } describe '.new' do it 'returns a new metric' do expect(subject).to be end it 'raises an exception if a reserved base label is used' do exception = Prometheus::Client::LabelSetValidator::ReservedLabelError expect do described_class.new(:foo, 'foo docstring', __name__: 'reserved') end.to raise_exception exception end it 'raises an exception if the given name is blank' do expect do described_class.new(nil, 'foo') end.to raise_exception ArgumentError end it 'raises an exception if docstring is missing' do expect do described_class.new(:foo, '') end.to raise_exception ArgumentError end end describe '#type' do it 'returns the metric type as symbol' do expect(subject.type).to be_a(Symbol) end end describe '#get' do it 'returns the current metric value' do expect(subject.get).to be_a(type) end it 'returns the current metric value for a given label set' do expect(subject.get(test: 'label')).to be_a(type) end end end prometheus-client-mmap-1.2.9/spec/prometheus/000077500000000000000000000000001475532144100212555ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/spec/prometheus/client/000077500000000000000000000000001475532144100225335ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/spec/prometheus/client/counter_spec.rb000066400000000000000000000033631475532144100255560ustar00rootroot00000000000000require 'prometheus/client/counter' require 'prometheus/client' require 'examples/metric_example' describe Prometheus::Client::Counter do before do allow(Prometheus::Client.configuration).to receive(:multiprocess_files_dir).and_return('tmp/') end let(:counter) { Prometheus::Client::Counter.new(:foo, 'foo description') } it_behaves_like Prometheus::Client::Metric do let(:type) { Float } end describe 'Memory Error tests' do it "creating many counters shouldn't cause a SIGBUS" do 4.times do |j| 9999.times do |i| counter = Prometheus::Client::Counter.new("foo#{j}_z#{i}".to_sym, 'some string') counter.increment end GC.start end end end describe '#increment' do it 'increments the counter' do expect { counter.increment }.to change { counter.get }.by(1) end it 'increments the counter for a given label set' do expect do expect do counter.increment(test: 'label') end.to change { counter.get(test: 'label') }.by(1) end.to_not change { counter.get(test: 'other_label') } end it 'increments the counter by a given value' do expect do counter.increment({}, 5) end.to change { counter.get }.by(5) end it 'raises an ArgumentError on negative increments' do expect do counter.increment({}, -1) end.to raise_error ArgumentError end it 'returns the new counter value' do expect(counter.increment).to eql(counter.get) end it 'is thread safe' do expect do Array.new(10) do Thread.new do 10.times { counter.increment } end end.each(&:join) end.to change { counter.get }.by(100) end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/formats/000077500000000000000000000000001475532144100242065ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/spec/prometheus/client/formats/text_spec.rb000066400000000000000000000211551475532144100265350ustar00rootroot00000000000000require 'spec_helper' require 'prometheus/client/formats/text' require 'prometheus/client/mmaped_value' describe Prometheus::Client::Formats::Text do context 'single process metrics' do let(:value_class) { Prometheus::Client::SimpleValue } let(:summary_value) do { 0.5 => 4.2, 0.9 => 8.32, 0.99 => 15.3 }.tap do |value| allow(value).to receive_messages(sum: 1243.21, total: 93) end end let(:histogram_value) do { 10 => 1, 20 => 2, 30 => 2 }.tap do |value| allow(value).to receive_messages(sum: 15.2, total: 2) end end let(:registry) do metrics = [ double( name: :foo, docstring: 'foo description', base_labels: { umlauts: 'Björn', utf: 'ä½–ä½¥' }, type: :counter, values: { { code: 'red' } => 42, { code: 'green' } => 3.14E42, { code: 'blue' } => -1.23e-45, }, ), double( name: :bar, docstring: "bar description\nwith newline", base_labels: { status: 'success' }, type: :gauge, values: { { code: 'pink' } => 15, }, ), double( name: :baz, docstring: 'baz "description" \\escaping', base_labels: {}, type: :counter, values: { { text: "with \"quotes\", \\escape \n and newline" } => 15, }, ), double( name: :qux, docstring: 'qux description', base_labels: { for: 'sake' }, type: :summary, values: { { code: '1' } => summary_value, }, ), double( name: :xuq, docstring: 'xuq description', base_labels: {}, type: :histogram, values: { { code: 'ah' } => histogram_value, }, ), ] metrics.each do |m| m.values.each do |k, v| m.values[k] = value_class.new(m.type, m.name, m.name, k) m.values[k].set(v) end end double(metrics: metrics) end describe '.marshal' do it 'returns a Text format version 0.0.4 compatible representation' do expect(subject.marshal(registry)).to eql <<-'TEXT'.gsub(/^\s+/, '') # HELP foo foo description # TYPE foo counter foo{umlauts="Björn",utf="ä½–ä½¥",code="red"} 42 foo{umlauts="Björn",utf="ä½–ä½¥",code="green"} 3.14e+42 foo{umlauts="Björn",utf="ä½–ä½¥",code="blue"} -1.23e-45 # HELP bar bar description\nwith newline # TYPE bar gauge bar{status="success",code="pink"} 15 # HELP baz baz "description" \\escaping # TYPE baz counter baz{text="with \"quotes\", \\escape \n and newline"} 15 # HELP qux qux description # TYPE qux summary qux{for="sake",code="1",quantile="0.5"} 4.2 qux{for="sake",code="1",quantile="0.9"} 8.32 qux{for="sake",code="1",quantile="0.99"} 15.3 qux_sum{for="sake",code="1"} 1243.21 qux_count{for="sake",code="1"} 93 # HELP xuq xuq description # TYPE xuq histogram xuq{code="ah",le="10"} 1 xuq{code="ah",le="20"} 2 xuq{code="ah",le="30"} 2 xuq{code="ah",le="+Inf"} 2 xuq_sum{code="ah"} 15.2 xuq_count{code="ah"} 2 TEXT end end end context 'multi process metrics', :temp_metrics_dir do let(:registry) { Prometheus::Client::Registry.new } before do allow(Prometheus::Client.configuration).to receive(:multiprocess_files_dir).and_return(temp_metrics_dir) # reset all current metrics Prometheus::Client::MmapedValue.class_variable_set(:@@files, {}) end context 'pid provider returns compound ID', :temp_metrics_dir, :sample_metrics do before do allow(Prometheus::Client.configuration).to receive(:pid_provider).and_return(-> { 'pid_provider_id_1' }) # Prometheus::Client::MmapedValue.class_variable_set(:@@files, {}) add_simple_metrics(registry) end it '.marshal_multiprocess' do expect(described_class.marshal_multiprocess(temp_metrics_dir)).to eq <<-'TEXT'.gsub(/^\s+/, '') # HELP counter Multiprocess metric # TYPE counter counter counter{a="1",b="1"} 1 counter{a="1",b="2"} 1 counter{a="2",b="1"} 1 # HELP gauge Multiprocess metric # TYPE gauge gauge gauge{b="1"} 1 gauge{b="2"} 1 # HELP gauge_with_big_value Multiprocess metric # TYPE gauge_with_big_value gauge gauge_with_big_value{a="0.12345678901234566"} 0.12345678901234566 gauge_with_big_value{a="12345678901234567"} 12345678901234568 # HELP gauge_with_null_labels Multiprocess metric # TYPE gauge_with_null_labels gauge gauge_with_null_labels{a="",b=""} 1 # HELP gauge_with_pid Multiprocess metric # TYPE gauge_with_pid gauge gauge_with_pid{b="1",c="1",pid="pid_provider_id_1"} 1 # HELP histogram Multiprocess metric # TYPE histogram histogram histogram_bucket{a="1",le="+Inf"} 1 histogram_bucket{a="1",le="0.005"} 0 histogram_bucket{a="1",le="0.01"} 0 histogram_bucket{a="1",le="0.025"} 0 histogram_bucket{a="1",le="0.05"} 0 histogram_bucket{a="1",le="0.1"} 0 histogram_bucket{a="1",le="0.25"} 0 histogram_bucket{a="1",le="0.5"} 0 histogram_bucket{a="1",le="1"} 1 histogram_bucket{a="1",le="10"} 1 histogram_bucket{a="1",le="2.5"} 1 histogram_bucket{a="1",le="5"} 1 histogram_count{a="1"} 1 histogram_sum{a="1"} 1 # HELP summary Multiprocess metric # TYPE summary summary summary_count{a="1",b="1"} 1 summary_sum{a="1",b="1"} 1 TEXT end end context 'pid provider returns numerical value', :temp_metrics_dir, :sample_metrics do before do allow(Prometheus::Client.configuration).to receive(:pid_provider).and_return(-> { -1 }) add_simple_metrics(registry) end it '.marshal_multiprocess' do expect(described_class.marshal_multiprocess(temp_metrics_dir)).to eq <<-'TEXT'.gsub(/^\s+/, '') # HELP counter Multiprocess metric # TYPE counter counter counter{a="1",b="1"} 1 counter{a="1",b="2"} 1 counter{a="2",b="1"} 1 # HELP gauge Multiprocess metric # TYPE gauge gauge gauge{b="1"} 1 gauge{b="2"} 1 # HELP gauge_with_big_value Multiprocess metric # TYPE gauge_with_big_value gauge gauge_with_big_value{a="0.12345678901234566"} 0.12345678901234566 gauge_with_big_value{a="12345678901234567"} 12345678901234568 # HELP gauge_with_null_labels Multiprocess metric # TYPE gauge_with_null_labels gauge gauge_with_null_labels{a="",b=""} 1 # HELP gauge_with_pid Multiprocess metric # TYPE gauge_with_pid gauge gauge_with_pid{b="1",c="1",pid="-1"} 1 # HELP histogram Multiprocess metric # TYPE histogram histogram histogram_bucket{a="1",le="+Inf"} 1 histogram_bucket{a="1",le="0.005"} 0 histogram_bucket{a="1",le="0.01"} 0 histogram_bucket{a="1",le="0.025"} 0 histogram_bucket{a="1",le="0.05"} 0 histogram_bucket{a="1",le="0.1"} 0 histogram_bucket{a="1",le="0.25"} 0 histogram_bucket{a="1",le="0.5"} 0 histogram_bucket{a="1",le="1"} 1 histogram_bucket{a="1",le="10"} 1 histogram_bucket{a="1",le="2.5"} 1 histogram_bucket{a="1",le="5"} 1 histogram_count{a="1"} 1 histogram_sum{a="1"} 1 # HELP summary Multiprocess metric # TYPE summary summary summary_count{a="1",b="1"} 1 summary_sum{a="1",b="1"} 1 TEXT end end context 'when OJ is available uses OJ to parse keys' do let(:oj) { double(oj) } before do stub_const 'Oj', oj allow(oj).to receive(:load) end end context 'with metric having whitespace and UTF chars', :temp_metrics_dir do before do registry.gauge(:gauge, "bar description\nwith newline", { umlauts: 'Björn', utf: 'ä½–ä½¥' }, :all).set({ umlauts: 'Björn', utf: 'ä½–ä½¥' }, 1) end xit '.marshall_multiprocess' do expect(described_class.marshal_multiprocess(temp_metrics_dir, use_rust: true)).to eq <<-'TEXT'.gsub(/^\s+/, '') TODO... TEXT end end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/gauge_spec.rb000066400000000000000000000034451475532144100251700ustar00rootroot00000000000000require 'prometheus/client' require 'prometheus/client/gauge' require 'examples/metric_example' describe Prometheus::Client::Gauge do let(:gauge) { Prometheus::Client::Gauge.new(:foo, 'foo description', test: nil) } before do allow(Prometheus::Client.configuration).to receive(:value_class).and_return(Prometheus::Client::SimpleValue) end it_behaves_like Prometheus::Client::Metric do let(:type) { Float } end describe '#set' do it 'sets a metric value' do expect do gauge.set({}, 42) end.to change { gauge.get }.from(0).to(42) end it 'sets a metric value for a given label set' do expect do expect do gauge.set({ test: 'value' }, 42) end.to(change { gauge.get(test: 'value') }.from(0).to(42)) end.to_not(change { gauge.get }) end end describe '#increment' do it 'increments a metric value' do gauge.set({}, 1) expect do gauge.increment({}, 42) end.to change { gauge.get }.from(1).to(43) end it 'sets a metric value for a given label set' do gauge.increment({ test: 'value' }, 1) expect do expect do gauge.increment({ test: 'value' }, 42) end.to(change { gauge.get(test: 'value') }.from(1).to(43)) end.to_not(change { gauge.get }) end end describe '#decrement' do it 'decrements a metric value' do gauge.set({}, 10) expect do gauge.decrement({}, 1) end.to change { gauge.get }.from(10).to(9) end it 'sets a metric value for a given label set' do gauge.set({ test: 'value' }, 10) expect do expect do gauge.decrement({ test: 'value' }, 5) end.to(change { gauge.get(test: 'value') }.from(10).to(5)) end.to_not(change { gauge.get }) end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/helpers/000077500000000000000000000000001475532144100241755ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/spec/prometheus/client/helpers/json_parser_spec.rb000066400000000000000000000014261475532144100300640ustar00rootroot00000000000000require 'spec_helper' require 'oj' require 'prometheus/client/helper/json_parser' describe Prometheus::Client::Helper::JsonParser do describe '.load' do let(:input) { %({ "a": 1 }) } shared_examples 'JSON parser' do it 'parses JSON' do expect(described_class.load(input)).to eq({ 'a' => 1 }) end it 'raises JSON::ParserError' do expect { described_class.load("{false}") }.to raise_error(JSON::ParserError) end end context 'with Oj' do it_behaves_like 'JSON parser' end context 'without Oj' do before(:all) do Object.send(:remove_const, 'Oj') load File.join(__dir__, "../../../../lib/prometheus/client/helper/json_parser.rb") end it_behaves_like 'JSON parser' end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/helpers/mmaped_file_spec.rb000066400000000000000000000073351475532144100300060ustar00rootroot00000000000000require 'spec_helper' require 'prometheus/client/helper/mmaped_file' require 'prometheus/client/page_size' describe Prometheus::Client::Helper::MmapedFile do let(:filename) { Dir::Tmpname.create('mmaped_file_') {} } after do File.delete(filename) if File.exist?(filename) end describe '.open' do it 'initialize PRIVATE mmaped file read only' do expect(described_class).to receive(:new).with(filename).and_call_original expect(described_class.open(filename)).to be_instance_of(described_class) end end context 'file does not exist' do let (:subject) { described_class.open(filename) } it 'creates and initializes file correctly' do expect(File.exist?(filename)).to be_falsey subject expect(File.exist?(filename)).to be_truthy end it 'creates a file with minimum initial size' do expect(File.size(subject.filepath)).to eq(subject.send(:initial_mmap_file_size)) end context 'when initial mmap size is larger' do let(:page_size) { Prometheus::Client::PageSize.page_size } let (:initial_mmap_file_size) { page_size + 1024 } before do allow_any_instance_of(described_class).to receive(:initial_mmap_file_size).and_return(initial_mmap_file_size) end it 'creates a file with increased minimum initial size' do expect(File.size(subject.filepath)).to eq(page_size * 2); end end end describe '.ensure_exclusive_file' do let(:tmpdir) { Dir.mktmpdir('mmaped_file') } let(:pid) { 'pid' } before do allow(Prometheus::Client.configuration).to receive(:multiprocess_files_dir).and_return(tmpdir) allow(Prometheus::Client.configuration).to receive(:pid_provider).and_return(pid.method(:to_s)) end context 'when no files are already locked' do it 'provides first possible filename' do expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-0\.db/) end it 'provides first and second possible filenames for two invocations' do expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-0\.db/) expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-1\.db/) end end context 'when first possible file exists for current file ID' do let(:first_mmaped_file) { described_class.ensure_exclusive_file('mmaped_file') } before do first_mmaped_file end context 'first file is unlocked' do before do Prometheus::Client::Helper::FileLocker.unlock(first_mmaped_file) end it 'provides first possible filename discarding the lock' do expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-0\.db/) end it 'provides second possible filename for second invocation' do expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-0\.db/) expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-1\.db/) end end context 'first file is not unlocked' do it 'provides second possible filename' do expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-1\.db/) end it 'provides second and third possible filename for two invocations' do expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-1\.db/) expect(described_class.ensure_exclusive_file('mmaped_file')) .to match(/.*mmaped_file_pid-2\.db/) end end end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/histogram_spec.rb000066400000000000000000000044611475532144100260740ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client' require 'prometheus/client/histogram' require 'examples/metric_example' describe Prometheus::Client::Histogram do before do allow(Prometheus::Client.configuration).to receive(:multiprocess_files_dir).and_return('tmp/') end let(:histogram) do described_class.new(:bar, 'bar description', {}, [2.5, 5, 10]) end it_behaves_like Prometheus::Client::Metric do let(:type) { Hash } end describe '#initialization' do it 'raise error for unsorted buckets' do expect do described_class.new(:bar, 'bar description', {}, [5, 2.5, 10]) end.to raise_error ArgumentError end it 'raise error for accidentally missing out an argument' do expect do described_class.new(:bar, 'bar description', [5, 2.5, 10]) end.to raise_error Prometheus::Client::LabelSetValidator::InvalidLabelSetError end end describe '#observe' do it 'records the given value' do expect do histogram.observe({}, 5) end.to change { histogram.get } end xit 'raise error for le labels' do expect do histogram.observe({ le: 1 }, 5) end.to raise_error ArgumentError end end describe '#get' do before do histogram.observe({ foo: 'bar' }, 3) histogram.observe({ foo: 'bar' }, 5.2) histogram.observe({ foo: 'bar' }, 13) histogram.observe({ foo: 'bar' }, 4) end xit 'returns a set of buckets values' do expect(histogram.get(foo: 'bar')).to eql(2.5 => 0, 5 => 2, 10 => 3) end xit 'returns a value which responds to #sum and #total' do value = histogram.get(foo: 'bar') expect(value.sum).to eql(25.2) expect(value.total).to eql(4) expect(value.total_inf).to eql(4) end xit 'uses zero as default value' do expect(histogram.get({})).to eql(2.5 => 0, 5 => 0, 10 => 0) end end xdescribe '#values' do it 'returns a hash of all recorded summaries' do histogram.observe({ status: 'bar' }, 3) histogram.observe({ status: 'foo' }, 6) expect(histogram.values).to eql( { status: 'bar' } => { 2.5 => 0, 5 => 1, 10 => 1 }, { status: 'foo' } => { 2.5 => 0, 5 => 0, 10 => 1 }, ) end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/label_set_validator_spec.rb000066400000000000000000000050261475532144100300740ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client/label_set_validator' describe Prometheus::Client::LabelSetValidator do let(:validator) { Prometheus::Client::LabelSetValidator.new reserved_labels } let(:reserved_labels) { [] } describe '.new' do it 'returns an instance of a LabelSetValidator' do expect(validator).to be_a(Prometheus::Client::LabelSetValidator) end end describe '#valid?' do it 'returns true for a valid label check' do expect(validator.valid?(version: 'alpha')).to eql(true) end it 'raises InvalidLabelError if a label value is an array' do expect do validator.valid?(version: [1, 2, 3]) end.to raise_exception(described_class::InvalidLabelError) end it 'raises Invaliddescribed_classError if a label set is not a hash' do expect do validator.valid?('invalid') end.to raise_exception(described_class::InvalidLabelSetError) end it 'raises InvalidLabelError if a label key is not a symbol' do expect do validator.valid?('key' => 'value') end.to raise_exception(described_class::InvalidLabelError) end it 'raises InvalidLabelError if a label key starts with __' do expect do validator.valid?(__reserved__: 'key') end.to raise_exception(described_class::ReservedLabelError) end context "when reserved labels were set" do let(:reserved_labels) { [:reserved] } it 'raises ReservedLabelError if a label key is reserved' do reserved_labels.each do |label| expect do validator.valid?(label => 'value') end.to raise_exception(described_class::ReservedLabelError) end end end end describe '#validate' do it 'returns a given valid label set' do hash = { version: 'alpha' } expect(validator.validate(hash)).to eql(hash) end it 'raises an exception if a given label set is not valid' do input = 'broken' expect(validator).to receive(:valid?).with(input).and_raise(described_class::InvalidLabelSetError) expect { validator.validate(input) }.to raise_exception(described_class::InvalidLabelSetError) end it 'raises InvalidLabelSetError for varying label sets' do validator.validate(method: 'get', code: '200') expect do validator.validate(method: 'get', exception: 'NoMethodError') end.to raise_exception(described_class::InvalidLabelSetError, "labels must have the same signature: (expected keys: [:code, :method], got: [:exception, :method])") end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/mmaped_dict_spec.rb000066400000000000000000000137601475532144100263470ustar00rootroot00000000000000require 'prometheus/client/mmaped_dict' require 'prometheus/client/page_size' require 'tempfile' describe Prometheus::Client::MmapedDict do let(:tmp_file) { Tempfile.new('mmaped_dict') } let(:tmp_mmaped_file) { Prometheus::Client::Helper::MmapedFile.open(tmp_file.path) } after do tmp_mmaped_file.close tmp_file.close tmp_file.unlink end describe '#initialize' do describe "empty mmap'ed file" do it 'is initialized with correct size' do described_class.new(tmp_mmaped_file) expect(File.size(tmp_file.path)).to eq(tmp_mmaped_file.send(:initial_mmap_file_size)) end end describe "mmap'ed file that is above minimum size" do let(:above_minimum_size) { Prometheus::Client::Helper::EntryParser::MINIMUM_SIZE + 1 } let(:page_size) { Prometheus::Client::PageSize.page_size } before do tmp_file.truncate(above_minimum_size) end it 'is initialized with the a page size' do described_class.new(tmp_mmaped_file) tmp_file.open expect(tmp_file.size).to eq(page_size); end end end describe 'read on boundary conditions' do let(:locked_file) { Prometheus::Client::Helper::MmapedFile.ensure_exclusive_file } let(:mmaped_file) { Prometheus::Client::Helper::MmapedFile.open(locked_file) } let(:page_size) { Prometheus::Client::PageSize.page_size } let(:target_size) { page_size } let(:iterations) { page_size / 32 } let(:dummy_key) { '1234' } let(:dummy_value) { 1.0 } let(:expected) { { dummy_key => dummy_value } } before do Prometheus::Client.configuration.multiprocess_files_dir = Dir.tmpdir data = described_class.new(Prometheus::Client::Helper::MmapedFile.open(locked_file)) # This test exercises the case when the value ends on the last byte. # To generate a file like this, we create entries that require 32 bytes # total to store with 7 bytes of padding at the end. # # To make things align evenly against the system page size, add a dummy # entry that will occupy the next 3 bytes to start on a 32-byte boundary. # The filestructure looks like: # # Bytes 0-3 : Total used size of file # Bytes 4-7 : Padding # Bytes 8-11 : Length of '1234' (4) # Bytes 12-15: '1234' # Bytes 24-31: 1.0 # Bytes 32-35: Length of '1000000000000' (13) # Bytes 36-48: '1000000000000' # Bytes 49-55: Padding # Bytes 56-63: 0.0 # Bytes 64-67: Length of '1000000000001' (13) # Bytes 68-80: '1000000000001' # Bytes 81-87: Padding # Bytes 88-95: 1.0 # ... data.write_value(dummy_key, dummy_value) (1..iterations - 1).each do |i| # Using a 13-byte string text = (1000000000000 + i).to_s expected[text] = i.to_f data.write_value(text, i) end data.close end it '#read_all_values' do values = described_class.read_all_values(locked_file) expect(values.count).to eq(iterations) expect(values).to match_array(expected.to_a) end end describe 'read and write values' do let(:locked_file) { Prometheus::Client::Helper::MmapedFile.ensure_exclusive_file } let(:mmaped_file) { Prometheus::Client::Helper::MmapedFile.open(locked_file) } before do Prometheus::Client.configuration.multiprocess_files_dir = Dir.tmpdir data = described_class.new(Prometheus::Client::Helper::MmapedFile.open(locked_file)) data.write_value('foo', 100) data.write_value('bar', 500) data.close end after do mmaped_file.close if File.exist?(mmaped_file.filepath) Prometheus::Client::Helper::FileLocker.unlock(locked_file) if File.exist?(mmaped_file.filepath) File.unlink(locked_file) if File.exist?(mmaped_file.filepath) end it '#inspect' do data = described_class.new(Prometheus::Client::Helper::MmapedFile.open(locked_file)) expect(data.inspect).to match(/#{described_class}:0x/) expect(data.inspect).not_to match(/@position/) end it '#read_all_values' do values = described_class.read_all_values(locked_file) expect(values.count).to eq(2) expect(values[0]).to eq(['foo', 100]) expect(values[1]).to eq(['bar', 500]) end it '#read_all_positions' do data = described_class.new(Prometheus::Client::Helper::MmapedFile.open(locked_file)) positions = data.positions # Generated via https://github.com/luismartingarcia/protocol: # protocol "Used:4,Pad:4,K1 Size:4,K1 Name:4,K1 Value:8,K2 Size:4,K2 Name:4,K2 Value:8" # # 0 1 2 3 # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ # | Used | Pad |K1 Size|K1 Name| K1 Value |K2 Size|K2 Name| # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ # | K2 Value | # +-+-+-+-+-+-+-+ expect(positions).to eq({ 'foo' => 16, 'bar' => 32 }) end describe '#write_value' do it 'writes values' do # Reload dictionary # data = described_class.new(mmaped_file) data.write_value('new value', 500) # Overwrite existing values data.write_value('foo', 200) data.write_value('bar', 300) values = described_class.read_all_values(locked_file) expect(values.count).to eq(3) expect(values[0]).to eq(['foo', 200]) expect(values[1]).to eq(['bar', 300]) expect(values[2]).to eq(['new value', 500]) end context 'when mmaped_file got deleted' do it 'is able to write to and expand metrics file' do data = described_class.new(mmaped_file) data.write_value('new value', 500) FileUtils.rm(mmaped_file.filepath) 1000.times do |i| data.write_value("new new value #{i}", 567) end expect(File.exist?(locked_file)).not_to be_truthy end end end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/mmaped_value_spec.rb000066400000000000000000000126721475532144100265410ustar00rootroot00000000000000require 'prometheus/client/mmaped_dict' require 'prometheus/client/page_size' require 'tempfile' describe Prometheus::Client::MmapedValue, :temp_metrics_dir do before do allow(Prometheus::Client.configuration).to receive(:multiprocess_files_dir).and_return(temp_metrics_dir) end describe '.reset_and_reinitialize' do let(:counter) { described_class.new(:counter, :counter, 'counter', {}) } before do # reset all current metrics Prometheus::Client::MmapedValue.class_variable_set(:@@files, {}) counter.increment(1) end it 'calls reinitialize on the counter' do expect(counter).to receive(:unsafe_reinitialize_file).with(false).and_call_original described_class.reset_and_reinitialize end context 'when metrics folder changes' do around do |example| Dir.mktmpdir('temp_metrics_dir') do |path| @tmp_path = path example.run end end before do allow(Prometheus::Client.configuration).to receive(:multiprocess_files_dir).and_return(@tmp_path) end it 'resets the counter to zero' do expect(counter).to receive(:unsafe_reinitialize_file).with(false).and_call_original expect { described_class.reset_and_reinitialize }.to(change { counter.get }.from(1).to(0)) end end end describe '#initialize' do let(:pid) { 1234 } before do described_class.class_variable_set(:@@files, {}) described_class.class_variable_set(:@@pid, pid) allow(Prometheus::Client.configuration).to receive(:pid_provider).and_return(-> { pid }) allow(Process).to receive(:pid).and_return(pid) end describe 'counter type object initialized' do let!(:counter) { described_class.new(:counter, :counter, 'counter', {}) } describe 'PID unchanged' do it 'initializing gauge MmapValue object type keeps old file data' do described_class.new(:gauge, :gauge, 'gauge', {}, :all) expect(described_class.class_variable_get(:@@files)).to have_key('counter') expect(described_class.class_variable_get(:@@files)).to have_key('gauge_all') end end describe 'PID changed' do let(:new_pid) { pid - 1 } let(:page_size) { Prometheus::Client::PageSize.page_size } before do counter.increment @old_value = counter.get allow(Prometheus::Client.configuration).to receive(:pid_provider).and_return(-> { new_pid }) allow(Process).to receive(:pid).and_return(new_pid) end it 'initializing gauge MmapValue object type keeps old file data' do described_class.new(:gauge, :gauge, 'gauge', {}, :all) expect(described_class.class_variable_get(:@@files)).not_to have_key('counter') expect(described_class.class_variable_get(:@@files)).to have_key('gauge_all') end it 'updates pid' do expect { described_class.new(:gauge, :gauge, 'gauge', {}, :all) } .to change { described_class.class_variable_get(:@@pid) }.from(pid).to(new_pid) end it '#increment updates pid' do expect { counter.increment } .to change { described_class.class_variable_get(:@@pid) }.from(pid).to(new_pid) end it '#increment updates pid' do expect { counter.increment } .to change { described_class.class_variable_get(:@@pid) }.from(pid).to(new_pid) end it '#get updates pid' do expect { counter.get } .to change { described_class.class_variable_get(:@@pid) }.from(pid).to(new_pid) end it '#set updates pid' do expect { counter.set(1) } .to change { described_class.class_variable_get(:@@pid) }.from(pid).to(new_pid) end it '#set logs an error' do counter.set(1) allow(counter.instance_variable_get(:@file)) .to receive(:write_value) .and_raise('error writing value') expect(Prometheus::Client.logger).to receive(:warn).and_call_original counter.set(1) end it 'reinitialize restores all used file references and resets data' do described_class.new(:gauge, :gauge, 'gauge', {}, :all) described_class.reinitialize_on_pid_change expect(described_class.class_variable_get(:@@files)).to have_key('counter') expect(described_class.class_variable_get(:@@files)).to have_key('gauge_all') expect(counter.get).not_to eq(@old_value) end it 'updates strings properly upon memory expansion', :page_size do described_class.new(:gauge, :gauge, 'gauge2', { label_1: 'x' * page_size * 2 }, :all) # This previously failed on Linux but not on macOS since mmap() may re-allocate the same region. ObjectSpace.each_object(String, &:valid_encoding?) end end context 'different label ordering' do it 'does not care about label ordering' do counter1 = described_class.new(:counter, :counter, 'ordered_counter', { label_1: 'hello', label_2: 'world', label_3: 'baz' }).increment counter2 = described_class.new(:counter, :counter, 'ordered_counter', { label_2: 'world', label_3: 'baz', label_1: 'hello' }).increment reading_counter = described_class.new(:counter, :counter, 'ordered_counter', { label_3: 'baz', label_1: 'hello', label_2: 'world' }) expect(reading_counter.get).to eq(2) end end end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/push_spec.rb000066400000000000000000000255721475532144100250640ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client/gauge' require 'prometheus/client/push' describe Prometheus::Client::Push do let(:gateway) { 'http://localhost:9091' } let(:registry) { Prometheus::Client::Registry.new } let(:grouping_key) { {} } let(:push) { Prometheus::Client::Push.new(job: 'test-job', gateway: gateway, grouping_key: grouping_key, open_timeout: 5, read_timeout: 30) } describe '.new' do it 'returns a new push instance' do expect(push).to be_a(Prometheus::Client::Push) end it 'uses localhost as default Pushgateway' do push = Prometheus::Client::Push.new(job: 'test-job') expect(push.gateway).to eql('http://localhost:9091') end it 'allows to specify a custom Pushgateway' do push = Prometheus::Client::Push.new(job: 'test-job', gateway: 'http://pu.sh:1234') expect(push.gateway).to eql('http://pu.sh:1234') end it 'raises an ArgumentError if the job is nil' do expect do Prometheus::Client::Push.new(job: nil) end.to raise_error ArgumentError end it 'raises an ArgumentError if the job is empty' do expect do Prometheus::Client::Push.new(job: "") end.to raise_error ArgumentError end it 'raises an ArgumentError if the given gateway URL is invalid' do ['inva.lid:1233', 'http://[invalid]'].each do |url| expect do Prometheus::Client::Push.new(job: 'test-job', gateway: url) end.to raise_error ArgumentError end end it 'raises InvalidLabelError if a grouping key label has an invalid name' do expect do Prometheus::Client::Push.new(job: "test-job", grouping_key: { "not_a_symbol" => "foo" }) end.to raise_error Prometheus::Client::LabelSetValidator::InvalidLabelError end end describe '#add' do it 'sends a given registry to via HTTP POST' do expect(push).to receive(:request).with(Net::HTTP::Post, registry) push.add(registry) end end describe '#replace' do it 'sends a given registry to via HTTP PUT' do expect(push).to receive(:request).with(Net::HTTP::Put, registry) push.replace(registry) end end describe '#delete' do it 'deletes existing metrics with HTTP DELETE' do expect(push).to receive(:request).with(Net::HTTP::Delete) push.delete end end describe '#path' do it 'uses the default metrics path if no grouping key given' do push = Prometheus::Client::Push.new(job: 'test-job') expect(push.path).to eql('/metrics/job/test-job') end it 'appends additional grouping labels to the path if specified' do push = Prometheus::Client::Push.new( job: 'test-job', grouping_key: { foo: "bar", baz: "qux"}, ) expect(push.path).to eql('/metrics/job/test-job/foo/bar/baz/qux') end it 'encodes grouping key label values containing `/` in url-safe base64' do push = Prometheus::Client::Push.new( job: 'test-job', grouping_key: { foo: "bar/baz"}, ) expect(push.path).to eql('/metrics/job/test-job/foo@base64/YmFyL2Jheg==') end it 'encodes empty grouping key label values as a single base64 padding character' do push = Prometheus::Client::Push.new( job: 'test-job', grouping_key: { foo: ""}, ) expect(push.path).to eql('/metrics/job/test-job/foo@base64/=') end it 'URL-encodes all other non-URL-safe characters' do push = Prometheus::Client::Push.new(job: '', grouping_key: { foo_label: '' }) expected = '/metrics/job/%3Cbar%20job%3E/foo_label/%3Cbar%20value%3E' expect(push.path).to eql(expected) end end describe '#request' do let(:content_type) { Prometheus::Client::Formats::Text::CONTENT_TYPE } let(:data) { Prometheus::Client::Formats::Text.marshal(registry) } let(:uri) { URI.parse("#{gateway}/metrics/job/test-job") } let(:response) do double( :response, code: '200', message: 'OK', body: 'Everything worked' ) end it 'sends marshalled registry to the specified gateway' do request = double(:request) expect(request).to receive(:content_type=).with(content_type) expect(request).to receive(:body=).with(data) expect(Net::HTTP::Post).to receive(:new).with(uri).and_return(request) http = double(:http) expect(http).to receive(:use_ssl=).with(false) expect(http).to receive(:open_timeout=).with(5) expect(http).to receive(:read_timeout=).with(30) expect(http).to receive(:request).with(request).and_return(response) expect(Net::HTTP).to receive(:new).with('localhost', 9091).and_return(http) push.send(:request, Net::HTTP::Post, registry) end context 'for a 3xx response' do let(:response) do double( :response, code: '301', message: 'Moved Permanently', body: 'Probably no body, but technically you can return one' ) end it 'raises a redirect error' do request = double(:request) allow(request).to receive(:content_type=) allow(request).to receive(:body=) allow(Net::HTTP::Post).to receive(:new).with(uri).and_return(request) http = double(:http) allow(http).to receive(:use_ssl=) allow(http).to receive(:open_timeout=) allow(http).to receive(:read_timeout=) allow(http).to receive(:request).with(request).and_return(response) allow(Net::HTTP).to receive(:new).with('localhost', 9091).and_return(http) expect { push.send(:request, Net::HTTP::Post, registry) }.to raise_error( Prometheus::Client::Push::HttpRedirectError ) end end context 'for a 4xx response' do let(:response) do double( :response, code: '400', message: 'Bad Request', body: 'Info on why the request was bad' ) end it 'raises a client error' do request = double(:request) allow(request).to receive(:content_type=) allow(request).to receive(:body=) allow(Net::HTTP::Post).to receive(:new).with(uri).and_return(request) http = double(:http) allow(http).to receive(:use_ssl=) allow(http).to receive(:open_timeout=) allow(http).to receive(:read_timeout=) allow(http).to receive(:request).with(request).and_return(response) allow(Net::HTTP).to receive(:new).with('localhost', 9091).and_return(http) expect { push.send(:request, Net::HTTP::Post, registry) }.to raise_error( Prometheus::Client::Push::HttpClientError ) end end context 'for a 5xx response' do let(:response) do double( :response, code: '500', message: 'Internal Server Error', body: 'Apology for the server code being broken' ) end it 'raises a server error' do request = double(:request) allow(request).to receive(:content_type=) allow(request).to receive(:body=) allow(Net::HTTP::Post).to receive(:new).with(uri).and_return(request) http = double(:http) allow(http).to receive(:use_ssl=) allow(http).to receive(:open_timeout=) allow(http).to receive(:read_timeout=) allow(http).to receive(:request).with(request).and_return(response) allow(Net::HTTP).to receive(:new).with('localhost', 9091).and_return(http) expect { push.send(:request, Net::HTTP::Post, registry) }.to raise_error( Prometheus::Client::Push::HttpServerError ) end end it 'deletes data from the registry' do request = double(:request) expect(request).to receive(:content_type=).with(content_type) expect(Net::HTTP::Delete).to receive(:new).with(uri).and_return(request) http = double(:http) expect(http).to receive(:use_ssl=).with(false) expect(http).to receive(:open_timeout=).with(5) expect(http).to receive(:read_timeout=).with(30) expect(http).to receive(:request).with(request).and_return(response) expect(Net::HTTP).to receive(:new).with('localhost', 9091).and_return(http) push.send(:request, Net::HTTP::Delete) end context 'HTTPS support' do let(:gateway) { 'https://localhost:9091' } it 'uses HTTPS when requested' do request = double(:request) expect(request).to receive(:content_type=).with(content_type) expect(request).to receive(:body=).with(data) expect(Net::HTTP::Post).to receive(:new).with(uri).and_return(request) http = double(:http) expect(http).to receive(:use_ssl=).with(true) expect(http).to receive(:open_timeout=).with(5) expect(http).to receive(:read_timeout=).with(30) expect(http).to receive(:request).with(request).and_return(response) expect(Net::HTTP).to receive(:new).with('localhost', 9091).and_return(http) push.send(:request, Net::HTTP::Post, registry) end end context 'Basic Auth support' do context 'when credentials are passed in the gateway URL' do let(:gateway) { 'https://super:secret@localhost:9091' } it "raises an ArgumentError explaining why we don't support that mechanism" do expect { push }.to raise_error ArgumentError, /in the gateway URL.*username `super`/m end end context 'when credentials are passed to the separate `basic_auth` method' do let(:gateway) { 'https://localhost:9091' } it 'passes the credentials on to the HTTP client' do request = double(:request) expect(request).to receive(:content_type=).with(content_type) expect(request).to receive(:basic_auth).with('super', 'secret') expect(request).to receive(:body=).with(data) expect(Net::HTTP::Put).to receive(:new).with(uri).and_return(request) http = double(:http) expect(http).to receive(:use_ssl=).with(true) expect(http).to receive(:open_timeout=).with(5) expect(http).to receive(:read_timeout=).with(30) expect(http).to receive(:request).with(request).and_return(response) expect(Net::HTTP).to receive(:new).with('localhost', 9091).and_return(http) push.basic_auth("super", "secret") push.send(:request, Net::HTTP::Put, registry) end end end context 'with a grouping key that clashes with a metric label' do let(:grouping_key) { { foo: "bar"} } before do gauge = Prometheus::Client::Gauge.new( :test_gauge, 'test docstring', foo: nil ) registry.register(gauge) gauge.set({ foo: "bar"}, 42) end it 'raises an error when grouping key labels conflict with metric labels' do expect { push.send(:request, Net::HTTP::Post, registry) }.to raise_error( Prometheus::Client::LabelSetValidator::InvalidLabelSetError ) end end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/rack/000077500000000000000000000000001475532144100234535ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/spec/prometheus/client/rack/collector_spec.rb000066400000000000000000000043361475532144100270060ustar00rootroot00000000000000# encoding: UTF-8 require 'rack/test' require 'prometheus/client/rack/collector' describe Prometheus::Client::Rack::Collector do include Rack::Test::Methods before do allow(Prometheus::Client.configuration).to receive(:value_class).and_return(Prometheus::Client::SimpleValue) end let(:registry) do Prometheus::Client::Registry.new end let(:original_app) do ->(_) { [200, { 'Content-Type' => 'text/html' }, ['OK']] } end let!(:app) do described_class.new(original_app, registry: registry) end it 'returns the app response' do get '/foo' expect(last_response).to be_ok expect(last_response.body).to eql('OK') end it 'propagates errors in the registry' do counter = registry.get(:http_requests_total) expect(counter).to receive(:increment).and_raise(NoMethodError) expect { get '/foo' }.to raise_error(NoMethodError) end it 'traces request information' do # expect(Time).to receive(:now).and_return(Time.at(0.0), Time.at(0.2)) labels = { method: 'get', host: 'example.org', path: '/foo', code: '200' } get '/foo' { http_requests_total: 1.0, # http_request_duration_seconds: { 0.5 => 0.2, 0.9 => 0.2, 0.99 => 0.2 }, # TODO: Fix summaries }.each do |metric, result| expect(registry.get(metric).get(labels)).to eql(result) end end context 'when the app raises an exception' do let(:original_app) do lambda do |env| raise NoMethodError if env['PATH_INFO'] == '/broken' [200, { 'Content-Type' => 'text/html' }, ['OK']] end end before do get '/foo' end it 'traces exceptions' do labels = { exception: 'NoMethodError' } expect { get '/broken' }.to raise_error NoMethodError expect(registry.get(:http_exceptions_total).get(labels)).to eql(1.0) end end context 'setting up with a block' do let(:app) do described_class.new(original_app, registry: registry) do |env| { method: env['REQUEST_METHOD'].downcase } # and ignore the path end end it 'allows labels configuration' do get '/foo/bar' labels = { method: 'get', code: '200' } expect(registry.get(:http_requests_total).get(labels)).to eql(1.0) end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/rack/exporter_spec.rb000066400000000000000000000055731475532144100266740ustar00rootroot00000000000000# encoding: UTF-8 require 'rack/test' require 'prometheus/client/rack/exporter' xdescribe Prometheus::Client::Rack::Exporter do include Rack::Test::Methods let(:registry) do Prometheus::Client::Registry.new end let(:app) do app = ->(_) { [200, { 'Content-Type' => 'text/html' }, ['OK']] } Prometheus::Client::Rack::Exporter.new(app, registry: registry) end context 'when requesting app endpoints' do it 'returns the app response' do get '/foo' expect(last_response).to be_ok expect(last_response.body).to eql('OK') end end context 'when requesting /metrics' do text = Prometheus::Client::Formats::Text shared_examples 'ok' do |headers, fmt| it "responds with 200 OK and Content-Type #{fmt::CONTENT_TYPE}" do registry.counter(:foo, 'foo counter').increment({}, 9) get '/metrics', nil, headers expect(last_response.status).to eql(200) expect(last_response.header['Content-Type']).to eql(fmt::CONTENT_TYPE) expect(last_response.body).to eql(fmt.marshal(registry)) end end shared_examples 'not acceptable' do |headers| it 'responds with 406 Not Acceptable' do message = 'Supported media types: text/plain' get '/metrics', nil, headers expect(last_response.status).to eql(406) expect(last_response.header['Content-Type']).to eql('text/plain') expect(last_response.body).to eql(message) end end context 'when client does not send a Accept header' do include_examples 'ok', {}, text end context 'when client accpets any media type' do include_examples 'ok', { 'HTTP_ACCEPT' => '*/*' }, text end context 'when client requests application/json' do include_examples 'not acceptable', 'HTTP_ACCEPT' => 'application/json' end context 'when client requests text/plain' do include_examples 'ok', { 'HTTP_ACCEPT' => 'text/plain' }, text end context 'when client uses different white spaces in Accept header' do accept = 'text/plain;q=1.0 ; version=0.0.4' include_examples 'ok', { 'HTTP_ACCEPT' => accept }, text end context 'when client does not include quality attribute' do accept = 'application/json;q=0.5, text/plain' include_examples 'ok', { 'HTTP_ACCEPT' => accept }, text end context 'when client accepts some unknown formats' do accept = 'text/plain;q=0.3, proto/buf;q=0.7' include_examples 'ok', { 'HTTP_ACCEPT' => accept }, text end context 'when client accepts only unknown formats' do accept = 'fancy/woo;q=0.3, proto/buf;q=0.7' include_examples 'not acceptable', 'HTTP_ACCEPT' => accept end context 'when client accepts unknown formats and wildcard' do accept = 'fancy/woo;q=0.3, proto/buf;q=0.7, */*;q=0.1' include_examples 'ok', { 'HTTP_ACCEPT' => accept }, text end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/registry_spec.rb000066400000000000000000000054501475532144100257460ustar00rootroot00000000000000# encoding: UTF-8 require 'thread' require 'prometheus/client/registry' describe Prometheus::Client::Registry do let(:registry) { Prometheus::Client::Registry.new } describe '.new' do it 'returns a new registry instance' do expect(registry).to be_a(Prometheus::Client::Registry) end end describe '#register' do it 'registers a new metric container and returns it' do metric = double(name: :test) expect(registry.register(metric)).to eql(metric) end it 'raises an exception if a metric name gets registered twice' do metric = double(name: :test) registry.register(metric) expect do registry.register(metric) end.to raise_exception described_class::AlreadyRegisteredError end it 'is thread safe' do mutex = Mutex.new containers = [] def registry.exist?(*args) super.tap { sleep(0.01) } end Array.new(5) do Thread.new do result = begin registry.register(double(name: :test)) rescue Prometheus::Client::Registry::AlreadyRegisteredError nil end mutex.synchronize { containers << result } end end.each(&:join) expect(containers.compact.size).to eql(1) end end describe '#counter' do it 'registers a new counter metric container and returns the counter' do metric = registry.counter(:test, 'test docstring') expect(metric).to be_a(Prometheus::Client::Counter) end end describe '#gauge' do it 'registers a new gauge metric container and returns the gauge' do metric = registry.gauge(:test, 'test docstring') expect(metric).to be_a(Prometheus::Client::Gauge) end end describe '#summary' do it 'registers a new summary metric container and returns the summary' do metric = registry.summary(:test, 'test docstring') expect(metric).to be_a(Prometheus::Client::Summary) end end describe '#histogram' do it 'registers a new histogram metric container and returns the histogram' do metric = registry.histogram(:test, 'test docstring') expect(metric).to be_a(Prometheus::Client::Histogram) end end describe '#exist?' do it 'returns true if a metric name has been registered' do registry.register(double(name: :test)) expect(registry.exist?(:test)).to eql(true) end it 'returns false if a metric name has not been registered yet' do expect(registry.exist?(:test)).to eql(false) end end describe '#get' do it 'returns a previously registered metric container' do registry.register(double(name: :test)) expect(registry.get(:test)).to be end it 'returns nil if the metric has not been registered yet' do expect(registry.get(:test)).to eql(nil) end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/summary_spec.rb000066400000000000000000000026651475532144100256000ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client/summary' require 'examples/metric_example' describe Prometheus::Client::Summary do let(:summary) { Prometheus::Client::Summary.new(:bar, 'bar description') } it_behaves_like Prometheus::Client::Metric do let(:type) { Float } end describe '#observe' do it 'records the given value' do expect do summary.observe({}, 5) end.to change { summary.get } end end xdescribe '#get' do before do summary.observe({ foo: 'bar' }, 3) summary.observe({ foo: 'bar' }, 5.2) summary.observe({ foo: 'bar' }, 13) summary.observe({ foo: 'bar' }, 4) end it 'returns a set of quantile values' do expect(summary.get(foo: 'bar')).to eql(0.5 => 4, 0.9 => 5.2, 0.99 => 5.2) end it 'returns a value which responds to #sum and #total' do value = summary.get(foo: 'bar') expect(value.sum).to eql(25.2) expect(value.total).to eql(4) end it 'uses nil as default value' do expect(summary.get({})).to eql(0.5 => nil, 0.9 => nil, 0.99 => nil) end end xdescribe '#values' do it 'returns a hash of all recorded summaries' do summary.observe({ status: 'bar' }, 3) summary.observe({ status: 'foo' }, 5) expect(summary.values).to eql( { status: 'bar' } => { 0.5 => 3, 0.9 => 3, 0.99 => 3 }, { status: 'foo' } => { 0.5 => 5, 0.9 => 5, 0.99 => 5 }, ) end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/support/000077500000000000000000000000001475532144100242475ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/spec/prometheus/client/support/puma_spec.rb000066400000000000000000000030051475532144100265460ustar00rootroot00000000000000require 'spec_helper' require 'prometheus/client/support/puma' class FakePumaWorker attr_reader :index def initialize(index) @index = index end end describe Prometheus::Client::Support::Puma do describe '.worker_pid_provider' do let(:worker_id) { '2' } let(:program_name) { $PROGRAM_NAME } subject(:worker_pid_provider) { described_class.worker_pid_provider } before do expect(described_class).to receive(:program_name) .at_least(:once) .and_return(program_name) end context 'when the current process is a Puma cluster worker' do context 'when the process name contains a worker id' do let(:program_name) { 'puma: cluster worker 2: 34740 [my-app]' } it { is_expected.to eq('puma_2') } end context 'when the process name does not include a worker id' do let(:worker_number) { 10 } let!(:worker) { FakePumaWorker.new(worker_number) } before do stub_const('Puma::Cluster::Worker', FakePumaWorker) end it { is_expected.to eq("puma_#{worker_number}") } end end context 'when the current process is the Puma master' do let(:program_name) { 'bin/puma' } it { is_expected.to eq('puma_master') } end context 'when it cannot be determined that Puma is running' do let(:process_id) { 10 } before do allow(Process).to receive(:pid).and_return(process_id) end it { is_expected.to eq("process_id_#{process_id}") } end end end prometheus-client-mmap-1.2.9/spec/prometheus/client/support/unicorn_spec.rb000066400000000000000000000055201475532144100272650ustar00rootroot00000000000000require 'spec_helper' require 'prometheus/client/support/unicorn' class FakeUnicornWorker attr_reader :nr def initialize(nr) @nr = nr end end describe Prometheus::Client::Support::Unicorn do describe '.worker_id' do let(:worker_id) { '09' } around do |example| old_name = $0 example.run $0 = old_name end context 'process name contains worker id' do before do $0 = "program worker[#{worker_id}] arguments" end it 'returns worker_id' do expect(subject.worker_id).to eq(worker_id) end end context 'process name is without worker id' do it 'calls .object_based_worker_id id provider' do expect(subject).to receive(:object_based_worker_id).and_return(worker_id) expect(subject.worker_id).to eq(worker_id) end end end describe '.object_based_worker_id' do context 'when Unicorn is defined' do before do stub_const('Unicorn::Worker', FakeUnicornWorker) end context 'Worker instance is present in ObjectSpace' do let(:worker_number) { 10 } let!(:unicorn_worker) { FakeUnicornWorker.new(worker_number) } it 'Unicorn::Worker to be defined' do expect(defined?(Unicorn::Worker)).to be_truthy end it 'returns worker id' do expect(described_class.object_based_worker_id).to eq(worker_number) end end context 'Worker instance is not present in ObjectSpace' do it 'Unicorn::Worker id defined' do expect(defined?(Unicorn::Worker)).to be_truthy end it 'returns no worker id' do expect(ObjectSpace).to receive(:each_object).with(::Unicorn::Worker).and_return(nil) expect(described_class.object_based_worker_id).to eq(nil) end end end context 'Unicorn::Worker is not defined' do it 'Unicorn::Worker not defined' do expect(defined?(Unicorn::Worker)).to be_falsey end it 'returns no worker_id' do expect(described_class.object_based_worker_id).to eq(nil) end end end describe '.worker_pid_provider' do context 'worker_id is provided' do let(:worker_id) { 2 } before do allow(described_class).to receive(:worker_id).and_return(worker_id) end it 'returns worker pid created from worker id' do expect(described_class.worker_pid_provider).to eq("worker_id_#{worker_id}") end end context 'worker_id is not provided' do let(:process_id) { 10 } before do allow(described_class).to receive(:worker_id).and_return(nil) allow(Process).to receive(:pid).and_return(process_id) end it 'returns worker pid created from Process ID' do expect(described_class.worker_pid_provider).to eq("process_id_#{process_id}") end end end end prometheus-client-mmap-1.2.9/spec/prometheus/client_spec.rb000066400000000000000000000043501475532144100240740ustar00rootroot00000000000000# encoding: UTF-8 require 'prometheus/client' describe Prometheus::Client do describe '.registry' do it 'returns a registry object' do expect(described_class.registry).to be_a(described_class::Registry) end it 'memorizes the returned object' do expect(described_class.registry).to eql(described_class.registry) end end context '.reset! and .reinitialize_on_pid_change' do let(:metric_name) { :room_temperature_celsius } let(:label) { { room: 'kitchen' } } let(:value) { 21 } let(:gauge) { Prometheus::Client::Gauge.new(metric_name, 'test') } before do described_class.cleanup! described_class.reset! # registering metrics will leak into other specs registry = described_class.registry gauge.set(label, value) registry.register(gauge) expect(registry.metrics.count).to eq(1) expect(registry.get(metric_name).get(label)).to eq(value) end describe '.reset!' do it 'resets registry and clears existing metrics' do described_class.cleanup! described_class.reset! registry = described_class.registry expect(registry.metrics.count).to eq(0) registry.register(gauge) expect(registry.get(metric_name).get(label)).not_to eq(value) end end describe '.reinitialize_on_pid_change' do context 'with force: false' do it 'calls `MmapedValue.reinitialize_on_pid_change`' do expect(Prometheus::Client::MmapedValue).to receive(:reinitialize_on_pid_change).and_call_original described_class.reinitialize_on_pid_change(force: false) end end context 'without explicit :force param' do it 'defaults to `false` and calls `MmapedValue.reinitialize_on_pid_change`' do expect(Prometheus::Client::MmapedValue).to receive(:reinitialize_on_pid_change).and_call_original described_class.reinitialize_on_pid_change end end context 'with force: true' do it 'calls `MmapedValue.reset_and_reinitialize`' do expect(Prometheus::Client::MmapedValue).to receive(:reset_and_reinitialize).and_call_original described_class.reinitialize_on_pid_change(force: true) end end end end end prometheus-client-mmap-1.2.9/spec/sample_metrics.rb000066400000000000000000000021261475532144100224170ustar00rootroot00000000000000module SampleMetrics def add_simple_metrics(registry) counter = registry.counter(:counter, 'counter', { b: 1 }) counter.increment(a: 1) counter.increment(a: 2) counter.increment(a: 1, b: 2) gauge = registry.gauge(:gauge, 'gauge', {}, :livesum) gauge.set({ b: 1 }, 1) gauge.set({ b: 2 }, 1) gauge_with_pid = registry.gauge(:gauge_with_pid, 'gauge_with_pid', b: 1) gauge_with_pid.set({ c: 1 }, 1) gauge_with_null_labels = registry.gauge(:gauge_with_null_labels, 'gauge_with_null_labels', { a: nil, b: nil }, :livesum) gauge_with_null_labels.set({ a: nil, b: nil }, 1) gauge_with_big_value = registry.gauge(:gauge_with_big_value, 'gauge_with_big_value', { a: 0 }, :livesum) gauge_with_big_value.set({ a: 12345678901234567 }, 12345678901234567) gauge_with_big_value.set({ a: 0.12345678901234567 }, 0.12345678901234567) registry.gauge(:gauge_without_measurements, 'gauge_without_measurements', b: 1) registry.histogram(:histogram, 'histogram', {}).observe({ a: 1 }, 1) registry.summary(:summary, 'summary', a: 1).observe({ b: 1 }, 1) end end prometheus-client-mmap-1.2.9/spec/spec_helper.rb000066400000000000000000000006111475532144100216760ustar00rootroot00000000000000require 'simplecov' require 'sample_metrics' require 'temp_metrics_dir' SimpleCov.start do add_filter 'fuzz' add_filter 'tmp' add_filter 'vendor/ruby' end RSpec.configure do |config| config.include SampleMetrics, :sample_metrics config.include TempMetricsDir, :temp_metrics_dir config.after(:all) do cleanup_temp_metrics_dir if defined?(cleanup_temp_metrics_dir) end end prometheus-client-mmap-1.2.9/spec/temp_metrics_dir.rb000066400000000000000000000005501475532144100227400ustar00rootroot00000000000000module TempMetricsDir def temp_metrics_dir @temp_metrics_dir ||= Dir.mktmpdir('temp_metrics_dir') end def cleanup_temp_metrics_dir return if @temp_metrics_dir.nil? begin FileUtils.rm_rf(@temp_metrics_dir) rescue StandardError => ex puts "Files cleanup caused #{ex}" ensure @temp_metrics_dir = nil end end end prometheus-client-mmap-1.2.9/tmp/000077500000000000000000000000001475532144100167305ustar00rootroot00000000000000prometheus-client-mmap-1.2.9/tmp/.gitkeep000066400000000000000000000000001475532144100203470ustar00rootroot00000000000000