faster-hex-0.9.0/.github/workflows/rust.yml000064400000000000000000000034101046102023000170130ustar 00000000000000 name: nervosnetwork/faster-hex CI on: [push, pull_request] jobs: fmt: name: Rustfmt runs-on: ubuntu-latest steps: - name: Checkout sources uses: actions/checkout@v2 - name: Install rust uses: actions-rs/toolchain@v1 with: toolchain: 1.73 components: rustfmt profile: minimal override: true - name: Run rustfmt uses: actions-rs/cargo@v1 with: command: fmt args: --all -- --check audit: name: Security audit runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions-rs/audit-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} test: name: ${{ matrix.build }} runs-on: ${{ matrix.os }} strategy: matrix: build: [Linux, macOS, windows-2019] include: - build: Linux os: ubuntu-22.04 - build: macOS os: macos-12 - build: windows-2019 os: windows-2019 steps: - name: Maximize build space if: runner.os == 'Linux' uses: easimon/maximize-build-space@master with: root-reserve-mb: 512 swap-size-mb: 1024 remove-dotnet: 'true' remove-android: 'true' remove-haskell: 'true' - name: Checkout sources uses: actions/checkout@v2 with: submodules: true - name: Install rust uses: actions-rs/toolchain@v1 with: toolchain: 1.73 target: ${{ matrix.target }} profile: minimal override: true - name: Run tests if: runner.os == 'Linux' || runner.os == 'Windows' uses: actions-rs/cargo@v1 with: command: test faster-hex-0.9.0/.gitignore000064400000000000000000000005471046102023000136760ustar 00000000000000# Generated by Cargo # will have compiled files and executables /target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk proptest-regressions/* rust-toolchain faster-hex-0.9.0/Cargo.toml0000644000000025550000000000100111150ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "faster-hex" version = "0.9.0" authors = ["zhangsoledad <787953403@qq.com>"] exclude = [ "afl/*", "benches/*", "fuzz/*", "CHANGELOG.md", ] description = "Fast hex encoding." homepage = "https://github.com/NervosFoundation/faster-hex" readme = "README.md" keywords = [ "simd", "hex", "no-std", ] license = "MIT" repository = "https://github.com/NervosFoundation/faster-hex" [dependencies.serde] version = "1.0" optional = true [dev-dependencies.bytes] version = "1.4.0" [dev-dependencies.criterion] version = "0.3" [dev-dependencies.hex] version = "0.3.2" [dev-dependencies.proptest] version = "1.0" [dev-dependencies.rustc-hex] version = "1.0" [dev-dependencies.serde] version = "1.0" features = ["derive"] [dev-dependencies.serde_json] version = "1.0" [features] alloc = [] default = [ "std", "serde", ] serde = ["dep:serde"] std = ["alloc"] faster-hex-0.9.0/Cargo.toml.orig000064400000000000000000000013761046102023000145760ustar 00000000000000[package] name = "faster-hex" version = "0.9.0" authors = ["zhangsoledad <787953403@qq.com>"] edition = "2018" keywords = ["simd", "hex", "no-std"] license = "MIT" description = "Fast hex encoding." repository = "https://github.com/NervosFoundation/faster-hex" homepage = "https://github.com/NervosFoundation/faster-hex" readme = "README.md" exclude = [ "afl/*", "benches/*", "fuzz/*", "CHANGELOG.md" ] [dependencies] serde = { version = "1.0", optional = true } [features] default = ["std", "serde"] std = ["alloc"] alloc = [] serde = ["dep:serde"] [dev-dependencies] criterion = "0.3" rustc-hex = "1.0" hex = "0.3.2" proptest = "1.0" serde = { version = "1.0", features = ["derive"]} bytes = {version = "1.4.0"} serde_json ={ version = "1.0"} faster-hex-0.9.0/LICENSE000064400000000000000000000020621046102023000127050ustar 00000000000000MIT License Copyright (c) 2018 Nervos Foundation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. faster-hex-0.9.0/LICENSE-THIRD-PARTY/Rust Project Developers000064400000000000000000000020571046102023000210730ustar 00000000000000Copyright (c) 2017 The Rust Project Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. faster-hex-0.9.0/LICENSE-THIRD-PARTY/fast-hex000064400000000000000000000020561046102023000161740ustar 00000000000000MIT License Copyright (c) 2017 Zach Bjornson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. faster-hex-0.9.0/README.md000064400000000000000000000053141046102023000131620ustar 00000000000000# faster-hex [![License]](#license) [![crate-badge]](https://crates.io/crates/faster-hex) [crate-badge]: https://img.shields.io/crates/v/faster-hex.svg [license]: https://img.shields.io/badge/License-MIT-green.svg This program implements hex encoding a slice into a predetermined destination using various different instruction sets. ## Benchmark ### Running Runs benchmark ``` cargo bench ``` ### Results Machine: MacBook Pro (Early 2015) (2.7 GHz Intel Core i5) Rust: rustc 1.31.0 (abe02cefd 2018-12-04) Compare with [hex](https://crates.io/crates/hex): * Encoding ~10x over * Decoding ~10x over Compare with [rustc-hex](https://crates.io/crates/rustc-hex): * Encoding ~2.5x over * Decoding ~7x over ## Examples Encode to hex ```rust use faster_hex::hex_string; let result = hex_string(b"Hello world!"); assert_eq!(result, "48656c6c6f20776f726c6421"); ``` Encode to upper case hex ```rust use faster_hex::hex_string_upper; let result = hex_string_upper(b"Hello world!"); assert_eq!(result, "48656C6C6F20776F726C6421"); ``` Decode ```rust use faster_hex::hex_decode; let src = b"48656c6c6f20776f726c6421"; let mut dst = vec![0; src.len() / 2]; hex_decode(src, &mut dst).unwrap(); assert_eq!(dst, b"Hello world!"); ``` Decode with case check ```rust use faster_hex::{hex_decode_with_case, CheckCase}; let src = b"48656c6c6f20776f726c6421"; let mut dst = vec![0; src.len() / 2]; assert!(hex_decode_with_case(src, &mut dst, CheckCase::Lower).is_ok()); assert_eq!(dst, b"Hello world!"); assert!(hex_decode_with_case(src, &mut dst, CheckCase::None).is_ok()); assert_eq!(dst, b"Hello world!"); assert!(hex_decode_with_case(src, &mut dst, CheckCase::Upper).is_err()); ``` Serde feature ```rust use serde::{Deserialize, Serialize}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct Simple { #[serde(with = "faster_hex")] foo: Vec, #[serde(with = "faster_hex::nopfx_lowercase")] bar: Vec, } ``` ## Notice Major version zero (0.y.z) is for initial development. Anything MAY change at any time. The public API SHOULD NOT be considered stable. MINOR version when make incompatible API changes before 1.0.0. ## License This project is licensed under the [MIT license](LICENSE). ### Third party software This product includes copies and modifications of software developed by third parties: * [src/encode.rs](src/encode.rs) is based on [stdsimd](https://github.com/rust-lang-nursery/stdsimd), licensed under the MIT license or the Apache License (Version 2.0). * [src/decode.rs](src/decode.rs) avx2 decode is modified from [fast-hex](https://github.com/zbjornson/fast-hex) See the source code files for more details. Copies of third party licenses can be found in [LICENSE-THIRD-PARTY](LICENSE-THIRD-PARTY). faster-hex-0.9.0/proptest-regressions/.gitkeep000064400000000000000000000000001046102023000175400ustar 00000000000000faster-hex-0.9.0/src/decode.rs000064400000000000000000000505441046102023000142700ustar 00000000000000// avx2 decode modified from https://github.com/zbjornson/fast-hex/blob/master/src/hex.cc #[cfg(target_arch = "x86")] use core::arch::x86::*; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; use crate::error::Error; const NIL: u8 = u8::MAX; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] const T_MASK: i32 = 65535; const fn init_unhex_array(check_case: CheckCase) -> [u8; 256] { let mut arr = [0; 256]; let mut i = 0; while i < 256 { arr[i] = match i as u8 { b'0'..=b'9' => i as u8 - b'0', b'a'..=b'f' => match check_case { CheckCase::Lower | CheckCase::None => i as u8 - b'a' + 10, _ => NIL, }, b'A'..=b'F' => match check_case { CheckCase::Upper | CheckCase::None => i as u8 - b'A' + 10, _ => NIL, }, _ => NIL, }; i += 1; } arr } const fn init_unhex4_array(check_case: CheckCase) -> [u8; 256] { let unhex_arr = init_unhex_array(check_case); let mut unhex4_arr = [NIL; 256]; let mut i = 0; while i < 256 { if unhex_arr[i] != NIL { unhex4_arr[i] = unhex_arr[i] << 4; } i += 1; } unhex4_arr } // ASCII -> hex pub(crate) static UNHEX: [u8; 256] = init_unhex_array(CheckCase::None); // ASCII -> hex, lower case pub(crate) static UNHEX_LOWER: [u8; 256] = init_unhex_array(CheckCase::Lower); // ASCII -> hex, upper case pub(crate) static UNHEX_UPPER: [u8; 256] = init_unhex_array(CheckCase::Upper); // ASCII -> hex << 4 pub(crate) static UNHEX4: [u8; 256] = init_unhex4_array(CheckCase::None); const _0213: i32 = 0b11011000; // lower nibble #[inline] fn unhex_b(x: usize) -> u8 { UNHEX[x] } // upper nibble, logically equivalent to unhex_b(x) << 4 #[inline] fn unhex_a(x: usize) -> u8 { UNHEX4[x] } #[inline] #[target_feature(enable = "avx2")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] unsafe fn unhex_avx2(value: __m256i) -> __m256i { let sr6 = _mm256_srai_epi16(value, 6); let and15 = _mm256_and_si256(value, _mm256_set1_epi16(0xf)); let mul = _mm256_maddubs_epi16(sr6, _mm256_set1_epi16(9)); _mm256_add_epi16(mul, and15) } // (a << 4) | b; #[inline] #[target_feature(enable = "avx2")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] unsafe fn nib2byte_avx2(a1: __m256i, b1: __m256i, a2: __m256i, b2: __m256i) -> __m256i { let a4_1 = _mm256_slli_epi16(a1, 4); let a4_2 = _mm256_slli_epi16(a2, 4); let a4orb_1 = _mm256_or_si256(a4_1, b1); let a4orb_2 = _mm256_or_si256(a4_2, b2); let pck1 = _mm256_packus_epi16(a4orb_1, a4orb_2); _mm256_permute4x64_epi64(pck1, _0213) } /// Check if the input is valid hex bytes slice pub fn hex_check(src: &[u8]) -> bool { hex_check_with_case(src, CheckCase::None) } /// Check if the input is valid hex bytes slice with case check pub fn hex_check_with_case(src: &[u8], check_case: CheckCase) -> bool { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { match crate::vectorization_support() { crate::Vectorization::AVX2 | crate::Vectorization::SSE41 => unsafe { hex_check_sse_with_case(src, check_case) }, crate::Vectorization::None => hex_check_fallback_with_case(src, check_case), } } #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] hex_check_fallback_with_case(src, check_case) } /// Check if the input is valid hex bytes slice pub fn hex_check_fallback(src: &[u8]) -> bool { hex_check_fallback_with_case(src, CheckCase::None) } /// Check if the input is valid hex bytes slice with case check pub fn hex_check_fallback_with_case(src: &[u8], check_case: CheckCase) -> bool { match check_case { CheckCase::None => src.iter().all(|&x| UNHEX[x as usize] != NIL), CheckCase::Lower => src.iter().all(|&x| UNHEX_LOWER[x as usize] != NIL), CheckCase::Upper => src.iter().all(|&x| UNHEX_UPPER[x as usize] != NIL), } } /// # Safety /// Check if a byte slice is valid. #[target_feature(enable = "sse4.1")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub unsafe fn hex_check_sse(src: &[u8]) -> bool { hex_check_sse_with_case(src, CheckCase::None) } #[derive(Eq, PartialEq)] pub enum CheckCase { None, Lower, Upper, } /// # Safety /// Check if a byte slice is valid on given check_case. #[target_feature(enable = "sse4.1")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub unsafe fn hex_check_sse_with_case(mut src: &[u8], check_case: CheckCase) -> bool { let ascii_zero = _mm_set1_epi8((b'0' - 1) as i8); let ascii_nine = _mm_set1_epi8((b'9' + 1) as i8); let ascii_ua = _mm_set1_epi8((b'A' - 1) as i8); let ascii_uf = _mm_set1_epi8((b'F' + 1) as i8); let ascii_la = _mm_set1_epi8((b'a' - 1) as i8); let ascii_lf = _mm_set1_epi8((b'f' + 1) as i8); while src.len() >= 16 { let unchecked = _mm_loadu_si128(src.as_ptr() as *const _); let gt0 = _mm_cmpgt_epi8(unchecked, ascii_zero); let lt9 = _mm_cmplt_epi8(unchecked, ascii_nine); let valid_digit = _mm_and_si128(gt0, lt9); let (valid_la_lf, valid_ua_uf) = match check_case { CheckCase::None => { let gtua = _mm_cmpgt_epi8(unchecked, ascii_ua); let ltuf = _mm_cmplt_epi8(unchecked, ascii_uf); let gtla = _mm_cmpgt_epi8(unchecked, ascii_la); let ltlf = _mm_cmplt_epi8(unchecked, ascii_lf); ( Some(_mm_and_si128(gtla, ltlf)), Some(_mm_and_si128(gtua, ltuf)), ) } CheckCase::Lower => { let gtla = _mm_cmpgt_epi8(unchecked, ascii_la); let ltlf = _mm_cmplt_epi8(unchecked, ascii_lf); (Some(_mm_and_si128(gtla, ltlf)), None) } CheckCase::Upper => { let gtua = _mm_cmpgt_epi8(unchecked, ascii_ua); let ltuf = _mm_cmplt_epi8(unchecked, ascii_uf); (None, Some(_mm_and_si128(gtua, ltuf))) } }; let valid_letter = match (valid_la_lf, valid_ua_uf) { (Some(valid_lower), Some(valid_upper)) => _mm_or_si128(valid_lower, valid_upper), (Some(valid_lower), None) => valid_lower, (None, Some(valid_upper)) => valid_upper, _ => unreachable!(), }; let ret = _mm_movemask_epi8(_mm_or_si128(valid_digit, valid_letter)); if ret != T_MASK { return false; } src = &src[16..]; } hex_check_fallback_with_case(src, check_case) } /// Hex decode src into dst. /// The length of src must be even, and it's allowed to decode a zero length src. /// The length of dst must be src.len() / 2. pub fn hex_decode(src: &[u8], dst: &mut [u8]) -> Result<(), Error> { hex_decode_with_case(src, dst, CheckCase::None) } /// Hex decode src into dst. /// The length of src must be even, and it's allowed to decode a zero length src. /// The length of dst must be src.len() / 2. /// when check_case is CheckCase::Lower, the hex string must be lower case. /// when check_case is CheckCase::Upper, the hex string must be upper case. /// when check_case is CheckCase::None, the hex string can be lower case or upper case. pub fn hex_decode_with_case( src: &[u8], dst: &mut [u8], check_case: CheckCase, ) -> Result<(), Error> { let len = dst.len().checked_mul(2).ok_or(Error::Overflow)?; if src.len() < len || ((src.len() & 1) != 0) { return Err(Error::InvalidLength(len)); } if !hex_check_with_case(src, check_case) { return Err(Error::InvalidChar); } hex_decode_unchecked(src, dst); Ok(()) } pub fn hex_decode_unchecked(src: &[u8], dst: &mut [u8]) { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { match crate::vectorization_support() { crate::Vectorization::AVX2 => unsafe { hex_decode_avx2(src, dst) }, crate::Vectorization::None | crate::Vectorization::SSE41 => { hex_decode_fallback(src, dst) } } } #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] hex_decode_fallback(src, dst); } #[target_feature(enable = "avx2")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] unsafe fn hex_decode_avx2(mut src: &[u8], mut dst: &mut [u8]) { // 0, -1, 2, -1, 4, -1, 6, -1, 8, -1, 10, -1, 12, -1, 14, -1, // 0, -1, 2, -1, 4, -1, 6, -1, 8, -1, 10, -1, 12, -1, 14, -1 let mask_a = _mm256_setr_epi8( 0, -1, 2, -1, 4, -1, 6, -1, 8, -1, 10, -1, 12, -1, 14, -1, 0, -1, 2, -1, 4, -1, 6, -1, 8, -1, 10, -1, 12, -1, 14, -1, ); // 1, -1, 3, -1, 5, -1, 7, -1, 9, -1, 11, -1, 13, -1, 15, -1, // 1, -1, 3, -1, 5, -1, 7, -1, 9, -1, 11, -1, 13, -1, 15, -1 let mask_b = _mm256_setr_epi8( 1, -1, 3, -1, 5, -1, 7, -1, 9, -1, 11, -1, 13, -1, 15, -1, 1, -1, 3, -1, 5, -1, 7, -1, 9, -1, 11, -1, 13, -1, 15, -1, ); while dst.len() >= 32 { let av1 = _mm256_loadu_si256(src.as_ptr() as *const _); let av2 = _mm256_loadu_si256(src[32..].as_ptr() as *const _); let mut a1 = _mm256_shuffle_epi8(av1, mask_a); let mut b1 = _mm256_shuffle_epi8(av1, mask_b); let mut a2 = _mm256_shuffle_epi8(av2, mask_a); let mut b2 = _mm256_shuffle_epi8(av2, mask_b); a1 = unhex_avx2(a1); a2 = unhex_avx2(a2); b1 = unhex_avx2(b1); b2 = unhex_avx2(b2); let bytes = nib2byte_avx2(a1, b1, a2, b2); //dst does not need to be aligned on any particular boundary _mm256_storeu_si256(dst.as_mut_ptr() as *mut _, bytes); dst = &mut dst[32..]; src = &src[64..]; } hex_decode_fallback(src, dst) } pub fn hex_decode_fallback(src: &[u8], dst: &mut [u8]) { for (slot, bytes) in dst.iter_mut().zip(src.chunks_exact(2)) { let a = unhex_a(bytes[0] as usize); let b = unhex_b(bytes[1] as usize); *slot = a | b; } } #[cfg(test)] mod tests { use crate::decode::NIL; use crate::{ decode::{ hex_check_fallback, hex_check_fallback_with_case, hex_decode_fallback, CheckCase, }, encode::hex_string, }; use proptest::proptest; fn _test_decode_fallback(s: &String) { let len = s.as_bytes().len(); let mut dst = Vec::with_capacity(len); dst.resize(len, 0); let hex_string = hex_string(s.as_bytes()); hex_decode_fallback(hex_string.as_bytes(), &mut dst); assert_eq!(&dst[..], s.as_bytes()); } proptest! { #[test] fn test_decode_fallback(ref s in ".+") { _test_decode_fallback(s); } } fn _test_check_fallback_true(s: &String) { assert!(hex_check_fallback(s.as_bytes())); match ( s.contains(char::is_lowercase), s.contains(char::is_uppercase), ) { (true, true) => { assert!(!hex_check_fallback_with_case( s.as_bytes(), CheckCase::Lower )); assert!(!hex_check_fallback_with_case( s.as_bytes(), CheckCase::Upper )); } (true, false) => { assert!(hex_check_fallback_with_case(s.as_bytes(), CheckCase::Lower)); assert!(!hex_check_fallback_with_case( s.as_bytes(), CheckCase::Upper )); } (false, true) => { assert!(!hex_check_fallback_with_case( s.as_bytes(), CheckCase::Lower )); assert!(hex_check_fallback_with_case(s.as_bytes(), CheckCase::Upper)); } (false, false) => { assert!(hex_check_fallback_with_case(s.as_bytes(), CheckCase::Lower)); assert!(hex_check_fallback_with_case(s.as_bytes(), CheckCase::Upper)); } } } proptest! { #[test] fn test_check_fallback_true(ref s in "[0-9a-fA-F]+") { _test_check_fallback_true(s); } } fn _test_check_fallback_false(s: &String) { assert!(!hex_check_fallback(s.as_bytes())); assert!(!hex_check_fallback_with_case( s.as_bytes(), CheckCase::Upper )); assert!(!hex_check_fallback_with_case( s.as_bytes(), CheckCase::Lower )); } proptest! { #[test] fn test_check_fallback_false(ref s in ".{16}[^0-9a-fA-F]+") { _test_check_fallback_false(s); } } #[test] fn test_init_static_array_is_right() { static OLD_UNHEX: [u8; 256] = [ NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NIL, NIL, NIL, NIL, NIL, NIL, NIL, 10, 11, 12, 13, 14, 15, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, 10, 11, 12, 13, 14, 15, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, ]; static OLD_UNHEX4: [u8; 256] = [ NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, NIL, NIL, NIL, NIL, NIL, NIL, NIL, 160, 176, 192, 208, 224, 240, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, 160, 176, 192, 208, 224, 240, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, NIL, ]; assert_eq!(OLD_UNHEX, crate::decode::UNHEX); assert_eq!(OLD_UNHEX4, crate::decode::UNHEX4); } } #[cfg(all(test, any(target_arch = "x86", target_arch = "x86_64")))] mod test_sse { use crate::decode::{ hex_check, hex_check_fallback, hex_check_fallback_with_case, hex_check_sse, hex_check_sse_with_case, hex_check_with_case, hex_decode, hex_decode_unchecked, hex_decode_with_case, CheckCase, }; use proptest::proptest; fn _test_check_sse_with_case(s: &String, check_case: CheckCase, expect_result: bool) { if is_x86_feature_detected!("sse4.1") { assert_eq!( unsafe { hex_check_sse_with_case(s.as_bytes(), check_case) }, expect_result ) } } fn _test_check_sse_true(s: &String) { if is_x86_feature_detected!("sse4.1") { assert!(unsafe { hex_check_sse(s.as_bytes()) }); } } proptest! { #[test] fn test_check_sse_true(ref s in "([0-9a-fA-F][0-9a-fA-F])+") { _test_check_sse_true(s); _test_check_sse_with_case(s, CheckCase::None, true); match (s.contains(char::is_lowercase), s.contains(char::is_uppercase)){ (true, true) => { _test_check_sse_with_case(s, CheckCase::Lower, false); _test_check_sse_with_case(s, CheckCase::Upper, false); }, (true, false) => { _test_check_sse_with_case(s, CheckCase::Lower, true); _test_check_sse_with_case(s, CheckCase::Upper, false); }, (false, true) => { _test_check_sse_with_case(s, CheckCase::Lower, false); _test_check_sse_with_case(s, CheckCase::Upper, true); }, (false, false) => { _test_check_sse_with_case(s, CheckCase::Lower, true); _test_check_sse_with_case(s, CheckCase::Upper, true); } } } } fn _test_check_sse_false(s: &String) { if is_x86_feature_detected!("sse4.1") { assert!(!unsafe { hex_check_sse(s.as_bytes()) }); } } proptest! { #[test] fn test_check_sse_false(ref s in ".{16}[^0-9a-fA-F]+") { _test_check_sse_false(s); _test_check_sse_with_case(s, CheckCase::None, false); _test_check_sse_with_case(s, CheckCase::Lower, false); _test_check_sse_with_case(s, CheckCase::Upper, false); } } #[test] fn test_decode_zero_length_src_should_not_be_ok() { let src = b""; let mut dst = [0u8; 10]; assert!( matches!(hex_decode(src, &mut dst), Err(crate::Error::InvalidLength(len)) if len == 20) ); assert!( matches!(hex_decode_with_case(src, &mut dst, CheckCase::None), Err(crate::Error::InvalidLength(len)) if len == 20) ); assert!(hex_check(src)); assert!(hex_check_with_case(src, CheckCase::None)); assert!(hex_check_fallback(src)); assert!(hex_check_fallback_with_case(src, CheckCase::None)); if is_x86_feature_detected!("sse4.1") { assert!(unsafe { hex_check_sse_with_case(src, CheckCase::None) }); assert!(unsafe { hex_check_sse(src) }); } // this function have no return value, so we just execute it and expect no panic hex_decode_unchecked(src, &mut dst); } // If `dst's length` is greater than `src's length * 2`, `hex_decode` should return error #[test] fn test_if_dst_len_gt_expect_len_should_return_error() { let short_str = b"8e40af02265360d59f4ecf9ae9ebf8f00a3118408f5a9cdcbcc9c0f93642f3"; // 62 bytes { let mut dst = [0u8; 31]; let result = hex_decode(short_str.as_slice(), &mut dst); assert!(result.is_ok()); } { let mut dst = [0u8; 32]; let result = hex_decode(short_str.as_slice(), &mut dst); assert!(matches!(result, Err(crate::Error::InvalidLength(len)) if len == 64)) } { let mut dst = [0u8; 33]; let result = hex_decode(short_str.as_slice(), &mut dst); assert!(matches!(result, Err(crate::Error::InvalidLength(len)) if len == 66)) } } // if both `src` and `dst` are empty, it's ok // if `src` is empty, but `dst` is not empty, it should be reported as error #[test] fn test_decode_zero_src() { let zero_src = b""; { let mut zero_dst = []; assert!(hex_decode(zero_src, &mut zero_dst).is_ok()); } { let mut non_zero_dst = [0u8; 1]; assert!( matches!(hex_decode(zero_src, &mut non_zero_dst), Err(crate::Error::InvalidLength(len)) if len == 2) ); } } } faster-hex-0.9.0/src/encode.rs000064400000000000000000000206211046102023000142730ustar 00000000000000#[cfg(target_arch = "x86")] use core::arch::x86::*; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; #[cfg(feature = "alloc")] use alloc::{string::String, vec}; use crate::error::Error; static TABLE_LOWER: &[u8] = b"0123456789abcdef"; static TABLE_UPPER: &[u8] = b"0123456789ABCDEF"; #[cfg(any(feature = "alloc", test))] fn hex_string_custom_case(src: &[u8], upper_case: bool) -> String { let mut buffer = vec![0; src.len() * 2]; if upper_case { hex_encode_upper(src, &mut buffer).expect("hex_string"); } else { hex_encode(src, &mut buffer).expect("hex_string"); } if cfg!(debug_assertions) { String::from_utf8(buffer).unwrap() } else { // Saftey: We just wrote valid utf8 hex string into the dst unsafe { String::from_utf8_unchecked(buffer) } } } #[cfg(any(feature = "alloc", test))] pub fn hex_string(src: &[u8]) -> String { hex_string_custom_case(src, false) } #[cfg(any(feature = "alloc", test))] pub fn hex_string_upper(src: &[u8]) -> String { hex_string_custom_case(src, true) } pub fn hex_encode_custom<'a>( src: &[u8], dst: &'a mut [u8], upper_case: bool, ) -> Result<&'a mut str, Error> { unsafe fn mut_str(buffer: &mut [u8]) -> &mut str { if cfg!(debug_assertions) { core::str::from_utf8_mut(buffer).unwrap() } else { core::str::from_utf8_unchecked_mut(buffer) } } let expect_dst_len = src .len() .checked_mul(2) .ok_or(Error::InvalidLength(src.len()))?; if dst.len() < expect_dst_len { return Err(Error::InvalidLength(expect_dst_len)); } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { match crate::vectorization_support() { crate::Vectorization::AVX2 => unsafe { hex_encode_avx2(src, dst, upper_case) }, crate::Vectorization::SSE41 => unsafe { hex_encode_sse41(src, dst, upper_case) }, crate::Vectorization::None => hex_encode_custom_case_fallback(src, dst, upper_case), } // Safety: We just wrote valid utf8 hex string into the dst return Ok(unsafe { mut_str(dst) }); } #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] { hex_encode_custom_case_fallback(src, dst, upper_case); // Saftey: We just wrote valid utf8 hex string into the dst Ok(unsafe { mut_str(dst) }) } } /// Hex encode src into dst. /// The length of dst must be at least src.len() * 2. pub fn hex_encode<'a>(src: &[u8], dst: &'a mut [u8]) -> Result<&'a mut str, Error> { hex_encode_custom(src, dst, false) } pub fn hex_encode_upper<'a>(src: &[u8], dst: &'a mut [u8]) -> Result<&'a mut str, Error> { hex_encode_custom(src, dst, true) } #[deprecated(since = "0.3.0", note = "please use `hex_encode` instead")] pub fn hex_to(src: &[u8], dst: &mut [u8]) -> Result<(), Error> { hex_encode(src, dst).map(|_| ()) } #[target_feature(enable = "avx2")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] unsafe fn hex_encode_avx2(mut src: &[u8], dst: &mut [u8], upper_case: bool) { let ascii_zero = _mm256_set1_epi8(b'0' as i8); let nines = _mm256_set1_epi8(9); let ascii_a = if upper_case { _mm256_set1_epi8((b'A' - 9 - 1) as i8) } else { _mm256_set1_epi8((b'a' - 9 - 1) as i8) }; let and4bits = _mm256_set1_epi8(0xf); let mut i = 0_isize; while src.len() >= 32 { // https://stackoverflow.com/questions/47425851/whats-the-difference-between-mm256-lddqu-si256-and-mm256-loadu-si256 let invec = _mm256_loadu_si256(src.as_ptr() as *const _); let masked1 = _mm256_and_si256(invec, and4bits); let masked2 = _mm256_and_si256(_mm256_srli_epi64(invec, 4), and4bits); // return 0xff corresponding to the elements > 9, or 0x00 otherwise let cmpmask1 = _mm256_cmpgt_epi8(masked1, nines); let cmpmask2 = _mm256_cmpgt_epi8(masked2, nines); // add '0' or the offset depending on the masks let masked1 = _mm256_add_epi8(masked1, _mm256_blendv_epi8(ascii_zero, ascii_a, cmpmask1)); let masked2 = _mm256_add_epi8(masked2, _mm256_blendv_epi8(ascii_zero, ascii_a, cmpmask2)); // interleave masked1 and masked2 bytes let res1 = _mm256_unpacklo_epi8(masked2, masked1); let res2 = _mm256_unpackhi_epi8(masked2, masked1); // Store everything into the right destination now let base = dst.as_mut_ptr().offset(i * 2); let base1 = base.offset(0) as *mut _; let base2 = base.offset(16) as *mut _; let base3 = base.offset(32) as *mut _; let base4 = base.offset(48) as *mut _; _mm256_storeu2_m128i(base3, base1, res1); _mm256_storeu2_m128i(base4, base2, res2); src = &src[32..]; i += 32; } let i = i as usize; hex_encode_sse41(src, &mut dst[i * 2..], upper_case); } // copied from https://github.com/Matherunner/bin2hex-sse/blob/master/base16_sse4.cpp #[target_feature(enable = "sse4.1")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] unsafe fn hex_encode_sse41(mut src: &[u8], dst: &mut [u8], upper_case: bool) { let ascii_zero = _mm_set1_epi8(b'0' as i8); let nines = _mm_set1_epi8(9); let ascii_a = if upper_case { _mm_set1_epi8((b'A' - 9 - 1) as i8) } else { _mm_set1_epi8((b'a' - 9 - 1) as i8) }; let and4bits = _mm_set1_epi8(0xf); let mut i = 0_isize; while src.len() >= 16 { let invec = _mm_loadu_si128(src.as_ptr() as *const _); let masked1 = _mm_and_si128(invec, and4bits); let masked2 = _mm_and_si128(_mm_srli_epi64(invec, 4), and4bits); // return 0xff corresponding to the elements > 9, or 0x00 otherwise let cmpmask1 = _mm_cmpgt_epi8(masked1, nines); let cmpmask2 = _mm_cmpgt_epi8(masked2, nines); // add '0' or the offset depending on the masks let masked1 = _mm_add_epi8(masked1, _mm_blendv_epi8(ascii_zero, ascii_a, cmpmask1)); let masked2 = _mm_add_epi8(masked2, _mm_blendv_epi8(ascii_zero, ascii_a, cmpmask2)); // interleave masked1 and masked2 bytes let res1 = _mm_unpacklo_epi8(masked2, masked1); let res2 = _mm_unpackhi_epi8(masked2, masked1); _mm_storeu_si128(dst.as_mut_ptr().offset(i * 2) as *mut _, res1); _mm_storeu_si128(dst.as_mut_ptr().offset(i * 2 + 16) as *mut _, res2); src = &src[16..]; i += 16; } let i = i as usize; hex_encode_custom_case_fallback(src, &mut dst[i * 2..], upper_case); } #[inline] fn hex_lower(byte: u8) -> u8 { TABLE_LOWER[byte as usize] } #[inline] fn hex_upper(byte: u8) -> u8 { TABLE_UPPER[byte as usize] } fn hex_encode_custom_case_fallback(src: &[u8], dst: &mut [u8], upper_case: bool) { if upper_case { for (byte, slots) in src.iter().zip(dst.chunks_exact_mut(2)) { slots[0] = hex_upper((*byte >> 4) & 0xf); slots[1] = hex_upper(*byte & 0xf); } } else { for (byte, slots) in src.iter().zip(dst.chunks_exact_mut(2)) { slots[0] = hex_lower((*byte >> 4) & 0xf); slots[1] = hex_lower(*byte & 0xf); } } } pub fn hex_encode_fallback(src: &[u8], dst: &mut [u8]) { hex_encode_custom_case_fallback(src, dst, false) } pub fn hex_encode_upper_fallback(src: &[u8], dst: &mut [u8]) { hex_encode_custom_case_fallback(src, dst, true) } #[cfg(test)] mod tests { use crate::encode::{hex_encode, hex_encode_custom_case_fallback}; use crate::hex_encode_fallback; use core::str; use proptest::proptest; fn _test_encode_fallback(s: &String, upper_case: bool) { let mut buffer = vec![0; s.as_bytes().len() * 2]; hex_encode_custom_case_fallback(s.as_bytes(), &mut buffer, upper_case); let encode = unsafe { str::from_utf8_unchecked(&buffer[..s.as_bytes().len() * 2]) }; if upper_case { assert_eq!(encode, hex::encode_upper(s)); } else { assert_eq!(encode, hex::encode(s)); } } proptest! { #[test] fn test_encode_fallback(ref s in ".*") { _test_encode_fallback(s, true); _test_encode_fallback(s, false); } } #[test] fn test_encode_zero_length_src_should_be_ok() { let src = b""; let mut dst = [0u8; 10]; assert!(hex_encode(src, &mut dst).is_ok()); // this function have no return value, so we just execute it and expect no panic hex_encode_fallback(src, &mut dst); } } faster-hex-0.9.0/src/error.rs000064400000000000000000000016161046102023000141720ustar 00000000000000#[derive(Clone, Copy)] pub enum Error { InvalidChar, InvalidLength(usize), Overflow, } impl ::core::fmt::Debug for Error { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { match *self { Error::InvalidLength(len) => write!(f, "Invalid input length {len}"), Error::InvalidChar => write!(f, "Invalid character"), Error::Overflow => write!(f, "Overflow"), } } } impl ::core::fmt::Display for Error { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { ::core::fmt::Debug::fmt(&self, f) } } #[cfg(feature = "std")] impl ::std::error::Error for Error { fn description(&self) -> &str { match *self { Error::InvalidChar => "invalid character", Error::InvalidLength(_) => "invalid length", Error::Overflow => "overflow", } } } faster-hex-0.9.0/src/lib.rs000064400000000000000000000156531046102023000136150ustar 00000000000000#![cfg_attr(not(any(test, feature = "std")), no_std)] #[cfg(feature = "alloc")] extern crate alloc; mod decode; mod encode; mod error; #[cfg(feature = "serde")] mod serde; pub use crate::decode::{ hex_check, hex_check_fallback, hex_check_with_case, hex_decode, hex_decode_fallback, hex_decode_unchecked, }; pub use crate::encode::{ hex_encode, hex_encode_fallback, hex_encode_upper, hex_encode_upper_fallback, }; #[cfg(feature = "alloc")] pub use crate::encode::{hex_string, hex_string_upper}; pub use crate::error::Error; #[cfg(feature = "serde")] pub use crate::serde::{ deserialize, nopfx_ignorecase, nopfx_lowercase, nopfx_uppercase, serialize, withpfx_ignorecase, withpfx_lowercase, withpfx_uppercase, }; #[allow(deprecated)] pub use crate::encode::hex_to; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub use crate::decode::{hex_check_sse, hex_check_sse_with_case}; #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub(crate) enum Vectorization { None = 0, SSE41 = 1, AVX2 = 2, } #[inline(always)] pub(crate) fn vectorization_support() -> Vectorization { #[cfg(all(any(target_arch = "x86", target_arch = "x86_64")))] { use core::sync::atomic::{AtomicU8, Ordering}; static FLAGS: AtomicU8 = AtomicU8::new(u8::MAX); // We're OK with relaxed, worst case scenario multiple threads checked the CPUID. let current_flags = FLAGS.load(Ordering::Relaxed); // u8::MAX means uninitialized. if current_flags != u8::MAX { return match current_flags { 0 => Vectorization::None, 1 => Vectorization::SSE41, 2 => Vectorization::AVX2, _ => unreachable!(), }; } let val = vectorization_support_no_cache_x86(); FLAGS.store(val as u8, Ordering::Relaxed); return val; } #[allow(unreachable_code)] Vectorization::None } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[cold] fn vectorization_support_no_cache_x86() -> Vectorization { #[cfg(target_arch = "x86")] use core::arch::x86::__cpuid_count; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::__cpuid_count; // SGX doesn't support CPUID, // If there's no SSE there might not be CPUID and there's no SSE4.1/AVX2 if cfg!(target_env = "sgx") || !cfg!(target_feature = "sse") { return Vectorization::None; } let proc_info_ecx = unsafe { __cpuid_count(1, 0) }.ecx; let have_sse4 = (proc_info_ecx >> 19) & 1 == 1; // If there's no SSE4 there can't be AVX2. if !have_sse4 { return Vectorization::None; } let have_xsave = (proc_info_ecx >> 26) & 1 == 1; let have_osxsave = (proc_info_ecx >> 27) & 1 == 1; let have_avx = (proc_info_ecx >> 27) & 1 == 1; if have_xsave && have_osxsave && have_avx { // # Safety: We checked that the processor supports xsave if unsafe { avx2_support_no_cache_x86() } { return Vectorization::AVX2; } } Vectorization::SSE41 } // We enable xsave so it can inline the _xgetbv call. // # Safety: Safe as long it's only called when xsave is supported #[target_feature(enable = "xsave")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[cold] unsafe fn avx2_support_no_cache_x86() -> bool { #[cfg(target_arch = "x86")] use core::arch::x86::{__cpuid_count, _xgetbv}; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{__cpuid_count, _xgetbv}; let xcr0 = _xgetbv(0); let os_avx_support = xcr0 & 6 == 6; if os_avx_support { let extended_features_ebx = __cpuid_count(7, 0).ebx; let have_avx2 = (extended_features_ebx >> 5) & 1 == 1; if have_avx2 { return true; } } false } #[cfg(test)] mod tests { use crate::decode::{hex_decode, hex_decode_with_case, CheckCase}; use crate::encode::{hex_encode, hex_string}; use crate::{hex_encode_upper, hex_string_upper, vectorization_support, Vectorization}; use proptest::proptest; #[test] fn test_feature_detection() { let vector_support = vectorization_support(); #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { match vector_support { Vectorization::AVX2 => assert!(is_x86_feature_detected!("avx2")), Vectorization::SSE41 => assert!(is_x86_feature_detected!("sse4.1")), Vectorization::None => assert!( !cfg!(target_feature = "sse") || !is_x86_feature_detected!("avx2") && !is_x86_feature_detected!("sse4.1") ), } } #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] assert_eq!(vector_support, Vectorization::None); } fn _test_hex_encode(s: &String) { let mut buffer = vec![0; s.as_bytes().len() * 2]; { let encode = &*hex_encode(s.as_bytes(), &mut buffer).unwrap(); let hex_string = hex_string(s.as_bytes()); assert_eq!(encode, hex::encode(s)); assert_eq!(hex_string, hex::encode(s)); } { let encode_upper = &*hex_encode_upper(s.as_bytes(), &mut buffer).unwrap(); let hex_string_upper = hex_string_upper(s.as_bytes()); assert_eq!(encode_upper, hex::encode_upper(s)); assert_eq!(hex_string_upper, hex::encode_upper(s)); } } proptest! { #[test] fn test_hex_encode(ref s in ".*") { _test_hex_encode(s); } } fn _test_hex_decode(s: &String) { let len = s.as_bytes().len(); { let mut dst = Vec::with_capacity(len); dst.resize(len, 0); let hex_string = hex_string(s.as_bytes()); hex_decode(hex_string.as_bytes(), &mut dst).unwrap(); hex_decode_with_case(hex_string.as_bytes(), &mut dst, CheckCase::Lower).unwrap(); assert_eq!(&dst[..], s.as_bytes()); } { let mut dst = Vec::with_capacity(len); dst.resize(len, 0); let hex_string_upper = hex_string_upper(s.as_bytes()); hex_decode_with_case(hex_string_upper.as_bytes(), &mut dst, CheckCase::Upper).unwrap(); assert_eq!(&dst[..], s.as_bytes()); } } proptest! { #[test] fn test_hex_decode(ref s in ".+") { _test_hex_decode(s); } } fn _test_hex_decode_check(s: &String, ok: bool) { let len = s.as_bytes().len(); let mut dst = Vec::with_capacity(len / 2); dst.resize(len / 2, 0); assert!(hex_decode(s.as_bytes(), &mut dst).is_ok() == ok); } proptest! { #[test] fn test_hex_decode_check(ref s in "([0-9a-fA-F][0-9a-fA-F])+") { _test_hex_decode_check(s, true); } } proptest! { #[test] fn test_hex_decode_check_odd(ref s in "[0-9a-fA-F]{11}") { _test_hex_decode_check(s, false); } } } faster-hex-0.9.0/src/serde.rs000064400000000000000000000322561046102023000141470ustar 00000000000000#![warn(missing_docs)] use std::iter::FromIterator; mod internal { use crate::{ decode::{hex_decode_with_case, CheckCase}, encode::hex_encode_custom, }; use alloc::borrow::Cow; use serde::{de::Error, Deserializer, Serializer}; use std::iter::FromIterator; pub(crate) fn serialize( data: T, serializer: S, with_prefix: bool, case: CheckCase, ) -> Result where S: Serializer, T: AsRef<[u8]>, { let src: &[u8] = data.as_ref(); let mut dst_length = data.as_ref().len() << 1; if with_prefix { dst_length += 2; } let mut dst = vec![0u8; dst_length]; let mut dst_start = 0; if with_prefix { dst[0] = b'0'; dst[1] = b'x'; dst_start = 2; } hex_encode_custom(src, &mut dst[dst_start..], matches!(case, CheckCase::Upper)) .map_err(serde::ser::Error::custom)?; serializer.serialize_str(unsafe { ::std::str::from_utf8_unchecked(&dst) }) } pub(crate) fn deserialize<'de, D, T>( deserializer: D, with_prefix: bool, check_case: CheckCase, ) -> Result where D: Deserializer<'de>, T: FromIterator, { let raw_src: Cow = serde::Deserialize::deserialize(deserializer)?; if with_prefix && !raw_src.starts_with("0x") { return Err(D::Error::custom("invalid prefix".to_string())); } let src: &[u8] = { if with_prefix { raw_src[2..].as_bytes() } else { raw_src.as_bytes() } }; if src.len() & 1 != 0 { return Err(D::Error::custom("invalid length".to_string())); } // we have already checked src's length, so src's length is a even integer let mut dst = vec![0; src.len() >> 1]; hex_decode_with_case(src, &mut dst, check_case) .map_err(|e| Error::custom(format!("{:?}", e)))?; Ok(dst.into_iter().collect()) } } /// Serde: Serialize with 0x-prefix and ignore case pub fn serialize(data: T, serializer: S) -> Result where S: serde::Serializer, T: AsRef<[u8]>, { withpfx_ignorecase::serialize(data, serializer) } /// Serde: Deserialize with 0x-prefix and ignore case pub fn deserialize<'de, D, T>(deserializer: D) -> Result where D: serde::Deserializer<'de>, T: FromIterator, { withpfx_ignorecase::deserialize(deserializer) } /// Generate module with serde methods macro_rules! faster_hex_serde_macros { ($mod_name:ident, $with_pfx:expr, $check_case:expr) => { /// Serialize and deserialize with or without 0x-prefix, /// and lowercase or uppercase or ignorecase pub mod $mod_name { use crate::decode::CheckCase; use crate::serde::internal; use std::iter::FromIterator; /// Serializes `data` as hex string pub fn serialize(data: T, serializer: S) -> Result where S: serde::Serializer, T: AsRef<[u8]>, { internal::serialize(data, serializer, $with_pfx, $check_case) } /// Deserializes a hex string into raw bytes. pub fn deserialize<'de, D, T>(deserializer: D) -> Result where D: serde::Deserializer<'de>, T: FromIterator, { internal::deserialize(deserializer, $with_pfx, $check_case) } } }; } // /// Serialize with 0x-prefix and lowercase // /// When deserialize, expect 0x-prefix and don't care case faster_hex_serde_macros!(withpfx_ignorecase, true, CheckCase::None); // /// Serialize without 0x-prefix and lowercase // /// When deserialize, expect without 0x-prefix and don't care case faster_hex_serde_macros!(nopfx_ignorecase, false, CheckCase::None); // /// Serialize with 0x-prefix and lowercase // /// When deserialize, expect with 0x-prefix and lower case faster_hex_serde_macros!(withpfx_lowercase, true, CheckCase::Lower); // /// Serialize without 0x-prefix and lowercase // /// When deserialize, expect without 0x-prefix and lower case faster_hex_serde_macros!(nopfx_lowercase, false, CheckCase::Lower); // /// Serialize with 0x-prefix and upper case // /// When deserialize, expect with 0x-prefix and upper case faster_hex_serde_macros!(withpfx_uppercase, true, CheckCase::Upper); // /// Serialize without 0x-prefix and upper case // /// When deserialize, expect without 0x-prefix and upper case faster_hex_serde_macros!(nopfx_uppercase, false, CheckCase::Upper); #[cfg(test)] mod tests { use super::{ nopfx_ignorecase, nopfx_lowercase, nopfx_uppercase, withpfx_ignorecase, withpfx_lowercase, withpfx_uppercase, }; use crate as faster_hex; use bytes::Bytes; use proptest::proptest; use serde::{Deserialize, Serialize}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct Simple { #[serde(with = "faster_hex")] bar: Vec, } #[test] fn test_deserialize_escaped() { // 0x03 but escaped. let x: Simple = serde_json::from_str( r#"{ "bar": "\u0030x\u00303" }"#, ) .unwrap(); assert_eq!(x.bar, b"\x03"); } fn _test_simple(src: &str) { let simple = Simple { bar: src.into() }; let result = serde_json::to_string(&simple); assert!(result.is_ok()); let result = result.unwrap(); // #[serde(with = "faster_hex")] should result with 0x prefix assert!(result.starts_with(r#"{"bar":"0x"#)); // #[serde(with = "faster_hex")] shouldn't contains uppercase assert!(result[7..].chars().all(|c| !c.is_uppercase())); let decode_simple = serde_json::from_str::(&result); assert!(decode_simple.is_ok()); assert_eq!(decode_simple.unwrap(), simple); } proptest! { #[test] fn test_simple(ref s in ".*") { _test_simple(s); } } #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct Foo { #[serde(with = "nopfx_lowercase")] bar_nopfx_lowercase_vec: Vec, #[serde(with = "nopfx_lowercase")] bar_nopfx_lowercase_bytes: Bytes, #[serde(with = "withpfx_lowercase")] bar_withpfx_lowercase_vec: Vec, #[serde(with = "withpfx_lowercase")] bar_withpfx_lowercase_bytes: Bytes, #[serde(with = "nopfx_uppercase")] bar_nopfx_uppercase_vec: Vec, #[serde(with = "nopfx_uppercase")] bar_nopfx_uppercase_bytes: Bytes, #[serde(with = "withpfx_uppercase")] bar_withpfx_uppercase_vec: Vec, #[serde(with = "withpfx_uppercase")] bar_withpfx_uppercase_bytes: Bytes, #[serde(with = "withpfx_ignorecase")] bar_withpfx_ignorecase_vec: Vec, #[serde(with = "withpfx_ignorecase")] bar_withpfx_ignorecase_bytes: Bytes, #[serde(with = "nopfx_ignorecase")] bar_nopfx_ignorecase_vec: Vec, #[serde(with = "nopfx_ignorecase")] bar_nopfx_ignorecase_bytes: Bytes, } #[test] fn test_serde_default() { { let foo_defuault = Foo { bar_nopfx_lowercase_vec: vec![], bar_nopfx_lowercase_bytes: Default::default(), bar_withpfx_lowercase_vec: vec![], bar_withpfx_lowercase_bytes: Default::default(), bar_nopfx_uppercase_vec: vec![], bar_nopfx_uppercase_bytes: Default::default(), bar_withpfx_uppercase_vec: vec![], bar_withpfx_uppercase_bytes: Default::default(), bar_withpfx_ignorecase_vec: vec![], bar_withpfx_ignorecase_bytes: Default::default(), bar_nopfx_ignorecase_vec: vec![], bar_nopfx_ignorecase_bytes: Default::default(), }; let serde_result = serde_json::to_string(&foo_defuault).unwrap(); let expect = "{\"bar_nopfx_lowercase_vec\":\"\",\"bar_nopfx_lowercase_bytes\":\"\",\"bar_withpfx_lowercase_vec\":\"0x\",\"bar_withpfx_lowercase_bytes\":\"0x\",\"bar_nopfx_uppercase_vec\":\"\",\"bar_nopfx_uppercase_bytes\":\"\",\"bar_withpfx_uppercase_vec\":\"0x\",\"bar_withpfx_uppercase_bytes\":\"0x\",\"bar_withpfx_ignorecase_vec\":\"0x\",\"bar_withpfx_ignorecase_bytes\":\"0x\",\"bar_nopfx_ignorecase_vec\":\"\",\"bar_nopfx_ignorecase_bytes\":\"\"}"; assert_eq!(serde_result, expect); let foo_src: Foo = serde_json::from_str(&serde_result).unwrap(); assert_eq!(foo_defuault, foo_src); } } fn _test_serde(src: &str) { let foo = Foo { bar_nopfx_lowercase_vec: Vec::from(src), bar_nopfx_lowercase_bytes: Bytes::from(Vec::from(src)), bar_withpfx_lowercase_vec: Vec::from(src), bar_withpfx_lowercase_bytes: Bytes::from(Vec::from(src)), bar_nopfx_uppercase_vec: Vec::from(src), bar_nopfx_uppercase_bytes: Bytes::from(Vec::from(src)), bar_withpfx_uppercase_vec: Vec::from(src), bar_withpfx_uppercase_bytes: Bytes::from(Vec::from(src)), bar_withpfx_ignorecase_vec: Vec::from(src), bar_withpfx_ignorecase_bytes: Bytes::from(Vec::from(src)), bar_nopfx_ignorecase_vec: Vec::from(src), bar_nopfx_ignorecase_bytes: Bytes::from(Vec::from(src)), }; let hex_str = hex::encode(src); let hex_str_upper = hex::encode_upper(src); let serde_result = serde_json::to_string(&foo).unwrap(); let expect = format!("{{\"bar_nopfx_lowercase_vec\":\"{}\",\"bar_nopfx_lowercase_bytes\":\"{}\",\"bar_withpfx_lowercase_vec\":\"0x{}\",\"bar_withpfx_lowercase_bytes\":\"0x{}\",\"bar_nopfx_uppercase_vec\":\"{}\",\"bar_nopfx_uppercase_bytes\":\"{}\",\"bar_withpfx_uppercase_vec\":\"0x{}\",\"bar_withpfx_uppercase_bytes\":\"0x{}\",\"bar_withpfx_ignorecase_vec\":\"0x{}\",\"bar_withpfx_ignorecase_bytes\":\"0x{}\",\"bar_nopfx_ignorecase_vec\":\"{}\",\"bar_nopfx_ignorecase_bytes\":\"{}\"}}", hex_str, hex_str, hex_str, hex_str, hex_str_upper, hex_str_upper, hex_str_upper, hex_str_upper, hex_str, hex_str, hex_str, hex_str, ); assert_eq!(serde_result, expect); let foo_src: Foo = serde_json::from_str(&serde_result).unwrap(); assert_eq!(foo, foo_src); } proptest! { #[test] fn test_serde(ref s in ".*") { _test_serde(s); } } fn _test_serde_deserialize(src: &str) { #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct FooNoPfxLower { #[serde(with = "nopfx_lowercase")] bar: Vec, } #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct FooWithPfxLower { #[serde(with = "withpfx_lowercase")] bar: Vec, } #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct FooNoPfxUpper { #[serde(with = "nopfx_uppercase")] bar: Vec, } #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct FooWithPfxUpper { #[serde(with = "withpfx_uppercase")] bar: Vec, } #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct FooNoPfxIgnoreCase { #[serde(with = "nopfx_ignorecase")] bar: Vec, } #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct FooWithPfxIgnoreCase { #[serde(with = "withpfx_ignorecase")] bar: Vec, } { let hex_foo = serde_json::to_string(&FooNoPfxLower { bar: src.into() }).unwrap(); let foo_pfx: serde_json::Result = serde_json::from_str(&hex_foo); // assert foo_pfx is Error, and contains "invalid prefix" assert!(foo_pfx.is_err()); assert!(foo_pfx.unwrap_err().to_string().contains("invalid prefix")); } { let foo_lower = serde_json::to_string(&FooNoPfxLower { bar: src.into() }).unwrap(); let foo_upper_result: serde_json::Result = serde_json::from_str(&foo_lower); if hex::encode(src).contains(char::is_lowercase) { // FooNoPfxLower's foo field is lowercase, so we can't deserialize it to FooNoPfxUpper assert!(foo_upper_result.is_err()); assert!(foo_upper_result .unwrap_err() .to_string() .contains("Invalid character")); } } } proptest! { #[test] fn test_serde_deserialize(ref s in ".*") { _test_serde_deserialize(s); } } }