lz4-compress-0.1.1/Cargo.toml010064400017500000144000000022701310271312300142120ustar0000000000000000# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] name = "lz4-compress" version = "0.1.1" authors = ["ticki "] build = false exclude = [ "target", "Cargo.lock", ] autolib = false autobins = false autoexamples = false autotests = false autobenches = false description = "Pure Rust implementation of raw LZ4 compression/decompression." documentation = "https://docs.rs/lz4-compress" readme = false keywords = [ "compression", "lz4", "compress", "decompression", "decompress", ] license = "MIT" repository = "https://github.com/ticki/tfs" [lib] name = "lz4_compress" path = "src/lib.rs" [[bin]] name = "lz4-compress" path = "src/main.rs" [dependencies.byteorder] version = "0.5" [dependencies.quick-error] version = "1" lz4-compress-0.1.1/Cargo.toml.orig010064400017500000144000000007051310271312300151520ustar0000000000000000[package] name = "lz4-compress" version = "0.1.1" authors = ["ticki "] description = "Pure Rust implementation of raw LZ4 compression/decompression." repository = "https://github.com/ticki/tfs" documentation = "https://docs.rs/lz4-compress" license = "MIT" keywords = ["compression", "lz4", "compress", "decompression", "decompress"] exclude = ["target", "Cargo.lock"] [dependencies] byteorder = "0.5" quick-error = "1" lz4-compress-0.1.1/src/decompress.rs010064400017500000144000000254221303500522700155730ustar0000000000000000//! The decompression algorithm. use byteorder::{LittleEndian, ByteOrder}; quick_error! { /// An error representing invalid compressed data. #[derive(Debug)] pub enum Error { /// Expected another byte, but none found. ExpectedAnotherByte { description("Expected another byte, found none.") } /// Deduplication offset out of bounds (not in buffer). OffsetOutOfBounds { description("The offset to copy is not contained in the decompressed buffer.") } } } /// A LZ4 decoder. /// /// This will decode in accordance to the LZ4 format. It represents a particular state of the /// decompressor. struct Decoder<'a> { /// The compressed input. input: &'a [u8], /// The decompressed output. output: &'a mut Vec, /// The current block's "token". /// /// This token contains to 4-bit "fields", a higher and a lower, representing the literals' /// length and the back reference's length, respectively. LSIC is used if either are their /// maximal values. token: u8, } impl<'a> Decoder<'a> { /// Internal (partial) function for `take`. #[inline] fn take_imp(input: &mut &'a [u8], n: usize) -> Result<&'a [u8], Error> { // Check if we have enough bytes left. if input.len() < n { // No extra bytes. This is clearly not expected, so we return an error. Err(Error::ExpectedAnotherByte) } else { // Take the first n bytes. let res = Ok(&input[..n]); // Shift the stream to left, so that it is no longer the first byte. *input = &input[n..]; // Return the former first byte. res } } /// Pop n bytes from the start of the input stream. fn take(&mut self, n: usize) -> Result<&[u8], Error> { Self::take_imp(&mut self.input, n) } /// Write a buffer to the output stream. /// /// The reason this doesn't take `&mut self` is that we need partial borrowing due to the rules /// of the borrow checker. For this reason, we instead take some number of segregated /// references so we can read and write them independently. fn output(output: &mut Vec, buf: &[u8]) { // We use simple memcpy to extend the vector. output.extend_from_slice(&buf[..buf.len()]); } /// Write an already decompressed match to the output stream. /// /// This is used for the essential part of the algorithm: deduplication. We start at some /// position `start` and then keep pushing the following element until we've added /// `match_length` elements. fn duplicate(&mut self, start: usize, match_length: usize) { // We cannot simply use memcpy or `extend_from_slice`, because these do not allow // self-referential copies: http://ticki.github.io/img/lz4_runs_encoding_diagram.svg for i in start..start + match_length { let b = self.output[i]; self.output.push(b); } } /// Read an integer LSIC (linear small integer code) encoded. /// /// In LZ4, we encode small integers in a way that we can have an arbitrary number of bytes. In /// particular, we add the bytes repeatedly until we hit a non-0xFF byte. When we do, we add /// this byte to our sum and terminate the loop. /// /// # Example /// /// ```notest /// 255, 255, 255, 4, 2, 3, 4, 6, 7 /// ``` /// /// is encoded to _255 + 255 + 255 + 4 = 769_. The bytes after the first 4 is ignored, because /// 4 is the first non-0xFF byte. #[inline] fn read_integer(&mut self) -> Result { // We start at zero and count upwards. let mut n = 0; // If this byte takes value 255 (the maximum value it can take), another byte is read // and added to the sum. This repeats until a byte lower than 255 is read. while { // We add the next byte until we get a byte which we add to the counting variable. let extra = self.take(1)?[0]; n += extra as usize; // We continue if we got 255. extra == 0xFF } {} Ok(n) } /// Read a little-endian 16-bit integer from the input stream. #[inline] fn read_u16(&mut self) -> Result { // We use byteorder to read an u16 in little endian. Ok(LittleEndian::read_u16(self.take(2)?)) } /// Read the literals section of a block. /// /// The literals section encodes some bytes which are to be copied to the output without any /// modification. /// /// It consists of two parts: /// /// 1. An LSIC integer extension to the literals length as defined by the first part of the /// token, if it takes the highest value (15). /// 2. The literals themself. fn read_literal_section(&mut self) -> Result<(), Error> { // The higher token is the literals part of the token. It takes a value from 0 to 15. let mut literal = (self.token >> 4) as usize; // If the initial value is 15, it is indicated that another byte will be read and added to // it. if literal == 15 { // The literal length took the maximal value, indicating that there is more than 15 // literal bytes. We read the extra integer. literal += self.read_integer()?; } // Now we know the literal length. The number will be used to indicate how long the // following literal copied to the output buffer is. // Read the literals segment and output them without processing. Self::output(&mut self.output, Self::take_imp(&mut self.input, literal)?); Ok(()) } /// Read the duplicates section of the block. /// /// The duplicates section serves to reference an already decoded segment. This consists of two /// parts: /// /// 1. A 16-bit little-endian integer defining the "offset", i.e. how long back we need to go /// in the decoded buffer and copy. /// 2. An LSIC integer extension to the duplicate length as defined by the first part of the /// token, if it takes the highest value (15). fn read_duplicate_section(&mut self) -> Result<(), Error> { // Now, we will obtain the offset which we will use to copy from the output. It is an // 16-bit integer. let offset = self.read_u16()?; // Obtain the initial match length. The match length is the length of the duplicate segment // which will later be copied from data previously decompressed into the output buffer. The // initial length is derived from the second part of the token (the lower nibble), we read // earlier. Since having a match length of less than 4 would mean negative compression // ratio, we start at 4. let mut match_length = (4 + (self.token & 0xF)) as usize; // The intial match length can maximally be 19. As with the literal length, this indicates // that there are more bytes to read. if match_length == 4 + 15 { // The match length took the maximal value, indicating that there is more bytes. We // read the extra integer. match_length += self.read_integer()?; } // We now copy from the already decompressed buffer. This allows us for storing duplicates // by simply referencing the other location. // Calculate the start of this duplicate segment. We use wrapping subtraction to avoid // overflow checks, which we will catch later. let start = self.output.len().wrapping_sub(offset as usize); // We'll do a bound check to avoid panicking. if start < self.output.len() { // Write the duplicate segment to the output buffer. self.duplicate(start, match_length); Ok(()) } else { Err(Error::OffsetOutOfBounds) } } /// Complete the decompression by reading all the blocks. /// /// # Decompressing a block /// /// Blocks consists of: /// - A 1 byte token /// * A 4 bit integer $t_1$. /// * A 4 bit integer $t_2$. /// - A $n$ byte sequence of 0xFF bytes (if $t_1 \neq 15$, then $n = 0$). /// - $x$ non-0xFF 8-bit integers, L (if $t_1 = 15$, $x = 1$, else $x = 0$). /// - $t_1 + 15n + L$ bytes of uncompressed data (literals). /// - 16-bits offset (little endian), $a$. /// - A $m$ byte sequence of 0xFF bytes (if $t_2 \neq 15$, then $m = 0$). /// - $y$ non-0xFF 8-bit integers, $c$ (if $t_2 = 15$, $y = 1$, else $y = 0$). /// /// First, the literals are copied directly and unprocessed to the output buffer, then (after /// the involved parameters are read) $t_2 + 15m + c$ bytes are copied from the output buffer /// at position $a + 4$ and appended to the output buffer. Note that this copy can be /// overlapping. #[inline] fn complete(&mut self) -> Result<(), Error> { // Exhaust the decoder by reading and decompressing all blocks until the remaining buffer // is empty. while !self.input.is_empty() { // Read the token. The token is the first byte in a block. It is divided into two 4-bit // subtokens, the higher and the lower. self.token = self.take(1)?[0]; // Now, we read the literals section. self.read_literal_section()?; // If the input stream is emptied, we break out of the loop. This is only the case // in the end of the stream, since the block is intact otherwise. if self.input.is_empty() { break; } // Now, we read the duplicates section. self.read_duplicate_section()?; } Ok(()) } } /// Decompress all bytes of `input` into `output`. pub fn decompress_into(input: &[u8], output: &mut Vec) -> Result<(), Error> { // Decode into our vector. Decoder { input: input, output: output, token: 0, }.complete()?; Ok(()) } /// Decompress all bytes of `input`. pub fn decompress(input: &[u8]) -> Result, Error> { // Allocate a vector to contain the decompressed stream. let mut vec = Vec::with_capacity(4096); decompress_into(input, &mut vec)?; Ok(vec) } #[cfg(test)] mod test { use super::*; #[test] fn aaaaaaaaaaa_lots_of_aaaaaaaaa() { assert_eq!(decompress(&[0x11, b'a', 1, 0]).unwrap(), b"aaaaaa"); } #[test] fn multiple_repeated_blocks() { assert_eq!(decompress(&[0x11, b'a', 1, 0, 0x22, b'b', b'c', 2, 0]).unwrap(), b"aaaaaabcbcbcbc"); } #[test] fn all_literal() { assert_eq!(decompress(&[0x30, b'a', b'4', b'9']).unwrap(), b"a49"); } #[test] fn offset_oob() { decompress(&[0x10, b'a', 2, 0]).unwrap_err(); decompress(&[0x40, b'a', 1, 0]).unwrap_err(); } } lz4-compress-0.1.1/src/compress.rs010064400017500000144000000234371310271372200152670ustar0000000000000000//! The compression algorithm. //! //! We make use of hash tables to find duplicates. This gives a reasonable compression ratio with a //! high performance. It has fixed memory usage, which contrary to other approachs, makes it less //! memory hungry. use byteorder::{NativeEndian, ByteOrder}; /// Duplication dictionary size. /// /// Every four bytes is assigned an entry. When this number is lower, fewer entries exists, and /// thus collisions are more likely, hurting the compression ratio. const DICTIONARY_SIZE: usize = 4096; /// A LZ4 block. /// /// This defines a single compression "unit", consisting of two parts, a number of raw literals, /// and possibly a pointer to the already encoded buffer from which to copy. #[derive(Debug)] struct Block { /// The length (in bytes) of the literals section. lit_len: usize, /// The duplicates section if any. /// /// Only the last block in a stream can lack of the duplicates section. dup: Option, } /// A consecutive sequence of bytes found in already encoded part of the input. #[derive(Copy, Clone, Debug)] struct Duplicate { /// The number of bytes before our cursor, where the duplicate starts. offset: u16, /// The length beyond the four first bytes. /// /// Adding four to this number yields the actual length. extra_bytes: usize, } /// An LZ4 encoder. struct Encoder<'a> { /// The raw uncompressed input. input: &'a [u8], /// The compressed output. output: &'a mut Vec, /// The number of bytes from the input that are encoded. cur: usize, /// The dictionary of previously encoded sequences. /// /// This is used to find duplicates in the stream so they are not written multiple times. /// /// Every four bytes are hashed, and in the resulting slot their position in the input buffer /// is placed. This way we can easily look up a candidate to back references. dict: [usize; DICTIONARY_SIZE], } impl<'a> Encoder<'a> { /// Go forward by some number of bytes. /// /// This will update the cursor and dictionary to reflect the now processed bytes. /// /// This returns `false` if all the input bytes are processed. fn go_forward(&mut self, steps: usize) -> bool { // Go over all the bytes we are skipping and update the cursor and dictionary. for _ in 0..steps { // Insert the cursor position into the dictionary. self.insert_cursor(); // Increment the cursor. self.cur += 1; } // Return `true` if there's more to read. self.cur <= self.input.len() } /// Insert the batch under the cursor into the dictionary. fn insert_cursor(&mut self) { // Make sure that there is at least one batch remaining. if self.remaining_batch() { // Insert the cursor into the table. self.dict[self.get_cur_hash()] = self.cur; } } /// Check if there are any remaining batches. fn remaining_batch(&self) -> bool { self.cur + 4 < self.input.len() } /// Get the hash of the current four bytes below the cursor. /// /// This is guaranteed to be below `DICTIONARY_SIZE`. fn get_cur_hash(&self) -> usize { // Use PCG transform to generate a relatively good hash of the four bytes batch at the // cursor. let mut x = self.get_batch_at_cursor().wrapping_mul(0xa4d94a4f); let a = x >> 16; let b = x >> 30; x ^= a >> b; x = x.wrapping_mul(0xa4d94a4f); x as usize % DICTIONARY_SIZE } /// Read a 4-byte "batch" from some position. /// /// This will read a native-endian 4-byte integer from some position. fn get_batch(&self, n: usize) -> u32 { debug_assert!(self.remaining_batch(), "Reading a partial batch."); NativeEndian::read_u32(&self.input[n..]) } /// Read the batch at the cursor. fn get_batch_at_cursor(&self) -> u32 { self.get_batch(self.cur) } /// Find a duplicate of the current batch. /// /// If any duplicate is found, a tuple `(position, size - 4)` is returned. fn find_duplicate(&self) -> Option { // If there is no remaining batch, we return none. if !self.remaining_batch() { return None; } // Find a candidate in the dictionary by hashing the current four bytes. let candidate = self.dict[self.get_cur_hash()]; // Three requirements to the candidate exists: // - The candidate is not the trap value (0xFFFFFFFF), which represents an empty bucket. // - We should not return a position which is merely a hash collision, so w that the // candidate actually matches what we search for. // - We can address up to 16-bit offset, hence we are only able to address the candidate if // its offset is less than or equals to 0xFFFF. if candidate != !0 && self.get_batch(candidate) == self.get_batch_at_cursor() && self.cur - candidate <= 0xFFFF { // Calculate the "extension bytes", i.e. the duplicate bytes beyond the batch. These // are the number of prefix bytes shared between the match and needle. let ext = self.input[self.cur + 4..] .iter() .zip(&self.input[candidate + 4..]) .take_while(|&(a, b)| a == b) .count(); Some(Duplicate { offset: (self.cur - candidate) as u16, extra_bytes: ext, }) } else { None } } /// Write an integer to the output in LSIC format. fn write_integer(&mut self, mut n: usize) { // Write the 0xFF bytes as long as the integer is higher than said value. while n >= 0xFF { n -= 0xFF; self.output.push(0xFF); } // Write the remaining byte. self.output.push(n as u8); } /// Read the block of the top of the stream. fn pop_block(&mut self) -> Block { // The length of the literals section. let mut lit = 0; loop { // Search for a duplicate. if let Some(dup) = self.find_duplicate() { // We found a duplicate, so the literals section is over... // Move forward. Note that `ext` is actually the steps minus 4, because of the // minimum matchlenght, so we need to add 4. self.go_forward(dup.extra_bytes + 4); return Block { lit_len: lit, dup: Some(dup), }; } // Try to move forward. if !self.go_forward(1) { // We reached the end of the stream, and no duplicates section follows. return Block { lit_len: lit, dup: None, }; } // No duplicates found yet, so extend the literals section. lit += 1; } } /// Complete the encoding into `self.output`. fn complete(&mut self) { // Construct one block at a time. loop { // The start of the literals section. let start = self.cur; // Read the next block into two sections, the literals and the duplicates. let block = self.pop_block(); // Generate the higher half of the token. let mut token = if block.lit_len < 0xF { // Since we can fit the literals length into it, there is no need for saturation. (block.lit_len as u8) << 4 } else { // We were unable to fit the literals into it, so we saturate to 0xF. We will later // write the extensional value through LSIC encoding. 0xF0 }; // Generate the lower half of the token, the duplicates length. let dup_extra_len = block.dup.map_or(0, |x| x.extra_bytes); token |= if dup_extra_len < 0xF { // We could fit it in. dup_extra_len as u8 } else { // We were unable to fit it in, so we default to 0xF, which will later be extended // by LSIC encoding. 0xF }; // Push the token to the output stream. self.output.push(token); // If we were unable to fit the literals length into the token, write the extensional // part through LSIC. if block.lit_len >= 0xF { self.write_integer(block.lit_len - 0xF); } // Now, write the actual literals. self.output.extend_from_slice(&self.input[start..start + block.lit_len]); if let Some(Duplicate { offset, .. }) = block.dup { // Wait! There's more. Now, we encode the duplicates section. // Push the offset in little endian. self.output.push(offset as u8); self.output.push((offset >> 8) as u8); // If we were unable to fit the duplicates length into the token, write the // extensional part through LSIC. if dup_extra_len >= 0xF { self.write_integer(dup_extra_len - 0xF); } } else { break; } } } } /// Compress all bytes of `input` into `output`. pub fn compress_into(input: &[u8], output: &mut Vec) { Encoder { input: input, output: output, cur: 0, dict: [!0; DICTIONARY_SIZE], }.complete(); } /// Compress all bytes of `input`. pub fn compress(input: &[u8]) -> Vec { // In most cases, the compression won't expand the size, so we set the input size as capacity. let mut vec = Vec::with_capacity(input.len()); compress_into(input, &mut vec); vec } lz4-compress-0.1.1/src/lib.rs010064400017500000144000000005561303500342700141760ustar0000000000000000//! Pure Rust implementation of LZ4 compression. //! //! A detailed explanation of the algorithm can be found [here](http://ticki.github.io/blog/how-lz4-works/). #![warn(missing_docs)] extern crate byteorder; #[macro_use] extern crate quick_error; mod decompress; mod compress; #[cfg(test)] mod tests; pub use decompress::decompress; pub use compress::compress; lz4-compress-0.1.1/src/tests.rs010064400017500000144000000056051301332546100145730ustar0000000000000000//! Tests. use std::str; use {decompress, compress}; /// Test that the compressed string decompresses to the original string. fn inverse(s: &str) { let compressed = compress(s.as_bytes()); println!("Compressed '{}' into {:?}", s, compressed); let decompressed = decompress(&compressed).unwrap(); println!("Decompressed it into {:?}", str::from_utf8(&decompressed).unwrap()); assert_eq!(decompressed, s.as_bytes()); } #[test] fn shakespear() { inverse("to live or not to live"); inverse("Love is a wonderful terrible thing"); inverse("There is nothing either good or bad, but thinking makes it so."); inverse("I burn, I pine, I perish."); } #[test] fn totally_not_antifa_propaganda() { inverse("The only good fascist is a dead fascist."); inverse("bash the fash"); inverse("the fash deserves no bash, only smash"); inverse("Dead fascists can't vote."); inverse("Good night, white pride."); inverse("Some say fascism started with gas chambers. I say that's where it ends."); } #[test] fn not_compressible() { inverse("as6yhol.;jrew5tyuikbfewedfyjltre22459ba"); inverse("jhflkdjshaf9p8u89ybkvjsdbfkhvg4ut08yfrr"); } #[test] fn short() { inverse("ahhd"); inverse("ahd"); inverse("x-29"); inverse("x"); inverse("k"); inverse("."); inverse("ajsdh"); } #[test] fn empty_string() { inverse(""); } #[test] fn nulls() { inverse("\0\0\0\0\0\0\0\0\0\0\0\0\0"); } #[test] fn compression_works() { let s = "micah (Micah Cohen, politics editor): Clinton’s lead has shrunk to a hair above 4 percentage points in our polls-only model, down from about 7 points two weeks ago. So we find ourselves in an odd position where Clinton still holds a clear lead, but it’s shrinking by the day. I’ve been getting questions from Clinton supporters wondering how panicked they should be, and while we advise everyone of all political stripes to always remain calm, let’s try to answer that question today. How safe is Clinton’s lead/how panicked should Democrats be? As tacky as it is to cite your own tweet, I’m going to do it anyway — here’s a handy scale: natesilver: It’s uncertain, in part, because of the risk of a popular vote-Electoral College split. And, in part, because there are various reasons to think polling error could be high this year, such as the number of undecided voters. You can see those forces at play in the recent tightening. Clinton hasn’t really declined very much in these latest polls. But she was at only 46 percent in national polls, and that left a little bit of wiggle room for Trump."; inverse(s); assert!(compress(s.as_bytes()).len() < s.len()); } #[test] fn big_compression() { let mut s = Vec::with_capacity(80_000000); for n in 0..80_000000 { s.push((n as u8).wrapping_mul(0xA).wrapping_add(33) ^ 0xA2); } assert_eq!(&decompress(&compress(&s)).unwrap(), &s); } lz4-compress-0.1.1/src/main.rs010064400017500000144000000034151306651375200143640ustar0000000000000000extern crate lz4_compress as lz4; use std::{env, process}; use std::io::{self, Write, Read}; /// The help page for this command. const HELP: &'static [u8] = br#" Introduction: lz4 - an utility to decompress or compress a raw, headerless LZ4 stream. Usage: lz4 [option] Options: -c : Compress stdin and write the result to stdout. -d : Decompress stdin and write the result to stdout. -h : Write this manpage to stderr. "#; fn main() { let mut iter = env::args().skip(1); let mut flag = iter.next().unwrap_or(String::new()); // If another argument is provided (e.g. the user passes a file name), we need to make sure we // issue an error properly, so we set back the flag to `""`. if iter.next().is_some() { flag = String::new(); } match &*flag { "-c" => { // Read stream from stdin. let mut vec = Vec::new(); io::stdin().read_to_end(&mut vec).expect("Failed to read stdin"); // Compress it and write the result to stdout. io::stdout().write(&lz4::compress(&vec)).expect("Failed to write to stdout"); }, "-d" => { // Read stream from stdin. let mut vec = Vec::new(); io::stdin().read_to_end(&mut vec).expect("Failed to read stdin"); // Decompress the input. let decompressed = lz4::decompress(&vec).expect("Compressed data contains errors"); // Write the decompressed buffer to stdout. io::stdout().write(&decompressed).expect("Failed to write to stdout"); }, // If no valid arguments are given, we print the help page. _ => { io::stdout().write(HELP).expect("Failed to write to stdout"); process::exit(1); }, } }