build: pure-Rust zlib-rs as the default deflate backend
The core crates (clawhdf5, -agent, -format, -io, -filters, -ann, -accel, -netcdf4, -cli) now build no C by default: deflate defaults to zlib-rs, a pure-Rust port of zlib-ng, and zlib-ng becomes the opt-in `fast-deflate`, which overrides zlib-rs wherever it is enabled. A default build no longer needs cmake or a C compiler. Measured on tank, both builds run alternately, three rounds, medians: zlib-rs is within 6% of zlib-ng on every HDF5 read and write (512x512 deflate-6 chunked write 1.458 vs 1.484 ms; 64 MB compressed read 64.4 vs 65.2 ms), and compressed output is byte-identical. Details in BENCHMARKS.md, "Deflate backend". Getting there took two fixes the first measurement exposed: - zlib-rs needs `std` to detect SIMD at runtime. flate2 enables it via its default `runtime_detection`, which `default-features = false` had switched off, leaving zlib-rs 3.5x slower on inflate. The `zlib-rs` features now enable it. - Both deflate paths streamed through flate2's 32 KiB read/write wrappers. They now hand the codec the whole chunk in one call, into a buffer sized up front (~5% on chunked writes). This also fixes a silent short read: the streaming reader returned a truncated stream's bytes without an error; a truncated chunk is now DecompressionError. In clawhdf5-filters, output longer than the stated size is now an error rather than silently cut off. CI: ci-test.sh lints and tests the zlib-ng path, and fails if a C-building crate (*-sys, cc, cmake) enters a core crate's default dependency tree. The arm64 job no longer installs cmake. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -32,7 +32,10 @@ name = "bench"
|
||||
harness = false
|
||||
|
||||
[features]
|
||||
default = ["std", "checksum", "deflate", "provenance", "fast-deflate", "system-zlib-decompress"]
|
||||
# Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects
|
||||
# zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one
|
||||
# is enabled, so turning it on anywhere in the build overrides the default.
|
||||
default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"]
|
||||
std = []
|
||||
checksum = []
|
||||
deflate = ["flate2"]
|
||||
@@ -42,7 +45,10 @@ fast-checksum = ["crc32fast"]
|
||||
fast-deflate = ["flate2/zlib-ng"]
|
||||
system-zlib = ["flate2/zlib-default"]
|
||||
system-zlib-decompress = []
|
||||
zlib-rs = ["flate2/zlib-rs"]
|
||||
# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use
|
||||
# SIMD at runtime. flate2 enables it by default, but we build flate2 with
|
||||
# default-features = false, and without it zlib-rs inflates 3.5x slower.
|
||||
zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"]
|
||||
lz4 = ["lz4_flex"]
|
||||
zstd = ["dep:zstd"]
|
||||
blake3_hash = ["blake3"]
|
||||
|
||||
@@ -629,21 +629,70 @@ fn deflate_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, For
|
||||
// Fall through to flate2 on error
|
||||
}
|
||||
|
||||
use std::io::Read;
|
||||
let decoder = flate2::read::ZlibDecoder::new(data);
|
||||
let mut result = Vec::with_capacity(limit.min(1 << 20));
|
||||
// Read one byte past the limit so an over-size stream is distinguishable
|
||||
// A chunk's decompressed size is known, so allocate it once; without one,
|
||||
// start from a multiple of the input and grow.
|
||||
let size_hint = if expected_bytes != 0 {
|
||||
expected_bytes
|
||||
} else {
|
||||
data.len().saturating_mul(4).min(1 << 20)
|
||||
};
|
||||
inflate_bounded(data, size_hint, limit).map_err(FormatError::DecompressionError)
|
||||
}
|
||||
|
||||
/// Inflate a zlib stream into a buffer sized up front, handing the decoder the
|
||||
/// whole input at once.
|
||||
///
|
||||
/// `flate2::read::ZlibDecoder` feeds its input through a 32 KiB buffer and
|
||||
/// grows the output as it goes; on single chunks that cost zlib-rs up to 3.7x
|
||||
/// against zlib-ng (`BENCHMARKS.md`, "Deflate backend"). Output beyond `limit`
|
||||
/// is an error, as is a stream that ends before its end-of-stream marker (the
|
||||
/// streaming reader returned the bytes it had and no error).
|
||||
#[cfg(feature = "deflate")]
|
||||
pub(crate) fn inflate_bounded(
|
||||
data: &[u8],
|
||||
size_hint: usize,
|
||||
limit: usize,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
use flate2::{Decompress, FlushDecompress, Status};
|
||||
|
||||
// One byte of headroom past the limit distinguishes an over-size stream
|
||||
// from one that legitimately ends exactly at the limit.
|
||||
decoder
|
||||
.take(limit as u64 + 1)
|
||||
.read_to_end(&mut result)
|
||||
.map_err(|e| FormatError::DecompressionError(e.to_string()))?;
|
||||
if result.len() > limit {
|
||||
return Err(FormatError::DecompressionError(
|
||||
"deflate: output exceeds size limit".into(),
|
||||
));
|
||||
let max_capacity = limit.saturating_add(1);
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(size_hint.clamp(1, max_capacity))
|
||||
.map_err(|e| format!("deflate: cannot allocate output: {e}"))?;
|
||||
|
||||
let mut inflater = Decompress::new(true);
|
||||
loop {
|
||||
let (in_before, out_before) = (inflater.total_in(), inflater.total_out());
|
||||
let status = inflater
|
||||
.decompress_vec(
|
||||
&data[in_before as usize..],
|
||||
&mut out,
|
||||
FlushDecompress::Finish,
|
||||
)
|
||||
.map_err(|e| format!("deflate: {e}"))?;
|
||||
if out.len() > limit {
|
||||
return Err("deflate: output exceeds size limit".into());
|
||||
}
|
||||
match status {
|
||||
Status::StreamEnd => return Ok(out),
|
||||
Status::Ok | Status::BufError if out.len() == out.capacity() => {
|
||||
// Out of room: double, up to the limit.
|
||||
let grow = out.capacity().min(max_capacity - out.capacity()).max(1);
|
||||
out.try_reserve_exact(grow)
|
||||
.map_err(|e| format!("deflate: cannot allocate output: {e}"))?;
|
||||
}
|
||||
Status::Ok | Status::BufError => {
|
||||
// Room left, so the decoder stopped for want of input.
|
||||
if inflater.total_in() as usize >= data.len()
|
||||
|| (inflater.total_in(), inflater.total_out()) == (in_before, out_before)
|
||||
{
|
||||
return Err("deflate: truncated stream".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Direct FFI to Apple's system libz for fast decompression.
|
||||
@@ -722,14 +771,41 @@ fn deflate_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, F
|
||||
/// Compress data with zlib.
|
||||
#[cfg(feature = "deflate")]
|
||||
fn deflate_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
use std::io::Write;
|
||||
let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::new(level));
|
||||
encoder
|
||||
.write_all(data)
|
||||
.map_err(|e| FormatError::CompressionError(e.to_string()))?;
|
||||
encoder
|
||||
.finish()
|
||||
.map_err(|e| FormatError::CompressionError(e.to_string()))
|
||||
deflate_bounded(data, level).map_err(FormatError::CompressionError)
|
||||
}
|
||||
|
||||
/// Deflate `data` into a zlib stream in one pass, into a buffer sized for the
|
||||
/// worst case up front (the same reasoning as [`inflate_bounded`]).
|
||||
#[cfg(feature = "deflate")]
|
||||
pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result<Vec<u8>, String> {
|
||||
use flate2::{Compress, Compression, FlushCompress, Status};
|
||||
|
||||
// zlib's compressBound, plus the zlib header and trailer.
|
||||
let bound = data.len() + (data.len() >> 12) + (data.len() >> 14) + (data.len() >> 25) + 13 + 6;
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(bound)
|
||||
.map_err(|e| format!("deflate: cannot allocate output: {e}"))?;
|
||||
|
||||
let mut deflater = Compress::new(Compression::new(level), true);
|
||||
loop {
|
||||
let (in_before, out_before) = (deflater.total_in(), deflater.total_out());
|
||||
let status = deflater
|
||||
.compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish)
|
||||
.map_err(|e| format!("deflate: {e}"))?;
|
||||
match status {
|
||||
Status::StreamEnd => return Ok(out),
|
||||
// The bound should make running out of room unreachable; grow
|
||||
// rather than fail if it happens.
|
||||
Status::Ok | Status::BufError if out.len() == out.capacity() => out
|
||||
.try_reserve(out.capacity().max(4096))
|
||||
.map_err(|e| format!("deflate: cannot allocate output: {e}"))?,
|
||||
Status::Ok | Status::BufError => {
|
||||
if (deflater.total_in(), deflater.total_out()) == (in_before, out_before) {
|
||||
return Err("deflate: encoder made no progress".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "deflate"))]
|
||||
@@ -1833,6 +1909,74 @@ mod tests {
|
||||
assert!(deflate_decompress(&compressed, 64).is_err());
|
||||
}
|
||||
|
||||
#[cfg(feature = "deflate")]
|
||||
fn noisy_bytes(n: usize) -> Vec<u8> {
|
||||
// Compressible but not trivially so.
|
||||
(0..n)
|
||||
.map(|i| ((i as f64 * 0.01).sin() * 127.0 + 128.0) as u8 ^ (i as u8 & 3))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "deflate")]
|
||||
fn deflate_decompress_accepts_output_exactly_at_chunk_size() {
|
||||
let data = noisy_bytes(100_000);
|
||||
let compressed = deflate_compress(&data, 6).unwrap();
|
||||
assert_eq!(deflate_decompress(&compressed, data.len()).unwrap(), data);
|
||||
// One byte short of the real size is over the limit.
|
||||
assert!(deflate_decompress(&compressed, data.len() - 1).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "deflate")]
|
||||
fn deflate_decompress_without_size_grows_the_buffer() {
|
||||
// No chunk size: the output starts at 4x the input and has to grow.
|
||||
let data = vec![7u8; 3 * 1024 * 1024];
|
||||
let compressed = deflate_compress(&data, 6).unwrap();
|
||||
assert!(compressed.len() * 4 < data.len());
|
||||
assert_eq!(deflate_decompress(&compressed, 0).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "deflate")]
|
||||
fn deflate_decompress_rejects_truncated_stream() {
|
||||
// The streaming reader this replaced returned the bytes it had and no
|
||||
// error, so a truncated chunk read back short.
|
||||
let data = noisy_bytes(100_000);
|
||||
let compressed = deflate_compress(&data, 6).unwrap();
|
||||
for cut in [compressed.len() - 1, compressed.len() / 2, 3] {
|
||||
assert!(
|
||||
deflate_decompress(&compressed[..cut], data.len()).is_err(),
|
||||
"truncated to {cut} of {} bytes",
|
||||
compressed.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "deflate")]
|
||||
fn deflate_compress_roundtrips_incompressible_data() {
|
||||
// Random-looking input compresses to slightly more than it started
|
||||
// as; the output must still fit the pre-sized buffer (or grow).
|
||||
let mut x = 0x9E37_79B9_7F4A_7C15u64;
|
||||
let data: Vec<u8> = (0..200_000)
|
||||
.map(|_| {
|
||||
x ^= x << 13;
|
||||
x ^= x >> 7;
|
||||
x ^= x << 17;
|
||||
x as u8
|
||||
})
|
||||
.collect();
|
||||
for level in [0, 1, 6, 9] {
|
||||
let compressed = deflate_compress(&data, level).unwrap();
|
||||
assert_eq!(deflate_decompress(&compressed, data.len()).unwrap(), data);
|
||||
}
|
||||
assert_eq!(
|
||||
deflate_decompress(&deflate_compress(&[], 6).unwrap(), 0).unwrap(),
|
||||
Vec::<u8>::new()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "zstd")]
|
||||
fn zstd_decompress_rejects_output_exceeding_chunk_size() {
|
||||
|
||||
Reference in New Issue
Block a user