Write chunks of 4 GiB or more; FileEditor refuses to rewrite them
The writer gives a chunk of more than u32::MAX bytes layout message version 5 and, filtered, index elements whose stored size takes the file's size of lengths, as libhdf5 2.x does (the Fixed and Extensible Array structures match libhdf5's byte for byte). Chunk dimensions of 2^32 or more and filters that cannot take such a chunk (LZF, bitshuffle, bzip2, Blosc, pcodec) are refused instead of truncated. Chunks are extracted row by row and one at a time; deflate no longer cuts input at 4 GiB - 1 bytes, nor holds the worst-case bound of a large chunk; an LZ4 chunk of 4 GiB or more is read as the registered framing. FileEditor refuses writing values into, or pruning/allocating, chunks of 4 GiB or more before anything is written; growing the extent and attributes still work. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -402,86 +402,83 @@ pub fn split_into_chunks(
|
||||
chunk_dims: &[u64],
|
||||
element_size: usize,
|
||||
) -> Vec<(Vec<u64>, Vec<u8>)> {
|
||||
let rank = shape.len();
|
||||
if rank == 0 {
|
||||
if shape.is_empty() {
|
||||
return vec![(vec![], raw_data.to_vec())];
|
||||
}
|
||||
(0..chunk_count(shape, chunk_dims))
|
||||
.map(|i| extract_chunk(raw_data, shape, chunk_dims, element_size, i))
|
||||
.collect()
|
||||
}
|
||||
|
||||
// Compute number of chunks per dimension
|
||||
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
||||
for d in 0..rank {
|
||||
num_chunks_per_dim.push(shape[d].div_ceil(chunk_dims[d]));
|
||||
/// Number of chunks of the current extent `shape`.
|
||||
fn chunk_count(shape: &[u64], chunk_dims: &[u64]) -> u64 {
|
||||
shape
|
||||
.iter()
|
||||
.zip(chunk_dims)
|
||||
.map(|(&s, &c)| s.div_ceil(c))
|
||||
.product()
|
||||
}
|
||||
|
||||
/// The `linear_idx`-th chunk (row-major over the chunks of the current
|
||||
/// extent) of the row-major dataset `raw_data`: its offset in dataset space
|
||||
/// and its bytes, a whole chunk with the part past the dataset's edge zero.
|
||||
/// Copied one row (a run along the last dimension) at a time.
|
||||
fn extract_chunk(
|
||||
raw_data: &[u8],
|
||||
shape: &[u64],
|
||||
chunk_dims: &[u64],
|
||||
element_size: usize,
|
||||
linear_idx: u64,
|
||||
) -> (Vec<u64>, Vec<u8>) {
|
||||
let rank = shape.len();
|
||||
let mut offsets = vec![0u64; rank];
|
||||
let mut remaining = linear_idx;
|
||||
for d in (0..rank).rev() {
|
||||
let n = shape[d].div_ceil(chunk_dims[d]);
|
||||
offsets[d] = (remaining % n) * chunk_dims[d];
|
||||
remaining /= n;
|
||||
}
|
||||
let total_chunks: u64 = num_chunks_per_dim.iter().product();
|
||||
let chunk_elements: usize = chunk_dims.iter().map(|&d| saturating_usize(d)).product();
|
||||
let mut chunk = vec![0u8; chunk_elements.saturating_mul(element_size)];
|
||||
|
||||
// Dataset strides (row-major)
|
||||
// Elements of the chunk inside the dataset, per dimension.
|
||||
let valid: Vec<usize> = (0..rank)
|
||||
.map(|d| saturating_usize(shape[d].saturating_sub(offsets[d]).min(chunk_dims[d])))
|
||||
.collect();
|
||||
if valid.contains(&0) {
|
||||
return (offsets, chunk);
|
||||
}
|
||||
let mut ds_strides = vec![1usize; rank];
|
||||
for i in (0..rank.saturating_sub(1)).rev() {
|
||||
ds_strides[i] = ds_strides[i + 1] * saturating_usize(shape[i + 1]);
|
||||
}
|
||||
|
||||
// Chunk strides
|
||||
let mut chunk_strides = vec![1usize; rank];
|
||||
for i in (0..rank.saturating_sub(1)).rev() {
|
||||
chunk_strides[i] = chunk_strides[i + 1] * saturating_usize(chunk_dims[i + 1]);
|
||||
for d in (0..rank - 1).rev() {
|
||||
ds_strides[d] = ds_strides[d + 1] * saturating_usize(shape[d + 1]);
|
||||
chunk_strides[d] = chunk_strides[d + 1] * saturating_usize(chunk_dims[d + 1]);
|
||||
}
|
||||
|
||||
let chunk_total_elements: usize = chunk_dims.iter().map(|&d| saturating_usize(d)).product();
|
||||
|
||||
let mut result = Vec::with_capacity(saturating_usize(total_chunks));
|
||||
|
||||
for linear_idx in 0..total_chunks {
|
||||
// Convert linear index to chunk grid coordinates
|
||||
let mut chunk_grid_coords = vec![0u64; rank];
|
||||
let mut remaining = linear_idx;
|
||||
for d in (0..rank).rev() {
|
||||
chunk_grid_coords[d] = remaining % num_chunks_per_dim[d];
|
||||
remaining /= num_chunks_per_dim[d];
|
||||
let row = valid[rank - 1] * element_size;
|
||||
let mut idx = vec![0usize; rank];
|
||||
loop {
|
||||
let src: usize = (0..rank)
|
||||
.map(|d| (saturating_usize(offsets[d]) + idx[d]) * ds_strides[d])
|
||||
.sum::<usize>()
|
||||
* element_size;
|
||||
let dst: usize = (0..rank).map(|d| idx[d] * chunk_strides[d]).sum::<usize>() * element_size;
|
||||
// Whole elements only, as far as `raw_data` reaches.
|
||||
let n = row.min(raw_data.len().saturating_sub(src)) / element_size * element_size;
|
||||
chunk[dst..dst + n].copy_from_slice(&raw_data[src..src + n]);
|
||||
// Next row: advance every dimension but the last.
|
||||
let mut d = rank - 1;
|
||||
loop {
|
||||
if d == 0 {
|
||||
return (offsets, chunk);
|
||||
}
|
||||
d -= 1;
|
||||
idx[d] += 1;
|
||||
if idx[d] < valid[d] {
|
||||
break;
|
||||
}
|
||||
idx[d] = 0;
|
||||
}
|
||||
|
||||
// Chunk offset in dataset space
|
||||
let offsets: Vec<u64> = (0..rank)
|
||||
.map(|d| chunk_grid_coords[d] * chunk_dims[d])
|
||||
.collect();
|
||||
|
||||
// Extract chunk data
|
||||
let mut chunk_bytes = vec![0u8; chunk_total_elements * element_size];
|
||||
|
||||
for flat_idx in 0..chunk_total_elements {
|
||||
let mut remaining_idx = flat_idx;
|
||||
let mut ds_flat = 0usize;
|
||||
let mut out_of_bounds = false;
|
||||
|
||||
for d in 0..rank {
|
||||
let coord_in_chunk = remaining_idx / chunk_strides[d];
|
||||
remaining_idx %= chunk_strides[d];
|
||||
|
||||
let global_coord = saturating_usize(offsets[d]) + coord_in_chunk;
|
||||
if global_coord >= saturating_usize(shape[d]) {
|
||||
out_of_bounds = true;
|
||||
break;
|
||||
}
|
||||
ds_flat += global_coord * ds_strides[d];
|
||||
}
|
||||
|
||||
if out_of_bounds {
|
||||
// Zero-filled (already initialized)
|
||||
continue;
|
||||
}
|
||||
|
||||
let src_start = ds_flat * element_size;
|
||||
let dst_start = flat_idx * element_size;
|
||||
|
||||
if src_start + element_size <= raw_data.len() {
|
||||
chunk_bytes[dst_start..dst_start + element_size]
|
||||
.copy_from_slice(&raw_data[src_start..src_start + element_size]);
|
||||
}
|
||||
}
|
||||
|
||||
result.push((offsets, chunk_bytes));
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Parallel compression threshold: use rayon when chunk count exceeds this.
|
||||
@@ -492,46 +489,68 @@ pub fn split_into_chunks(
|
||||
#[cfg(feature = "parallel")]
|
||||
const PARALLEL_COMPRESS_THRESHOLD: usize = 2;
|
||||
|
||||
/// Compress all chunks, using parallel compression when beneficial, and
|
||||
/// return each chunk's stored bytes with its filter mask.
|
||||
/// Largest chunk compressed in parallel: every thread holds a chunk and its
|
||||
/// compressed copy at once, so chunks larger than this (up to 4 GiB and
|
||||
/// more) are compressed one after another.
|
||||
#[cfg(feature = "parallel")]
|
||||
const PARALLEL_COMPRESS_MAX_CHUNK_BYTES: u64 = 64 << 20;
|
||||
|
||||
/// Extract and compress every chunk of the dataset, returning each chunk's
|
||||
/// raw size, stored bytes and filter mask, in chunk order.
|
||||
///
|
||||
/// Chunks run through the pipeline as libhdf5 runs them
|
||||
/// ([`compress_chunk_masked`]): an optional filter that fails — LZF or Blosc
|
||||
/// output no smaller than its input — is skipped and its mask bit set.
|
||||
///
|
||||
/// With the `parallel` feature and more than [`PARALLEL_COMPRESS_THRESHOLD`]
|
||||
/// filtered chunks, compression runs across rayon threads; otherwise it is
|
||||
/// sequential. Output order matches input order, so per-chunk bytes are
|
||||
/// identical to the sequential path.
|
||||
/// Each chunk is extracted just before it is compressed and dropped after,
|
||||
/// so at most one raw chunk per thread is held. With the `parallel` feature,
|
||||
/// more than [`PARALLEL_COMPRESS_THRESHOLD`] filtered chunks, and chunks of
|
||||
/// at most [`PARALLEL_COMPRESS_MAX_CHUNK_BYTES`], compression runs across
|
||||
/// rayon threads; otherwise it is sequential. Output order matches chunk
|
||||
/// order, so per-chunk bytes are identical to the sequential path.
|
||||
fn compress_all_chunks(
|
||||
chunks: &[(Vec<u64>, Vec<u8>)],
|
||||
raw_data: &[u8],
|
||||
shape: &[u64],
|
||||
chunk_dims: &[u64],
|
||||
element_size: usize,
|
||||
chunk_bytes: u64,
|
||||
pipeline: &Option<FilterPipeline>,
|
||||
element_size: u32,
|
||||
) -> Result<Vec<(Vec<u8>, u32)>, FormatError> {
|
||||
) -> Result<Vec<(u64, Vec<u8>, u32)>, FormatError> {
|
||||
let one = |i: u64| -> Result<(u64, Vec<u8>, u32), FormatError> {
|
||||
let (_, raw) = if shape.is_empty() {
|
||||
(Vec::new(), raw_data.to_vec())
|
||||
} else {
|
||||
extract_chunk(raw_data, shape, chunk_dims, element_size, i)
|
||||
};
|
||||
let raw_size = raw.len() as u64;
|
||||
match pipeline {
|
||||
Some(pl) => {
|
||||
let (stored, mask) = compress_chunk_masked(&raw, pl, element_size as u32)?;
|
||||
Ok((raw_size, stored, mask))
|
||||
}
|
||||
None => Ok((raw_size, raw, 0)),
|
||||
}
|
||||
};
|
||||
let n = if shape.is_empty() {
|
||||
1
|
||||
} else {
|
||||
chunk_count(shape, chunk_dims)
|
||||
};
|
||||
#[cfg(feature = "parallel")]
|
||||
{
|
||||
if let Some(pl) = pipeline
|
||||
&& chunks.len() > PARALLEL_COMPRESS_THRESHOLD
|
||||
if pipeline.is_some()
|
||||
&& n > PARALLEL_COMPRESS_THRESHOLD as u64
|
||||
&& chunk_bytes <= PARALLEL_COMPRESS_MAX_CHUNK_BYTES
|
||||
{
|
||||
use rayon::prelude::*;
|
||||
return chunks
|
||||
.par_iter()
|
||||
.map(|(_offsets, chunk_bytes)| compress_chunk_masked(chunk_bytes, pl, element_size))
|
||||
.collect();
|
||||
return (0..n).into_par_iter().map(one).collect();
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "parallel"))]
|
||||
let _ = chunk_bytes;
|
||||
|
||||
// Sequential fallback
|
||||
chunks
|
||||
.iter()
|
||||
.map(|(_offsets, chunk_bytes)| {
|
||||
if let Some(pl) = pipeline {
|
||||
compress_chunk_masked(chunk_bytes, pl, element_size)
|
||||
} else {
|
||||
Ok((chunk_bytes.clone(), 0))
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
(0..n).map(one).collect()
|
||||
}
|
||||
|
||||
/// Build the complete chunked dataset blob (chunk data + index) and return
|
||||
@@ -552,10 +571,12 @@ pub fn serialize_v4_single_chunk_pub(
|
||||
filter_mask,
|
||||
offset_size,
|
||||
element_size,
|
||||
4,
|
||||
)
|
||||
}
|
||||
|
||||
/// Serialize a v4 single chunk layout message.
|
||||
/// Serialize a v4 (or, for a chunk of 4 GiB or more, v5) single chunk
|
||||
/// layout message.
|
||||
fn serialize_v4_single_chunk(
|
||||
chunk_dims: &[u32],
|
||||
chunk_address: u64,
|
||||
@@ -563,9 +584,10 @@ fn serialize_v4_single_chunk(
|
||||
filter_mask: Option<u32>,
|
||||
offset_size: u8,
|
||||
element_size: u32,
|
||||
version: u8,
|
||||
) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
buf.push(4); // version
|
||||
buf.push(version);
|
||||
buf.push(2); // class = chunked
|
||||
|
||||
// flags: bit 0 = unknown meaning in some files, bit 1 = filters for single chunk
|
||||
@@ -605,8 +627,9 @@ fn serialize_v4_fixed_array(
|
||||
offset_size: u8,
|
||||
element_size: u32,
|
||||
max_bits: u8,
|
||||
version: u8,
|
||||
) -> Vec<u8> {
|
||||
let mut buf = layout_v4_chunked_prefix(chunk_dims, element_size);
|
||||
let mut buf = layout_v4_chunked_prefix(chunk_dims, element_size, version);
|
||||
|
||||
// chunk index type = 3 (Fixed Array)
|
||||
buf.push(3);
|
||||
@@ -645,9 +668,9 @@ pub(crate) fn push_v4_chunk_dims(buf: &mut Vec<u8>, chunk_dims: &[u32], element_
|
||||
}
|
||||
}
|
||||
|
||||
fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec<u8> {
|
||||
fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32, version: u8) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
buf.push(4); // version
|
||||
buf.push(version);
|
||||
buf.push(2); // class = chunked
|
||||
|
||||
let flags: u8 = 0x00;
|
||||
@@ -673,21 +696,53 @@ pub(crate) fn push_addr(buf: &mut Vec<u8>, addr: u64, offset_size: u8) {
|
||||
|
||||
/// Width of the chunk-size field of a filtered chunk index element. Must
|
||||
/// match the library's `H5D_FARRAY_FILT_COMPUTE_CHUNK_SIZE_LEN` (the EA and
|
||||
/// B-tree v2 indexes use the same formula):
|
||||
/// `1 + ((log2(unfiltered chunk bytes) + 8) / 8)`, capped at 8.
|
||||
pub(crate) fn filtered_chunk_size_len(slots: &[Option<WrittenChunk>]) -> usize {
|
||||
/// B-tree v2 indexes use the same formula): see [`chunk_size_len`]. Chunks
|
||||
/// of more than `u32::MAX` bytes are written with layout version 5
|
||||
/// ([`layout_version_for`]).
|
||||
pub(crate) fn filtered_chunk_size_len(slots: &[Option<WrittenChunk>], length_size: u8) -> usize {
|
||||
let max_raw = slots
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|c| c.raw_size)
|
||||
.max()
|
||||
.unwrap_or(1);
|
||||
let log2_val = if max_raw <= 1 {
|
||||
chunk_size_len(max_raw, layout_version_for(max_raw), length_size)
|
||||
}
|
||||
|
||||
/// Largest chunk, in bytes, a layout message of version 4 or lower may
|
||||
/// describe: libhdf5 writes a larger one with version 5
|
||||
/// (`H5D__chunk_construct`: "chunk size > 4GB requires H5F_LIBVER_V200"),
|
||||
/// which libhdf5 before 2.0 cannot read.
|
||||
pub const MAX_V4_CHUNK_BYTES: u64 = u32::MAX as u64;
|
||||
|
||||
/// The layout message version clawhdf5 writes for chunks of `chunk_bytes`
|
||||
/// bytes: 4, or 5 for a chunk larger than [`MAX_V4_CHUNK_BYTES`] (what
|
||||
/// libhdf5 2.x writes for it; the chunk index is chosen as for version 4).
|
||||
pub fn layout_version_for(chunk_bytes: u64) -> u8 {
|
||||
if chunk_bytes > MAX_V4_CHUNK_BYTES {
|
||||
5
|
||||
} else {
|
||||
4
|
||||
}
|
||||
}
|
||||
|
||||
/// Width libhdf5 gives the stored-size field of a filtered chunk index
|
||||
/// element (Fixed Array, Extensible Array, v2 B-tree) for chunks of
|
||||
/// `chunk_bytes` bytes under layout message `layout_version`
|
||||
/// (`H5D_FARRAY_FILT_COMPUTE_CHUNK_SIZE_LEN` and its EA and B-tree twins):
|
||||
/// up to version 4, one byte more than the chunk's size needs,
|
||||
/// `1 + ((log2(chunk_bytes) + 8) / 8)` capped at 8; from version 5 (HDF5
|
||||
/// 2.0), the file's size of lengths (`length_size`), whatever the chunk.
|
||||
pub fn chunk_size_len(chunk_bytes: u64, layout_version: u8, length_size: u8) -> usize {
|
||||
if layout_version >= 5 {
|
||||
return usize::from(length_size);
|
||||
}
|
||||
let log2 = if chunk_bytes <= 1 {
|
||||
0
|
||||
} else {
|
||||
63 - max_raw.leading_zeros()
|
||||
63 - chunk_bytes.leading_zeros()
|
||||
};
|
||||
(1 + ((log2_val + 8) / 8) as usize).min(8)
|
||||
(1 + ((log2 + 8) / 8) as usize).min(8)
|
||||
}
|
||||
|
||||
/// Append one chunk index element: the chunk's address, plus its stored size
|
||||
@@ -734,7 +789,7 @@ pub fn build_fixed_array_at(
|
||||
let os = offset_size as usize;
|
||||
let num_elements = slots.len();
|
||||
|
||||
let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots));
|
||||
let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots, length_size));
|
||||
let elem_size = os + chunk_size_bytes.map_or(0, |n| n + 4);
|
||||
let client_id: u8 = if has_filters { 1 } else { 0 };
|
||||
|
||||
@@ -829,23 +884,23 @@ pub fn precompress_chunks(
|
||||
element_size: usize,
|
||||
options: &ChunkOptions,
|
||||
) -> Result<PrecompressedChunks, FormatError> {
|
||||
let chunk_bytes = chunk_dims
|
||||
.iter()
|
||||
.try_fold(element_size as u64, |acc, &d| acc.checked_mul(d))
|
||||
.and_then(|b| u32::try_from(b).ok())
|
||||
.unwrap_or(0);
|
||||
let pipeline = options.build_pipeline_for_chunk(element_size as u32, chunk_bytes);
|
||||
let (_, chunk_bytes) = checked_chunk_dims(chunk_dims, element_size)?;
|
||||
if chunk_bytes > MAX_V4_CHUNK_BYTES {
|
||||
check_huge_chunk_filters(options, chunk_bytes)?;
|
||||
}
|
||||
let pipeline = options
|
||||
.build_pipeline_for_chunk(element_size as u32, u32::try_from(chunk_bytes).unwrap_or(0));
|
||||
let has_filters = pipeline.is_some();
|
||||
let pipeline_message = pipeline.as_ref().map(|pl| pl.serialize());
|
||||
|
||||
let raw_chunks = split_into_chunks(raw_data, shape, chunk_dims, element_size);
|
||||
let compressed = compress_all_chunks(&raw_chunks, &pipeline, element_size as u32)?;
|
||||
|
||||
let chunks = raw_chunks
|
||||
.into_iter()
|
||||
.zip(compressed)
|
||||
.map(|((_offsets, raw_bytes), (c, mask))| (raw_bytes.len() as u64, c, mask))
|
||||
.collect();
|
||||
let chunks = compress_all_chunks(
|
||||
raw_data,
|
||||
shape,
|
||||
chunk_dims,
|
||||
element_size,
|
||||
chunk_bytes,
|
||||
&pipeline,
|
||||
)?;
|
||||
|
||||
Ok(PrecompressedChunks {
|
||||
chunks,
|
||||
@@ -857,6 +912,67 @@ pub fn precompress_chunks(
|
||||
})
|
||||
}
|
||||
|
||||
/// The chunk dimensions as the layout message stores them (each below
|
||||
/// 2^32), and one chunk's size in bytes. A chunk dimension of 2^32 or more
|
||||
/// (which HDF5 2.0 can store, in wider fields) is refused, as it is when
|
||||
/// read: clawhdf5 holds chunk dimensions as `u32`. So is a chunk whose size
|
||||
/// overflows 64 bits, or that this platform cannot hold in memory (a chunk
|
||||
/// of 4 GiB or more on a 32-bit target).
|
||||
fn checked_chunk_dims(
|
||||
chunk_dims: &[u64],
|
||||
element_size: usize,
|
||||
) -> Result<(Vec<u32>, u64), FormatError> {
|
||||
let dims = chunk_dims
|
||||
.iter()
|
||||
.map(|&d| {
|
||||
u32::try_from(d).map_err(|_| {
|
||||
FormatError::InvalidChunkDimensions(format!(
|
||||
"chunk dimension {d} is 2^32 or more, which clawhdf5 does not support"
|
||||
))
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<u32>, _>>()?;
|
||||
let bytes = chunk_dims
|
||||
.iter()
|
||||
.try_fold(element_size as u64, |acc, &d| acc.checked_mul(d))
|
||||
.filter(|&b| usize::try_from(b).is_ok())
|
||||
.ok_or_else(|| {
|
||||
FormatError::Overflow(format!(
|
||||
"a chunk of {chunk_dims:?} x {element_size} bytes exceeds this platform's address space"
|
||||
))
|
||||
})?;
|
||||
Ok((dims, bytes))
|
||||
}
|
||||
|
||||
/// The filters clawhdf5 can apply to a chunk of more than `u32::MAX` bytes:
|
||||
/// shuffle, deflate, Zstandard, LZ4 (whose HDF5 framing records the size in
|
||||
/// 64 bits and splits the chunk into blocks) and Fletcher32. The others
|
||||
/// record the chunk size or their block lengths in 32 bits, or cannot take
|
||||
/// a buffer that large (h5py's LZF, bitshuffle, bzip2, Blosc), and pcodec is
|
||||
/// clawhdf5's own; they are refused rather than written into a chunk
|
||||
/// libhdf5 could not decode.
|
||||
fn check_huge_chunk_filters(options: &ChunkOptions, chunk_bytes: u64) -> Result<(), FormatError> {
|
||||
let refused = if let Some(plugin) = &options.plugin {
|
||||
Some(match plugin {
|
||||
PluginFilter::Lzf => "LZF",
|
||||
PluginFilter::Bitshuffle { .. } => "bitshuffle",
|
||||
PluginFilter::Bzip2 { .. } => "bzip2",
|
||||
PluginFilter::Blosc { .. } => "Blosc",
|
||||
})
|
||||
} else if options.pcodec {
|
||||
Some("pcodec")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match refused {
|
||||
Some(name) => Err(FormatError::FilterError(format!(
|
||||
"{name} cannot compress a chunk of {chunk_bytes} bytes (4 GiB or more); \
|
||||
use smaller chunks, or deflate, Zstandard or LZ4"
|
||||
))),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Lay out precompressed chunks at `base_address` and build index structures.
|
||||
///
|
||||
/// This is the address-dependent half of chunk writing. Call it in Pass 1
|
||||
@@ -910,7 +1026,8 @@ pub fn build_chunked_data_from_precompressed_libver(
|
||||
});
|
||||
}
|
||||
|
||||
let chunk_dims_u32: Vec<u32> = pre.chunk_dims.iter().map(|&d| d as u32).collect();
|
||||
let (chunk_dims_u32, chunk_bytes) = checked_chunk_dims(&pre.chunk_dims, element_size)?;
|
||||
let version = layout_version_for(chunk_bytes);
|
||||
|
||||
let aligned_idx = align_to_cache_line(data_buf.len());
|
||||
if aligned_idx > data_buf.len() {
|
||||
@@ -934,6 +1051,7 @@ pub fn build_chunked_data_from_precompressed_libver(
|
||||
ea_address,
|
||||
offset_size,
|
||||
element_size as u32,
|
||||
version,
|
||||
)
|
||||
}
|
||||
ChunkIndexPlan::SingleChunk => {
|
||||
@@ -951,6 +1069,7 @@ pub fn build_chunked_data_from_precompressed_libver(
|
||||
filter_mask,
|
||||
offset_size,
|
||||
element_size as u32,
|
||||
version,
|
||||
)
|
||||
}
|
||||
ChunkIndexPlan::FixedArray(grid, nslots) => {
|
||||
@@ -976,6 +1095,7 @@ pub fn build_chunked_data_from_precompressed_libver(
|
||||
offset_size,
|
||||
element_size as u32,
|
||||
FA_PAGE_BITS,
|
||||
version,
|
||||
)
|
||||
}
|
||||
ChunkIndexPlan::BTreeV2 => {
|
||||
@@ -1000,6 +1120,7 @@ pub fn build_chunked_data_from_precompressed_libver(
|
||||
offset_size,
|
||||
element_size as u32,
|
||||
node_size,
|
||||
version,
|
||||
)
|
||||
}
|
||||
};
|
||||
@@ -1237,7 +1358,7 @@ fn build_btree_v2_chunk_index_at(
|
||||
let chunk_size_bytes = has_filters.then(|| {
|
||||
let slots: Vec<Option<WrittenChunk>> =
|
||||
records.iter().map(|(_, c)| Some((*c).clone())).collect();
|
||||
filtered_chunk_size_len(&slots)
|
||||
filtered_chunk_size_len(&slots, length_size)
|
||||
});
|
||||
let record_size = os + chunk_size_bytes.map_or(0, |n| n + 4) + 8 * rank;
|
||||
let record_size_u16 = u16::try_from(record_size)
|
||||
@@ -1287,8 +1408,9 @@ fn serialize_v4_btree_v2(
|
||||
offset_size: u8,
|
||||
element_size: u32,
|
||||
node_size: u32,
|
||||
version: u8,
|
||||
) -> Vec<u8> {
|
||||
let mut buf = layout_v4_chunked_prefix(chunk_dims, element_size);
|
||||
let mut buf = layout_v4_chunked_prefix(chunk_dims, element_size, version);
|
||||
buf.push(5); // chunk index type = 5 (version-2 B-tree)
|
||||
buf.extend_from_slice(&node_size.to_le_bytes());
|
||||
buf.push(BT2_SPLIT_PERCENT);
|
||||
@@ -1868,7 +1990,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn serialize_v4_single_chunk_no_filters_roundtrip() {
|
||||
let msg = serialize_v4_single_chunk(&[20], 0x1000, None, None, 8, 8);
|
||||
let msg = serialize_v4_single_chunk(&[20], 0x1000, None, None, 8, 8, 4);
|
||||
let layout = DataLayout::parse(&msg, 8, 8).unwrap();
|
||||
match layout {
|
||||
DataLayout::Chunked {
|
||||
@@ -1893,7 +2015,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn serialize_v4_single_chunk_with_filters_roundtrip() {
|
||||
let msg = serialize_v4_single_chunk(&[100], 0x2000, Some(500), Some(0), 8, 8);
|
||||
let msg = serialize_v4_single_chunk(&[100], 0x2000, Some(500), Some(0), 8, 8, 4);
|
||||
let layout = DataLayout::parse(&msg, 8, 8).unwrap();
|
||||
match layout {
|
||||
DataLayout::Chunked {
|
||||
@@ -1912,7 +2034,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn serialize_v4_fixed_array_roundtrip() {
|
||||
let msg = serialize_v4_fixed_array(&[20], 0x3000, 8, 8, 4);
|
||||
let msg = serialize_v4_fixed_array(&[20], 0x3000, 8, 8, 4, 4);
|
||||
let layout = DataLayout::parse(&msg, 8, 8).unwrap();
|
||||
match layout {
|
||||
DataLayout::Chunked {
|
||||
@@ -1956,11 +2078,213 @@ mod tests {
|
||||
assert_eq!(&fa[28..32], b"FADB");
|
||||
}
|
||||
|
||||
// ---- Chunks of 4 GiB or more (layout message version 5) ----
|
||||
|
||||
/// 2^29 + 1 `f64`: 4 GiB + 8 bytes, the smallest `f64` chunk past
|
||||
/// `u32::MAX`.
|
||||
const HUGE_DIM: u64 = (1 << 29) + 1;
|
||||
const HUGE_BYTES: u64 = HUGE_DIM * 8;
|
||||
|
||||
fn huge_chunk(address: u64, compressed_size: u64) -> WrittenChunk {
|
||||
WrittenChunk {
|
||||
address,
|
||||
compressed_size,
|
||||
raw_size: HUGE_BYTES,
|
||||
filter_mask: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunks_past_u32_max_take_layout_version_5() {
|
||||
assert_eq!(layout_version_for(0), 4);
|
||||
assert_eq!(layout_version_for(u64::from(u32::MAX)), 4);
|
||||
assert_eq!(layout_version_for(u64::from(u32::MAX) + 1), 5);
|
||||
assert_eq!(layout_version_for(HUGE_BYTES), 5);
|
||||
}
|
||||
|
||||
/// `H5D_FARRAY_FILT_COMPUTE_CHUNK_SIZE_LEN` (and its EA and v2 B-tree
|
||||
/// twins) in libhdf5 2.2.0: one byte more than the chunk size needs up to
|
||||
/// layout version 4, the size of lengths from version 5.
|
||||
#[test]
|
||||
fn chunk_size_len_follows_libhdf5() {
|
||||
assert_eq!(chunk_size_len(1, 4, 8), 2);
|
||||
assert_eq!(chunk_size_len(160, 4, 8), 2);
|
||||
assert_eq!(chunk_size_len(255, 4, 8), 2);
|
||||
assert_eq!(chunk_size_len(256, 4, 8), 3);
|
||||
assert_eq!(chunk_size_len(u64::from(u32::MAX), 4, 8), 5);
|
||||
assert_eq!(chunk_size_len(1 << 32, 4, 8), 6);
|
||||
assert_eq!(chunk_size_len(u64::MAX, 4, 8), 8);
|
||||
assert_eq!(chunk_size_len(HUGE_BYTES, 5, 8), 8);
|
||||
assert_eq!(chunk_size_len(160, 5, 8), 8);
|
||||
assert_eq!(chunk_size_len(HUGE_BYTES, 5, 4), 4);
|
||||
let slots = [Some(huge_chunk(0x1000, 20_000)), None];
|
||||
assert_eq!(filtered_chunk_size_len(&slots, 8), 8);
|
||||
let small = [Some(WrittenChunk {
|
||||
raw_size: 160,
|
||||
..huge_chunk(0x1000, 100)
|
||||
})];
|
||||
assert_eq!(filtered_chunk_size_len(&small, 8), 2);
|
||||
}
|
||||
|
||||
/// A 4 GiB + 8 byte chunk: every layout message is version 5 with the
|
||||
/// dimensions libhdf5 writes (4 bytes each: 0x20000001 and 8), and a
|
||||
/// filtered index element stores the chunk's size in 8 bytes.
|
||||
#[test]
|
||||
fn huge_chunk_layout_messages_and_index_elements() {
|
||||
let dims = [HUGE_DIM as u32];
|
||||
let parsed = |msg: &[u8]| {
|
||||
assert_eq!(msg[0], 5, "layout message version");
|
||||
// Class chunked, then (after the flags) 2 dimensions of 4 bytes.
|
||||
assert_eq!(&msg[1..2], &[2]);
|
||||
assert_eq!(&msg[3..5], &[2, 4]);
|
||||
assert_eq!(&msg[5..13], &[1, 0, 0, 0x20, 8, 0, 0, 0]);
|
||||
match DataLayout::parse(msg, 8, 8).unwrap() {
|
||||
DataLayout::Chunked {
|
||||
chunk_dimensions,
|
||||
chunk_index_type,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(chunk_dimensions, vec![HUGE_DIM as u32, 8]);
|
||||
chunk_index_type.unwrap()
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
};
|
||||
let single = serialize_v4_single_chunk(&dims, 0x800, Some(20_000), Some(0), 8, 8, 5);
|
||||
assert_eq!(parsed(&single), 1);
|
||||
// Filtered size (8 bytes), filter mask, address.
|
||||
assert_eq!(single.len(), 13 + 1 + 8 + 4 + 8);
|
||||
assert_eq!(
|
||||
parsed(&serialize_v4_fixed_array(&dims, 0x800, 8, 8, 10, 5)),
|
||||
3
|
||||
);
|
||||
let ea = ea_writer::serialize_v4_extensible_array(&dims, 0x800, 8, 8, 5);
|
||||
assert_eq!(parsed(&ea), 4);
|
||||
assert_eq!(
|
||||
parsed(&serialize_v4_btree_v2(&dims, 0x800, 8, 8, 2048, 5)),
|
||||
5
|
||||
);
|
||||
|
||||
let slots = [Some(huge_chunk(0x1000, 20_000)), None];
|
||||
// FAHD: element size (address 8 + size 8 + mask 4) at byte 6.
|
||||
let fa = build_fixed_array_at(&slots, 8, 8, true, 0x2000);
|
||||
assert_eq!(fa[6], 20);
|
||||
// FADB element 0: address, then the stored size in 8 bytes.
|
||||
let fadb = 28;
|
||||
let prefix = 4 + 1 + 1 + 8;
|
||||
assert_eq!(
|
||||
&fa[fadb + prefix..fadb + prefix + 8],
|
||||
&0x1000u64.to_le_bytes()
|
||||
);
|
||||
assert_eq!(
|
||||
&fa[fadb + prefix + 8..fadb + prefix + 16],
|
||||
&20_000u64.to_le_bytes()
|
||||
);
|
||||
// AEHD: element size at byte 6 as well.
|
||||
let ea = ea_writer::build_extensible_array_at(&slots, 8, 8, true, 0x2000);
|
||||
assert_eq!(&ea[..4], b"EAHD");
|
||||
assert_eq!(ea[6], 20);
|
||||
// BTHD: record size (element + 8-byte scaled offset) at bytes 10-11.
|
||||
let chunk = huge_chunk(0x1000, 20_000);
|
||||
let (bt, _) =
|
||||
build_btree_v2_chunk_index_at(1, &[(vec![0], &chunk)], 8, 8, true, 0x2000).unwrap();
|
||||
assert_eq!(&bt[..4], b"BTHD");
|
||||
assert_eq!(u16::from_le_bytes([bt[10], bt[11]]), 28);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn huge_chunks_refused_where_unsupported() {
|
||||
// A chunk dimension of 2^32 or more.
|
||||
assert!(matches!(
|
||||
checked_chunk_dims(&[1 << 32], 1),
|
||||
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("2^32")
|
||||
));
|
||||
// A chunk size that overflows u64.
|
||||
assert!(matches!(
|
||||
checked_chunk_dims(&[u32::MAX.into(), u32::MAX.into(), 2], 8),
|
||||
Err(FormatError::Overflow(_))
|
||||
));
|
||||
assert_eq!(
|
||||
checked_chunk_dims(&[HUGE_DIM], 8).unwrap(),
|
||||
(vec![HUGE_DIM as u32], HUGE_BYTES)
|
||||
);
|
||||
// Filters that cannot take a chunk that large.
|
||||
for plugin in [
|
||||
PluginFilter::Lzf,
|
||||
PluginFilter::Bzip2 { level: 9 },
|
||||
PluginFilter::Blosc {
|
||||
codec: BloscCodec::Lz4,
|
||||
level: 5,
|
||||
shuffle: BloscShuffle::Byte,
|
||||
},
|
||||
PluginFilter::Bitshuffle {
|
||||
block_size: 0,
|
||||
compression: BitshuffleCompression::Lz4,
|
||||
},
|
||||
] {
|
||||
let options = ChunkOptions {
|
||||
plugin: Some(plugin),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(
|
||||
check_huge_chunk_filters(&options, HUGE_BYTES),
|
||||
Err(FormatError::FilterError(m)) if m.contains("4 GiB")
|
||||
));
|
||||
}
|
||||
for options in [
|
||||
ChunkOptions {
|
||||
deflate_level: Some(6),
|
||||
shuffle: true,
|
||||
fletcher32: true,
|
||||
..Default::default()
|
||||
},
|
||||
ChunkOptions {
|
||||
zstd_level: Some(3),
|
||||
..Default::default()
|
||||
},
|
||||
ChunkOptions {
|
||||
lz4: true,
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
check_huge_chunk_filters(&options, HUGE_BYTES).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Chunks are extracted row by row: the result is the element-by-element
|
||||
/// split, edge padding included.
|
||||
#[test]
|
||||
fn extract_chunk_matches_elementwise_split() {
|
||||
let shape = [5u64, 7, 3];
|
||||
let chunks = [2u64, 3, 2];
|
||||
let data: Vec<u8> = (0..5 * 7 * 3 * 2).map(|i| i as u8).collect();
|
||||
let n = chunk_count(&shape, &chunks);
|
||||
assert_eq!(n, 3 * 3 * 2);
|
||||
for i in 0..n {
|
||||
let (offsets, chunk) = extract_chunk(&data, &shape, &chunks, 2, i);
|
||||
assert_eq!(chunk.len(), 2 * 3 * 2 * 2);
|
||||
for (e, pair) in chunk.chunks_exact(2).enumerate() {
|
||||
let c = [e / 6, (e / 2) % 3, e % 2];
|
||||
let g: Vec<u64> = (0..3).map(|d| offsets[d] + c[d] as u64).collect();
|
||||
let expect = if (0..3).all(|d| g[d] < shape[d]) {
|
||||
let flat = ((g[0] * 7 + g[1]) * 3 + g[2]) as usize;
|
||||
[data[2 * flat], data[2 * flat + 1]]
|
||||
} else {
|
||||
[0, 0]
|
||||
};
|
||||
assert_eq!(pair, expect, "chunk {i} element {e}");
|
||||
}
|
||||
}
|
||||
// Data shorter than the shape: the missing elements stay zero.
|
||||
let (_, chunk) = extract_chunk(&data[..5], &[4], &[4], 2, 0);
|
||||
assert_eq!(chunk, [0, 1, 2, 3, 0, 0, 0, 0]);
|
||||
}
|
||||
|
||||
// ---- Extensible Array tests ----
|
||||
|
||||
#[test]
|
||||
fn serialize_v4_extensible_array_roundtrip() {
|
||||
let msg = ea_writer::serialize_v4_extensible_array(&[10], 0x4000, 8, 8);
|
||||
let msg = ea_writer::serialize_v4_extensible_array(&[10], 0x4000, 8, 8, 4);
|
||||
let layout = DataLayout::parse(&msg, 8, 8).unwrap();
|
||||
match layout {
|
||||
DataLayout::Chunked {
|
||||
|
||||
Reference in New Issue
Block a user