quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 1 Wiki Activity

Chunk dimensions of 2^32 or more; unfiltered 4 GiB chunks written without copies

Browse Source 
- DataLayout::Chunked::chunk_dimensions is Vec<u64> (was Vec<u32>), and
  the chunk index readers, writers and serializers take &[u64]: layout
  messages of version 4/5 store each dimension in up to 8 bytes, and
  libhdf5 2.x writes dimensions of 2^32 or more (layout version 5). Such
  dimensions were refused on read (InvalidChunkDimensions) and write. A
  version-3 layout (4-byte dimensions) is never written for them; a chunk
  whose size overflows 64 bits is refused when opened.
- The file writer lays chunked datasets out as pieces referring to the
  chunks instead of copying them into one buffer per pass, and an
  unfiltered chunk that is a contiguous run of the dataset's data (a
  dataset stored as one chunk of its shape, row blocks) borrows it; a
  filtered one is compressed straight from it. Contiguous datasets are not
  copied either. FileWriter::finish_with streams the file to a callback;
  FileBuilder::write uses it, so the file is never assembled in memory.
  DatasetBuilder::with_u8_data_owned takes the data without a copy.
  Peak RSS writing one unfiltered 1 GiB chunk (with_u8_data_owned +
  write): 5.0 GiB before, 1.0 GiB after; with_u8_data: 6.0 -> 2.0 GiB.
- extract_chunk no longer panics on data shorter than the shape.
- Tests: huge_chunk_dims.h5 fixture (libhdf5 2.0.0 via h5py 3.16, u8
  chunks of 2^32 + 7), and opt-in end-to-end tests of chunk dims >= 2^32
  (filtered and unfiltered, read and written, h5py and h5dump 2.2.0), of
  an unfiltered 4 GiB+ chunk written by clawhdf5, and of LZ4/Zstd chunks
  of that size; example write_one_chunk for memory measurements.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-29 20:47:33 -05:00
co-authored by Claude Opus 5.5
parent e4ba09946f
commit e4b3a76dc7
17 changed files with 1156 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_read.rs
+42 -30
View File
@@ -555,7 +555,7 @@ pub(crate) fn checked_byte_len(elements: u64, elem_size: usize) -> Result<usize,
/// HDF5 2.0 writes them with layout version 5. A zero chunk dimension used
/// to read as all fill values, and a huge one to hang the reader.
pub(crate) fn chunk_geometry(
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
layout_version: u8,
dataspace: &Dataspace,
elem_size: usize,
@@ -576,15 +576,26 @@ pub(crate) fn chunk_geometry(
"chunk size must be > 0, dim = {d}"
)));
}
let bytes = spatial
.iter()
.fold(elem_size as u128, |acc, &c| acc * u128::from(c));
// Saturating: 33 dimensions of up to 64 bits each can overflow u128.
let bytes = spatial.iter().fold(elem_size as u128, |acc, &c| {
acc.saturating_mul(u128::from(c))
});
// So the indexes can compute a chunk's size in 64 bits.
if bytes > u128::from(u64::MAX) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk size overflows 64 bits (chunk {spatial:?} of {elem_size}-byte elements)"
)));
}
if layout_version < 4 && bytes > u128::from(u32::MAX) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk size must be < 4GB with v1 b-tree index (chunk {spatial:?} of {elem_size}-byte elements)"
)));
}
Ok((rank, spatial.iter().map(|&c| c as usize).collect()))
let spatial = spatial
.iter()
.map(|&c| to_usize(c))
.collect::<Result<Vec<usize>, _>>()?;
Ok((rank, spatial))
}
/// The size of one element of `dt` as stored in the file: a
@@ -625,7 +636,7 @@ pub fn check_chunk_element_size(
return Ok(());
};
let expected = stored_element_size(datatype, offset_size);
if u64::from(stored) != expected {
if stored != expected {
return Err(FormatError::InvalidChunkDimensions(format!(
"stored datatype size in chunk layout does not match datatype description \
(layout {stored} bytes, datatype {expected})"
@@ -759,7 +770,7 @@ pub fn collect_chunk_info_in<S: Storage + ?Sized>(
pub fn collect_chunk_info_checked(
file_data: &[u8],
btree_address: u64,
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
@@ -776,7 +787,7 @@ pub fn collect_chunk_info_checked(
pub fn collect_chunk_info_checked_in<S: Storage + ?Sized>(
file_data: &S,
btree_address: u64,
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
@@ -808,7 +819,7 @@ pub fn collect_chunk_info_checked_in<S: Storage + ?Sized>(
.iter_mut()
.zip(chunk.offsets.iter().zip(chunk_dimensions))
{
*w = o / u64::from(d);
*w = o / d;
}
wanted[ndims - 1] = 0;
if let Some(i) = root.find(&wanted) {
@@ -849,9 +860,9 @@ impl ChunkNode {
}
}
fn scale_keys(&mut self, dims: &[u32]) {
fn scale_keys(&mut self, dims: &[u64]) {
for (k, &d) in self.keys.iter_mut().zip(dims.iter().cycle()) {
*k /= u64::from(d);
*k /= d;
}
if let ChunkChildren::Nodes(nodes) = &mut self.children {
for n in nodes {
@@ -919,9 +930,9 @@ fn btree_cmp3(lt: &[u64], scaled: &[u64], rt: &[u64]) -> core::cmp::Ordering {
/// Check one v1 B-tree chunk key's offsets (see
/// [`collect_chunk_info_checked`]).
fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), FormatError> {
fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u64]) -> Result<(), FormatError> {
for (&offset, &dim) in offsets.iter().zip(chunk_dimensions) {
if dim == 0 || offset % u64::from(dim) != 0 {
if dim == 0 || offset % dim != 0 {
return Err(FormatError::ChunkedReadError(format!(
"bad coordinate offset {offsets:?} for chunk dimensions {chunk_dimensions:?}"
)));
@@ -937,7 +948,7 @@ fn read_key_offsets(
file_data: &[u8],
pos: usize,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
chunk_dimensions: Option<&[u64]>,
out: &mut Vec<u64>,
) -> Result<(), FormatError> {
let start = out.len();
@@ -966,7 +977,7 @@ fn parse_chunk_node<S: Storage + ?Sized>(
file_data: &S,
btree_address: u64,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
chunk_dimensions: Option<&[u64]>,
offset_size: u8,
depth: usize,
stored: &mut Vec<ChunkInfo>,
@@ -1075,7 +1086,7 @@ fn parse_chunk_node<S: Storage + ?Sized>(
pub fn generate_implicit_chunks(
base_address: u64,
dataset_dims: &[u64],
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
element_size: u32,
) -> Vec<ChunkInfo> {
generate_implicit_chunks_in_grid(
@@ -1097,17 +1108,18 @@ pub fn generate_implicit_chunks_in_grid(
base_address: u64,
dataset_dims: &[u64],
max_dims: &[u64],
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
element_size: u32,
) -> Vec<ChunkInfo> {
let rank = chunk_dimensions.len();
let chunk_byte_size: u64 =
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
let chunk_byte_size: u64 = chunk_dimensions
.iter()
.fold(u64::from(element_size), |acc, &d| acc.saturating_mul(d));
let mut num_chunks_per_dim = Vec::with_capacity(rank);
let mut grid_per_dim = Vec::with_capacity(rank);
for d in 0..rank {
let ch = chunk_dimensions[d] as u64;
let ch = chunk_dimensions[d];
let n = dataset_dims[d].div_ceil(ch);
num_chunks_per_dim.push(n);
grid_per_dim.push(max_dims.get(d).map_or(n, |m| m.div_ceil(ch)).max(n));
@@ -1125,7 +1137,7 @@ pub fn generate_implicit_chunks_in_grid(
let nchunks = num_chunks_per_dim[d];
let chunk_idx = remaining % nchunks;
remaining /= nchunks;
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
offsets[d] = chunk_idx * chunk_dimensions[d];
grid_idx = grid_idx.saturating_add(chunk_idx.saturating_mul(down));
down = down.saturating_mul(grid_per_dim[d]);
}
@@ -1350,7 +1362,7 @@ pub fn list_chunks_in<S: Storage + ?Sized>(
}
(4, Some(2)) => {
// Implicit index — use spatial chunk dims only
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
let spatial_chunk_dims: &[u64] = &chunk_dimensions[..rank];
generate_implicit_chunks_in_grid(
addr,
&dataspace.dimensions,
@@ -1364,7 +1376,7 @@ pub fn list_chunks_in<S: Storage + ?Sized>(
}
(4, Some(3)) => {
// Fixed Array — use spatial chunk dims only
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
let spatial_chunk_dims: &[u64] = &chunk_dimensions[..rank];
let header = FixedArrayHeader::parse_in(
file_data,
checked_addr(addr)?,
@@ -1384,7 +1396,7 @@ pub fn list_chunks_in<S: Storage + ?Sized>(
}
(4, Some(4)) => {
// Extensible Array — use spatial chunk dims only
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
let spatial_chunk_dims: &[u64] = &chunk_dimensions[..rank];
let header = ExtensibleArrayHeader::parse_in(
file_data,
checked_addr(addr)?,
@@ -2435,12 +2447,12 @@ mod tests {
/// A leaf whose final key is the one libhdf5 writes past the last chunk:
/// each coordinate of the last chunk plus its chunk dimension (the
/// element size last).
fn build_chunk_btree_leaf_dims(chunks: &[ChunkInfo], dims: &[u32], offset_size: u8) -> Vec<u8> {
fn build_chunk_btree_leaf_dims(chunks: &[ChunkInfo], dims: &[u64], offset_size: u8) -> Vec<u8> {
let last = &chunks.last().expect("a chunk").offsets;
let end: Vec<u64> = dims
.iter()
.enumerate()
.map(|(d, &c)| last.get(d).copied().unwrap_or(0) + u64::from(c))
.map(|(d, &c)| last.get(d).copied().unwrap_or(0) + c)
.collect();
build_chunk_btree_leaf_to(chunks, &end, offset_size)
}
@@ -2771,7 +2783,7 @@ mod tests {
}
// Build B-tree at offset 0x100
let dims = [chunk_size_elems as u32, elem_size as u32];
let dims = [chunk_size_elems as u64, elem_size as u64];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
@@ -3145,7 +3157,7 @@ mod tests {
data_offset += compressed.len() + 16; // some padding
}
let dims = [chunk_elems as u32, elem_size as u32];
let dims = [chunk_elems as u64, elem_size as u64];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
@@ -3228,7 +3240,7 @@ mod tests {
}
}
let dims = [chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32];
let dims = [chunk_dims[0] as u64, chunk_dims[1] as u64, elem_size as u64];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
@@ -3408,7 +3420,7 @@ mod tests {
}
let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_elems as u32, elem_size as u32],
chunk_dimensions: vec![chunk_elems as u64, elem_size as u64],
btree_address: Some(data_addr as u64),
version: 4,
chunk_index_type: Some(1),
crates/clawhdf5-format/src/chunked_write.rs
+351 -102
View File
@@ -5,12 +5,14 @@ extern crate alloc;
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use alloc::{borrow::Cow, format, vec, vec::Vec};
#[cfg(feature = "std")]
use std::borrow::Cow;
use crate::btree_v1_write;
use crate::btree_v2_write::{BTreeV2Params, build_btree_v2};
use crate::checksum::jenkins_lookup3;
use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line};
use crate::chunk_cache::CACHE_LINE_SIZE;
use crate::chunk_grid::ChunkGrid;
use crate::ea_writer;
use crate::error::FormatError;
@@ -464,7 +466,9 @@ fn extract_chunk(
let dst: usize = (0..rank).map(|d| idx[d] * chunk_strides[d]).sum::<usize>() * element_size;
// Whole elements only, as far as `raw_data` reaches.
let n = row.min(raw_data.len().saturating_sub(src)) / element_size * element_size;
chunk[dst..dst + n].copy_from_slice(&raw_data[src..src + n]);
if n > 0 {
chunk[dst..dst + n].copy_from_slice(&raw_data[src..src + n]);
}
// Next row: advance every dimension but the last.
let mut d = rank - 1;
loop {
@@ -481,6 +485,66 @@ fn extract_chunk(
}
}
/// The bytes of the `linear_idx`-th chunk (see [`extract_chunk`]),
/// borrowed from `raw_data` when the chunk is one contiguous run of it: a
/// chunk inside the dataset (not past its edge) whose dimensions after the
/// first one above 1 equal the dataset's. A dataset stored as one chunk of
/// its own shape is such a run, so its chunk is never copied. Other chunks
/// are copied out, as [`extract_chunk`] does.
fn chunk_bytes_of<'a>(
raw_data: &'a [u8],
shape: &[u64],
chunk_dims: &[u64],
element_size: usize,
linear_idx: u64,
) -> Cow<'a, [u8]> {
if shape.is_empty() {
return Cow::Borrowed(raw_data);
}
if let Some(range) = contiguous_chunk(shape, chunk_dims, element_size, linear_idx)
&& let (Ok(start), Ok(end)) = (usize::try_from(range.start), usize::try_from(range.end))
&& end <= raw_data.len()
{
return Cow::Borrowed(&raw_data[start..end]);
}
Cow::Owned(extract_chunk(raw_data, shape, chunk_dims, element_size, linear_idx).1)
}
/// The byte range of the `linear_idx`-th chunk in the row-major dataset,
/// when the chunk is a contiguous run of it (see [`chunk_bytes_of`]).
fn contiguous_chunk(
shape: &[u64],
chunk_dims: &[u64],
element_size: usize,
linear_idx: u64,
) -> Option<core::ops::Range<u64>> {
let rank = shape.len();
// Past the first chunk dimension above 1, every chunk dimension must
// span the dataset's.
let first_wide = chunk_dims.iter().position(|&c| c > 1).unwrap_or(rank);
if (first_wide + 1..rank).any(|d| chunk_dims[d] != shape[d]) {
return None;
}
let mut remaining = linear_idx;
let mut start = 0u64;
let mut stride = element_size as u64;
for d in (0..rank).rev() {
let n = shape[d].div_ceil(chunk_dims[d]);
let offset = (remaining % n).checked_mul(chunk_dims[d])?;
remaining /= n;
// A chunk past the dataset's edge is padded: not a run of it.
if offset.checked_add(chunk_dims[d])? > shape[d] {
return None;
}
start = start.checked_add(offset.checked_mul(stride)?)?;
stride = stride.checked_mul(shape[d])?;
}
let len = chunk_dims
.iter()
.try_fold(element_size as u64, |acc, &c| acc.checked_mul(c))?;
Some(start..start.checked_add(len)?)
}
/// Parallel compression threshold: use rayon when chunk count exceeds this.
///
/// Lowered to 2 to enable parallel compression for typical 4-chunk workloads
@@ -508,25 +572,21 @@ const PARALLEL_COMPRESS_MAX_CHUNK_BYTES: u64 = 64 << 20;
/// at most [`PARALLEL_COMPRESS_MAX_CHUNK_BYTES`], compression runs across
/// rayon threads; otherwise it is sequential. Output order matches chunk
/// order, so per-chunk bytes are identical to the sequential path.
fn compress_all_chunks(
raw_data: &[u8],
fn compress_all_chunks<'a>(
raw_data: &'a [u8],
shape: &[u64],
chunk_dims: &[u64],
element_size: usize,
chunk_bytes: u64,
pipeline: &Option<FilterPipeline>,
) -> Result<Vec<(u64, Vec<u8>, u32)>, FormatError> {
let one = |i: u64| -> Result<(u64, Vec<u8>, u32), FormatError> {
let (_, raw) = if shape.is_empty() {
(Vec::new(), raw_data.to_vec())
} else {
extract_chunk(raw_data, shape, chunk_dims, element_size, i)
};
) -> Result<Vec<PreparedChunk<'a>>, FormatError> {
let one = |i: u64| -> Result<PreparedChunk<'a>, FormatError> {
let raw = chunk_bytes_of(raw_data, shape, chunk_dims, element_size, i);
let raw_size = raw.len() as u64;
match pipeline {
Some(pl) => {
let (stored, mask) = compress_chunk_masked(&raw, pl, element_size as u32)?;
Ok((raw_size, stored, mask))
Ok((raw_size, Cow::Owned(stored), mask))
}
None => Ok((raw_size, raw, 0)),
}
@@ -557,7 +617,7 @@ fn compress_all_chunks(
/// layout/pipeline messages. `base_address` is where the blob will be placed in the file.
/// Serialize a v4 single chunk layout message (public for OH size estimation).
pub fn serialize_v4_single_chunk_pub(
chunk_dims: &[u32],
chunk_dims: &[u64],
chunk_address: u64,
filtered_size: Option<u64>,
filter_mask: Option<u32>,
@@ -578,7 +638,7 @@ pub fn serialize_v4_single_chunk_pub(
/// Serialize a v4 (or, for a chunk of 4 GiB or more, v5) single chunk
/// layout message.
fn serialize_v4_single_chunk(
chunk_dims: &[u32],
chunk_dims: &[u64],
chunk_address: u64,
filtered_size: Option<u64>,
filter_mask: Option<u32>,
@@ -622,7 +682,7 @@ fn serialize_v4_single_chunk(
/// Serialize a v4 Fixed Array layout message.
fn serialize_v4_fixed_array(
chunk_dims: &[u32],
chunk_dims: &[u64],
fixed_array_address: u64,
offset_size: u8,
element_size: u32,
@@ -653,7 +713,8 @@ fn serialize_v4_fixed_array(
/// dimensions, then the element size). Each takes the fewest bytes that hold
/// the largest, as libhdf5 computes it (`H5D__chunk_set_sizes`:
/// `(log2(dim) + 8) / 8`); HDF5 2.0.0 refuses any other width.
pub(crate) fn push_v4_chunk_dims(buf: &mut Vec<u8>, chunk_dims: &[u32], element_size: u32) {
pub(crate) fn push_v4_chunk_dims(buf: &mut Vec<u8>, chunk_dims: &[u64], element_size: u32) {
let element_size = u64::from(element_size);
let max_dim = chunk_dims
.iter()
.copied()
@@ -661,14 +722,14 @@ pub(crate) fn push_v4_chunk_dims(buf: &mut Vec<u8>, chunk_dims: &[u32], element_
.max()
.unwrap_or(1)
.max(1);
let width = (32 - max_dim.leading_zeros()).div_ceil(8) as usize;
let width = (64 - max_dim.leading_zeros()).div_ceil(8) as usize;
buf.push(width as u8);
for &d in chunk_dims.iter().chain(core::iter::once(&element_size)) {
buf.extend_from_slice(&d.to_le_bytes()[..width]);
}
}
fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32, version: u8) -> Vec<u8> {
fn layout_v4_chunked_prefix(chunk_dims: &[u64], element_size: u32, version: u8) -> Vec<u8> {
let mut buf = Vec::new();
buf.push(version);
buf.push(2); // class = chunked
@@ -884,7 +945,66 @@ pub fn precompress_chunks(
element_size: usize,
options: &ChunkOptions,
) -> Result<PrecompressedChunks, FormatError> {
let (_, chunk_bytes) = checked_chunk_dims(chunk_dims, element_size)?;
let p = prepare_chunks(raw_data, shape, chunk_dims, element_size, options)?;
Ok(PrecompressedChunks {
chunks: p
.chunks
.into_iter()
.map(|(raw, stored, mask)| (raw, stored.into_owned(), mask))
.collect(),
has_filters: p.has_filters,
element_size: p.element_size,
shape: p.shape,
chunk_dims: p.chunk_dims,
pipeline_message: p.pipeline_message,
})
}
/// One chunk of [`PreparedChunks`]: its raw size, its stored bytes and its
/// filter mask.
pub(crate) type PreparedChunk<'a> = (u64, Cow<'a, [u8]>, u32);
/// [`PrecompressedChunks`] whose stored bytes may borrow the dataset's data:
/// an unfiltered chunk that is a contiguous run of it (a dataset stored as
/// one chunk, say) is not copied. What the file writer lays out.
pub(crate) struct PreparedChunks<'a> {
pub(crate) chunks: Vec<PreparedChunk<'a>>,
pub(crate) has_filters: bool,
pub(crate) element_size: usize,
pub(crate) shape: Vec<u64>,
pub(crate) chunk_dims: Vec<u64>,
pub(crate) pipeline_message: Option<Vec<u8>>,
}
impl PrecompressedChunks {
/// A view of these chunks as [`PreparedChunks`] (their bytes borrowed).
fn prepared(&self) -> PreparedChunks<'_> {
PreparedChunks {
chunks: self
.chunks
.iter()
.map(|(raw, stored, mask)| (*raw, Cow::Borrowed(&stored[..]), *mask))
.collect(),
has_filters: self.has_filters,
element_size: self.element_size,
shape: self.shape.clone(),
chunk_dims: self.chunk_dims.clone(),
pipeline_message: self.pipeline_message.clone(),
}
}
}
/// [`precompress_chunks`], borrowing what it can (see [`PreparedChunks`]).
/// A filtered chunk that is a contiguous run of `raw_data` is compressed
/// straight from it, without a copy of the raw chunk.
pub(crate) fn prepare_chunks<'a>(
raw_data: &'a [u8],
shape: &[u64],
chunk_dims: &[u64],
element_size: usize,
options: &ChunkOptions,
) -> Result<PreparedChunks<'a>, FormatError> {
let chunk_bytes = checked_chunk_dims(chunk_dims, element_size)?;
if chunk_bytes > MAX_V4_CHUNK_BYTES {
check_huge_chunk_filters(options, chunk_bytes)?;
}
@@ -902,7 +1022,7 @@ pub fn precompress_chunks(
&pipeline,
)?;
Ok(PrecompressedChunks {
Ok(PreparedChunks {
chunks,
has_filters,
element_size,
@@ -912,26 +1032,87 @@ pub fn precompress_chunks(
})
}
/// The chunk dimensions as the layout message stores them (each below
/// 2^32), and one chunk's size in bytes. A chunk dimension of 2^32 or more
/// (which HDF5 2.0 can store, in wider fields) is refused, as it is when
/// read: clawhdf5 holds chunk dimensions as `u32`. So is a chunk whose size
/// overflows 64 bits, or that this platform cannot hold in memory (a chunk
/// of 4 GiB or more on a 32-bit target).
fn checked_chunk_dims(
chunk_dims: &[u64],
element_size: usize,
) -> Result<(Vec<u32>, u64), FormatError> {
let dims = chunk_dims
.iter()
.map(|&d| {
u32::try_from(d).map_err(|_| {
FormatError::InvalidChunkDimensions(format!(
"chunk dimension {d} is 2^32 or more, which clawhdf5 does not support"
))
})
})
.collect::<Result<Vec<u32>, _>>()?;
/// A piece of a chunked dataset's data blob as laid out by
/// [`place_chunked_data`]: zero padding, a chunk's stored bytes (by index
/// into [`PreparedChunks::chunks`]), or index structures.
#[derive(Debug)]
pub(crate) enum Piece {
Zeros(usize),
Chunk(usize),
Bytes(Vec<u8>),
}
/// A chunked dataset laid out at an address, without its chunk bytes
/// copied: the pieces of its data blob, in order, and its messages.
pub(crate) struct ChunkedPlacement {
pub(crate) pieces: Vec<Piece>,
/// Length of the data blob in bytes.
pub(crate) len: u64,
pub(crate) layout_message: Vec<u8>,
pub(crate) pipeline_message: Option<Vec<u8>>,
}
impl ChunkedPlacement {
/// The data blob as one buffer (the chunks copied in).
fn into_result(self, pre: &PreparedChunks<'_>) -> ChunkedDataResult {
let mut data_bytes = Vec::with_capacity(saturating_usize(self.len));
for piece in &self.pieces {
match piece {
Piece::Zeros(n) => data_bytes.resize(data_bytes.len() + n, 0),
Piece::Chunk(i) => data_bytes.extend_from_slice(&pre.chunks[*i].1),
Piece::Bytes(b) => data_bytes.extend_from_slice(b),
}
}
ChunkedDataResult {
data_bytes,
layout_message: self.layout_message,
pipeline_message: self.pipeline_message,
}
}
}
/// Builds the pieces of a data blob, tracking its length.
#[derive(Default)]
struct Pieces {
pieces: Vec<Piece>,
len: u64,
}
impl Pieces {
/// Pad to the next cache line.
fn align(&mut self) {
let aligned = align_chunk_offset(self.len);
if aligned > self.len {
self.pieces
.push(Piece::Zeros(saturating_usize(aligned - self.len)));
self.len = aligned;
}
}
fn chunk(&mut self, i: usize, len: usize) {
self.pieces.push(Piece::Chunk(i));
self.len += len as u64;
}
fn bytes(&mut self, b: Vec<u8>) {
self.len += b.len() as u64;
self.pieces.push(Piece::Bytes(b));
}
}
/// One chunk's size in bytes. A chunk dimension of 0 is refused, and so is
/// a chunk whose size overflows 64 bits, or that this platform cannot hold
/// in memory (a chunk of 4 GiB or more on a 32-bit target). Chunk
/// dimensions of 2^32 or more are allowed, as in libhdf5 2.x: such a chunk
/// is over 4 GiB, so it takes layout message version 5, whose dimension
/// fields are up to 8 bytes wide (a version-3 layout, with 4-byte fields,
/// is never written for it).
fn checked_chunk_dims(chunk_dims: &[u64], element_size: usize) -> Result<u64, FormatError> {
if chunk_dims.contains(&0) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk dimensions {chunk_dims:?} include 0"
)));
}
let bytes = chunk_dims
.iter()
.try_fold(element_size as u64, |acc, &d| acc.checked_mul(d))
@@ -941,7 +1122,7 @@ fn checked_chunk_dims(
"a chunk of {chunk_dims:?} x {element_size} bytes exceeds this platform's address space"
))
})?;
Ok((dims, bytes))
Ok(bytes)
}
/// The filters clawhdf5 can apply to a chunk of more than `u32::MAX` bytes:
@@ -1010,7 +1191,22 @@ pub fn build_chunked_data_from_precompressed_libver(
low: LibVer,
high: LibVer,
) -> Result<ChunkedDataResult, FormatError> {
let (_, chunk_bytes) = checked_chunk_dims(&pre.chunk_dims, pre.element_size)?;
let pre = pre.prepared();
let placement = place_chunked_data(&pre, base_address, maxshape, low, high)?;
Ok(placement.into_result(&pre))
}
/// [`build_chunked_data_from_precompressed_libver`] without copying the
/// chunks into one buffer: the data blob as [`Piece`]s referring to
/// `pre`'s chunks. The file writer writes the pieces out one by one.
pub(crate) fn place_chunked_data(
pre: &PreparedChunks<'_>,
base_address: u64,
maxshape: Option<&[u64]>,
low: LibVer,
high: LibVer,
) -> Result<ChunkedPlacement, FormatError> {
let chunk_bytes = checked_chunk_dims(&pre.chunk_dims, pre.element_size)?;
if chunk_bytes > MAX_V4_CHUNK_BYTES {
if high < LibVer::V200 {
return Err(FormatError::LibverBound {
@@ -1020,7 +1216,7 @@ pub fn build_chunked_data_from_precompressed_libver(
});
}
} else if low < LibVer::V110 {
return build_btree_v1_chunked_data(pre, base_address, maxshape);
return place_btree_v1_chunked_data(pre, base_address, maxshape);
}
let index = ChunkIndexPlan::new(&pre.shape, maxshape, &pre.chunk_dims)?;
let offset_size: u8 = 8;
@@ -1028,17 +1224,14 @@ pub fn build_chunked_data_from_precompressed_libver(
let num_chunks = pre.chunks.len();
let element_size = pre.element_size;
let mut data_buf = Vec::new();
let mut blob = Pieces::default();
let mut written_chunks = Vec::with_capacity(num_chunks);
for (raw_size, compressed, filter_mask) in &pre.chunks {
let aligned_offset = align_to_cache_line(data_buf.len());
if aligned_offset > data_buf.len() {
data_buf.resize(aligned_offset, 0u8);
}
let address = base_address + data_buf.len() as u64;
for (i, (raw_size, compressed, filter_mask)) in pre.chunks.iter().enumerate() {
blob.align();
let address = base_address + blob.len;
let compressed_size = compressed.len() as u64;
data_buf.extend_from_slice(compressed);
blob.chunk(i, compressed.len());
written_chunks.push(WrittenChunk {
address,
compressed_size,
@@ -1047,28 +1240,22 @@ pub fn build_chunked_data_from_precompressed_libver(
});
}
let (chunk_dims_u32, chunk_bytes) = checked_chunk_dims(&pre.chunk_dims, element_size)?;
let version = layout_version_for(chunk_bytes);
let aligned_idx = align_to_cache_line(data_buf.len());
if aligned_idx > data_buf.len() {
data_buf.resize(aligned_idx, 0u8);
}
blob.align();
let layout_message = match &index {
ChunkIndexPlan::ExtensibleArray(grid) => {
let ea_address = base_address + data_buf.len() as u64;
let ea_address = base_address + blob.len;
let slots = index_slots(grid, &pre.shape, &pre.chunk_dims, &written_chunks, None)?;
let ea_bytes = ea_writer::build_extensible_array_at(
blob.bytes(ea_writer::build_extensible_array_at(
&slots,
offset_size,
length_size,
pre.has_filters,
ea_address,
);
data_buf.extend_from_slice(&ea_bytes);
));
ea_writer::serialize_v4_extensible_array(
&chunk_dims_u32,
&pre.chunk_dims,
ea_address,
offset_size,
element_size as u32,
@@ -1084,7 +1271,7 @@ pub fn build_chunked_data_from_precompressed_libver(
};
let filter_mask = pre.has_filters.then_some(written_chunks[0].filter_mask);
serialize_v4_single_chunk(
&chunk_dims_u32,
&pre.chunk_dims,
chunk_addr,
filtered_size,
filter_mask,
@@ -1094,7 +1281,7 @@ pub fn build_chunked_data_from_precompressed_libver(
)
}
ChunkIndexPlan::FixedArray(grid, nslots) => {
let fa_address = base_address + data_buf.len() as u64;
let fa_address = base_address + blob.len;
let slots = index_slots(
grid,
&pre.shape,
@@ -1102,16 +1289,15 @@ pub fn build_chunked_data_from_precompressed_libver(
&written_chunks,
Some(*nslots),
)?;
let fa_bytes = build_fixed_array_at(
blob.bytes(build_fixed_array_at(
&slots,
offset_size,
length_size,
pre.has_filters,
fa_address,
);
data_buf.extend_from_slice(&fa_bytes);
));
serialize_v4_fixed_array(
&chunk_dims_u32,
&pre.chunk_dims,
fa_address,
offset_size,
element_size as u32,
@@ -1120,7 +1306,7 @@ pub fn build_chunked_data_from_precompressed_libver(
)
}
ChunkIndexPlan::BTreeV2 => {
let bt_address = base_address + data_buf.len() as u64;
let bt_address = base_address + blob.len;
let records: Vec<(Vec<u64>, &WrittenChunk)> = written_chunks
.iter()
.enumerate()
@@ -1134,9 +1320,9 @@ pub fn build_chunked_data_from_precompressed_libver(
pre.has_filters,
bt_address,
)?;
data_buf.extend_from_slice(&bt_bytes);
blob.bytes(bt_bytes);
serialize_v4_btree_v2(
&chunk_dims_u32,
&pre.chunk_dims,
bt_address,
offset_size,
element_size as u32,
@@ -1146,8 +1332,9 @@ pub fn build_chunked_data_from_precompressed_libver(
}
};
Ok(ChunkedDataResult {
data_bytes: data_buf,
Ok(ChunkedPlacement {
pieces: blob.pieces,
len: blob.len,
layout_message,
pipeline_message: pre.pipeline_message.clone(),
})
@@ -1156,11 +1343,11 @@ pub fn build_chunked_data_from_precompressed_libver(
/// Lay out precompressed chunks at `base_address` followed by a version-1
/// B-tree chunk index, with a version-3 layout message: what libhdf5 writes
/// for a chunked dataset under a low bound of 1.8.
fn build_btree_v1_chunked_data(
pre: &PrecompressedChunks,
fn place_btree_v1_chunked_data(
pre: &PreparedChunks<'_>,
base_address: u64,
maxshape: Option<&[u64]>,
) -> Result<ChunkedDataResult, FormatError> {
) -> Result<ChunkedPlacement, FormatError> {
if let Some(ms) = maxshape {
let bad = |what: &str| FormatError::ChunkedReadError(format!("maxshape: {what}"));
if ms.len() != pre.shape.len() {
@@ -1171,20 +1358,17 @@ fn build_btree_v1_chunked_data(
}
}
let offset_size: u8 = 8;
let mut data_buf = Vec::new();
let mut blob = Pieces::default();
let mut entries = Vec::with_capacity(pre.chunks.len());
for (i, (_raw_size, stored, filter_mask)) in pre.chunks.iter().enumerate() {
let aligned_offset = align_to_cache_line(data_buf.len());
if aligned_offset > data_buf.len() {
data_buf.resize(aligned_offset, 0u8);
}
blob.align();
entries.push(btree_v1_write::ChunkEntry {
scaled: scaled_coords(&pre.shape, &pre.chunk_dims, i),
nbytes: stored.len() as u64,
filter_mask: *filter_mask,
address: base_address + data_buf.len() as u64,
address: base_address + blob.len,
});
data_buf.extend_from_slice(stored);
blob.chunk(i, stored.len());
}
let element_size = u32::try_from(pre.element_size)
.map_err(|_| FormatError::Overflow("element size".into()))?;
@@ -1193,11 +1377,8 @@ fn build_btree_v1_chunked_data(
let btree_address = if entries.is_empty() {
u64::MAX
} else {
let aligned_idx = align_to_cache_line(data_buf.len());
if aligned_idx > data_buf.len() {
data_buf.resize(aligned_idx, 0u8);
}
let addr = base_address + data_buf.len() as u64;
blob.align();
let addr = base_address + blob.len;
let tree = btree_v1_write::build_chunk_btree_v1_at(
&entries,
&pre.chunk_dims,
@@ -1205,13 +1386,14 @@ fn build_btree_v1_chunked_data(
addr,
offset_size,
)?;
data_buf.extend_from_slice(&tree);
blob.bytes(tree);
addr
};
let layout_message =
serialize_v3_chunked(&pre.chunk_dims, btree_address, offset_size, element_size)?;
Ok(ChunkedDataResult {
data_bytes: data_buf,
Ok(ChunkedPlacement {
pieces: blob.pieces,
len: blob.len,
layout_message,
pipeline_message: pre.pipeline_message.clone(),
})
@@ -1424,7 +1606,7 @@ fn build_btree_v2_chunk_index_at(
/// Serialize a v4 layout message for a version-2 B-tree chunk index.
fn serialize_v4_btree_v2(
chunk_dims: &[u32],
chunk_dims: &[u64],
btree_address: u64,
offset_size: u8,
element_size: u32,
@@ -2184,7 +2366,7 @@ mod tests {
/// filtered index element stores the chunk's size in 8 bytes.
#[test]
fn huge_chunk_layout_messages_and_index_elements() {
let dims = [HUGE_DIM as u32];
let dims = [HUGE_DIM];
let parsed = |msg: &[u8]| {
assert_eq!(msg[0], 5, "layout message version");
// Class chunked, then (after the flags) 2 dimensions of 4 bytes.
@@ -2197,7 +2379,7 @@ mod tests {
chunk_index_type,
..
} => {
assert_eq!(chunk_dimensions, vec![HUGE_DIM as u32, 8]);
assert_eq!(chunk_dimensions, vec![HUGE_DIM, 8]);
chunk_index_type.unwrap()
}
other => panic!("{other:?}"),
@@ -2245,22 +2427,89 @@ mod tests {
assert_eq!(u16::from_le_bytes([bt[10], bt[11]]), 28);
}
/// A chunk dimension of 2^32 or more (a 1-byte element, 2^32 + 7 per
/// chunk): the dimensions take 5 bytes each, as libhdf5 2.x writes
/// them (`(log2(dim) + 8) / 8`), and parse back whole.
#[test]
fn chunk_dimensions_of_2_pow_32_or_more_take_5_bytes() {
const DIM: u64 = (1 << 32) + 7;
let msg = serialize_v4_single_chunk(&[DIM], 0x800, None, None, 8, 1, 5);
assert_eq!(&msg[..5], &[5, 2, 0, 2, 5]);
assert_eq!(&msg[5..10], &[7, 0, 0, 0, 1]);
assert_eq!(&msg[10..15], &[1, 0, 0, 0, 0]);
assert_eq!(msg[15], 1, "Single Chunk index");
match DataLayout::parse(&msg, 8, 8).unwrap() {
DataLayout::Chunked {
chunk_dimensions, ..
} => assert_eq!(chunk_dimensions, [DIM, 1]),
other => panic!("{other:?}"),
}
// The largest dimension takes all 8 bytes.
let mut buf = Vec::new();
push_v4_chunk_dims(&mut buf, &[u64::MAX, 3], 4);
assert_eq!(buf[0], 8);
assert_eq!(buf.len(), 1 + 3 * 8);
// A version-3 layout has 4-byte dimensions: refused, not truncated.
assert!(serialize_v3_chunked(&[DIM], 0x800, 8, 1).is_err());
}
/// Which chunks are contiguous runs of the dataset's data (borrowed,
/// not copied, when unfiltered).
#[test]
fn contiguous_chunks_are_borrowed() {
// Rows of a 2-D dataset: chunk (2, 5) of shape (5, 5).
assert_eq!(contiguous_chunk(&[5, 5], &[2, 5], 4, 0), Some(0..40));
assert_eq!(contiguous_chunk(&[5, 5], &[2, 5], 4, 1), Some(40..80));
// The last row chunk runs past the edge: padded, so copied.
assert_eq!(contiguous_chunk(&[5, 5], &[2, 5], 4, 2), None);
// Columns split: not contiguous.
assert_eq!(contiguous_chunk(&[4, 6], &[2, 3], 1, 0), None);
// Leading dimensions of 1, then a partial row: contiguous.
assert_eq!(contiguous_chunk(&[2, 3, 8], &[1, 1, 4], 2, 5), Some(40..48));
// The whole dataset as one chunk.
assert_eq!(contiguous_chunk(&[3, 4], &[3, 4], 8, 0), Some(0..96));
let raw: Vec<u8> = (0..100).collect();
let whole = chunk_bytes_of(&raw, &[10, 10], &[10, 10], 1, 0);
assert!(matches!(whole, Cow::Borrowed(b) if b.as_ptr() == raw.as_ptr()));
let rows = chunk_bytes_of(&raw, &[10, 10], &[3, 10], 1, 1);
assert!(matches!(rows, Cow::Borrowed(b) if b == &raw[30..60]));
// Data shorter than the shape is copied (and zero-padded) as before.
let short = chunk_bytes_of(&raw[..50], &[10, 10], &[10, 10], 1, 0);
assert!(matches!(&short, Cow::Owned(v) if v.len() == 100));
// Borrowed or copied, the bytes are what extract_chunk gives.
for (shape, chunk) in [
(&[10u64, 10][..], &[3u64, 10][..]),
(&[10, 10], &[3, 4]),
(&[4, 5, 5], &[1, 5, 5]),
(&[4, 5, 5], &[1, 2, 5]),
] {
for i in 0..chunk_count(shape, chunk) {
let got = chunk_bytes_of(&raw, shape, chunk, 1, i);
assert_eq!(&got[..], &extract_chunk(&raw, shape, chunk, 1, i).1[..]);
}
}
}
#[test]
fn huge_chunks_refused_where_unsupported() {
// A chunk dimension of 2^32 or more.
// A chunk dimension of 0.
assert!(matches!(
checked_chunk_dims(&[1 << 32], 1),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("2^32")
checked_chunk_dims(&[4, 0], 1),
Err(FormatError::InvalidChunkDimensions(_))
));
// A chunk dimension of 2^32 or more is fine (on a 64-bit target).
#[cfg(target_pointer_width = "64")]
assert_eq!(
checked_chunk_dims(&[(1 << 32) + 5], 1).unwrap(),
(1 << 32) + 5
);
// A chunk size that overflows u64.
assert!(matches!(
checked_chunk_dims(&[u32::MAX.into(), u32::MAX.into(), 2], 8),
Err(FormatError::Overflow(_))
));
assert_eq!(
checked_chunk_dims(&[HUGE_DIM], 8).unwrap(),
(vec![HUGE_DIM as u32], HUGE_BYTES)
);
assert_eq!(checked_chunk_dims(&[HUGE_DIM], 8).unwrap(), HUGE_BYTES);
// Filters that cannot take a chunk that large.
for plugin in [
PluginFilter::Lzf,
crates/clawhdf5-format/src/data_layout.rs
+21 -21
View File
@@ -34,7 +34,7 @@ const MAX_LAYOUT_NDIMS: usize = 33;
/// (`H5O__layout_decode`): at most [`MAX_LAYOUT_NDIMS`], no dimension 0, and
/// before version 4 at least one dataspace dimension plus the element size.
/// A zero chunk dimension used to read the dataset as all fill values.
fn check_chunk_dims(dims: Vec<u32>, layout_version: u8) -> Result<Vec<u32>, FormatError> {
fn check_chunk_dims(dims: Vec<u64>, layout_version: u8) -> Result<Vec<u64>, FormatError> {
if dims.len() > MAX_LAYOUT_NDIMS {
return Err(FormatError::InvalidChunkDimensions(
"dimensionality is too large".into(),
@@ -72,7 +72,7 @@ pub enum DataLayout {
/// Chunked: data stored in chunks via a B-tree.
Chunked {
/// Chunk dimension sizes.
chunk_dimensions: Vec<u32>,
chunk_dimensions: Vec<u64>,
/// B-tree address, or `None` if undefined.
btree_address: Option<u64>,
/// Layout version (3 or 4). Version 1/2 messages (HDF5 1.4/1.6-era)
@@ -404,11 +404,11 @@ impl DataLayout {
_ => return Err(FormatError::InvalidLayoutClass(layout_class)),
};
ensure_len(data, p, dimensionality * 4)?;
let dims: Vec<u32> = data[p..p + dimensionality * 4]
let dims: Vec<u64> = data[p..p + dimensionality * 4]
.as_chunks::<4>()
.0
.iter()
.map(|c| u32::from_le_bytes(*c))
.map(|c| u64::from(u32::from_le_bytes(*c)))
.collect();
p += dimensionality * 4;
match layout_class {
@@ -424,7 +424,7 @@ impl DataLayout {
1 => {
let size = dims
.iter()
.try_fold(1u64, |acc, &d| acc.checked_mul(d as u64))
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
.ok_or_else(|| {
FormatError::Overflow(format!("contiguous layout size {dims:?}"))
})?;
@@ -489,7 +489,12 @@ impl DataLayout {
ensure_len(data, p, dimensionality * 4)?;
let mut chunk_dimensions = Vec::with_capacity(dimensionality);
for _ in 0..dimensionality {
let dim = u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]);
let dim = u64::from(u32::from_le_bytes([
data[p],
data[p + 1],
data[p + 2],
data[p + 3],
]));
chunk_dimensions.push(dim);
p += 4;
}
@@ -565,14 +570,8 @@ impl DataLayout {
.iter()
.rev()
.fold(0u64, |acc, &b| (acc << 8) | u64::from(b));
// Chunk dimensions are held as u32; HDF5 2.0 can write
// larger ones (layout version 5), which are refused
// rather than truncated.
let val = u32::try_from(val).map_err(|_| {
FormatError::InvalidChunkDimensions(format!(
"chunk dimension {val} is larger than 2^32 - 1, which is not supported"
))
})?;
// HDF5 2.0 writes dimensions of 2^32 or more (layout
// version 5, chunks over 4 GiB) in 5 to 8 bytes.
chunk_dimensions.push(val);
p += dim_size_encoded_length;
}
@@ -858,7 +857,7 @@ mod tests {
.unwrap_or_else(|e| panic!("width {width}: {e:?}"));
assert!(
matches!(&layout, DataLayout::Chunked { chunk_dimensions, .. }
if chunk_dimensions.iter().map(|&d| u64::from(d)).eq(dims)),
if chunk_dimensions[..] == dims),
"width {width}: {layout:?}"
);
}
@@ -871,12 +870,13 @@ mod tests {
)
);
}
// A dimension past u32 cannot be represented and is refused, not
// truncated.
assert!(matches!(
DataLayout::parse(&v4_chunked_msg(5, &[1 << 32, 8]), 8, 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("2^32")
));
// HDF5 2.0 writes dimensions of 2^32 or more (up to 8 bytes each).
for (width, dim) in [(5u8, (1u64 << 32) + 3), (8, u64::MAX)] {
assert!(matches!(
DataLayout::parse(&v4_chunked_msg(width, &[dim, 1]), 8, 8),
Ok(DataLayout::Chunked { chunk_dimensions, .. }) if chunk_dimensions == [dim, 1]
));
}
}
#[test]
crates/clawhdf5-format/src/ea_writer.rs
+1 -1
View File
@@ -14,7 +14,7 @@ use crate::chunked_write::{
/// Serialize a v4 Extensible Array layout message.
pub(crate) fn serialize_v4_extensible_array(
chunk_dims: &[u32],
chunk_dims: &[u64],
ea_address: u64,
offset_size: u8,
element_size: u32,
crates/clawhdf5-format/src/extensible_array.rs
+8 -8
View File
@@ -454,7 +454,7 @@ pub fn read_extensible_array_chunks(
header: &ExtensibleArrayHeader,
dataset_dims: &[u64],
max_dims: Option<&[u64]>,
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
element_size: u32,
offset_size: u8,
length_size: u8,
@@ -480,7 +480,7 @@ pub fn read_extensible_array_chunks_in<S: Storage + ?Sized>(
header: &ExtensibleArrayHeader,
dataset_dims: &[u64],
max_dims: Option<&[u64]>,
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
element_size: u32,
offset_size: u8,
_length_size: u8,
@@ -489,12 +489,12 @@ pub fn read_extensible_array_chunks_in<S: Storage + ?Sized>(
// Linear indexes follow the maximum dimensions, with the unlimited
// dimension swizzled to the slowest position (see `chunk_grid`).
let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
let grid = ChunkGrid::extensible_array(dataset_dims, max_dims, &dims_u64)?;
let grid = ChunkGrid::extensible_array(dataset_dims, max_dims, chunk_dimensions)?;
let grid = &grid;
let chunk_byte_size: u64 =
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
let chunk_byte_size: u64 = chunk_dimensions
.iter()
.fold(u64::from(element_size), |acc, &d| acc.saturating_mul(d));
// Parse index block (EAIB): signature(4) + version(1) + client_id(1)
// + header address(offset_size), then the inline elements, then the
@@ -930,7 +930,7 @@ mod tests {
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
let ds_dims = vec![40u64]; // 2 chunks × 20 elements
let chunk_dims = vec![20u32];
let chunk_dims = vec![20u64];
let chunks = read_extensible_array_chunks(
&file_data,
&header,
@@ -1057,7 +1057,7 @@ mod tests {
let file_data = build_inline_plus_data_blocks();
let header = ExtensibleArrayHeader::parse(&file_data, 0x100, os, ls).unwrap();
let ds_dims = vec![40u64];
let chunk_dims = vec![10u32];
let chunk_dims = vec![10u64];
let chunks = read_extensible_array_chunks(
&file_data,
&header,
crates/clawhdf5-format/src/file_writer.rs
+233 -58
View File
@@ -3,15 +3,14 @@
//! Produces valid HDF5 files with v3 superblock, v2 object headers,
//! link messages, contiguous datasets, inline and dense attributes.
use crate::addr::saturating_usize;
use crate::addr::{saturating_usize, to_usize};
#[cfg(not(feature = "std"))]
use alloc::{format, string::String, vec, vec::Vec};
use crate::attribute::AttributeMessage;
use crate::btree_v2_write::{BTreeV2Params, build_btree_v2};
use crate::chunked_write::{
ChunkOptions, PrecompressedChunks, build_chunked_data_from_precompressed_libver,
precompress_chunks,
ChunkOptions, Piece, PreparedChunks, place_chunked_data, prepare_chunks,
};
use crate::data_layout::VdsMapping;
use crate::dataspace::{Dataspace, DataspaceType};
@@ -1612,7 +1611,30 @@ impl FileWriter {
self
}
/// The file, in memory. Its datasets' data is copied into the buffer
/// once (a dataset stored as one chunk of its own shape, or unfiltered
/// chunks that are contiguous runs of its data, are not copied before);
/// to write a file without holding it all, use [`Self::finish_with`].
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
let mut buf = Vec::new();
self.finish_into(&mut buf)?;
Ok(buf)
}
/// The file, handed to `put` in order, a piece at a time, without
/// assembling it in memory: an unfiltered chunk or contiguous dataset
/// is passed straight from the data the dataset was given. Writing a
/// dataset stored as one unfiltered chunk this way needs little memory
/// beyond the dataset's own data. `put`'s first error stops the write
/// and is returned.
pub fn finish_with<E: From<FormatError>>(
self,
put: impl FnMut(&[u8]) -> Result<(), E>,
) -> Result<(), E> {
self.finish_into(&mut FnSink(put))
}
fn finish_into<S: Sink>(self, sink: &mut S) -> Result<(), S::Error> {
let page_size = self.page_size;
if let Some(ps) = page_size
&& !(MIN_FILE_SPACE_PAGE_SIZE..=MAX_FILE_SPACE_PAGE_SIZE).contains(&ps)
@@ -1620,7 +1642,8 @@ impl FileWriter {
return Err(FormatError::SerializationError(format!(
"file space page size {ps} is outside libhdf5's \
{MIN_FILE_SPACE_PAGE_SIZE}..={MAX_FILE_SPACE_PAGE_SIZE} bytes"
)));
))
.into());
}
let (low, high) = (self.low, self.high);
@@ -1774,15 +1797,16 @@ impl FileWriter {
})
.collect::<Result<_, _>>()?;
struct DataBlob {
struct DataBlob<'a> {
data: Vec<u8>,
oh_bytes: Vec<u8>,
/// Cached compressed chunks for chunked datasets; reused in Pass 2
/// to avoid re-compressing the same data.
precompressed: Option<PrecompressedChunks>,
/// to avoid re-compressing the same data. Unfiltered chunks
/// borrow the dataset's data where they can.
precompressed: Option<PreparedChunks<'a>>,
}
let mut dummy_blobs: Vec<DataBlob> = Vec::new();
let mut dummy_blobs: Vec<DataBlob<'_>> = Vec::new();
let mut dummy_cursor = 0u64;
for (i, d) in all_ds.iter().enumerate() {
let dense_blob = ds_dense[i]
@@ -1820,21 +1844,16 @@ impl FileWriter {
.resolve_chunk_dims_for(&d.ds.dimensions, elem_size);
// Compress once in Pass 1; cache the result so Pass 2 can skip
// re-compression and just rebuild the index with real addresses.
let pre = precompress_chunks(
let pre = prepare_chunks(
&d.raw,
&d.ds.dimensions,
&chunk_dims,
elem_size,
&d.chunk_options,
)?;
let result = build_chunked_data_from_precompressed_libver(
&pre,
dummy_cursor,
d.maxshape.as_deref(),
low,
high,
)?;
dummy_cursor += result.data_bytes.len() as u64;
let result =
place_chunked_data(&pre, dummy_cursor, d.maxshape.as_deref(), low, high)?;
dummy_cursor += result.len;
let oh = build_chunked_dataset_oh(
&d.dt,
&d.ds,
@@ -1849,7 +1868,7 @@ impl FileWriter {
d.refcount,
)?;
dummy_blobs.push(DataBlob {
data: result.data_bytes,
data: Vec::new(),
oh_bytes: oh,
precompressed: Some(pre),
});
@@ -1955,7 +1974,7 @@ impl FileWriter {
})
.collect::<Result<_, FormatError>>()?;
let mut ds_blobs2: Vec<DataBlob> = Vec::new();
let mut ds_blobs2: Vec<OutBlob<'_>> = Vec::new();
let global_align_threshold = self.alignment_threshold;
let global_align_bytes = self.alignment_bytes;
for (i, d) in all_ds.iter().enumerate() {
@@ -1977,26 +1996,21 @@ impl FileWriter {
&d.fill_message,
d.refcount,
)?;
ds_blobs2.push(DataBlob {
data: gcol_bytes.clone(),
ds_blobs2.push(OutBlob {
data: vec![Out::Slice(gcol_bytes)],
oh_bytes: oh,
precompressed: None,
});
} else if is_chunked[i] {
let base_address = cursor2 as u64;
// Reuse precompressed chunks from Pass 1 — avoids re-compressing
// the same data a second time.
let result = build_chunked_data_from_precompressed_libver(
dummy_blobs[i]
.precompressed
.as_ref()
.expect("chunked dataset missing precompressed cache"),
base_address,
d.maxshape.as_deref(),
low,
high,
)?;
cursor2 += result.data_bytes.len();
let pre = dummy_blobs[i]
.precompressed
.as_ref()
.expect("chunked dataset missing precompressed cache");
let result =
place_chunked_data(pre, base_address, d.maxshape.as_deref(), low, high)?;
cursor2 += to_usize(result.len)?;
let oh = build_chunked_dataset_oh(
&d.dt,
&d.ds,
@@ -2010,11 +2024,16 @@ impl FileWriter {
&d.fill_message,
d.refcount,
)?;
ds_blobs2.push(DataBlob {
data: result.data_bytes,
oh_bytes: oh,
precompressed: None,
});
let data = result
.pieces
.into_iter()
.map(|p| match p {
Piece::Zeros(n) => Out::Zeros(n),
Piece::Chunk(c) => Out::Slice(&pre.chunks[c].1),
Piece::Bytes(b) => Out::Bytes(b),
})
.collect();
ds_blobs2.push(OutBlob { data, oh_bytes: oh });
} else if is_compact[i] {
// Compact: data is inline in the object header, no external blob
let oh = build_compact_dataset_oh(
@@ -2030,10 +2049,9 @@ impl FileWriter {
d.refcount,
layout_version,
)?;
ds_blobs2.push(DataBlob {
ds_blobs2.push(OutBlob {
data: vec![],
oh_bytes: oh,
precompressed: None,
});
} else {
// Determine alignment: per-dataset overrides global
@@ -2060,13 +2078,10 @@ impl FileWriter {
d.refcount,
layout_version,
)?;
let mut data = vec![0u8; padding];
data.extend_from_slice(&d.raw);
cursor2 += d.raw.len();
ds_blobs2.push(DataBlob {
data,
ds_blobs2.push(OutBlob {
data: vec![Out::Zeros(padding), Out::Slice(&d.raw)],
oh_bytes: oh,
precompressed: None,
});
}
}
@@ -2080,7 +2095,8 @@ impl FileWriter {
cursor2 = cursor2.next_multiple_of(ps as usize);
}
let eof_addr2 = cursor2 as u64;
let mut buf = Vec::with_capacity(cursor2);
sink.reserve(cursor2);
let mut out = Counted { sink, written: 0 };
let sb = Superblock {
version: superblock_version,
@@ -2103,9 +2119,9 @@ impl FileWriter {
checksum: None,
page_size: None,
};
buf.extend_from_slice(&sb.serialize());
out.put(&sb.serialize())?;
if let Some(ref ext) = sb_ext {
buf.extend_from_slice(ext);
out.put(ext)?;
}
// Group OHs + dense blobs (link blob, then attr blob, matching pass 2)
@@ -2132,32 +2148,120 @@ impl FileWriter {
g.refcount,
)?;
debug_assert_eq!(oh.len(), group_oh_sizes[gi]);
debug_assert_eq!(buf.len() as u64, group_addrs2[gi]);
buf.extend_from_slice(&oh);
debug_assert_eq!(out.written as u64, group_addrs2[gi]);
out.put(&oh)?;
if let Some(ref b) = link_blob {
buf.extend_from_slice(&b.blob);
out.put(&b.blob)?;
}
if let Some(ref blob) = group_dense_blobs[gi] {
buf.extend_from_slice(&blob.blob);
out.put(&blob.blob)?;
}
}
// Dataset OHs + dense blobs
for (i, blob) in ds_blobs2.iter().enumerate() {
buf.extend_from_slice(&blob.oh_bytes);
out.put(&blob.oh_bytes)?;
if let Some(ref dense) = ds_dense_blobs[i] {
buf.extend_from_slice(&dense.blob);
out.put(&dense.blob)?;
}
}
// Data
for blob in &ds_blobs2 {
buf.extend_from_slice(&blob.data);
for piece in &blob.data {
match piece {
Out::Bytes(b) => out.put(b)?,
Out::Slice(s) => out.put(s)?,
Out::Zeros(n) => out.zeros(*n)?,
}
}
}
debug_assert_eq!(buf.len(), data_end);
buf.resize(cursor2, 0);
Ok(buf)
debug_assert_eq!(out.written, data_end);
out.zeros(cursor2 - data_end)?;
Ok(())
}
}
/// A piece of a dataset's data as [`FileWriter`] writes it out.
enum Out<'a> {
Bytes(Vec<u8>),
/// Borrowed from the dataset's data or its prepared chunks.
Slice(&'a [u8]),
Zeros(usize),
}
/// A dataset's object header and the pieces of its data.
struct OutBlob<'a> {
oh_bytes: Vec<u8>,
data: Vec<Out<'a>>,
}
/// Where [`FileWriter`] puts a file's bytes, in order.
trait Sink {
type Error: From<FormatError>;
/// The file will be `_len` bytes long.
fn reserve(&mut self, _len: usize) {}
fn put(&mut self, bytes: &[u8]) -> Result<(), Self::Error>;
fn zeros(&mut self, n: usize) -> Result<(), Self::Error> {
const ZEROS: [u8; 4096] = [0; 4096];
let mut left = n;
while left > 0 {
let k = left.min(ZEROS.len());
self.put(&ZEROS[..k])?;
left -= k;
}
Ok(())
}
}
impl Sink for Vec<u8> {
type Error = FormatError;
fn reserve(&mut self, len: usize) {
self.reserve_exact(len.saturating_sub(self.len()));
}
fn put(&mut self, bytes: &[u8]) -> Result<(), FormatError> {
self.extend_from_slice(bytes);
Ok(())
}
fn zeros(&mut self, n: usize) -> Result<(), FormatError> {
self.resize(self.len() + n, 0);
Ok(())
}
}
/// A [`Sink`] calling a function ([`FileWriter::finish_with`]).
struct FnSink<F>(F);
impl<E: From<FormatError>, F: FnMut(&[u8]) -> Result<(), E>> Sink for FnSink<F> {
type Error = E;
fn put(&mut self, bytes: &[u8]) -> Result<(), E> {
(self.0)(bytes)
}
}
/// A [`Sink`] and the number of bytes put into it.
struct Counted<'s, S> {
sink: &'s mut S,
written: usize,
}
impl<S: Sink> Counted<'_, S> {
fn put(&mut self, bytes: &[u8]) -> Result<(), S::Error> {
self.written += bytes.len();
self.sink.put(bytes)
}
fn zeros(&mut self, n: usize) -> Result<(), S::Error> {
self.written += n;
self.sink.zeros(n)
}
}
@@ -3015,4 +3119,75 @@ mod tests {
assert_eq!(Superblock::parse(&bytes, 0).unwrap().version, 3);
assert_eq!(layout_of(&bytes, "x")[0], 3);
}
/// `finish_with` hands out exactly the bytes `finish` returns, for every
/// storage (contiguous, compact, chunked filtered and not, chunks
/// borrowed and copied, a version-1 B-tree, a paged file), and stops at
/// the sink's first error.
#[test]
fn finish_with_matches_finish() {
let build = |low: LibVer, paged: bool| {
let mut fw = FileWriter::new();
fw.libver_bounds(low, LibVer::Latest);
if paged {
fw.with_page_size(4096);
}
let v: Vec<f64> = (0..300).map(f64::from).collect();
fw.create_dataset("contig").with_f64_data(&v);
fw.create_dataset("compact")
.with_f64_data(&[1.0, 2.0])
.compact();
fw.create_dataset("one_chunk")
.with_f64_data(&v)
.with_shape(&[30, 10])
.with_chunks(&[30, 10]);
fw.create_dataset("rows")
.with_f64_data(&v)
.with_shape(&[30, 10])
.with_chunks(&[7, 10]);
fw.create_dataset("tiles")
.with_f64_data(&v)
.with_shape(&[30, 10])
.with_chunks(&[7, 3]);
fw.create_dataset("deflated")
.with_f64_data(&v)
.with_chunks(&[64])
.with_shuffle()
.with_deflate(4);
fw.create_dataset("grows")
.with_u8_data_owned((0..=255).collect())
.with_chunks(&[100])
.with_maxshape(&[u64::MAX]);
fw
};
for (low, paged) in [
(LibVer::V18, false),
(LibVer::V110, false),
(LibVer::Latest, true),
] {
let bytes = build(low, paged).finish().unwrap();
let mut streamed = Vec::new();
build(low, paged)
.finish_with(|b: &[u8]| {
streamed.extend_from_slice(b);
Ok::<(), FormatError>(())
})
.unwrap();
assert_eq!(streamed, bytes, "{low:?} paged {paged}");
}
let mut calls = 0;
let err = build(LibVer::V110, false)
.finish_with(|_| {
calls += 1;
if calls == 3 {
Err(FormatError::SerializationError("full".into()))
} else {
Ok(())
}
})
.unwrap_err();
assert_eq!(err, FormatError::SerializationError("full".into()));
assert_eq!(calls, 3);
}
}
crates/clawhdf5-format/src/fixed_array.rs
+9 -9
View File
@@ -155,7 +155,7 @@ pub fn read_fixed_array_chunks(
header: &FixedArrayHeader,
dataset_dims: &[u64],
max_dims: Option<&[u64]>,
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
element_size: u32,
offset_size: u8,
length_size: u8,
@@ -180,7 +180,7 @@ pub fn read_fixed_array_chunks_in<S: Storage + ?Sized>(
header: &FixedArrayHeader,
dataset_dims: &[u64],
max_dims: Option<&[u64]>,
chunk_dimensions: &[u32],
chunk_dimensions: &[u64],
element_size: u32,
offset_size: u8,
_length_size: u8,
@@ -227,11 +227,11 @@ pub fn read_fixed_array_chunks_in<S: Storage + ?Sized>(
// The index is laid out over the chunk grid of the *maximum* dimensions
// (row-major), so a dataset smaller than its maxshape has gaps.
let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, &dims_u64)?;
let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, chunk_dimensions)?;
let chunk_byte_size: u64 =
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
let chunk_byte_size: u64 = chunk_dimensions
.iter()
.fold(u64::from(element_size), |acc, &d| acc.saturating_mul(d));
let mut chunks = Vec::new();
// `rel` is relative to the data block, whose bytes are in `w`.
@@ -670,7 +670,7 @@ mod tests {
let header =
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
let ds_dims = vec![100u64];
let chunk_dims = vec![20u32];
let chunk_dims = vec![20u64];
let chunks = read_fixed_array_chunks(
&file_data,
&header,
@@ -747,7 +747,7 @@ mod tests {
let header =
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
let ds_dims = vec![60u64];
let chunk_dims = vec![20u32];
let chunk_dims = vec![20u64];
let chunks = read_fixed_array_chunks(
&file_data,
&header,
@@ -848,7 +848,7 @@ mod tests {
assert_eq!(header.num_elements, 11);
let ds_dims = vec![11u64 * 20];
let chunk_dims = vec![20u32];
let chunk_dims = vec![20u64];
let chunks = read_fixed_array_chunks(
&file_data,
&header,
crates/clawhdf5-format/src/type_builders.rs
+7 -1
View File
@@ -663,11 +663,17 @@ impl DatasetBuilder {
}
pub fn with_u8_data(&mut self, data: &[u8]) -> &mut Self {
self.with_u8_data_owned(data.to_vec())
}
/// [`Self::with_u8_data`] without the copy: the builder takes the
/// vector. For large data this halves the memory a write needs.
pub fn with_u8_data_owned(&mut self, data: Vec<u8>) -> &mut Self {
self.datatype = Some(make_u8_type());
self.data = Some(data.to_vec());
if self.shape.is_none() {
self.shape = Some(vec![data.len() as u64]);
}
self.data = Some(data);
self
}