format: raw data, VDS and VL data over Storage
Every raw-data path has a generic *_in core, with the &[u8] functions as thin wrappers: data_read (read_raw_data*, read_raw_data_selection, read_chunked_native), chunked_read (the v1 B-tree chunk index, list_chunks, the full, cached, sweep and indexed reads), parallel_read, partial_read, fill_value (read_full_with_fill, apply_to_unallocated_chunks; and dataset_fill_value_from_storage is now generic), vds (the virtual file through Storage, external sources still through the resolver), vl_data (VlResolver<'a, S = [u8]>, read_vl_strings_in, read_vl_bytes_in), AttributeMessage::read_vl_strings_in and provenance::verify_dataset_in. With the whole file in memory nothing changes: chunks and contiguous data are sliced from it as before. Otherwise a chunked read lists its chunks, fetches their stored bytes with one Storage::read_ranges call per 64 MiB batch (chunks the cache already holds are not fetched), then decodes as today; a selection fetches only the chunks it overlaps, and a contiguous selection only its runs. Each extent's bounds error is the one the slice code gave, reported when that extent is reached, so errors keep their order. Tests: the equivalence harness now reads every dataset's values (whole, fill-aware, cached, indexed, three selections, VDS, VL strings and sequences) through the read_at-only storage and requires the slice results (all 653 corpus files agree); a misbehaving storage (a failing Nth read, short reads) only ever yields errors or the right values; and chunked reads are checked to use one read_ranges call. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -37,14 +37,25 @@ use clawhdf5_format::btree_v2::{
|
||||
BTreeV2Header, collect_btree_v2_records, collect_btree_v2_records_in, find_btree_v2_records,
|
||||
find_btree_v2_records_in,
|
||||
};
|
||||
use clawhdf5_format::chunk_cache::ChunkCache;
|
||||
use clawhdf5_format::chunked_read::{list_chunks, list_chunks_in};
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::data_read::{
|
||||
read_raw_data_cached, read_raw_data_cached_in, read_raw_data_full, read_raw_data_full_in,
|
||||
read_raw_data_indexed, read_raw_data_indexed_in, read_raw_data_selection,
|
||||
read_raw_data_selection_in,
|
||||
};
|
||||
use clawhdf5_format::dataspace::Dataspace;
|
||||
use clawhdf5_format::datatype::Datatype;
|
||||
use clawhdf5_format::error::FormatError;
|
||||
use clawhdf5_format::extensible_array::{
|
||||
ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in,
|
||||
};
|
||||
use clawhdf5_format::fill_value::{dataset_fill_value_from_storage, dataset_fill_value_in};
|
||||
use clawhdf5_format::fill_value::{
|
||||
dataset_fill_value_from_storage, dataset_fill_value_in, read_full_with_fill,
|
||||
read_full_with_fill_in,
|
||||
};
|
||||
use clawhdf5_format::filter_pipeline::FilterPipeline;
|
||||
use clawhdf5_format::fixed_array::{
|
||||
FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in,
|
||||
};
|
||||
@@ -54,6 +65,7 @@ use clawhdf5_format::link_info::LinkInfoMessage;
|
||||
use clawhdf5_format::local_heap::LocalHeap;
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use clawhdf5_format::selection::Selection;
|
||||
use clawhdf5_format::shared_message::{
|
||||
self, load_sohm_table, load_sohm_table_in, message_data_with_sohm, message_data_with_sohm_in,
|
||||
parse_sohm_btree_entries, parse_sohm_btree_entries_in, parse_sohm_list, parse_sohm_list_in,
|
||||
@@ -66,11 +78,18 @@ use clawhdf5_format::superblock_ext::{
|
||||
read_superblock_extension_in,
|
||||
};
|
||||
use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode};
|
||||
use clawhdf5_format::vds::{
|
||||
read_virtual_dataset, read_virtual_dataset_in, virtual_dataset_extent,
|
||||
virtual_dataset_extent_in,
|
||||
};
|
||||
use clawhdf5_format::vl_data::{VlResolver, read_vl_bytes, read_vl_bytes_in};
|
||||
|
||||
/// Objects visited per file, heap objects read per heap: enough to cover
|
||||
/// every structure kind while keeping a 35 000-group file fast.
|
||||
const MAX_OBJECTS: usize = 1500;
|
||||
const MAX_HEAP_IDS: usize = 200;
|
||||
/// Datasets larger than this are not read (their chunk indexes still are).
|
||||
const MAX_DATA_BYTES: u64 = 16 << 20;
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
struct Tally {
|
||||
@@ -89,6 +108,8 @@ struct Walk<'a> {
|
||||
slice: &'a [u8],
|
||||
storage: &'a CountingStorage,
|
||||
name: String,
|
||||
/// The file's directory, for external VDS sources.
|
||||
dir: Option<PathBuf>,
|
||||
tally: &'a mut Tally,
|
||||
}
|
||||
|
||||
@@ -264,6 +285,7 @@ impl Walk<'_> {
|
||||
}
|
||||
}
|
||||
self.check_layout(&header, os, ls);
|
||||
self.check_data(&header, os, ls);
|
||||
}
|
||||
|
||||
/// A symbol-table group: its local heap, B-tree, nodes and names.
|
||||
@@ -352,6 +374,179 @@ impl Walk<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A dataset's values through every raw-data path: whole reads (plain,
|
||||
/// cached, indexed, fill-aware, virtual), chunk listings, selections
|
||||
/// (a box, a strided hyperslab, points), VL strings and sequences.
|
||||
fn check_data(&mut self, header: &ObjectHeader, os: u8, ls: u8) {
|
||||
let slice = self.slice;
|
||||
let find = |t: MessageType| {
|
||||
header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == t)
|
||||
.and_then(|m| shared_message::message_data_with_sohm(slice, m, os, ls).ok())
|
||||
};
|
||||
let (Some(dt), Some(ds), Some(dl)) = (
|
||||
find(MessageType::Datatype),
|
||||
find(MessageType::Dataspace),
|
||||
find(MessageType::DataLayout),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let (Ok((dt, _)), Ok(ds), Ok(dl)) = (
|
||||
Datatype::parse(&dt),
|
||||
Dataspace::parse(&ds, ls),
|
||||
DataLayout::parse(&dl, os, ls),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let pipeline = match find(MessageType::FilterPipeline).map(|p| FilterPipeline::parse(&p)) {
|
||||
Some(Ok(p)) => Some(p),
|
||||
Some(Err(_)) => return,
|
||||
None => None,
|
||||
};
|
||||
let pl = pipeline.as_ref();
|
||||
let elem = dt.type_size() as u64;
|
||||
let bytes = ds
|
||||
.dimensions
|
||||
.iter()
|
||||
.try_fold(elem, |a, &d| a.checked_mul(d));
|
||||
if bytes.is_none_or(|b| b > MAX_DATA_BYTES) {
|
||||
return;
|
||||
}
|
||||
|
||||
if matches!(dl, DataLayout::Virtual { .. }) {
|
||||
let resolver = self.resolver();
|
||||
let r: &clawhdf5_format::vds::VdsFileResolver = &resolver;
|
||||
let want = virtual_dataset_extent(slice, &dl, &ds, os, ls, Some(r));
|
||||
let got = virtual_dataset_extent_in(self.st(), &dl, &ds, os, ls, Some(r));
|
||||
self.same("VDS extent", &want, &got);
|
||||
let want = read_virtual_dataset(slice, &dl, &ds, &dt, None, os, ls, Some(r));
|
||||
let got = read_virtual_dataset_in(self.st(), &dl, &ds, &dt, None, os, ls, Some(r));
|
||||
self.same("VDS read", &want, &got);
|
||||
return;
|
||||
}
|
||||
|
||||
let want = read_raw_data_full(slice, &dl, &ds, &dt, pl, os, ls);
|
||||
let got = read_raw_data_full_in(self.st(), &dl, &ds, &dt, pl, os, ls);
|
||||
self.same("raw data", &want, &got);
|
||||
let want_fill = read_full_with_fill(
|
||||
&header.messages,
|
||||
slice,
|
||||
&dl,
|
||||
&ds,
|
||||
elem as usize,
|
||||
os,
|
||||
ls,
|
||||
|| read_raw_data_full(slice, &dl, &ds, &dt, pl, os, ls),
|
||||
);
|
||||
let got_fill = read_full_with_fill_in(
|
||||
&header.messages,
|
||||
self.st(),
|
||||
&dl,
|
||||
&ds,
|
||||
elem as usize,
|
||||
os,
|
||||
ls,
|
||||
|| read_raw_data_full_in(self.st(), &dl, &ds, &dt, pl, os, ls),
|
||||
);
|
||||
self.same("raw data with fill", &want_fill, &got_fill);
|
||||
|
||||
if matches!(dl, DataLayout::Chunked { .. }) {
|
||||
let want = list_chunks(slice, &dl, &ds, elem as usize, os, ls);
|
||||
let got = list_chunks_in(self.st(), &dl, &ds, elem as usize, os, ls);
|
||||
self.same("chunk list", &want, &got);
|
||||
// Through a chunk cache, twice (the second read is served from
|
||||
// it), and through the indexed path.
|
||||
let (c1, c2) = (ChunkCache::new(), ChunkCache::new());
|
||||
for _ in 0..2 {
|
||||
let want = read_raw_data_cached(slice, &dl, &ds, &dt, pl, os, ls, &c1);
|
||||
let got = read_raw_data_cached_in(self.st(), &dl, &ds, &dt, pl, os, ls, &c2);
|
||||
// A cache lists the chunks in hash-map order (see below).
|
||||
if want.is_err() && got.is_err() {
|
||||
self.tally.checks += 1;
|
||||
} else {
|
||||
self.same("raw data (cached)", &want, &got);
|
||||
}
|
||||
}
|
||||
let (c1, c2) = (ChunkCache::new(), ChunkCache::new());
|
||||
let want = read_raw_data_indexed(slice, &dl, &ds, &dt, pl, os, ls, &c1);
|
||||
let got = read_raw_data_indexed_in(self.st(), &dl, &ds, &dt, pl, os, ls, &c2);
|
||||
// The indexed path decodes chunks in hash-map order, so which
|
||||
// failing chunk it reports varies between two caches (with the
|
||||
// slice alone, too): only whether it fails must agree.
|
||||
if want.is_err() && got.is_err() {
|
||||
self.tally.checks += 1;
|
||||
} else {
|
||||
self.same("raw data (indexed)", &want, &got);
|
||||
}
|
||||
}
|
||||
|
||||
let dims = &ds.dimensions;
|
||||
if !dims.is_empty() && dims.iter().all(|&d| d > 0) {
|
||||
let rank = dims.len();
|
||||
let ones = vec![1u64; rank];
|
||||
let quarter = Selection::Hyperslab {
|
||||
start: dims.iter().map(|&d| d / 4).collect(),
|
||||
stride: ones.clone(),
|
||||
count: dims.iter().map(|&d| (d / 3).max(1)).collect(),
|
||||
block: ones.clone(),
|
||||
};
|
||||
let mut stride = ones.clone();
|
||||
stride[rank - 1] = 2;
|
||||
let mut count = dims.clone();
|
||||
count[rank - 1] = dims[rank - 1].div_ceil(2);
|
||||
let strided = Selection::Hyperslab {
|
||||
start: vec![0; rank],
|
||||
stride,
|
||||
count,
|
||||
block: ones.clone(),
|
||||
};
|
||||
let points = Selection::Points(vec![
|
||||
dims.iter().map(|&d| d - 1).collect(),
|
||||
vec![0; rank],
|
||||
dims.iter().map(|&d| d / 2).collect(),
|
||||
]);
|
||||
for (what, sel) in [
|
||||
("selection (box)", &quarter),
|
||||
("selection (strided)", &strided),
|
||||
("selection (points)", &points),
|
||||
] {
|
||||
let want = read_raw_data_selection(slice, &dl, &ds, &dt, pl, os, ls, sel);
|
||||
let got = read_raw_data_selection_in(self.st(), &dl, &ds, &dt, pl, os, ls, sel);
|
||||
self.same(what, &want, &got);
|
||||
}
|
||||
}
|
||||
|
||||
// Variable-length strings and sequences, resolved in the global heap.
|
||||
if let (Datatype::VariableLength { base_type, .. }, Ok(raw)) = (&dt, &want) {
|
||||
let n = raw.len() / clawhdf5_format::vl_data::element_size(os).max(1);
|
||||
let raw = &raw[..n * clawhdf5_format::vl_data::element_size(os)];
|
||||
let want = VlResolver::new(slice, os, ls).string_bytes(raw);
|
||||
let got = VlResolver::new_in(self.st(), os, ls).string_bytes(raw);
|
||||
self.same("VL strings", &want, &got);
|
||||
let base = base_type.type_size() as usize;
|
||||
let want = VlResolver::new(slice, os, ls).sequences(raw, base);
|
||||
let got = VlResolver::new_in(self.st(), os, ls).sequences(raw, base);
|
||||
self.same("VL sequences", &want, &got);
|
||||
let want = read_vl_bytes(slice, raw, n as u64, os, ls);
|
||||
let got = read_vl_bytes_in(self.st(), raw, n as u64, os, ls);
|
||||
self.same("VL bytes", &want, &got);
|
||||
}
|
||||
}
|
||||
|
||||
/// External VDS source files: siblings of the file being walked.
|
||||
fn resolver(&self) -> impl Fn(&str) -> Result<Option<Vec<u8>>, FormatError> + use<> {
|
||||
let dir = self.dir.clone();
|
||||
move |name: &str| {
|
||||
let (Some(dir), false) = (dir.as_ref(), name.contains("..") || name.starts_with('/'))
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(std::fs::read(dir.join(name)).ok())
|
||||
}
|
||||
}
|
||||
|
||||
/// A dataset's layout: VDS mappings, and fixed/extensible array chunk
|
||||
/// indexes.
|
||||
fn check_layout(&mut self, header: &ObjectHeader, os: u8, ls: u8) {
|
||||
@@ -461,10 +656,19 @@ fn check_file(path: &Path, tally: &mut Tally) {
|
||||
let Ok(bytes) = std::fs::read(path) else {
|
||||
return;
|
||||
};
|
||||
check_bytes(&path.display().to_string(), &bytes, tally);
|
||||
check_bytes_in(
|
||||
&path.display().to_string(),
|
||||
&bytes,
|
||||
path.parent().map(Path::to_path_buf),
|
||||
tally,
|
||||
);
|
||||
}
|
||||
|
||||
fn check_bytes(name: &str, bytes: &[u8], tally: &mut Tally) {
|
||||
check_bytes_in(name, bytes, None, tally);
|
||||
}
|
||||
|
||||
fn check_bytes_in(name: &str, bytes: &[u8], dir: Option<PathBuf>, tally: &mut Tally) {
|
||||
let Ok((_, hdf5)) = split_user_block(bytes) else {
|
||||
return;
|
||||
};
|
||||
@@ -474,6 +678,7 @@ fn check_bytes(name: &str, bytes: &[u8], tally: &mut Tally) {
|
||||
slice: hdf5,
|
||||
storage: &storage,
|
||||
name: name.to_string(),
|
||||
dir,
|
||||
tally,
|
||||
};
|
||||
walk.run();
|
||||
@@ -548,6 +753,311 @@ fn corpus_parses_identically_through_storage() {
|
||||
assert!(tally.files > 0);
|
||||
}
|
||||
|
||||
/// A storage that misbehaves: fails its `fail_at`-th read (1-based, `0`
|
||||
/// never), and, with `short`, serves one byte less than asked for inside
|
||||
/// the file (a truncated response).
|
||||
struct Adversary {
|
||||
data: Vec<u8>,
|
||||
reads: std::sync::atomic::AtomicUsize,
|
||||
fail_at: usize,
|
||||
short: bool,
|
||||
}
|
||||
|
||||
impl Storage for Adversary {
|
||||
fn read_at(&self, offset: u64, len: usize) -> Result<std::borrow::Cow<'_, [u8]>, FormatError> {
|
||||
let n = self
|
||||
.reads
|
||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
||||
+ 1;
|
||||
if n == self.fail_at {
|
||||
return Err(FormatError::Storage(format!(
|
||||
"injected failure of read {n}"
|
||||
)));
|
||||
}
|
||||
let got = self.data.as_slice().read_at(offset, len)?;
|
||||
let mut v = got.into_owned();
|
||||
if self.short && v.len() > 1 {
|
||||
v.pop();
|
||||
}
|
||||
Ok(std::borrow::Cow::Owned(v))
|
||||
}
|
||||
|
||||
fn len(&self) -> u64 {
|
||||
self.data.len() as u64
|
||||
}
|
||||
}
|
||||
|
||||
/// Every group listing and every dataset's values (whole, fill-aware and
|
||||
/// through a selection) read through a storage that fails or serves short
|
||||
/// reads: each result is an error or exactly the in-memory result, never
|
||||
/// other data; and a failing read is reported as that failure.
|
||||
#[test]
|
||||
fn misbehaving_storage_never_returns_wrong_data() {
|
||||
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
|
||||
let mut files = Vec::new();
|
||||
hdf5_files(&dir, &mut files);
|
||||
files.sort();
|
||||
let (mut compared, mut failures_seen) = (0usize, 0usize);
|
||||
for path in &files {
|
||||
let Ok(bytes) = std::fs::read(path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok((_, hdf5)) = split_user_block(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(sb) = Superblock::parse(hdf5, 0) else {
|
||||
continue;
|
||||
};
|
||||
// The whole read of every object, as one result to compare.
|
||||
let everything = |file: &dyn Storage| -> Result<String, FormatError> {
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
let mut out = String::new();
|
||||
let mut queue = VecDeque::from([sb.root_group_address]);
|
||||
let mut seen = HashSet::new();
|
||||
while let Some(addr) = queue.pop_front() {
|
||||
if seen.len() > 200 || !seen.insert(addr) {
|
||||
continue;
|
||||
}
|
||||
let header = ObjectHeader::parse_in(file, addr, os, ls)?;
|
||||
let find = |t: MessageType| {
|
||||
header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == t)
|
||||
.map(|m| message_data_with_sohm_in(file, m, os, ls))
|
||||
.transpose()
|
||||
};
|
||||
if let (Some(dt), Some(ds), Some(dl)) = (
|
||||
find(MessageType::Datatype)?,
|
||||
find(MessageType::Dataspace)?,
|
||||
find(MessageType::DataLayout)?,
|
||||
) {
|
||||
let dt = Datatype::parse(&dt)?.0;
|
||||
let ds = Dataspace::parse(&ds, ls)?;
|
||||
let dl = DataLayout::parse(&dl, os, ls)?;
|
||||
let pl = find(MessageType::FilterPipeline)?
|
||||
.map(|p| FilterPipeline::parse(&p))
|
||||
.transpose()?;
|
||||
let data = read_full_with_fill_in(
|
||||
&header.messages,
|
||||
file,
|
||||
&dl,
|
||||
&ds,
|
||||
dt.type_size() as usize,
|
||||
os,
|
||||
ls,
|
||||
|| read_raw_data_full_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls),
|
||||
);
|
||||
out.push_str(&format!("{addr}: {data:?}\n"));
|
||||
if let Some(&d0) = ds.dimensions.first() {
|
||||
let rank = ds.dimensions.len();
|
||||
let sel = Selection::Hyperslab {
|
||||
start: vec![0; rank],
|
||||
stride: vec![1; rank],
|
||||
count: std::iter::once(d0.div_ceil(2))
|
||||
.chain(ds.dimensions[1..].iter().copied())
|
||||
.collect(),
|
||||
block: vec![1; rank],
|
||||
};
|
||||
let part = read_raw_data_selection_in(
|
||||
file,
|
||||
&dl,
|
||||
&ds,
|
||||
&dt,
|
||||
pl.as_ref(),
|
||||
os,
|
||||
ls,
|
||||
&sel,
|
||||
);
|
||||
out.push_str(&format!("{addr} half: {part:?}\n"));
|
||||
}
|
||||
}
|
||||
let children = group_v2::resolve_group_children_in(file, &sb, addr);
|
||||
out.push_str(&format!("{addr} children: {children:?}\n"));
|
||||
if let Ok(c) = children {
|
||||
queue.extend(c.iter().map(|c| c.object_header_address));
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
};
|
||||
let want = everything(&hdf5);
|
||||
let counting = CountingStorage::new(hdf5.to_vec());
|
||||
assert_eq!(
|
||||
format!("{:?}", everything(&counting)),
|
||||
format!("{want:?}"),
|
||||
"{}",
|
||||
path.display()
|
||||
);
|
||||
let total = counting.reads() as usize;
|
||||
let step = (total / 25).max(1);
|
||||
for fail_at in (1..=total).step_by(step) {
|
||||
for short in [false, true] {
|
||||
if short && fail_at != 1 {
|
||||
continue;
|
||||
}
|
||||
let adv = Adversary {
|
||||
data: hdf5.to_vec(),
|
||||
reads: Default::default(),
|
||||
fail_at: if short { 0 } else { fail_at },
|
||||
short,
|
||||
};
|
||||
let got = everything(&adv);
|
||||
compared += 1;
|
||||
match (&got, &want) {
|
||||
(Ok(g), Ok(w)) => {
|
||||
// Per-object results inside may be errors; values
|
||||
// that were read must be the right ones.
|
||||
for (gl, wl) in g.lines().zip(w.lines()) {
|
||||
if gl != wl {
|
||||
assert!(
|
||||
gl.contains("Err("),
|
||||
"{}: fail_at {fail_at} short {short}:\n got {gl}\n want {wl}",
|
||||
path.display()
|
||||
);
|
||||
failures_seen += 1;
|
||||
// A listing that failed ends the walk
|
||||
// differently from here on.
|
||||
if gl.contains("children: Err(") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(Err(FormatError::Storage(_)), _) => failures_seen += 1,
|
||||
(Err(e), Ok(_)) => panic!(
|
||||
"{}: fail_at {fail_at} short {short}: {e:?} instead of a storage error",
|
||||
path.display()
|
||||
),
|
||||
(Err(_), Err(_)) => {}
|
||||
// A listing failed, so the walk never reached the
|
||||
// object that fails in memory.
|
||||
(Ok(g), Err(e)) => assert!(
|
||||
g.contains("children: Err(Storage"),
|
||||
"{}: fail_at {fail_at} short {short}: read where memory fails ({e:?})",
|
||||
path.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!("misbehaving storage: {compared} runs, {failures_seen} failures reported");
|
||||
assert!(
|
||||
compared > 500 && failures_seen > 100,
|
||||
"{compared} {failures_seen}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A read_at-only storage that also counts `read_ranges` calls and ranges.
|
||||
struct BatchCounting {
|
||||
inner: CountingStorage,
|
||||
batches: std::sync::atomic::AtomicUsize,
|
||||
ranges: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
impl Storage for BatchCounting {
|
||||
fn read_at(&self, offset: u64, len: usize) -> Result<std::borrow::Cow<'_, [u8]>, FormatError> {
|
||||
self.inner.read_at(offset, len)
|
||||
}
|
||||
|
||||
fn len(&self) -> u64 {
|
||||
self.inner.len()
|
||||
}
|
||||
|
||||
fn read_ranges(
|
||||
&self,
|
||||
ranges: &[std::ops::Range<u64>],
|
||||
) -> Result<Vec<std::borrow::Cow<'_, [u8]>>, FormatError> {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
self.batches.fetch_add(1, Relaxed);
|
||||
self.ranges.fetch_add(ranges.len(), Relaxed);
|
||||
ranges
|
||||
.iter()
|
||||
.map(|r| self.inner.read_at(r.start, (r.end - r.start) as usize))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// A chunked read lists its chunks, then fetches all their bytes with one
|
||||
/// `read_ranges` call (a remote backend coalesces and parallelises it), and
|
||||
/// a selection fetches only the chunks it overlaps, in one call too.
|
||||
#[test]
|
||||
fn chunked_reads_fetch_their_chunks_in_one_batch() {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
|
||||
let mut datasets = 0;
|
||||
for name in [
|
||||
"chunked_large.h5",
|
||||
"chunked_deflate.h5",
|
||||
"chunked_2d.h5",
|
||||
"v4_fixed_array.h5",
|
||||
] {
|
||||
let bytes = std::fs::read(dir.join(name)).unwrap();
|
||||
let sb = Superblock::parse(&bytes, 0).unwrap();
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
let st = BatchCounting {
|
||||
inner: CountingStorage::new(bytes.clone()),
|
||||
batches: Default::default(),
|
||||
ranges: Default::default(),
|
||||
};
|
||||
for child in group_v2::resolve_group_children(&bytes, &sb, sb.root_group_address).unwrap() {
|
||||
let header =
|
||||
ObjectHeader::parse(&bytes, child.object_header_address as usize, os, ls).unwrap();
|
||||
let msg = |t: MessageType| {
|
||||
header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == t)
|
||||
.map(|m| m.data.clone())
|
||||
};
|
||||
let Some(dl) = msg(MessageType::DataLayout) else {
|
||||
continue;
|
||||
};
|
||||
let dl = DataLayout::parse(&dl, os, ls).unwrap();
|
||||
if !matches!(dl, DataLayout::Chunked { .. }) {
|
||||
continue;
|
||||
}
|
||||
let dt = Datatype::parse(&msg(MessageType::Datatype).unwrap())
|
||||
.unwrap()
|
||||
.0;
|
||||
let ds = Dataspace::parse(&msg(MessageType::Dataspace).unwrap(), ls).unwrap();
|
||||
let pl = msg(MessageType::FilterPipeline).map(|p| FilterPipeline::parse(&p).unwrap());
|
||||
let es = dt.type_size() as usize;
|
||||
let (chunks, _) = list_chunks(&bytes, &dl, &ds, es, os, ls).unwrap();
|
||||
let want = read_raw_data_full(&bytes, &dl, &ds, &dt, pl.as_ref(), os, ls).unwrap();
|
||||
st.batches.store(0, Relaxed);
|
||||
st.ranges.store(0, Relaxed);
|
||||
let got = read_raw_data_full_in(&st, &dl, &ds, &dt, pl.as_ref(), os, ls).unwrap();
|
||||
assert_eq!(got, want, "{name} {}", child.name);
|
||||
assert_eq!(st.batches.load(Relaxed), 1, "{name} {}", child.name);
|
||||
assert_eq!(
|
||||
st.ranges.load(Relaxed),
|
||||
chunks.len(),
|
||||
"{name} {}",
|
||||
child.name
|
||||
);
|
||||
// The first chunk only.
|
||||
let rank = ds.dimensions.len();
|
||||
let sel = Selection::Hyperslab {
|
||||
start: vec![0; rank],
|
||||
stride: vec![1; rank],
|
||||
count: vec![1; rank],
|
||||
block: vec![1; rank],
|
||||
};
|
||||
let want = read_raw_data_selection(&bytes, &dl, &ds, &dt, pl.as_ref(), os, ls, &sel);
|
||||
st.batches.store(0, Relaxed);
|
||||
st.ranges.store(0, Relaxed);
|
||||
let got = read_raw_data_selection_in(&st, &dl, &ds, &dt, pl.as_ref(), os, ls, &sel);
|
||||
assert_eq!(got, want, "{name} {}", child.name);
|
||||
if chunks.len() > 2 {
|
||||
assert_eq!(st.batches.load(Relaxed), 1, "{name} {}", child.name);
|
||||
assert_eq!(st.ranges.load(Relaxed), 1, "{name} {}", child.name);
|
||||
}
|
||||
datasets += 1;
|
||||
}
|
||||
}
|
||||
assert!(datasets >= 4, "{datasets}");
|
||||
}
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user