Merge branch 'feat/p2-writer-groups-links' into feat/p2-perf-coverage
# Conflicts: # CHANGELOG.md # crates/clawhdf5-tools/tests/h5rs_interop.rs
This commit is contained in:
osobh
File diff suppressed because it is too large
Load Diff
@@ -132,6 +132,7 @@ mod test_fuzz;
|
||||
pub mod type_builders;
|
||||
pub mod vds;
|
||||
pub mod vl_data;
|
||||
mod writer_tree;
|
||||
|
||||
#[cfg(feature = "provenance")]
|
||||
pub mod provenance;
|
||||
|
||||
@@ -695,8 +695,13 @@ impl DatasetBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set attribute `name`. Setting it again replaces the earlier value,
|
||||
/// as `attrs[name] = v` does in h5py.
|
||||
pub fn set_attr(&mut self, name: &str, value: AttrValue) -> &mut Self {
|
||||
self.attrs.push((name.to_string(), value));
|
||||
match self.attrs.iter_mut().find(|(n, _)| n == name) {
|
||||
Some(slot) => slot.1 = value,
|
||||
None => self.attrs.push((name.to_string(), value)),
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
@@ -903,34 +908,117 @@ impl DatasetBuilder {
|
||||
|
||||
// ---- Group builder ----
|
||||
|
||||
/// Builder for groups.
|
||||
/// One entry of a [`GroupBuilder`], kept in the order it was added (the
|
||||
/// order a group that tracks creation order lists its links in).
|
||||
pub(crate) enum GroupItem {
|
||||
Dataset(Box<DatasetBuilder>),
|
||||
Group(GroupBuilder),
|
||||
/// A soft link: `name` resolves to whatever `target` names when read.
|
||||
Soft {
|
||||
name: String,
|
||||
target: String,
|
||||
},
|
||||
/// An extra hard link to the object at `target` (a path in this file).
|
||||
Hard {
|
||||
name: String,
|
||||
target: String,
|
||||
},
|
||||
/// An external link to `path` in the file `file`.
|
||||
External {
|
||||
name: String,
|
||||
file: String,
|
||||
path: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// Builder for a group: its datasets, subgroups, links and attributes.
|
||||
///
|
||||
/// Names are paths relative to the group: `create_dataset("a/b/x")` creates
|
||||
/// the groups `a` and `a/b` as needed, as h5py does. A group added where a
|
||||
/// group of the same path already exists (added by another builder, or
|
||||
/// created as an intermediate group) is merged into it, like h5py's
|
||||
/// `require_group`; any other name used twice in a group is an error when the
|
||||
/// file is written. A path component must not be empty or `"."`.
|
||||
pub struct GroupBuilder {
|
||||
pub(crate) name: String,
|
||||
pub(crate) datasets: Vec<DatasetBuilder>,
|
||||
pub(crate) items: Vec<GroupItem>,
|
||||
pub(crate) attrs: Vec<(String, AttrValue)>,
|
||||
/// (link_name, target_file, target_path)
|
||||
pub(crate) external_links: Vec<(String, String, String)>,
|
||||
/// Track (and index) link creation order; `None` follows the file's
|
||||
/// default (`FileWriter::track_order`).
|
||||
pub(crate) track_order: Option<bool>,
|
||||
}
|
||||
|
||||
impl GroupBuilder {
|
||||
pub(crate) fn new(name: &str) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
datasets: Vec::new(),
|
||||
items: Vec::new(),
|
||||
attrs: Vec::new(),
|
||||
external_links: Vec::new(),
|
||||
track_order: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a dataset in this group. `name` may be a relative path
|
||||
/// (`"a/b/x"`); missing intermediate groups are created.
|
||||
pub fn create_dataset(&mut self, name: &str) -> &mut DatasetBuilder {
|
||||
self.datasets.push(DatasetBuilder::new(name));
|
||||
self.datasets.last_mut().unwrap()
|
||||
self.items
|
||||
.push(GroupItem::Dataset(Box::new(DatasetBuilder::new(name))));
|
||||
match self.items.last_mut() {
|
||||
Some(GroupItem::Dataset(d)) => d,
|
||||
_ => unreachable!("just pushed a dataset"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Start a subgroup of this group. Like `FileWriter::create_group`, the
|
||||
/// builder is detached: fill it, then pass `finish()`'s result to
|
||||
/// [`Self::add_group`]. `name` may be a relative path.
|
||||
pub fn create_group(&self, name: &str) -> GroupBuilder {
|
||||
GroupBuilder::new(name)
|
||||
}
|
||||
|
||||
/// Add a finished subgroup to this group.
|
||||
pub fn add_group(&mut self, group: FinishedGroup) -> &mut Self {
|
||||
self.items.push(GroupItem::Group(group.group));
|
||||
self
|
||||
}
|
||||
|
||||
pub fn set_attr(&mut self, name: &str, value: AttrValue) {
|
||||
self.attrs.push((name.to_string(), value));
|
||||
}
|
||||
|
||||
/// Track the creation order of this group's links, and index it, as
|
||||
/// h5py's `track_order=True` does: libhdf5 (and h5py) then list the
|
||||
/// group's members in the order they were added rather than by name.
|
||||
/// Applies to links only, not to attributes.
|
||||
pub fn track_order(&mut self, track: bool) -> &mut Self {
|
||||
self.track_order = Some(track);
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a soft link `name` to the path `target` (absolute, or relative to
|
||||
/// this group), like h5py's `grp[name] = h5py.SoftLink(target)`. The
|
||||
/// target need not exist.
|
||||
pub fn add_soft_link(&mut self, name: &str, target: &str) -> &mut Self {
|
||||
self.items.push(GroupItem::Soft {
|
||||
name: name.to_string(),
|
||||
target: target.to_string(),
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add another hard link `name` to the group or dataset at `target`
|
||||
/// (absolute, or relative to this group), like h5py's
|
||||
/// `grp[name] = f[target]`. The target must be written in the same file;
|
||||
/// its path may go through other hard links, but not through soft or
|
||||
/// external links.
|
||||
pub fn add_hard_link(&mut self, name: &str, target: &str) -> &mut Self {
|
||||
self.items.push(GroupItem::Hard {
|
||||
name: name.to_string(),
|
||||
target: target.to_string(),
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add an external link: a named pointer to an object in another HDF5 file.
|
||||
pub fn add_external_link(
|
||||
&mut self,
|
||||
@@ -938,30 +1026,21 @@ impl GroupBuilder {
|
||||
target_file: &str,
|
||||
target_path: &str,
|
||||
) -> &mut Self {
|
||||
self.external_links.push((
|
||||
name.to_string(),
|
||||
target_file.to_string(),
|
||||
target_path.to_string(),
|
||||
));
|
||||
self.items.push(GroupItem::External {
|
||||
name: name.to_string(),
|
||||
file: target_file.to_string(),
|
||||
path: target_path.to_string(),
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Consume the builder, returning a FinishedGroup to add to FileWriter.
|
||||
pub fn finish(self) -> FinishedGroup {
|
||||
FinishedGroup {
|
||||
name: self.name,
|
||||
datasets: self.datasets,
|
||||
attrs: self.attrs,
|
||||
external_links: self.external_links,
|
||||
}
|
||||
FinishedGroup { group: self }
|
||||
}
|
||||
}
|
||||
|
||||
/// A finished group ready for the file writer.
|
||||
pub struct FinishedGroup {
|
||||
pub(crate) name: String,
|
||||
pub(crate) datasets: Vec<DatasetBuilder>,
|
||||
pub(crate) attrs: Vec<(String, AttrValue)>,
|
||||
/// (link_name, target_file, target_path)
|
||||
pub(crate) external_links: Vec<(String, String, String)>,
|
||||
pub(crate) group: GroupBuilder,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,494 @@
|
||||
//! The group hierarchy `FileWriter` writes: builders flattened into a tree
|
||||
//! of groups, datasets and links, with path names expanded into
|
||||
//! intermediate groups, hard links resolved to objects, reference counts
|
||||
//! counted, and everything put in layout order.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{
|
||||
collections::BTreeMap,
|
||||
format,
|
||||
string::{String, ToString},
|
||||
vec,
|
||||
vec::Vec,
|
||||
};
|
||||
#[cfg(feature = "std")]
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::error::FormatError;
|
||||
use crate::type_builders::{AttrValue, DatasetBuilder, GroupBuilder, GroupItem};
|
||||
|
||||
/// Depth of the chain of unresolved hard links followed while resolving one
|
||||
/// hard-link target path (a bound on recursion; cycles are found exactly).
|
||||
const MAX_LINK_DEPTH: usize = 64;
|
||||
|
||||
fn err(msg: String) -> FormatError {
|
||||
FormatError::SerializationError(msg)
|
||||
}
|
||||
|
||||
/// A link name must be one path component: not empty, not ".", and without
|
||||
/// '/' (a '/' separates components, so it cannot be part of a name).
|
||||
fn check_link_name(name: &str, path: &str) -> Result<(), FormatError> {
|
||||
if name.is_empty() || name == "." || name.contains('/') {
|
||||
return Err(err(format!(
|
||||
"invalid object name {path:?}: every path component must be a \
|
||||
non-empty name other than \".\""
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What a link in the final tree points at.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) enum LinkTo {
|
||||
/// A group, by index into [`Tree::groups`] (layout order).
|
||||
Group(usize),
|
||||
/// A dataset, by index into [`Tree::datasets`] (layout order).
|
||||
Dataset(usize),
|
||||
Soft(String),
|
||||
External {
|
||||
file: String,
|
||||
path: String,
|
||||
},
|
||||
}
|
||||
|
||||
pub(crate) struct Link {
|
||||
pub(crate) name: String,
|
||||
pub(crate) to: LinkTo,
|
||||
/// Set when the group tracks creation order.
|
||||
pub(crate) creation_order: Option<u64>,
|
||||
}
|
||||
|
||||
pub(crate) struct Group {
|
||||
pub(crate) attrs: Vec<(String, AttrValue)>,
|
||||
/// Links in the order they are written.
|
||||
pub(crate) links: Vec<Link>,
|
||||
pub(crate) track_order: bool,
|
||||
/// Number of hard links to this group (the root counts one for the
|
||||
/// superblock's reference).
|
||||
pub(crate) refcount: u32,
|
||||
}
|
||||
|
||||
/// The flattened file: groups (root first) and datasets, both in the order
|
||||
/// they are laid out in the file.
|
||||
pub(crate) struct Tree {
|
||||
pub(crate) groups: Vec<Group>,
|
||||
pub(crate) datasets: Vec<(DatasetBuilder, u32)>,
|
||||
}
|
||||
|
||||
// ---- construction ----
|
||||
|
||||
enum Target {
|
||||
Group(usize),
|
||||
Dataset(usize),
|
||||
Soft(String),
|
||||
Hard(String),
|
||||
External { file: String, path: String },
|
||||
}
|
||||
|
||||
struct BuildGroup {
|
||||
/// Full path, for messages.
|
||||
path: String,
|
||||
attrs: Vec<(String, AttrValue)>,
|
||||
links: Vec<(String, Target)>,
|
||||
by_name: BTreeMap<String, usize>,
|
||||
track_order: Option<bool>,
|
||||
}
|
||||
|
||||
struct Builder {
|
||||
groups: Vec<BuildGroup>,
|
||||
datasets: Vec<DatasetBuilder>,
|
||||
}
|
||||
|
||||
fn join(parent: &str, name: &str) -> String {
|
||||
if parent == "/" {
|
||||
format!("/{name}")
|
||||
} else {
|
||||
format!("{parent}/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
impl Builder {
|
||||
fn new_group(&mut self, path: String) -> usize {
|
||||
self.groups.push(BuildGroup {
|
||||
path,
|
||||
attrs: Vec::new(),
|
||||
links: Vec::new(),
|
||||
by_name: BTreeMap::new(),
|
||||
track_order: None,
|
||||
});
|
||||
self.groups.len() - 1
|
||||
}
|
||||
|
||||
/// Split `path` (relative to group `g`) into the group holding its last
|
||||
/// component, creating missing intermediate groups, and that component.
|
||||
fn parent_of<'p>(&mut self, g: usize, path: &'p str) -> Result<(usize, &'p str), FormatError> {
|
||||
// An absolute path is accepted at the root only.
|
||||
let rel = match path.strip_prefix('/') {
|
||||
Some(rest) if g == 0 => rest,
|
||||
Some(_) => {
|
||||
return Err(err(format!(
|
||||
"invalid object name {path:?} in {}: absolute paths are accepted \
|
||||
only at the root",
|
||||
self.groups[g].path
|
||||
)));
|
||||
}
|
||||
None => path,
|
||||
};
|
||||
let mut comps: Vec<&str> = rel.split('/').collect();
|
||||
let last = comps.pop().unwrap_or("");
|
||||
check_link_name(last, path)?;
|
||||
let mut cur = g;
|
||||
for c in comps {
|
||||
check_link_name(c, path)?;
|
||||
cur = match self.groups[cur].by_name.get(c).copied() {
|
||||
Some(i) => match self.groups[cur].links[i].1 {
|
||||
Target::Group(child) => child,
|
||||
_ => {
|
||||
return Err(err(format!(
|
||||
"cannot create {path:?} in {}: {c:?} exists and is not a group",
|
||||
self.groups[g].path
|
||||
)));
|
||||
}
|
||||
},
|
||||
None => {
|
||||
let child = self.new_group(join(&self.groups[cur].path, c));
|
||||
self.push_link(cur, c, Target::Group(child))?;
|
||||
child
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok((cur, last))
|
||||
}
|
||||
|
||||
fn push_link(&mut self, g: usize, name: &str, to: Target) -> Result<(), FormatError> {
|
||||
let grp = &mut self.groups[g];
|
||||
if grp.by_name.contains_key(name) {
|
||||
return Err(err(format!("{:?} already exists", join(&grp.path, name))));
|
||||
}
|
||||
grp.by_name.insert(name.to_string(), grp.links.len());
|
||||
grp.links.push((name.to_string(), to));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add `item` to group `g`.
|
||||
fn add_item(&mut self, g: usize, item: GroupItem) -> Result<(), FormatError> {
|
||||
match item {
|
||||
GroupItem::Dataset(db) => {
|
||||
let (parent, name) = self.parent_of(g, &db.name)?;
|
||||
let name = name.to_string();
|
||||
self.push_link(parent, &name, Target::Dataset(self.datasets.len()))?;
|
||||
self.datasets.push(*db);
|
||||
}
|
||||
GroupItem::Group(gb) => self.add_group(g, gb)?,
|
||||
GroupItem::Soft { name, target } => {
|
||||
if target.is_empty() {
|
||||
return Err(err(format!("soft link {name:?} has an empty target")));
|
||||
}
|
||||
let (parent, last) = self.parent_of(g, &name)?;
|
||||
self.push_link(parent, last, Target::Soft(target))?;
|
||||
}
|
||||
GroupItem::Hard { name, target } => {
|
||||
let (parent, last) = self.parent_of(g, &name)?;
|
||||
self.push_link(parent, last, Target::Hard(target))?;
|
||||
}
|
||||
GroupItem::External { name, file, path } => {
|
||||
if file.is_empty() || path.is_empty() {
|
||||
return Err(err(format!(
|
||||
"external link {name:?} needs a file name and an object path"
|
||||
)));
|
||||
}
|
||||
let (parent, last) = self.parent_of(g, &name)?;
|
||||
self.push_link(parent, last, Target::External { file, path })?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add the group `gb` (named by a path relative to group `g`), merging it
|
||||
/// into a group already at that path.
|
||||
fn add_group(&mut self, g: usize, gb: GroupBuilder) -> Result<(), FormatError> {
|
||||
let (parent, last) = self.parent_of(g, &gb.name)?;
|
||||
let idx = match self.groups[parent].by_name.get(last).copied() {
|
||||
Some(i) => match self.groups[parent].links[i].1 {
|
||||
Target::Group(child) => child,
|
||||
_ => {
|
||||
return Err(err(format!(
|
||||
"{:?} already exists and is not a group",
|
||||
join(&self.groups[parent].path, last)
|
||||
)));
|
||||
}
|
||||
},
|
||||
None => {
|
||||
let child = self.new_group(join(&self.groups[parent].path, last));
|
||||
self.push_link(parent, last, Target::Group(child))?;
|
||||
child
|
||||
}
|
||||
};
|
||||
self.merge_into(idx, gb)
|
||||
}
|
||||
|
||||
/// Merge a builder's attributes, setting and items into group `idx`.
|
||||
fn merge_into(&mut self, idx: usize, gb: GroupBuilder) -> Result<(), FormatError> {
|
||||
// An attribute set again (by this builder or a merged one) takes the
|
||||
// new value, as assigning `attrs[name]` in h5py does.
|
||||
for (name, value) in gb.attrs {
|
||||
let attrs = &mut self.groups[idx].attrs;
|
||||
match attrs.iter_mut().find(|(n, _)| *n == name) {
|
||||
Some(slot) => slot.1 = value,
|
||||
None => attrs.push((name, value)),
|
||||
}
|
||||
}
|
||||
if let Some(t) = gb.track_order {
|
||||
match self.groups[idx].track_order {
|
||||
Some(old) if old != t => {
|
||||
return Err(err(format!(
|
||||
"conflicting track_order settings for {}",
|
||||
self.groups[idx].path
|
||||
)));
|
||||
}
|
||||
_ => self.groups[idx].track_order = Some(t),
|
||||
}
|
||||
}
|
||||
for item in gb.items {
|
||||
self.add_item(idx, item)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The object a hard link's `target` path names, from group `from`.
|
||||
///
|
||||
/// Hard links met on the way are resolved once and remembered in
|
||||
/// `memo` (by group and link index), so a target that goes through
|
||||
/// other hard links costs time linear in the links, not exponential; a
|
||||
/// hard link met again while it is being resolved is a cycle.
|
||||
fn resolve(
|
||||
&self,
|
||||
memo: &mut [Vec<Resolution>],
|
||||
from: usize,
|
||||
target: &str,
|
||||
depth: usize,
|
||||
) -> Result<Obj, FormatError> {
|
||||
if depth > MAX_LINK_DEPTH {
|
||||
return Err(err(format!(
|
||||
"hard link target {target:?}: more than {MAX_LINK_DEPTH} hard links \
|
||||
to follow"
|
||||
)));
|
||||
}
|
||||
let (mut cur, rest) = match target.strip_prefix('/') {
|
||||
Some(rest) => (0, rest),
|
||||
None => (from, target),
|
||||
};
|
||||
if target.is_empty() {
|
||||
return Err(err("a hard link needs a target path".to_string()));
|
||||
}
|
||||
let comps: Vec<&str> = rest
|
||||
.split('/')
|
||||
.filter(|c| !c.is_empty() && *c != ".")
|
||||
.collect();
|
||||
let mut obj = Obj::Group(cur);
|
||||
for (i, c) in comps.iter().enumerate() {
|
||||
let Obj::Group(g) = obj else {
|
||||
return Err(err(format!(
|
||||
"hard link target {target:?}: {:?} is not a group",
|
||||
comps[..i].join("/")
|
||||
)));
|
||||
};
|
||||
cur = g;
|
||||
let grp = &self.groups[cur];
|
||||
let Some(&li) = grp.by_name.get(*c) else {
|
||||
return Err(err(format!(
|
||||
"hard link target {target:?} does not exist in the file"
|
||||
)));
|
||||
};
|
||||
obj = match &grp.links[li].1 {
|
||||
Target::Group(child) => Obj::Group(*child),
|
||||
Target::Dataset(d) => Obj::Dataset(*d),
|
||||
Target::Hard(p) => match memo[cur][li] {
|
||||
Resolution::Done(o) => o,
|
||||
Resolution::InProgress => {
|
||||
return Err(err(format!(
|
||||
"hard link target {target:?}: the hard link {:?} leads \
|
||||
back to itself (a cycle)",
|
||||
join(&grp.path, c)
|
||||
)));
|
||||
}
|
||||
Resolution::Todo => {
|
||||
memo[cur][li] = Resolution::InProgress;
|
||||
let o = self.resolve(memo, cur, p, depth + 1)?;
|
||||
memo[cur][li] = Resolution::Done(o);
|
||||
o
|
||||
}
|
||||
},
|
||||
Target::Soft(_) | Target::External { .. } => {
|
||||
return Err(err(format!(
|
||||
"hard link target {target:?} goes through a soft or external \
|
||||
link ({:?}); name the object by its hard-link path",
|
||||
join(&grp.path, c)
|
||||
)));
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(obj)
|
||||
}
|
||||
}
|
||||
|
||||
/// Where resolving one hard link has got to.
|
||||
#[derive(Clone, Copy)]
|
||||
enum Resolution {
|
||||
Todo,
|
||||
InProgress,
|
||||
Done(Obj),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Obj {
|
||||
Group(usize),
|
||||
Dataset(usize),
|
||||
}
|
||||
|
||||
/// Flatten the root group builder into a [`Tree`]. `default_track_order`
|
||||
/// applies to every group that does not set its own.
|
||||
pub(crate) fn build(root: GroupBuilder, default_track_order: bool) -> Result<Tree, FormatError> {
|
||||
let mut b = Builder {
|
||||
groups: Vec::new(),
|
||||
datasets: Vec::new(),
|
||||
};
|
||||
b.new_group("/".to_string());
|
||||
b.merge_into(0, root)?;
|
||||
|
||||
// Resolve hard links and count references.
|
||||
let mut group_refs = vec![0u32; b.groups.len()];
|
||||
let mut ds_refs = vec![0u32; b.datasets.len()];
|
||||
group_refs[0] = 1; // the superblock's reference to the root
|
||||
let mut memo: Vec<Vec<Resolution>> = b
|
||||
.groups
|
||||
.iter()
|
||||
.map(|g| vec![Resolution::Todo; g.links.len()])
|
||||
.collect();
|
||||
let mut resolved: Vec<Vec<Option<Obj>>> = Vec::with_capacity(b.groups.len());
|
||||
for (gi, g) in b.groups.iter().enumerate() {
|
||||
let mut row = Vec::with_capacity(g.links.len());
|
||||
for (li, (_, t)) in g.links.iter().enumerate() {
|
||||
let obj = match t {
|
||||
Target::Group(i) => Some(Obj::Group(*i)),
|
||||
Target::Dataset(d) => Some(Obj::Dataset(*d)),
|
||||
Target::Hard(p) => Some(match memo[gi][li] {
|
||||
Resolution::Done(o) => o,
|
||||
_ => {
|
||||
memo[gi][li] = Resolution::InProgress;
|
||||
let o = b.resolve(&mut memo, gi, p, 0)?;
|
||||
memo[gi][li] = Resolution::Done(o);
|
||||
o
|
||||
}
|
||||
}),
|
||||
Target::Soft(_) | Target::External { .. } => None,
|
||||
};
|
||||
match obj {
|
||||
Some(Obj::Group(i)) => group_refs[i] += 1,
|
||||
Some(Obj::Dataset(d)) => ds_refs[d] += 1,
|
||||
None => {}
|
||||
}
|
||||
row.push(obj);
|
||||
}
|
||||
resolved.push(row);
|
||||
}
|
||||
|
||||
// The order each group's links are written in: creation order when
|
||||
// tracked; otherwise datasets, then groups, then other links (the order
|
||||
// earlier versions wrote, so one-level files keep their layout).
|
||||
let tracked: Vec<bool> = b
|
||||
.groups
|
||||
.iter()
|
||||
.map(|g| g.track_order.unwrap_or(default_track_order))
|
||||
.collect();
|
||||
let link_order: Vec<Vec<usize>> = b
|
||||
.groups
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(gi, g)| {
|
||||
let mut idx: Vec<usize> = (0..g.links.len()).collect();
|
||||
if !tracked[gi] {
|
||||
idx.sort_by_key(|&i| match g.links[i].1 {
|
||||
Target::Dataset(_) => 0,
|
||||
Target::Group(_) => 1,
|
||||
_ => 2,
|
||||
});
|
||||
}
|
||||
idx
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Layout order: groups depth-first from the root, following the links
|
||||
// that created them; datasets group by group in that order.
|
||||
let mut group_order = Vec::with_capacity(b.groups.len());
|
||||
let mut stack = vec![0usize];
|
||||
while let Some(g) = stack.pop() {
|
||||
group_order.push(g);
|
||||
let children: Vec<usize> = link_order[g]
|
||||
.iter()
|
||||
.filter_map(|&i| match b.groups[g].links[i].1 {
|
||||
Target::Group(c) => Some(c),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
stack.extend(children.into_iter().rev());
|
||||
}
|
||||
let mut ds_order = Vec::with_capacity(b.datasets.len());
|
||||
for &g in &group_order {
|
||||
for &i in &link_order[g] {
|
||||
if let Target::Dataset(d) = b.groups[g].links[i].1 {
|
||||
ds_order.push(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut group_pos = vec![0usize; b.groups.len()];
|
||||
for (pos, &g) in group_order.iter().enumerate() {
|
||||
group_pos[g] = pos;
|
||||
}
|
||||
let mut ds_pos = vec![0usize; b.datasets.len()];
|
||||
for (pos, &d) in ds_order.iter().enumerate() {
|
||||
ds_pos[d] = pos;
|
||||
}
|
||||
|
||||
let mut groups_by_id: Vec<Option<BuildGroup>> = b.groups.into_iter().map(Some).collect();
|
||||
let mut groups = Vec::with_capacity(group_order.len());
|
||||
for &g in &group_order {
|
||||
let bg = groups_by_id[g].take().expect("each group is laid out once");
|
||||
let mut targets: Vec<Option<(String, Target)>> = bg.links.into_iter().map(Some).collect();
|
||||
let links = link_order[g]
|
||||
.iter()
|
||||
.map(|&i| {
|
||||
let (name, t) = targets[i].take().expect("each link is written once");
|
||||
let to = match (resolved[g][i], t) {
|
||||
(Some(Obj::Group(c)), _) => LinkTo::Group(group_pos[c]),
|
||||
(Some(Obj::Dataset(d)), _) => LinkTo::Dataset(ds_pos[d]),
|
||||
(None, Target::Soft(s)) => LinkTo::Soft(s),
|
||||
(None, Target::External { file, path }) => LinkTo::External { file, path },
|
||||
(None, _) => unreachable!("hard links are resolved"),
|
||||
};
|
||||
Link {
|
||||
name,
|
||||
to,
|
||||
creation_order: tracked[g].then_some(i as u64),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
groups.push(Group {
|
||||
attrs: bg.attrs,
|
||||
links,
|
||||
track_order: tracked[g],
|
||||
refcount: group_refs[g],
|
||||
});
|
||||
}
|
||||
let mut ds_by_id: Vec<Option<DatasetBuilder>> = b.datasets.into_iter().map(Some).collect();
|
||||
let datasets = ds_order
|
||||
.iter()
|
||||
.map(|&d| {
|
||||
(
|
||||
ds_by_id[d].take().expect("each dataset is laid out once"),
|
||||
ds_refs[d],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
Ok(Tree { groups, datasets })
|
||||
}
|
||||
@@ -528,33 +528,50 @@ fn h5py_reads_all_attributes_next_to_an_empty_string() {
|
||||
// ---- 6. path-like names ----
|
||||
|
||||
#[test]
|
||||
fn slash_in_a_group_or_dataset_name_is_an_error() {
|
||||
// Measured: create_group("a/b") wrote one link literally named "a/b",
|
||||
// which h5py cannot reach ("component not found"). The writer has no
|
||||
// nested groups, so such names are refused.
|
||||
fn path_names_create_nested_groups() {
|
||||
// create_group("a/b") used to write one link literally named "a/b",
|
||||
// which h5py cannot reach ("component not found"); then such names were
|
||||
// refused. Now a path creates its missing intermediate groups, as h5py
|
||||
// does.
|
||||
let mut fw = FileWriter::new();
|
||||
let mut g = fw.create_group("a/b");
|
||||
g.create_dataset("c").with_f64_data(&[1.0]);
|
||||
fw.add_group(g.finish());
|
||||
assert!(fw.finish().is_err());
|
||||
|
||||
let mut fw = FileWriter::new();
|
||||
fw.create_dataset("x/y").with_f64_data(&[1.0]);
|
||||
assert!(fw.finish().is_err());
|
||||
|
||||
let mut fw = FileWriter::new();
|
||||
fw.create_dataset("x/y").with_f64_data(&[2.0]);
|
||||
fw.create_dataset("/a/b/z").with_f64_data(&[3.0]);
|
||||
let mut g = fw.create_group("g");
|
||||
g.create_dataset("x/y").with_f64_data(&[1.0]);
|
||||
g.create_dataset("x/y").with_f64_data(&[4.0]);
|
||||
fw.add_group(g.finish());
|
||||
assert!(fw.finish().is_err());
|
||||
let bytes = fw.finish().unwrap();
|
||||
for path in ["a", "a/b", "a/b/c", "a/b/z", "x", "x/y", "g/x", "g/x/y"] {
|
||||
header_at(&bytes, path);
|
||||
}
|
||||
}
|
||||
|
||||
for bad in ["", "."] {
|
||||
#[test]
|
||||
fn names_that_are_not_valid_link_names_are_errors() {
|
||||
for bad in ["", ".", "a//b", "a/", "a/./b", "/"] {
|
||||
let mut fw = FileWriter::new();
|
||||
fw.create_dataset(bad).with_f64_data(&[1.0]);
|
||||
assert!(fw.finish().is_err(), "{bad:?}");
|
||||
}
|
||||
// An absolute path inside a group, and a name used twice.
|
||||
let mut fw = FileWriter::new();
|
||||
let mut g = fw.create_group("g");
|
||||
g.create_dataset("/x").with_f64_data(&[1.0]);
|
||||
fw.add_group(g.finish());
|
||||
assert!(fw.finish().is_err());
|
||||
let mut fw = FileWriter::new();
|
||||
fw.create_dataset("x").with_f64_data(&[1.0]);
|
||||
fw.create_dataset("x").with_f64_data(&[1.0]);
|
||||
assert!(fw.finish().is_err());
|
||||
// A dataset in the way of a path.
|
||||
let mut fw = FileWriter::new();
|
||||
fw.create_dataset("x").with_f64_data(&[1.0]);
|
||||
fw.create_dataset("x/y").with_f64_data(&[1.0]);
|
||||
assert!(fw.finish().is_err());
|
||||
|
||||
// One level of groups still works, and '/' stays legal in attribute names.
|
||||
// '/' stays legal in attribute names.
|
||||
let mut fw = FileWriter::new();
|
||||
let mut g = fw.create_group("g");
|
||||
g.create_dataset("c").with_f64_data(&[1.0]);
|
||||
|
||||
@@ -908,3 +908,119 @@ fn check_data_flags_mis_sized_vl_heap_objects() {
|
||||
assert!(s.contains("undefined global heap address"), "bad{tag}: {s}");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// files clawhdf5 writes: nested groups and links
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Nested groups (4 levels, by builders and by path names), soft, hard and
|
||||
/// external links, creation-order tracking, and dense link and attribute
|
||||
/// storage, as `FileBuilder` writes them.
|
||||
fn write_nested_links(dir: &Path) -> Vec<String> {
|
||||
use clawhdf5::{AttrValue, FileBuilder};
|
||||
let mut b = FileBuilder::new();
|
||||
b.set_attr("title", AttrValue::String("links".into()));
|
||||
b.create_dataset("x/y").with_f64_data(&[1.0, 2.0]);
|
||||
b.create_dataset("a/b/c/d/leaf")
|
||||
.with_i32_data(&[4, 5])
|
||||
.set_attr("depth", AttrValue::I64(5));
|
||||
b.add_soft_link("soft", "/x/y");
|
||||
b.add_soft_link("dangling", "/nowhere");
|
||||
b.add_hard_link("alias", "/x/y");
|
||||
b.add_external_link("ext", "other.h5", "/data");
|
||||
let mut g = b.create_group("a/b");
|
||||
g.set_attr("merged", AttrValue::I64(1));
|
||||
for i in 0..10 {
|
||||
g.set_attr(&format!("attr{i}"), AttrValue::F64(i as f64));
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("ordered");
|
||||
g.track_order(true);
|
||||
for i in (0..40).rev() {
|
||||
g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]);
|
||||
}
|
||||
g.add_hard_link("back", "/a/b/c");
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("compact_ordered");
|
||||
g.track_order(true);
|
||||
g.create_dataset("z").with_i32_data(&[1]);
|
||||
g.create_dataset("a").with_i32_data(&[2]);
|
||||
b.add_group(g.finish());
|
||||
let nested = dir.join("nested.h5");
|
||||
b.write(&nested).unwrap();
|
||||
|
||||
let mut b = FileBuilder::new();
|
||||
let mut g = b.create_group("many");
|
||||
for i in 0..10_000 {
|
||||
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let many = dir.join("many.h5");
|
||||
b.write(&many).unwrap();
|
||||
[nested, many]
|
||||
.iter()
|
||||
.map(|p| p.to_string_lossy().into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_and_dump_files_with_nested_groups_and_links() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let files = write_nested_links(dir.path());
|
||||
// `--data` reads every dataset by path, and a lookup in a dense group
|
||||
// scans all of its links: 10 000 datasets take minutes in a debug
|
||||
// build, so the big file is checked structurally only (and not dumped).
|
||||
for (p, data) in [(&files[0], true), (&files[1], false)] {
|
||||
let args: &[&str] = if data {
|
||||
&["check", "--data", p]
|
||||
} else {
|
||||
&["check", p]
|
||||
};
|
||||
let o = h5rs(args);
|
||||
assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o));
|
||||
assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o));
|
||||
}
|
||||
if missing(tool_available("h5dump"), "h5dump") {
|
||||
return;
|
||||
}
|
||||
for p in &files[..1] {
|
||||
let name = Path::new(p).file_name().unwrap().to_string_lossy();
|
||||
let ours = h5rs(&["dump", p]);
|
||||
assert!(ours.status.success(), "{p}: {ours:?}");
|
||||
let reference = run("h5dump", &[p]);
|
||||
assert!(reference.status.success(), "h5dump {p}: {reference:?}");
|
||||
let r = stdout(&reference).replacen(p.as_str(), &name, 1);
|
||||
assert_eq!(stdout(&ours), r, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_files_with_big_dense_storage() {
|
||||
// Dense links and attributes past the 512 KiB the root indirect block's
|
||||
// direct blocks hold: the heap then needs child indirect blocks, which
|
||||
// the writer used to write as direct blocks ("fractal heap indirect
|
||||
// block: bad signature").
|
||||
use clawhdf5::{AttrValue, FileBuilder};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
let x = b.create_dataset("x");
|
||||
x.with_i32_data(&[7]);
|
||||
for i in 0..150usize {
|
||||
let len = if i % 3 == 0 { 7_000 } else { 1 + i };
|
||||
x.set_attr(
|
||||
&format!("a{i:03}"),
|
||||
AttrValue::F64Array(vec![i as f64; len]),
|
||||
);
|
||||
}
|
||||
let mut g = b.create_group("g");
|
||||
for i in 0..40_000 {
|
||||
g.add_hard_link(&format!("link_{i:06}_{}", "x".repeat(88)), "/x");
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let p = dir.path().join("big.h5").to_string_lossy().into_owned();
|
||||
b.write(&p).unwrap();
|
||||
// Structure only: `--data` looks every link up by a linear scan.
|
||||
let o = h5rs(&["check", &p]);
|
||||
assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o));
|
||||
assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o));
|
||||
}
|
||||
|
||||
@@ -42,14 +42,17 @@ impl FileBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a dataset at the root level. Returns a mutable reference to
|
||||
/// a `DatasetBuilder` for configuring data, shape, and attributes.
|
||||
/// Create a dataset. Returns a mutable reference to a `DatasetBuilder`
|
||||
/// for configuring data, shape, and attributes. `name` may be a path
|
||||
/// (`"a/b/x"`): missing intermediate groups are created, as in h5py.
|
||||
pub fn create_dataset(&mut self, name: &str) -> &mut FormatDatasetBuilder {
|
||||
self.writer.create_dataset(name)
|
||||
}
|
||||
|
||||
/// Create a group builder. Call `.finish()` on the returned builder
|
||||
/// to complete it, then pass to `add_group()`.
|
||||
/// to complete it, then pass to `add_group()`. `name` may be a path;
|
||||
/// groups nest to any depth (see `GroupBuilder::add_group`), and a group
|
||||
/// added at a path that already holds a group is merged into it.
|
||||
pub fn create_group(&mut self, name: &str) -> FormatGroupBuilder {
|
||||
self.writer.create_group(name)
|
||||
}
|
||||
@@ -59,6 +62,40 @@ impl FileBuilder {
|
||||
self.writer.add_group(group);
|
||||
}
|
||||
|
||||
/// Add a soft link `name` to the path `target`, like h5py's
|
||||
/// `f[name] = h5py.SoftLink(target)`. The target need not exist.
|
||||
pub fn add_soft_link(&mut self, name: &str, target: &str) -> &mut Self {
|
||||
self.writer.add_soft_link(name, target);
|
||||
self
|
||||
}
|
||||
|
||||
/// Add another hard link `name` to the object at `target`, like h5py's
|
||||
/// `f[name] = f[target]`. The target must be written in this file.
|
||||
pub fn add_hard_link(&mut self, name: &str, target: &str) -> &mut Self {
|
||||
self.writer.add_hard_link(name, target);
|
||||
self
|
||||
}
|
||||
|
||||
/// Add an external link `name` to `target_path` in `target_file`.
|
||||
pub fn add_external_link(
|
||||
&mut self,
|
||||
name: &str,
|
||||
target_file: &str,
|
||||
target_path: &str,
|
||||
) -> &mut Self {
|
||||
self.writer
|
||||
.add_external_link(name, target_file, target_path);
|
||||
self
|
||||
}
|
||||
|
||||
/// Track link creation order in every group that does not set its own
|
||||
/// (`GroupBuilder::track_order`), as h5py's `track_order=True`: libhdf5
|
||||
/// then lists members in the order they were added.
|
||||
pub fn track_order(&mut self, track: bool) -> &mut Self {
|
||||
self.writer.track_order(track);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set an attribute on the root group.
|
||||
pub fn set_attr(&mut self, name: &str, value: AttrValue) {
|
||||
self.writer.set_root_attr(name, value);
|
||||
|
||||
@@ -0,0 +1,940 @@
|
||||
//! Groups and links written by `FileBuilder`, read back by h5py (libhdf5)
|
||||
//! and h5dump, and by clawhdf5 itself.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::{AttrValue, File, FileBuilder, Group};
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn h5dump_available() -> bool {
|
||||
Command::new("h5dump")
|
||||
.arg("--version")
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn run_python(script: &str) -> String {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// Run `body` under h5py with `path` bound to the file's path.
|
||||
fn h5py(path: &str, body: &str) -> String {
|
||||
run_python(&format!(
|
||||
"import h5py, numpy as np, json\npath = r'{path}'\n{body}"
|
||||
))
|
||||
}
|
||||
|
||||
fn write(dir: &tempfile::TempDir, name: &str, b: FileBuilder) -> String {
|
||||
let path = dir.path().join(name).display().to_string();
|
||||
b.write(&path).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
/// h5dump must read the whole file without an error.
|
||||
fn h5dump_ok(path: &str) -> String {
|
||||
if !h5dump_available() {
|
||||
assert!(!interop_required(), "h5dump is not available");
|
||||
return String::new();
|
||||
}
|
||||
let o = Command::new("h5dump").arg(path).output().unwrap();
|
||||
let out = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
assert!(
|
||||
o.status.success(),
|
||||
"h5dump failed:\n{out}{}",
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
// ---- libhdf5 can modify the groups we write ----
|
||||
|
||||
#[test]
|
||||
fn h5py_can_add_links_to_groups_we_wrote() {
|
||||
skip_if_no_python!();
|
||||
// Measured before the fix: h5py in "r+" mode could not add a link to any
|
||||
// group we wrote ("Unable to create link (message type not found)"):
|
||||
// libhdf5 reads a group's Group Info message before inserting a link, and
|
||||
// the writer wrote none.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("x").with_f64_data(&[1.0, 2.0]);
|
||||
let mut g = b.create_group("small");
|
||||
g.create_dataset("a").with_i32_data(&[1]);
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("big"); // dense link storage
|
||||
for i in 0..20 {
|
||||
g.create_dataset(&format!("d{i:02}")).with_i32_data(&[i]);
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let path = write(&dir, "modify.h5", b);
|
||||
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 f['alias'] = f['x']\n\
|
||||
\x20 f['small']['new'] = np.arange(3)\n\
|
||||
\x20 f['big']['new'] = np.arange(4)\n\
|
||||
\x20 f.create_group('added/deeper')\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([sorted(f), sorted(f['small']), len(f['big']),\n\
|
||||
\x20 f['alias'][()].tolist(), f['big/new'][()].tolist(), f['big/d07'][()].tolist()]))",
|
||||
);
|
||||
assert_eq!(
|
||||
out,
|
||||
r#"[["added", "alias", "big", "small", "x"], ["a", "new"], 21, [1.0, 2.0], [0, 1, 2, 3], [7]]"#
|
||||
);
|
||||
h5dump_ok(&path);
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
f.dataset("big/new").unwrap().read_i64().unwrap(),
|
||||
[0, 1, 2, 3]
|
||||
);
|
||||
assert_eq!(f.dataset("alias").unwrap().read_f64().unwrap(), [1.0, 2.0]);
|
||||
}
|
||||
|
||||
// ---- the whole tree, as h5py and as clawhdf5 read it ----
|
||||
|
||||
fn fmt_num(x: f64) -> String {
|
||||
format!("{x:.6}")
|
||||
}
|
||||
|
||||
fn fmt_attr(v: &AttrValue) -> String {
|
||||
let join = |v: Vec<String>| v.join(",");
|
||||
match v {
|
||||
AttrValue::F64(x) => fmt_num(*x),
|
||||
AttrValue::I64(x) => fmt_num(*x as f64),
|
||||
AttrValue::U64(x) => fmt_num(*x as f64),
|
||||
AttrValue::F64Array(a) => join(a.iter().map(|x| fmt_num(*x)).collect()),
|
||||
AttrValue::I64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()),
|
||||
AttrValue::U64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()),
|
||||
AttrValue::String(s) => s.clone(),
|
||||
AttrValue::StringArray(a) => a.join(","),
|
||||
AttrValue::Raw { .. } => "raw".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn fmt_attrs(attrs: std::collections::HashMap<String, AttrValue>) -> String {
|
||||
let mut v: Vec<_> = attrs.into_iter().collect();
|
||||
v.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
v.iter()
|
||||
.map(|(k, a)| format!("{k}={}", fmt_attr(a)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(";")
|
||||
}
|
||||
|
||||
fn child_path(path: &str, name: &str) -> String {
|
||||
if path == "/" {
|
||||
format!("/{name}")
|
||||
} else {
|
||||
format!("{path}/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
/// Every group and dataset reachable from `g` (following hard and soft
|
||||
/// links; the tree must be acyclic), one line each: path, kind, attributes
|
||||
/// and (datasets) values.
|
||||
fn walk(g: &Group<'_>, path: &str, out: &mut Vec<String>) {
|
||||
out.push(format!("{path}|group|{}", fmt_attrs(g.attrs().unwrap())));
|
||||
let mut names: Vec<(String, bool)> = g
|
||||
.datasets()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|n| (n, false))
|
||||
.chain(g.groups().unwrap().into_iter().map(|n| (n, true)))
|
||||
.collect();
|
||||
names.sort();
|
||||
for (name, is_group) in names {
|
||||
let p = child_path(path, &name);
|
||||
if is_group {
|
||||
walk(&g.group(&name).unwrap(), &p, out);
|
||||
} else {
|
||||
let ds = g.dataset(&name).unwrap();
|
||||
let values: Vec<String> = ds.read_f64().unwrap().into_iter().map(fmt_num).collect();
|
||||
out.push(format!(
|
||||
"{p}|dataset|{}|{}",
|
||||
fmt_attrs(ds.attrs().unwrap()),
|
||||
values.join(",")
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn clawhdf5_tree(path: &str) -> String {
|
||||
let f = File::open(path).unwrap();
|
||||
let mut out = Vec::new();
|
||||
walk(&f.root(), "/", &mut out);
|
||||
out.join("\n")
|
||||
}
|
||||
|
||||
/// The same listing as [`walk`], from h5py. External and dangling soft
|
||||
/// links are skipped, as clawhdf5's group listings skip them.
|
||||
const H5PY_WALK: &str = r#"
|
||||
def fmt(v):
|
||||
if isinstance(v, bytes): return v.decode()
|
||||
if isinstance(v, str): return v
|
||||
a = np.asarray(v)
|
||||
if a.dtype.kind in 'SUO':
|
||||
return ','.join(x.decode() if isinstance(x, bytes) else str(x) for x in a.ravel())
|
||||
if a.ndim == 0: return '%.6f' % float(a)
|
||||
return ','.join('%.6f' % float(x) for x in a.ravel())
|
||||
def attrs(o): return ';'.join(f'{k}={fmt(o.attrs[k])}' for k in sorted(o.attrs))
|
||||
out = []
|
||||
def walk(g, path):
|
||||
out.append(f'{path}|group|{attrs(g)}')
|
||||
for k in sorted(g.keys()):
|
||||
if isinstance(g.get(k, getlink=True), h5py.ExternalLink): continue
|
||||
o = g.get(k)
|
||||
if o is None: continue
|
||||
p = '/' + k if path == '/' else path + '/' + k
|
||||
if isinstance(o, h5py.Group): walk(o, p)
|
||||
else:
|
||||
vals = ','.join('%.6f' % float(x) for x in np.asarray(o[()]).ravel())
|
||||
out.append(f'{p}|dataset|{attrs(o)}|{vals}')
|
||||
with h5py.File(path, 'r') as f:
|
||||
walk(f, '/')
|
||||
print('\n'.join(out))
|
||||
"#;
|
||||
|
||||
fn h5py_tree(path: &str) -> String {
|
||||
h5py(path, H5PY_WALK)
|
||||
}
|
||||
|
||||
/// A four-level tree with attributes on every object: nested builders,
|
||||
/// path names (with intermediate groups made on the way) and a group added
|
||||
/// twice (merged), with dense attribute storage at one level and dense link
|
||||
/// storage at another.
|
||||
fn nested_builder() -> FileBuilder {
|
||||
let mut b = FileBuilder::new();
|
||||
b.set_attr("title", AttrValue::String("nested".into()));
|
||||
let mut l1 = b.create_group("l1");
|
||||
l1.set_attr("depth", AttrValue::I64(1));
|
||||
l1.create_dataset("d1")
|
||||
.with_f64_data(&[1.0, 1.5])
|
||||
.set_attr("unit", AttrValue::String("m".into()));
|
||||
let mut l2 = l1.create_group("l2");
|
||||
l2.set_attr("depth", AttrValue::I64(2));
|
||||
l2.create_dataset("d2").with_i32_data(&[2, 3, 4]);
|
||||
let mut l3 = l2.create_group("l3");
|
||||
for i in 0..10 {
|
||||
l3.set_attr(&format!("a{i}"), AttrValue::F64(i as f64 / 4.0)); // dense
|
||||
}
|
||||
for i in 0..12 {
|
||||
l3.create_dataset(&format!("x{i:02}")) // dense links
|
||||
.with_i64_data(&[i, -i])
|
||||
.set_attr("i", AttrValue::I64(i));
|
||||
}
|
||||
let mut l4 = l3.create_group("l4");
|
||||
l4.set_attr("depth", AttrValue::I64(4));
|
||||
l4.create_dataset("leaf")
|
||||
.with_f64_data(&[4.0, 4.25, 4.5])
|
||||
.set_attr(
|
||||
"tags",
|
||||
AttrValue::StringArray(vec!["a".into(), "bc".into()]),
|
||||
);
|
||||
l3.add_group(l4.finish());
|
||||
l2.add_group(l3.finish());
|
||||
l1.add_group(l2.finish());
|
||||
b.add_group(l1.finish());
|
||||
// Path names: /p, /p/q and /p/q/r are made on the way to the dataset.
|
||||
b.create_dataset("p/q/r/s")
|
||||
.with_f64_data(&[7.0])
|
||||
.set_attr("deep", AttrValue::I64(4));
|
||||
// A group at an existing path is merged into it.
|
||||
let mut pq = b.create_group("p/q");
|
||||
pq.set_attr("merged", AttrValue::I64(1));
|
||||
pq.create_dataset("t").with_i32_data(&[8]);
|
||||
b.add_group(pq.finish());
|
||||
let mut l1b = b.create_group("l1/l2/l3/l4/l5");
|
||||
l1b.set_attr("depth", AttrValue::I64(5));
|
||||
b.add_group(l1b.finish());
|
||||
b
|
||||
}
|
||||
|
||||
const NESTED_TREE: &str = "\
|
||||
/|group|title=nested
|
||||
/l1|group|depth=1.000000
|
||||
/l1/d1|dataset|unit=m|1.000000,1.500000
|
||||
/l1/l2|group|depth=2.000000
|
||||
/l1/l2/d2|dataset||2.000000,3.000000,4.000000";
|
||||
|
||||
#[test]
|
||||
fn nested_groups_read_the_same_in_h5py_and_clawhdf5() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = write(&dir, "nested.h5", nested_builder());
|
||||
let ours = clawhdf5_tree(&path);
|
||||
let theirs = h5py_tree(&path);
|
||||
assert_eq!(ours, theirs);
|
||||
assert!(ours.starts_with(NESTED_TREE), "{ours}");
|
||||
for line in [
|
||||
"/l1/l2/l3|group|a0=0.000000;a1=0.250000;a2=0.500000;a3=0.750000;a4=1.000000;\
|
||||
a5=1.250000;a6=1.500000;a7=1.750000;a8=2.000000;a9=2.250000",
|
||||
"/l1/l2/l3/l4|group|depth=4.000000",
|
||||
"/l1/l2/l3/l4/l5|group|depth=5.000000",
|
||||
"/l1/l2/l3/l4/leaf|dataset|tags=a,bc|4.000000,4.250000,4.500000",
|
||||
"/l1/l2/l3/x11|dataset|i=11.000000|11.000000,-11.000000",
|
||||
"/p|group|",
|
||||
"/p/q|group|merged=1.000000",
|
||||
"/p/q/r/s|dataset|deep=4.000000|7.000000",
|
||||
"/p/q/t|dataset||8.000000",
|
||||
] {
|
||||
assert!(
|
||||
ours.lines().any(|l| l == line),
|
||||
"missing {line:?} in\n{ours}"
|
||||
);
|
||||
}
|
||||
assert_eq!(ours.lines().count(), 26, "{ours}");
|
||||
let dump = h5dump_ok(&path);
|
||||
if !dump.is_empty() {
|
||||
assert!(dump.contains("GROUP \"l5\""), "{dump}");
|
||||
assert!(dump.contains("DATASET \"leaf\""), "{dump}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn soft_hard_and_external_links() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut other = FileBuilder::new();
|
||||
other.create_dataset("data").with_i32_data(&[42, 43]);
|
||||
write(&dir, "other.h5", other);
|
||||
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("x/y").with_f64_data(&[1.0, 2.0, 3.0]);
|
||||
b.create_dataset("x/z").with_i32_data(&[9]);
|
||||
b.add_soft_link("soft_abs", "/x/y");
|
||||
b.add_soft_link("dangling", "/nowhere");
|
||||
b.add_hard_link("alias", "/x/y");
|
||||
b.add_hard_link("x_again", "x");
|
||||
b.add_external_link("ext", "other.h5", "/data");
|
||||
let mut g = b.create_group("a/b/c");
|
||||
g.add_soft_link("rel", "sib"); // relative to /a/b/c
|
||||
g.create_dataset("sib").with_i32_data(&[5]);
|
||||
g.add_hard_link("deep_alias", "/x_again/z"); // through a hard link
|
||||
g.add_soft_link("to_group", "/x");
|
||||
b.add_group(g.finish());
|
||||
let path = write(&dir, "links.h5", b);
|
||||
|
||||
let out = h5py(
|
||||
&path,
|
||||
"import os\nos.chdir(os.path.dirname(path))\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 def kind(g, k):\n\
|
||||
\x20 l = g.get(k, getlink=True)\n\
|
||||
\x20 if isinstance(l, h5py.SoftLink): return 'soft:' + l.path\n\
|
||||
\x20 if isinstance(l, h5py.ExternalLink): return 'ext:' + l.filename + ':' + l.path\n\
|
||||
\x20 return 'hard'\n\
|
||||
\x20 print(json.dumps({\n\
|
||||
\x20 'root': {k: kind(f, k) for k in f},\n\
|
||||
\x20 'abc': {k: kind(f['a/b/c'], k) for k in f['a/b/c']},\n\
|
||||
\x20 'same': [f['alias'].id == f['x/y'].id, f['x_again'].id == f['x'].id,\n\
|
||||
\x20 f['a/b/c/deep_alias'].id == f['x/z'].id],\n\
|
||||
\x20 'rc': [h5py.h5o.get_info(f['x/y'].id).rc, h5py.h5o.get_info(f['x'].id).rc,\n\
|
||||
\x20 h5py.h5o.get_info(f['x/z'].id).rc, h5py.h5o.get_info(f['a'].id).rc],\n\
|
||||
\x20 'vals': [f['soft_abs'][()].tolist(), f['ext'][()].tolist(),\n\
|
||||
\x20 f['a/b/c/rel'][()].tolist(), sorted(f['a/b/c/to_group'])],\n\
|
||||
\x20 'dangling': f.get('dangling') is None,\n\
|
||||
\x20 }, sort_keys=True))",
|
||||
);
|
||||
assert_eq!(
|
||||
out,
|
||||
r#"{"abc": {"deep_alias": "hard", "rel": "soft:sib", "sib": "hard", "to_group": "soft:/x"}, "dangling": true, "rc": [2, 2, 2, 1], "root": {"a": "hard", "alias": "hard", "dangling": "soft:/nowhere", "ext": "ext:other.h5:/data", "soft_abs": "soft:/x/y", "x": "hard", "x_again": "hard"}, "same": [true, true, true], "vals": [[1.0, 2.0, 3.0], [42, 43], [5], ["y", "z"]]}"#
|
||||
);
|
||||
assert_eq!(clawhdf5_tree(&path), h5py_tree(&path));
|
||||
h5dump_ok(&path);
|
||||
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
f.dataset("alias").unwrap().read_f64().unwrap(),
|
||||
[1.0, 2.0, 3.0]
|
||||
);
|
||||
assert_eq!(
|
||||
f.dataset("soft_abs").unwrap().read_f64().unwrap(),
|
||||
[1.0, 2.0, 3.0]
|
||||
);
|
||||
assert_eq!(f.dataset("a/b/c/rel").unwrap().read_i32().unwrap(), [5]);
|
||||
assert_eq!(
|
||||
f.dataset("a/b/c/deep_alias").unwrap().read_i32().unwrap(),
|
||||
[9]
|
||||
);
|
||||
assert_eq!(
|
||||
f.dataset("x_again/y").unwrap().read_f64().unwrap(),
|
||||
[1.0, 2.0, 3.0]
|
||||
);
|
||||
drop(f);
|
||||
|
||||
// The reference counts let libhdf5 delete one of two hard links and
|
||||
// keep the object; with a count of 1 it would free an object still
|
||||
// linked from elsewhere.
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 del f['alias']\n\
|
||||
\x20 del f['x_again']\n\
|
||||
\x20 f.create_dataset('filler', data=np.arange(1000))\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([f['x/y'][()].tolist(), sorted(f['x']), h5py.h5o.get_info(f['x/y'].id).rc]))",
|
||||
);
|
||||
assert_eq!(out, r#"[[1.0, 2.0, 3.0], ["y", "z"], 1]"#);
|
||||
h5dump_ok(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hard_link_can_make_a_cycle() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
let mut g = b.create_group("g");
|
||||
g.create_dataset("v").with_i32_data(&[1]);
|
||||
g.add_hard_link("up", "/");
|
||||
g.add_hard_link("me", ".");
|
||||
b.add_group(g.finish());
|
||||
let path = write(&dir, "cycle.h5", b);
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([sorted(f['g/up/g']), f['g/up/g/me/me/v'][()].tolist(),\n\
|
||||
\x20 h5py.h5o.get_info(f.id).rc, h5py.h5o.get_info(f['g'].id).rc]))",
|
||||
);
|
||||
assert_eq!(out, r#"[["me", "up", "v"], [1], 2, 2]"#);
|
||||
h5dump_ok(&path);
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(f.dataset("g/up/g/me/v").unwrap().read_i32().unwrap(), [1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_links_are_errors() {
|
||||
for setup in [
|
||||
|b: &mut FileBuilder| {
|
||||
b.add_hard_link("h", "/missing");
|
||||
},
|
||||
|b: &mut FileBuilder| {
|
||||
b.create_dataset("x").with_i32_data(&[1]);
|
||||
b.add_soft_link("s", "/x");
|
||||
b.add_hard_link("h", "/s"); // through a soft link
|
||||
},
|
||||
|b: &mut FileBuilder| {
|
||||
b.add_hard_link("h1", "/h2");
|
||||
b.add_hard_link("h2", "/h1");
|
||||
},
|
||||
|b: &mut FileBuilder| {
|
||||
b.create_dataset("x").with_i32_data(&[1]);
|
||||
b.add_hard_link("h", "/x/y"); // a dataset is not a group
|
||||
},
|
||||
|b: &mut FileBuilder| {
|
||||
b.add_soft_link("s", "");
|
||||
},
|
||||
|b: &mut FileBuilder| {
|
||||
b.add_external_link("e", "", "/x");
|
||||
},
|
||||
|b: &mut FileBuilder| {
|
||||
b.create_dataset("x").with_i32_data(&[1]);
|
||||
b.add_soft_link("x", "/y"); // name taken
|
||||
},
|
||||
] {
|
||||
let mut b = FileBuilder::new();
|
||||
setup(&mut b);
|
||||
assert!(b.finish().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ten_thousand_links_in_one_group() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
let mut g = b.create_group("many");
|
||||
for i in 0..10_000 {
|
||||
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
||||
}
|
||||
g.set_attr("n", AttrValue::I64(10_000));
|
||||
b.add_group(g.finish());
|
||||
// The same in creation order, added in reverse name order, with soft
|
||||
// links among them.
|
||||
let mut g = b.create_group("ordered");
|
||||
g.track_order(true);
|
||||
for i in (0..10_000).rev() {
|
||||
if i % 1000 == 0 {
|
||||
g.add_soft_link(&format!("s{i:05}"), &format!("/many/d{i:05}"));
|
||||
}
|
||||
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let path = write(&dir, "many.h5", b);
|
||||
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 m, o = f['many'], f['ordered']\n\
|
||||
\x20 names = list(m)\n\
|
||||
\x20 onames = list(o)\n\
|
||||
\x20 print(json.dumps([len(names), names == sorted(names), names[:2], int(m.attrs['n']),\n\
|
||||
\x20 [int(m['d%05d' % i][0]) for i in (0, 1, 4096, 9999)],\n\
|
||||
\x20 len(onames), onames[:3], onames[-2:], int(o['s05000'][0]),\n\
|
||||
\x20 o.id.get_create_plist().get_link_creation_order()]))",
|
||||
);
|
||||
assert_eq!(
|
||||
out,
|
||||
r#"[10000, true, ["d00000", "d00001"], 10000, [0, 1, 4096, 9999], 10010, ["d09999", "d09998", "d09997"], ["s00000", "d00000"], 5000, 3]"#
|
||||
);
|
||||
h5dump_ok(&path);
|
||||
let f = File::open(&path).unwrap();
|
||||
let g = f.group("many").unwrap();
|
||||
assert_eq!(g.datasets().unwrap().len(), 10_000);
|
||||
assert_eq!(g.dataset("d09999").unwrap().read_i32().unwrap(), [9999]);
|
||||
assert_eq!(
|
||||
f.dataset("ordered/s05000").unwrap().read_i32().unwrap(),
|
||||
[5000]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn more_links_than_one_index_leaf_holds_is_an_error() {
|
||||
let mut b = FileBuilder::new();
|
||||
for i in 0..70_000 {
|
||||
b.add_soft_link(&format!("s{i}"), "/x");
|
||||
}
|
||||
let err = b.finish().unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("70000 links in one group: at most 65535"),
|
||||
"{err}"
|
||||
);
|
||||
// Dense attributes have the same one-leaf index. Their count used to
|
||||
// be written modulo 65 536.
|
||||
let mut b = FileBuilder::new();
|
||||
let x = b.create_dataset("x");
|
||||
x.with_i32_data(&[1]);
|
||||
for i in 0..70_000 {
|
||||
x.set_attr(&format!("a{i}"), AttrValue::I64(i));
|
||||
}
|
||||
let err = b.finish().unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("70000 attributes on one object: at most 65535"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn track_order_lists_members_in_creation_order() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let names = ["zeta", "alpha", "mid", "beta"];
|
||||
let mut b = FileBuilder::new();
|
||||
b.track_order(true); // the root and every group without its own setting
|
||||
for n in names {
|
||||
b.create_dataset(n).with_i32_data(&[1]);
|
||||
}
|
||||
let mut g = b.create_group("by_name");
|
||||
g.track_order(false);
|
||||
for n in names {
|
||||
g.create_dataset(n).with_i32_data(&[2]);
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("dense");
|
||||
for i in (0..20).rev() {
|
||||
g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]);
|
||||
}
|
||||
g.add_soft_link("soft", "/zeta");
|
||||
b.add_group(g.finish());
|
||||
b.create_dataset("made/on/the/way").with_i32_data(&[3]);
|
||||
let path = write(&dir, "order.h5", b);
|
||||
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([list(f), list(f['by_name']), list(f['dense'])[:3],\n\
|
||||
\x20 list(f['dense'])[-2:], list(f['made/on'])]))",
|
||||
);
|
||||
assert_eq!(
|
||||
out,
|
||||
r#"[["zeta", "alpha", "mid", "beta", "by_name", "dense", "made"], ["alpha", "beta", "mid", "zeta"], ["n19", "n18", "n17"], ["n00", "soft"], ["the"]]"#
|
||||
);
|
||||
h5dump_ok(&path);
|
||||
assert_eq!(clawhdf5_tree(&path), h5py_tree(&path));
|
||||
|
||||
// libhdf5 keeps the order when it adds to (and converts) these groups.
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 f['aaa'] = np.arange(2)\n\
|
||||
\x20 f['dense']['aaa'] = np.arange(2)\n\
|
||||
\x20 del f['dense/n10']\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([list(f)[-1], list(f['dense'])[-2:], len(f['dense'])]))",
|
||||
);
|
||||
assert_eq!(out, r#"["aaa", ["soft", "aaa"], 21]"#);
|
||||
h5dump_ok(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_ascii_names_are_utf8() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("größe/wert").with_i32_data(&[1]);
|
||||
let path = write(&dir, "utf8.h5", b);
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 l = f.id.links.get_info('größe'.encode())\n\
|
||||
\x20 print(json.dumps([list(f), list(f['größe']), l.cset], ensure_ascii=False))",
|
||||
);
|
||||
assert_eq!(out, r#"[["größe"], ["wert"], 1]"#);
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(f.dataset("größe/wert").unwrap().read_i32().unwrap(), [1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_group_attribute_set_again_takes_the_new_value() {
|
||||
skip_if_no_python!();
|
||||
// Setting a group attribute twice wrote two attribute messages with one
|
||||
// name. Now the later value replaces the earlier, as `attrs[name] = v`
|
||||
// does in h5py — also across a group merged from two builders.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
b.set_attr("v", AttrValue::I64(1));
|
||||
b.set_attr("v", AttrValue::I64(2));
|
||||
let mut g = b.create_group("g");
|
||||
g.set_attr("w", AttrValue::I64(1));
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("g");
|
||||
g.set_attr("w", AttrValue::String("two".into()));
|
||||
b.add_group(g.finish());
|
||||
let path = write(&dir, "attrs.h5", b);
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([list(f.attrs), int(f.attrs['v']), list(f['g'].attrs),\n\
|
||||
\x20 f['g'].attrs['w'].decode()]))",
|
||||
);
|
||||
assert_eq!(out, r#"[["v"], 2, ["w"], "two"]"#);
|
||||
let f = File::open(&path).unwrap();
|
||||
assert!(matches!(f.root().attrs().unwrap()["v"], AttrValue::I64(2)));
|
||||
}
|
||||
|
||||
// ---- big dense storage: child indirect blocks in the fractal heap ----
|
||||
|
||||
/// A name `len` bytes long, unique per `i`.
|
||||
fn long_name(i: usize, len: usize) -> String {
|
||||
let n = format!("link_{i:06}_");
|
||||
format!("{n}{}", "x".repeat(len - n.len()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_links_past_the_direct_blocks_of_the_root() {
|
||||
skip_if_no_python!();
|
||||
// A dense group's links live in a fractal heap whose root indirect
|
||||
// block holds direct blocks up to 64 KiB: 512 KiB of link messages.
|
||||
// Rows past that are child indirect blocks. The writer used to write
|
||||
// them as direct blocks, which libhdf5 cannot read ("incorrect metadata
|
||||
// checksum"), from about 17 000 links with 20-byte names.
|
||||
// `g` crosses the first boundary (0.6 MB of links); `deep` has 65 535
|
||||
// links of about 110 bytes (7 MB), so its heap reaches the child indirect
|
||||
// blocks that hold indirect blocks themselves.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("x").with_i32_data(&[7]);
|
||||
let mut g = b.create_group("g");
|
||||
for i in 0..20_000 {
|
||||
g.create_dataset(&format!("dataset_number_{i:06}"))
|
||||
.with_i32_data(&[i]);
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("deep");
|
||||
g.track_order(true);
|
||||
for i in 0..usize::from(u16::MAX) {
|
||||
g.add_hard_link(&long_name(i, 100), "/x");
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let path = write(&dir, "big_links.h5", b);
|
||||
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 g, d = f['g'], f['deep']\n\
|
||||
\x20 names = list(g)\n\
|
||||
\x20 dn = list(d)\n\
|
||||
\x20 print(json.dumps([len(names), names[-1], int(g[names[-1]][0]),\n\
|
||||
\x20 sum(int(g[n][0]) for n in names), len(dn), dn[0][:12], dn[-1][:12],\n\
|
||||
\x20 int(d[dn[-1]][0]), h5py.h5o.get_info(f['x'].id).rc]))",
|
||||
);
|
||||
assert_eq!(
|
||||
out,
|
||||
r#"[20000, "dataset_number_019999", 19999, 199990000, 65535, "link_000000_", "link_065534_", 7, 65536]"#
|
||||
);
|
||||
h5dump_ok(&path);
|
||||
let f = File::open(&path).unwrap();
|
||||
let g = f.group("g").unwrap();
|
||||
assert_eq!(g.datasets().unwrap().len(), 20_000);
|
||||
assert_eq!(
|
||||
g.dataset("dataset_number_019999")
|
||||
.unwrap()
|
||||
.read_i32()
|
||||
.unwrap(),
|
||||
[19999]
|
||||
);
|
||||
let d = f.group("deep").unwrap();
|
||||
assert_eq!(d.datasets().unwrap().len(), usize::from(u16::MAX));
|
||||
assert_eq!(
|
||||
d.dataset(&long_name(65_534, 100))
|
||||
.unwrap()
|
||||
.read_i32()
|
||||
.unwrap(),
|
||||
[7]
|
||||
);
|
||||
|
||||
// libhdf5 can add to and delete from the heap. It could not when the
|
||||
// header's block allocation offset was 0: its next block overwrote the
|
||||
// first ("bad version number for message"). Adding to `deep` also
|
||||
// needs its index's leaf node to have room for at most 65 535 records:
|
||||
// a bigger node made libhdf5 overflow the leaf's 2-byte record count
|
||||
// (a crash, or "unknown link class" when listing).
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 f['g']['zz_new'] = np.arange(3)\n\
|
||||
\x20 f['deep']['zz_new'] = np.arange(4)\n\
|
||||
\x20 del f['g/dataset_number_000005']\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([len(f['g']), int(f['g/zz_new'][2]), len(f['deep']),\n\
|
||||
\x20 list(f['deep'])[-1], int(f['g/dataset_number_019998'][0])]))",
|
||||
);
|
||||
assert_eq!(out, r#"[20000, 2, 65536, "zz_new", 19998]"#);
|
||||
h5dump_ok(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_attributes_past_the_direct_blocks_of_the_root() {
|
||||
skip_if_no_python!();
|
||||
// Dense attributes share the heap writer. 150 attributes of up to 56 KB
|
||||
// (8 MB) need child indirect blocks, and a big attribute after small
|
||||
// ones must skip the small blocks rather than overrun one.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
let ds = b.create_dataset("x");
|
||||
ds.with_i32_data(&[1]);
|
||||
for i in 0..150usize {
|
||||
let len = if i % 3 == 0 { 7_000 } else { 1 + i };
|
||||
let v: Vec<f64> = (0..len).map(|k| (i * 100_000 + k) as f64).collect();
|
||||
ds.set_attr(&format!("a{i:03}"), AttrValue::F64Array(v));
|
||||
}
|
||||
let path = write(&dir, "big_attrs.h5", b);
|
||||
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 a = f['x'].attrs\n\
|
||||
\x20 ok = all(np.array_equal(a['a%03d' % i],\n\
|
||||
\x20 np.arange(7000 if i % 3 == 0 else 1 + i) + i * 100000) for i in range(150))\n\
|
||||
\x20 print(json.dumps([len(a), ok]))",
|
||||
);
|
||||
assert_eq!(out, "[150, true]");
|
||||
h5dump_ok(&path);
|
||||
let f = File::open(&path).unwrap();
|
||||
let attrs = f.dataset("x").unwrap().attrs().unwrap();
|
||||
assert_eq!(attrs.len(), 150);
|
||||
for i in [0usize, 1, 147, 149] {
|
||||
let len = if i % 3 == 0 { 7_000 } else { 1 + i };
|
||||
let want: Vec<f64> = (0..len).map(|k| (i * 100_000 + k) as f64).collect();
|
||||
match &attrs[&format!("a{i:03}")] {
|
||||
AttrValue::F64Array(v) => assert_eq!(*v, want, "a{i:03}"),
|
||||
other => panic!("a{i:03}: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_link_too_big_for_dense_storage_is_an_error() {
|
||||
// A link message must fit one fractal heap direct block (64 KiB less
|
||||
// its header); the writer has no huge-object path. It used to be
|
||||
// written anyway, cut off, and libhdf5 could not list the group.
|
||||
let mut b = FileBuilder::new();
|
||||
for i in 0..10 {
|
||||
b.create_dataset(&format!("d{i}")).with_i32_data(&[i]);
|
||||
}
|
||||
b.add_soft_link("s", &"/y".repeat(40_000));
|
||||
let err = b.finish().unwrap_err().to_string();
|
||||
assert!(err.contains("fractal heap object holds at most"), "{err}");
|
||||
// The same for a dense attribute.
|
||||
let mut b = FileBuilder::new();
|
||||
let x = b.create_dataset("x");
|
||||
x.with_i32_data(&[1]);
|
||||
for i in 0..9 {
|
||||
x.set_attr(&format!("a{i}"), AttrValue::I64(i));
|
||||
}
|
||||
x.set_attr("big", AttrValue::F64Array(vec![0.5; 9_000]));
|
||||
let err = b.finish().unwrap_err().to_string();
|
||||
assert!(err.contains("fractal heap object holds at most"), "{err}");
|
||||
|
||||
// Just under the limit is fine, and libhdf5 reads it back.
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
for i in 0..10 {
|
||||
b.create_dataset(&format!("d{i}")).with_i32_data(&[i]);
|
||||
}
|
||||
let target = format!("/{}", "y".repeat(65_000));
|
||||
b.add_soft_link("s", &target);
|
||||
let path = write(&dir, "long_soft.h5", b);
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([len(f), len(f.get('s', getlink=True).path)]))",
|
||||
);
|
||||
assert_eq!(out, "[11, 65001]");
|
||||
h5dump_ok(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chained_hard_links_resolve_in_linear_time() {
|
||||
skip_if_no_python!();
|
||||
// Each link's target goes through the previous link twice. Resolving
|
||||
// them without remembering resolved links doubled the work per link:
|
||||
// 26 links took 46 s in a debug build, so 60 would never finish.
|
||||
fn chain(reverse: bool) -> FileBuilder {
|
||||
let mut b = FileBuilder::new();
|
||||
let mut g = b.create_group("g");
|
||||
g.create_dataset("v").with_i32_data(&[5]);
|
||||
b.add_group(g.finish());
|
||||
let mut order: Vec<usize> = (0..60).collect();
|
||||
if reverse {
|
||||
order.reverse();
|
||||
}
|
||||
for i in order {
|
||||
if i == 0 {
|
||||
b.add_hard_link("g/s0", "/g");
|
||||
} else {
|
||||
b.add_hard_link(&format!("g/s{i}"), &format!("/g/s{}/s{}", i - 1, i - 1));
|
||||
}
|
||||
}
|
||||
b
|
||||
}
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let bytes = [false, true].map(|r| chain(r).finish().unwrap());
|
||||
tx.send(bytes).unwrap();
|
||||
});
|
||||
let [forward, reverse] = rx
|
||||
.recv_timeout(std::time::Duration::from_secs(60))
|
||||
.expect("resolving 60 chained hard links took over a minute");
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
for (name, bytes) in [("forward.h5", forward), ("reverse.h5", reverse)] {
|
||||
let path = dir.path().join(name).display().to_string();
|
||||
std::fs::write(&path, bytes).unwrap();
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([h5py.h5o.get_info(f['g'].id).rc, len(f['g']),\n\
|
||||
\x20 int(f['g/s59/s30/s0/v'][0]), f['g/s59'] == f['g']]))",
|
||||
);
|
||||
assert_eq!(out, "[61, 61, 5, true]", "{name}");
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(f.dataset("g/s59/s0/v").unwrap().read_i32().unwrap(), [5]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dataset_attribute_set_again_takes_the_new_value() {
|
||||
skip_if_no_python!();
|
||||
// Setting a dataset attribute twice wrote two attribute messages with
|
||||
// one name, and h5py read back the first value. Also with dense
|
||||
// attribute storage (more than 8).
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("x")
|
||||
.with_f64_data(&[1.0])
|
||||
.set_attr("a", AttrValue::I64(1))
|
||||
.set_attr("a", AttrValue::I64(2));
|
||||
let d = b.create_dataset("dense");
|
||||
d.with_i32_data(&[1]);
|
||||
for i in 0..12 {
|
||||
d.set_attr(&format!("k{i:02}"), AttrValue::I64(i));
|
||||
}
|
||||
d.set_attr("k03", AttrValue::String("three".into()));
|
||||
let path = write(&dir, "ds_attrs.h5", b);
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 a, d = f['x'].attrs, f['dense'].attrs\n\
|
||||
\x20 print(json.dumps([list(a), int(a['a']), len(d), d['k03'].decode(), int(d['k04'])]))",
|
||||
);
|
||||
assert_eq!(out, r#"[["a"], 2, 12, "three", 4]"#);
|
||||
let f = File::open(&path).unwrap();
|
||||
assert!(matches!(
|
||||
f.dataset("x").unwrap().attrs().unwrap()["a"],
|
||||
AttrValue::I64(2)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(feature = "provenance")]
|
||||
#[test]
|
||||
fn provenance_attributes_replace_ones_set_by_hand() {
|
||||
skip_if_no_python!();
|
||||
// A hand-set attribute with a provenance attribute's name was written
|
||||
// next to the computed one, and h5py read the hand-set value.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("p")
|
||||
.with_i32_data(&[1, 2])
|
||||
.with_provenance("me", "2026-09-26T00:00:00Z", None)
|
||||
.set_attr("_provenance_sha256", AttrValue::String("forged".into()));
|
||||
let path = write(&dir, "prov.h5", b);
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r') as f:\n\
|
||||
\x20 a = f['p'].attrs\n\
|
||||
\x20 h = a['_provenance_sha256']\n\
|
||||
\x20 h = h.decode() if isinstance(h, bytes) else h\n\
|
||||
\x20 print(json.dumps([list(a).count('_provenance_sha256'), h != 'forged']))",
|
||||
);
|
||||
assert_eq!(out, "[1, true]");
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
f.dataset("p").unwrap().verify_provenance().unwrap(),
|
||||
clawhdf5_format::provenance::VerifyResult::Ok
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user