diff --git a/CHANGELOG.md b/CHANGELOG.md index 95da497..b3ba430 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -327,6 +327,16 @@ cannot be followed without the application that registered the class, so they are now left out of `datasets()`/`groups()` and path lookup, as h5py leaves out links it cannot open; reserved link types are still an error. +- `clawhdf5` — soft links are listed, as h5py lists them: `datasets()` and + `groups()` on `Group`/`MmapGroup`/`LazyGroup` include each soft link under + its own name as the kind of object it resolves to, and `dataset(name)` / + `group(name)` open through it. Relative targets resolve from the group + holding the link. Dangling or cyclic soft links, external links and + user-defined links are left out (h5py lists their names but cannot open + them). Previously soft links were missing from the listings, and in + old-style (symbol table) groups a soft link made the listing fail. New + `group_v2::resolve_group_children` / `resolve_path_from` and + `group_v1::v1_soft_links` in `clawhdf5-format`. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/group_v1.rs b/crates/clawhdf5-format/src/group_v1.rs index 989f826..edb84f6 100644 --- a/crates/clawhdf5-format/src/group_v1.rs +++ b/crates/clawhdf5-format/src/group_v1.rs @@ -73,6 +73,53 @@ pub fn find_v1_soft_link( offset_size: u8, length_size: u8, ) -> Result, FormatError> { + let mut found = None; + for_each_v1_soft_link( + file_data, + sym_table_msg, + offset_size, + length_size, + |link_name| link_name == name, + |_, target| { + found = Some(target); + false + }, + )?; + Ok(found) +} + +/// Every soft link in a v1 group, as `(name, target path)`. +pub fn v1_soft_links( + file_data: &[u8], + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let mut links = Vec::new(); + for_each_v1_soft_link( + file_data, + sym_table_msg, + offset_size, + length_size, + |_| true, + |name, target| { + links.push((String::from(name), target)); + true + }, + )?; + Ok(links) +} + +/// Visit the soft links of a v1 group whose name passes `wanted`, with their +/// target paths, until `visit` returns false. +fn for_each_v1_soft_link( + file_data: &[u8], + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, + wanted: impl Fn(&str) -> bool, + mut visit: impl FnMut(&str, String) -> bool, +) -> Result<(), FormatError> { let heap = LocalHeap::parse( file_data, sym_table_msg.local_heap_address as usize, @@ -91,7 +138,8 @@ pub fn find_v1_soft_link( if entry.cache_type != CACHE_TYPE_SOFT_LINK { continue; } - if heap.read_string(file_data, entry.link_name_offset)? != name { + let name = heap.read_string(file_data, entry.link_name_offset)?; + if !wanted(&name) { continue; } let value_offset = u32::from_le_bytes([ @@ -100,12 +148,19 @@ pub fn find_v1_soft_link( entry.scratch_pad[2], entry.scratch_pad[3], ]); - return heap - .read_string(file_data, u64::from(value_offset)) - .map(Some); + let target = heap.read_string(file_data, u64::from(value_offset))?; + if !visit(&name, target) { + return Ok(()); + } } } - Ok(None) + Ok(()) +} + +/// Whether a v1 symbol-table entry is a soft link (no object header of its +/// own; its target path is in the local heap). +pub fn is_v1_soft_link(entry: &GroupEntry) -> bool { + entry.cache_type == CACHE_TYPE_SOFT_LINK } /// Extract the SymbolTableMessage from an object header's messages. diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index bcfa1be..57119c9 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -255,32 +255,135 @@ pub fn resolve_path_any( superblock: &Superblock, path: &str, ) -> Result { - resolve_path_following_links(file_data, superblock, path, 0) + resolve_path_following_links( + file_data, + superblock, + superblock.root_group_address, + path, + 0, + ) +} + +/// Resolve `path` relative to the group at `group_address` (an absolute path +/// starts at the root group instead), following soft links. This is how a +/// relative soft link's target is resolved: from the group holding the link. +pub fn resolve_path_from( + file_data: &[u8], + superblock: &Superblock, + group_address: u64, + path: &str, +) -> Result { + let start = if path.starts_with('/') { + superblock.root_group_address + } else { + group_address + }; + resolve_path_following_links(file_data, superblock, start, path, 0) +} + +/// The children of the group at `group_address` that can be opened, as h5py +/// lists them: hard links, and soft links resolved to the object they point +/// at (under the soft link's own name). Links that cannot be followed are +/// left out rather than failing the listing — a dangling or cyclic soft link +/// (h5py lists its name but cannot open it), an external link (another +/// file), and a user-defined link. An object header that is not a group has +/// no children. +/// +/// Any other error, such as a corrupt structure met while resolving a soft +/// link, is returned. +pub fn resolve_group_children( + file_data: &[u8], + superblock: &Superblock, + group_address: u64, +) -> Result, FormatError> { + let os = superblock.offset_size; + let ls = superblock.length_size; + let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?; + + let mut entries = Vec::new(); + let mut soft = Vec::new(); + if is_v1_group(&header) { + let sym_msg = header + .messages + .iter() + .find(|m| m.msg_type == MessageType::SymbolTable) + .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?; + let stm = SymbolTableMessage::parse(&sym_msg.data, os)?; + let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?; + if all.iter().any(group_v1::is_v1_soft_link) { + soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?; + } + entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e))); + } else if is_v2_group(&header) { + let mut visit = |link: LinkMessage| match link.link_target { + LinkTarget::Hard { + object_header_address, + } => entries.push(GroupEntry { + name: link.name, + object_header_address, + cache_type: 0, + }), + LinkTarget::Soft { target_path } => soft.push((link.name, target_path)), + LinkTarget::External { .. } => {} + }; + let link_info = find_link_info(&header, os)?; + if let Some(fh_addr) = link_info.fractal_heap_address { + for_each_dense_link(file_data, &link_info, fh_addr, os, ls, visit)?; + } else { + for msg in &header.messages { + if msg.msg_type == MessageType::Link + && let Some(link) = parse_link(&msg.data, os)? + { + visit(link); + } + } + } + } + + for (name, target) in soft { + match resolve_path_from(file_data, superblock, group_address, &target) { + Ok(object_header_address) => entries.push(GroupEntry { + name, + object_header_address, + cache_type: 0, + }), + // Dangling, cyclic, or ending in another file: not openable here. + Err( + FormatError::PathNotFound(_) + | FormatError::NestingDepthExceeded + | FormatError::ExternalLinkUnsupported { .. }, + ) => {} + Err(e) => return Err(e), + } + } + Ok(entries) } /// Soft links followed while resolving one path. Guards against link cycles /// (`a -> b -> a`), which are legal to create. const MAX_SOFT_LINK_DEPTH: u8 = 16; +/// Walk `path` from the group at `start`, following soft links. fn resolve_path_following_links( file_data: &[u8], superblock: &Superblock, + start: u64, path: &str, depth: u8, ) -> Result { - let components: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + let components: Vec<&str> = path + .split('/') + .filter(|s| !s.is_empty() && *s != ".") + .collect(); if components.is_empty() { - return Ok(superblock.root_group_address); + return Ok(start); } let os = superblock.offset_size; let ls = superblock.length_size; - let root_header = - ObjectHeader::parse(file_data, superblock.root_group_address as usize, os, ls)?; - - let mut current_addr = superblock.root_group_address; - let mut current_header = root_header; + let mut current_addr = start; + let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?; for (i, component) in components.iter().enumerate() { let entries = resolve_group_entries(file_data, ¤t_header, os, ls)?; @@ -304,20 +407,17 @@ fn resolve_path_following_links( } // A relative target is relative to the group holding // the link; then the rest of the original path. - let mut full = String::new(); - if !target_path.starts_with('/') { - for parent in &components[..i] { - full.push('/'); - full.push_str(parent); - } - } - full.push('/'); - full.push_str(&target_path); + let from = if target_path.starts_with('/') { + superblock.root_group_address + } else { + current_addr + }; + let mut full = target_path; for rest in &components[i + 1..] { full.push('/'); full.push_str(rest); } - resolve_path_following_links(file_data, superblock, &full, depth + 1) + resolve_path_following_links(file_data, superblock, from, &full, depth + 1) } Some(LinkTarget::External { filename, diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index 4a7421e..314234c 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -19,13 +19,12 @@ use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; use clawhdf5_format::filter_pipeline::FilterPipeline; -use clawhdf5_format::group_v1::{self, GroupEntry}; +use clawhdf5_format::group_v1::GroupEntry; use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; -use clawhdf5_format::symbol_table::SymbolTableMessage; use clawhdf5_io::HDF5Read; @@ -275,12 +274,14 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { }) } + /// This group's links that can be opened: hard links, and soft links + /// resolved to their targets (see + /// [`group_v2::resolve_group_children`]); dangling, external and + /// user-defined links are left out. fn children(&self) -> Result, Error> { - let hdr = self.file.get_or_parse_header(self.address)?; let data = self.file.reader.as_bytes(); - let os = self.file.offset_size(); - let ls = self.file.length_size(); - resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format) + group_v2::resolve_group_children(data, &self.file.superblock, self.address) + .map_err(Error::Format) } } @@ -530,33 +531,3 @@ fn is_group(header: &ObjectHeader) -> bool { || m.msg_type == MessageType::SymbolTable }) } - -fn resolve_group_entries( - file_data: &[u8], - object_header: &ObjectHeader, - offset_size: u8, - length_size: u8, -) -> Result, FormatError> { - let is_v1 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::SymbolTable); - let is_v2 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link); - - if is_v1 { - let sym_msg = object_header - .messages - .iter() - .find(|m| m.msg_type == MessageType::SymbolTable) - .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?; - let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size) - } else if is_v2 { - group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size) - } else { - Ok(Vec::new()) - } -} diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 9119cdf..bf098cf 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -14,13 +14,12 @@ use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; use clawhdf5_format::filter_pipeline::FilterPipeline; -use clawhdf5_format::group_v1::{self, GroupEntry}; +use clawhdf5_format::group_v1::GroupEntry; use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; -use clawhdf5_format::symbol_table::SymbolTableMessage; use clawhdf5_io::MmapReader; @@ -197,12 +196,14 @@ impl<'f> MmapGroup<'f> { }) } + /// This group's links that can be opened: hard links, and soft links + /// resolved to their targets (see + /// [`group_v2::resolve_group_children`]); dangling, external and + /// user-defined links are left out. fn children(&self) -> Result, Error> { let data = self.file.reader.as_bytes(); - let hdr = self.file.parse_header(self.address)?; - let os = self.file.offset_size(); - let ls = self.file.length_size(); - resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format) + group_v2::resolve_group_children(data, &self.file.superblock, self.address) + .map_err(Error::Format) } } @@ -470,33 +471,3 @@ fn is_group(header: &ObjectHeader) -> bool { || m.msg_type == MessageType::SymbolTable }) } - -fn resolve_group_entries( - file_data: &[u8], - object_header: &ObjectHeader, - offset_size: u8, - length_size: u8, -) -> Result, FormatError> { - let is_v1 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::SymbolTable); - let is_v2 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link); - - if is_v1 { - let sym_msg = object_header - .messages - .iter() - .find(|m| m.msg_type == MessageType::SymbolTable) - .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?; - let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size) - } else if is_v2 { - group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size) - } else { - Ok(Vec::new()) - } -} diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index bb6a8fd..9c9a805 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -15,13 +15,12 @@ use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; use clawhdf5_format::error::FormatError; use clawhdf5_format::filter_pipeline::FilterPipeline; -use clawhdf5_format::group_v1::{self, GroupEntry}; +use clawhdf5_format::group_v1::GroupEntry; use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; -use clawhdf5_format::symbol_table::SymbolTableMessage; use crate::error::Error; use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype}; @@ -337,12 +336,14 @@ impl<'f> Group<'f> { }) } + /// This group's links that can be opened: hard links, and soft links + /// resolved to their targets (see + /// [`group_v2::resolve_group_children`]); dangling, external and + /// user-defined links are left out. fn children(&self) -> Result, Error> { let data = self.file.data.as_bytes(); - let hdr = self.file.parse_header(self.address)?; - let os = self.file.offset_size(); - let ls = self.file.length_size(); - resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format) + group_v2::resolve_group_children(data, &self.file.superblock, self.address) + .map_err(Error::Format) } } @@ -959,37 +960,6 @@ fn is_group(header: &ObjectHeader) -> bool { }) } -fn resolve_group_entries( - file_data: &[u8], - object_header: &ObjectHeader, - offset_size: u8, - length_size: u8, -) -> Result, FormatError> { - let is_v1 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::SymbolTable); - let is_v2 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link); - - if is_v1 { - let sym_msg = object_header - .messages - .iter() - .find(|m| m.msg_type == MessageType::SymbolTable) - .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?; - let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size) - } else if is_v2 { - group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size) - } else { - // Empty group or unrecognized — return empty - Ok(Vec::new()) - } -} - #[cfg(test)] mod sibling_file_name_tests { use super::sibling_file_name; diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index 92af6ee..cbb3b63 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -298,3 +298,69 @@ fn user_defined_links_do_not_break_the_listing() { assert!(f.root().datasets().unwrap().is_empty()); assert!(f.root().groups().unwrap().is_empty()); } + +/// Soft links (absolute, relative, to a dataset and to a group), a dangling +/// one, a cycle and an external link, in an old-style (symbol table) or +/// new-style (link message) group. +fn soft_link_file(libver: &str) -> (tempfile::TempDir, String) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("soft.h5").display().to_string(); + run_python(&format!( + "import h5py, numpy as np\n\ + with h5py.File(r'{path}', 'w', libver='{libver}') as f:\n\ + \x20 f.create_dataset('a/b/deep', data=np.arange(3.0))\n\ + \x20 f.create_dataset('plain', data=[1.0])\n\ + \x20 f['soft_ds'] = h5py.SoftLink('/a/b/deep')\n\ + \x20 f['soft_grp'] = h5py.SoftLink('/a')\n\ + \x20 f['a/rel'] = h5py.SoftLink('b/deep')\n\ + \x20 f['a/rel_grp'] = h5py.SoftLink('b')\n\ + \x20 f['dangling'] = h5py.SoftLink('/nope')\n\ + \x20 f['loop1'] = h5py.SoftLink('/loop2')\n\ + \x20 f['loop2'] = h5py.SoftLink('/loop1')\n\ + \x20 f['ext'] = h5py.ExternalLink('elsewhere.h5', '/x')\n" + )); + (dir, path) +} + +fn check_soft_links(libver: &str) { + let (_dir, path) = soft_link_file(libver); + assert_same_listing(&path, "/"); + assert_same_listing(&path, "a"); + + let f = File::open(&path).unwrap(); + let root = f.root(); + let deep = vec![0.0, 1.0, 2.0]; + assert_eq!(root.dataset("soft_ds").unwrap().read_f64().unwrap(), deep); + let a = root.group("soft_grp").unwrap(); + assert_eq!(a.dataset("rel").unwrap().read_f64().unwrap(), deep); + assert_eq!(a.group("rel_grp").unwrap().datasets().unwrap(), ["deep"]); + // A dangling link is not listed and cannot be opened. + assert!(root.dataset("dangling").is_err()); + assert!(root.dataset("loop1").is_err()); + + // The memory-mapped and lazy handles list the same way. + let (ds, gs) = h5py_listing(&path, "/"); + let m = clawhdf5::MmapFile::open(&path).unwrap(); + let mut mds = m.root().datasets().unwrap(); + let mut mgs = m.root().groups().unwrap(); + mds.sort(); + mgs.sort(); + assert_eq!((mds, mgs), (ds.clone(), gs.clone())); + let l = clawhdf5::LazyFile::from_bytes(std::fs::read(&path).unwrap()).unwrap(); + let mut lds = l.root().datasets().unwrap(); + let mut lgs = l.root().groups().unwrap(); + lds.sort(); + lgs.sort(); + assert_eq!((lds, lgs), (ds, gs)); +} + +/// Soft links were left out of `datasets()`/`groups()` (and could not be +/// opened by name from a group handle); in old-style groups, where a soft +/// link has no object header address, they made the listing fail. h5py +/// lists a soft link under its own name as whatever it points at. +#[test] +fn soft_links_are_listed_as_their_targets() { + skip_if_no_python!(); + check_soft_links("latest"); + check_soft_links("earliest"); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index a437db4..3dba53d 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -82,7 +82,8 @@ the VDS item, which is marked. **Fixed 2026-09-25:** two bugs — fractal-heap child indirect blocks had the wrong row count, and v2 B-tree internal nodes at depth 3+ were read with the wrong pointer widths. - - Soft links are left out of `datasets()`. + - Soft links are left out of `datasets()`. **Fixed 2026-09-25:** soft + links are listed as their targets; dangling ones are left out. - **Dense attributes:** a large attribute stored as a fractal-heap "huge" object makes every attribute on the object fail. This affects real NetCDF files (`issue671.nc`). **Fixed 2026-09-25:** huge and tiny heap objects,