Merge pull request 'docs(clawhdf5): document DType variants, fix unresolved doc links' (#17) from sdlc-docs/clawhdf5-types-20260514-165210 into main
This commit is contained in:
redclawsystems
@@ -0,0 +1,352 @@
|
||||
//! Sub-filing support for extreme-scale parallel I/O.
|
||||
//!
|
||||
//! Sub-filing splits a single logical HDF5 file into multiple physical
|
||||
//! files (sub-files), one per node or I/O aggregator. This avoids
|
||||
//! contention on a single file system object and enables each node to
|
||||
//! write independently.
|
||||
//!
|
||||
//! # Architecture
|
||||
//!
|
||||
//! ```text
|
||||
//! ┌──────────────────────────────┐
|
||||
//! │ Logical HDF5 File │
|
||||
//! │ ┌────────┬────────┬───────┐ │
|
||||
//! │ │ Sub 0 │ Sub 1 │ Sub 2 │ │ ← stripe_count = 3
|
||||
//! │ │ node-0 │ node-1 │ node-2│ │
|
||||
//! │ └────────┴────────┴───────┘ │
|
||||
//! │ stripe_size = 32 MiB │
|
||||
//! └──────────────────────────────┘
|
||||
//! ```
|
||||
//!
|
||||
//! Data is distributed across sub-files in a round-robin fashion with
|
||||
//! a configurable stripe size.
|
||||
//!
|
||||
//! # Status
|
||||
//!
|
||||
//! This module provides the configuration types and address mapping logic.
|
||||
//! Full integration with the file writer requires additional work to
|
||||
//! coordinate metadata across sub-files.
|
||||
|
||||
use std::fmt;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// Default stripe size: 32 MiB.
|
||||
pub const DEFAULT_STRIPE_SIZE: u64 = 32 * 1024 * 1024;
|
||||
|
||||
/// Configuration for sub-filing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SubfileConfig {
|
||||
/// Number of sub-files (typically one per I/O aggregator or node).
|
||||
pub stripe_count: u32,
|
||||
/// Size of each stripe in bytes (data is round-robin'd across sub-files).
|
||||
pub stripe_size: u64,
|
||||
/// Base path for sub-files. Sub-files are named `{base}.subfile.{N}`.
|
||||
pub base_path: PathBuf,
|
||||
}
|
||||
|
||||
impl SubfileConfig {
|
||||
/// Create a new sub-file configuration.
|
||||
pub fn new(base_path: impl AsRef<Path>, stripe_count: u32) -> Self {
|
||||
Self {
|
||||
stripe_count,
|
||||
stripe_size: DEFAULT_STRIPE_SIZE,
|
||||
base_path: base_path.as_ref().to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the stripe size.
|
||||
pub fn with_stripe_size(mut self, size: u64) -> Self {
|
||||
self.stripe_size = size;
|
||||
self
|
||||
}
|
||||
|
||||
/// Generate the path for a specific sub-file index.
|
||||
pub fn subfile_path(&self, index: u32) -> PathBuf {
|
||||
let base = self.base_path.display();
|
||||
PathBuf::from(format!("{base}.subfile.{index}"))
|
||||
}
|
||||
|
||||
/// Generate paths for all sub-files.
|
||||
pub fn all_subfile_paths(&self) -> Vec<PathBuf> {
|
||||
(0..self.stripe_count)
|
||||
.map(|i| self.subfile_path(i))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Map a logical file offset to a (subfile_index, local_offset) pair.
|
||||
///
|
||||
/// The mapping is round-robin: offset 0..stripe_size goes to sub-file 0,
|
||||
/// stripe_size..2*stripe_size goes to sub-file 1, etc.
|
||||
pub fn map_offset(&self, logical_offset: u64) -> (u32, u64) {
|
||||
let stripe_index = logical_offset / self.stripe_size;
|
||||
let subfile_index = (stripe_index % self.stripe_count as u64) as u32;
|
||||
let stripe_in_subfile = stripe_index / self.stripe_count as u64;
|
||||
let offset_in_stripe = logical_offset % self.stripe_size;
|
||||
let local_offset = stripe_in_subfile * self.stripe_size + offset_in_stripe;
|
||||
(subfile_index, local_offset)
|
||||
}
|
||||
|
||||
/// Map a logical byte range to a set of (subfile_index, local_offset, length) operations.
|
||||
///
|
||||
/// A single logical range may span multiple sub-files, so this returns
|
||||
/// a vector of I/O operations, each targeting a specific sub-file.
|
||||
pub fn map_range(&self, logical_offset: u64, length: u64) -> Vec<SubfileIo> {
|
||||
let mut ops = Vec::new();
|
||||
let mut remaining = length;
|
||||
let mut offset = logical_offset;
|
||||
|
||||
while remaining > 0 {
|
||||
let (subfile_idx, local_off) = self.map_offset(offset);
|
||||
// How many bytes until the end of the current stripe?
|
||||
let offset_in_stripe = offset % self.stripe_size;
|
||||
let bytes_in_stripe = (self.stripe_size - offset_in_stripe).min(remaining);
|
||||
|
||||
ops.push(SubfileIo {
|
||||
subfile_index: subfile_idx,
|
||||
local_offset: local_off,
|
||||
length: bytes_in_stripe,
|
||||
});
|
||||
|
||||
offset += bytes_in_stripe;
|
||||
remaining -= bytes_in_stripe;
|
||||
}
|
||||
|
||||
ops
|
||||
}
|
||||
}
|
||||
|
||||
/// A single I/O operation targeting a specific sub-file.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SubfileIo {
|
||||
/// Index of the sub-file (0-based).
|
||||
pub subfile_index: u32,
|
||||
/// Offset within the sub-file.
|
||||
pub local_offset: u64,
|
||||
/// Number of bytes for this operation.
|
||||
pub length: u64,
|
||||
}
|
||||
|
||||
impl fmt::Display for SubfileIo {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"SubfileIo(file={}, offset={}, len={})",
|
||||
self.subfile_index, self.local_offset, self.length
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Manager for sub-file I/O operations.
|
||||
///
|
||||
/// Coordinates reads and writes across multiple sub-files.
|
||||
#[derive(Debug)]
|
||||
pub struct SubfileManager {
|
||||
config: SubfileConfig,
|
||||
}
|
||||
|
||||
impl SubfileManager {
|
||||
/// Create a new sub-file manager with the given configuration.
|
||||
pub fn new(config: SubfileConfig) -> Self {
|
||||
Self { config }
|
||||
}
|
||||
|
||||
/// Access the configuration.
|
||||
pub fn config(&self) -> &SubfileConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Create all sub-files on disk (empty files).
|
||||
pub fn create_subfiles(&self) -> io::Result<()> {
|
||||
for path in self.config.all_subfile_paths() {
|
||||
std::fs::File::create(path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove all sub-files from disk.
|
||||
pub fn remove_subfiles(&self) -> io::Result<()> {
|
||||
for path in self.config.all_subfile_paths() {
|
||||
if path.exists() {
|
||||
std::fs::remove_file(path)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if all sub-files exist.
|
||||
pub fn all_subfiles_exist(&self) -> bool {
|
||||
self.config.all_subfile_paths().iter().all(|p| p.exists())
|
||||
}
|
||||
|
||||
/// Write data at a logical offset, distributing across sub-files.
|
||||
pub fn write_at(&self, logical_offset: u64, data: &[u8]) -> io::Result<()> {
|
||||
use std::io::Write;
|
||||
|
||||
let ops = self.config.map_range(logical_offset, data.len() as u64);
|
||||
let mut data_offset = 0usize;
|
||||
|
||||
for op in &ops {
|
||||
let path = self.config.subfile_path(op.subfile_index);
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(path)?;
|
||||
std::io::Seek::seek(&mut file, std::io::SeekFrom::Start(op.local_offset))?;
|
||||
let end = data_offset + op.length as usize;
|
||||
file.write_all(&data[data_offset..end])?;
|
||||
data_offset = end;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read data from a logical offset, gathering from sub-files.
|
||||
pub fn read_at(&self, logical_offset: u64, length: u64) -> io::Result<Vec<u8>> {
|
||||
use std::io::Read;
|
||||
|
||||
let ops = self.config.map_range(logical_offset, length);
|
||||
let mut result = Vec::with_capacity(length as usize);
|
||||
|
||||
for op in &ops {
|
||||
let path = self.config.subfile_path(op.subfile_index);
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
std::io::Seek::seek(&mut file, std::io::SeekFrom::Start(op.local_offset))?;
|
||||
let mut buf = vec![0u8; op.length as usize];
|
||||
file.read_exact(&mut buf)?;
|
||||
result.extend_from_slice(&buf);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn subfile_path_generation() {
|
||||
let config = SubfileConfig::new("/tmp/data.h5", 3);
|
||||
assert_eq!(
|
||||
config.subfile_path(0),
|
||||
PathBuf::from("/tmp/data.h5.subfile.0")
|
||||
);
|
||||
assert_eq!(
|
||||
config.subfile_path(2),
|
||||
PathBuf::from("/tmp/data.h5.subfile.2")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_subfile_paths() {
|
||||
let config = SubfileConfig::new("/tmp/data.h5", 2);
|
||||
let paths = config.all_subfile_paths();
|
||||
assert_eq!(paths.len(), 2);
|
||||
assert_eq!(paths[0], PathBuf::from("/tmp/data.h5.subfile.0"));
|
||||
assert_eq!(paths[1], PathBuf::from("/tmp/data.h5.subfile.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn map_offset_single_stripe() {
|
||||
let config = SubfileConfig::new("/tmp/data.h5", 3).with_stripe_size(1024);
|
||||
|
||||
// Offset 0 -> sub-file 0, local offset 0
|
||||
assert_eq!(config.map_offset(0), (0, 0));
|
||||
|
||||
// Offset 512 -> sub-file 0, local offset 512
|
||||
assert_eq!(config.map_offset(512), (0, 512));
|
||||
|
||||
// Offset 1024 -> sub-file 1, local offset 0
|
||||
assert_eq!(config.map_offset(1024), (1, 0));
|
||||
|
||||
// Offset 2048 -> sub-file 2, local offset 0
|
||||
assert_eq!(config.map_offset(2048), (2, 0));
|
||||
|
||||
// Offset 3072 -> sub-file 0 again (wrap), local offset 1024
|
||||
assert_eq!(config.map_offset(3072), (0, 1024));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn map_range_within_single_stripe() {
|
||||
let config = SubfileConfig::new("/tmp/data.h5", 2).with_stripe_size(1024);
|
||||
|
||||
let ops = config.map_range(0, 512);
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert_eq!(ops[0].subfile_index, 0);
|
||||
assert_eq!(ops[0].local_offset, 0);
|
||||
assert_eq!(ops[0].length, 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn map_range_spanning_stripes() {
|
||||
let config = SubfileConfig::new("/tmp/data.h5", 2).with_stripe_size(1024);
|
||||
|
||||
// Reading 1536 bytes starting at offset 512 should span two stripes
|
||||
let ops = config.map_range(512, 1536);
|
||||
assert_eq!(ops.len(), 2);
|
||||
// First: sub-file 0, offset 512, length 512 (fills rest of stripe 0)
|
||||
assert_eq!(ops[0].subfile_index, 0);
|
||||
assert_eq!(ops[0].local_offset, 512);
|
||||
assert_eq!(ops[0].length, 512);
|
||||
// Second: sub-file 1, offset 0, length 1024 (entire stripe 1)
|
||||
assert_eq!(ops[1].subfile_index, 1);
|
||||
assert_eq!(ops[1].local_offset, 0);
|
||||
assert_eq!(ops[1].length, 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn map_range_empty() {
|
||||
let config = SubfileConfig::new("/tmp/data.h5", 2).with_stripe_size(1024);
|
||||
let ops = config.map_range(0, 0);
|
||||
assert!(ops.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subfile_io_display() {
|
||||
let io = SubfileIo {
|
||||
subfile_index: 1,
|
||||
local_offset: 4096,
|
||||
length: 1024,
|
||||
};
|
||||
let s = io.to_string();
|
||||
assert!(s.contains("file=1"));
|
||||
assert!(s.contains("offset=4096"));
|
||||
assert!(s.contains("len=1024"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_and_read_roundtrip() {
|
||||
let dir = std::env::temp_dir();
|
||||
let base = dir.join("clawhdf5_subfiling_test");
|
||||
|
||||
let config = SubfileConfig::new(&base, 2).with_stripe_size(16);
|
||||
let mgr = SubfileManager::new(config);
|
||||
|
||||
// Create sub-files
|
||||
mgr.create_subfiles().unwrap();
|
||||
assert!(mgr.all_subfiles_exist());
|
||||
|
||||
// Write data that spans multiple sub-files
|
||||
let data: Vec<u8> = (0..64).collect();
|
||||
mgr.write_at(0, &data).unwrap();
|
||||
|
||||
// Read it back
|
||||
let read_back = mgr.read_at(0, 64).unwrap();
|
||||
assert_eq!(read_back, data);
|
||||
|
||||
// Read a partial range
|
||||
let partial = mgr.read_at(8, 16).unwrap();
|
||||
assert_eq!(partial, &data[8..24]);
|
||||
|
||||
// Cleanup
|
||||
mgr.remove_subfiles().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_stripe_size() {
|
||||
let config = SubfileConfig::new("/tmp/data.h5", 4);
|
||||
assert_eq!(config.stripe_size, 32 * 1024 * 1024);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user