osobhandClaude Opus 5.5
c513f7e6d7
With the `remote` feature (`remote-https` for https://), ls, dump, stat and diff take an http(s):// (or s3://, gs://, az:// with those clawhdf5-remote features) URL wherever they take a file, and read it by range requests through clawhdf5-remote's block cache. check validates every byte, so it downloads a remote file whole and checks it as before. Without the feature a URL is a clean error naming it. The tools read the file through File::storage instead of as_bytes: object headers, shared messages, attributes, v1 and v2 group links, dense storage (fractal heaps and v2 B-trees), path resolution, chunk listings and variable-length values go through the format crate's *_in functions, and the fractal-heap block verifier reads each block through the storage (a read failure of a remote file is reported as a problem, not as "past the end of the file"). A local file's storage is its mapped bytes, so its reads are still slices. stat's file size comes from the opened file, so it is right for a URL. Tests: tests/remote.rs serves fixtures (old and new formats, a paged file, a metadata cache image, a multi-block fractal heap, compounds, v1 groups) with the clawhdf5-remote test server and requires every subcommand's output and exit status for the URL to equal the local file's, and diff of the two to be clean; 404s, non-HDF5 bodies and https without its feature are clean errors. Local output is unchanged: the old and new h5rs print the same for ls -r -v, dump, stat and check --data on the 747 conformance and CVE corpus files (tank, 2026-09-26; the dumps of h5diff_hyper1/2.h5 were too large for the comparison script, their ls, stat and check agree), except cve-2025-2310.h5, whose dump error messages differ between runs of the old binary too (which failing chunk is reported first). ci-test.sh lints h5rs with remote-https, runs the URL tests and checks h5rs with remote for C. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
101 lines
3.3 KiB
Rust
101 lines
3.3 KiB
Rust
//! `h5rs` on URLs (feature `remote`): every subcommand prints for
|
|
//! `http://…/file.h5` what it prints for the local file (the name aside).
|
|
//! The files are served by the range-request test server of
|
|
//! clawhdf5-remote on 127.0.0.1.
|
|
|
|
#![cfg(feature = "remote")]
|
|
|
|
#[path = "../../clawhdf5-remote/tests/common/server.rs"]
|
|
mod server;
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
|
|
fn h5rs(args: &[&str]) -> (String, i32) {
|
|
let out = Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
|
.args(args)
|
|
.output()
|
|
.expect("run h5rs");
|
|
let text = format!(
|
|
"{}{}",
|
|
String::from_utf8_lossy(&out.stdout),
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
(text, out.status.code().unwrap_or(-1))
|
|
}
|
|
|
|
fn fixtures() -> Vec<PathBuf> {
|
|
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
|
[
|
|
"../clawhdf5/tests/fixtures/tall.h5",
|
|
"../clawhdf5/tests/fixtures/written_by_v2_7_0.h5",
|
|
"../clawhdf5/tests/fixtures/written_by_v2_7_0_paged.h5",
|
|
"../clawhdf5/tests/fixtures/h5clear_mdc_image.h5",
|
|
"../clawhdf5-format/tests/fixtures/fractal_heap_multiblock.h5",
|
|
"../clawhdf5-format/tests/fixtures/legacy/tcompound.h5",
|
|
"../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5",
|
|
]
|
|
.iter()
|
|
.map(|p| root.join(p))
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn every_subcommand_reads_a_url_like_the_local_file() {
|
|
let files = fixtures();
|
|
let served: Vec<(String, Vec<u8>)> = files
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, p)| {
|
|
let name = p.file_name().unwrap().to_str().unwrap();
|
|
(format!("/{i}/{name}"), std::fs::read(p).unwrap())
|
|
})
|
|
.collect();
|
|
let server = server::Server::start(served.clone());
|
|
for (p, (url_path, _)) in files.iter().zip(&served) {
|
|
let url = server.url(url_path);
|
|
let local = p.to_str().unwrap();
|
|
for cmd in [
|
|
&["ls", "-r", "-v"][..],
|
|
&["dump"],
|
|
&["dump", "--json"],
|
|
&["stat"],
|
|
&["check", "--data"],
|
|
] {
|
|
fn args<'a>(cmd: &[&'a str], f: &'a str) -> Vec<&'a str> {
|
|
cmd.iter().copied().chain([f]).collect()
|
|
}
|
|
let (want, want_rc) = h5rs(&args(cmd, local));
|
|
let (got, got_rc) = h5rs(&args(cmd, &url));
|
|
assert_eq!(
|
|
got.replace(&url, local),
|
|
want,
|
|
"h5rs {} {url}",
|
|
cmd.join(" ")
|
|
);
|
|
assert_eq!(got_rc, want_rc, "h5rs {} {url}", cmd.join(" "));
|
|
}
|
|
let (a, rc) = h5rs(&["diff", local, &url]);
|
|
assert_eq!(rc, 0, "h5rs diff {local} {url}: {a}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn url_errors_are_clean() {
|
|
let server = server::Server::start(vec![("/x.h5".into(), vec![1u8; 100])]);
|
|
let (out, rc) = h5rs(&["ls", &server.url("/missing.h5")]);
|
|
assert_eq!(rc, 2, "{out}");
|
|
assert!(out.contains("404"), "{out}");
|
|
let (out, rc) = h5rs(&["ls", &server.url("/x.h5")]);
|
|
assert_eq!(rc, 2, "{out}");
|
|
assert!(out.contains("not an HDF5 file"), "{out}");
|
|
let (out, rc) = h5rs(&["check", &server.url("/missing.h5")]);
|
|
assert_eq!(rc, 2, "{out}");
|
|
#[cfg(not(feature = "remote-https"))]
|
|
{
|
|
let (out, rc) = h5rs(&["ls", "https://example.com/a.h5"]);
|
|
assert_eq!(rc, 2, "{out}");
|
|
assert!(out.contains("`https` feature"), "{out}");
|
|
}
|
|
}
|