quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

clawhdf5-remote, h5rs: URLs' credentials are never shown

Browse Source 
Every RemoteError message and HttpStorage's Debug output held the URL as
given, with any user:password@ and the query string — for a presigned
S3/GCS/Azure URL, its signature or token. An application logging the
error leaked the credential.

- New clawhdf5_remote::redact_url: no userinfo, no fragment, query values
  replaced by REDACTED (plain key names kept).
- HttpStorage formats every message with the redacted URL, and scrubs the
  URL's secret parts from errors of the HTTP client (whose texts can echo
  the URI); Debug shows the redacted URL. storage_for_url's and the object
  store URL errors are redacted too. HttpStorage::url() still returns the
  URL as given, documented as not for logging.
- h5rs prints FILE arguments that are URLs redacted: in errors and in
  dump/stat/check/diff output.
- The test server can force a status and send a wrong Content-Range.

Tests: 404, 403 (at open and on a read), wrong Content-Range (at open and
on a read), no range support, encoded body, ETag change, timeout,
connection closed and bad scheme errors, Display and Debug, contain none
of the secrets; h5rs likewise for every subcommand.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 18:30:02 -05:00
co-authored by Claude Opus 5.5
parent 8df5b209a7
commit c04e34620e
12 changed files with 421 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-remote/src/http.rs
+58 -26
View File
@@ -36,7 +36,7 @@ use std::time::Duration;
use clawhdf5_format::error::FormatError;
use clawhdf5_format::storage::Storage;
use crate::error::RemoteError;
use crate::error::{Redactor, RemoteError};
/// Settings of an [`HttpStorage`].
#[derive(Debug, Clone)]
@@ -107,7 +107,10 @@ enum Validator {
/// documentation](self).
pub struct HttpStorage {
agent: ureq::Agent,
/// The URL as given, credentials and all: only ever sent to the server.
url: String,
/// Shows the URL without its credentials, in every message.
redactor: Redactor,
len: u64,
validator: Validator,
options: HttpOptions,
@@ -122,7 +125,7 @@ pub struct HttpStorage {
impl std::fmt::Debug for HttpStorage {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HttpStorage")
.field("url", &self.url)
.field("url", &self.redactor.shown())
.field("len", &self.len)
.field("validator", &self.validator)
.field("full_download", &self.full.is_some())
@@ -181,15 +184,25 @@ impl HttpStorage {
/// and the bytes that request fetched (the file's start), for a
/// [`BlockCache`](crate::BlockCache) to keep.
pub fn open(url: &str, options: HttpOptions) -> Result<(HttpStorage, Vec<u8>), RemoteError> {
let redactor = Redactor::new(url);
HttpStorage::open_inner(url, options, &redactor).map_err(|e| e.scrubbed(&redactor))
}
fn open_inner(
url: &str,
options: HttpOptions,
redactor: &Redactor,
) -> Result<(HttpStorage, Vec<u8>), RemoteError> {
let shown = redactor.shown();
let lower = url.to_ascii_lowercase();
if lower.starts_with("https://") {
if !cfg!(feature = "https") {
return Err(RemoteError::UnsupportedScheme(format!(
"{url}: https:// needs the `https` feature of clawhdf5-remote"
"{shown}: https:// needs the `https` feature of clawhdf5-remote"
)));
}
} else if !lower.starts_with("http://") {
return Err(RemoteError::UnsupportedScheme(url.to_string()));
return Err(RemoteError::UnsupportedScheme(shown.to_string()));
}
let config = ureq::Agent::config_builder()
.http_status_as_error(false)
@@ -198,6 +211,7 @@ impl HttpStorage {
let mut storage = HttpStorage {
agent: ureq::Agent::new_with_config(config),
url: url.to_string(),
redactor: redactor.clone(),
len: 0,
validator: Validator::None,
options,
@@ -216,14 +230,15 @@ impl HttpStorage {
}
if storage.options.require_validator && storage.validator == Validator::None {
return Err(RemoteError::Usage(format!(
"{url}: the server sends neither a strong ETag nor Last-Modified, so a change \
"{shown}: the server sends neither a strong ETag nor Last-Modified, so a change \
of the file could not be detected (HttpOptions::require_validator)"
)));
}
Ok((storage, bytes))
}
/// The URL.
/// The URL as given — with any credentials it carries, so do not log
/// it; [`redact_url`](crate::redact_url) gives a form that can be.
pub fn url(&self) -> &str {
&self.url
}
@@ -307,18 +322,20 @@ impl HttpStorage {
let got = reader
.take(cap.saturating_add(1))
.read_to_end(&mut buf)
.map_err(|e| RemoteError::Transport(format!("{}: reading the body: {e}", self.url)));
.map_err(|e| {
RemoteError::Transport(format!("{}: reading the body: {e}", self.redactor.shown()))
});
self.bytes.fetch_add(buf.len() as u64, Ordering::Relaxed);
got?;
match want {
Some(n) if buf.len() as u64 != n => Err(RemoteError::BadResponse(format!(
"{}: body of {} bytes, expected {n}",
self.url,
self.redactor.shown(),
buf.len()
))),
None if buf.len() as u64 > limit => Err(RemoteError::Usage(format!(
"{}: the file is larger than HttpOptions::max_full_download ({limit} bytes)",
self.url
self.redactor.shown()
))),
_ => Ok(buf),
}
@@ -332,7 +349,7 @@ impl HttpStorage {
self.requests.fetch_add(1, Ordering::Relaxed);
let resp = self.request(Some((0, n - 1))).call().map_err(transport)?;
let status = resp.status().as_u16();
check_identity(&self.url, &resp)?;
check_identity(self.redactor.shown(), &resp)?;
let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) {
(Some(e), _) if !e.starts_with("W/") => Validator::ETag(e.to_string()),
(_, Some(t)) => Validator::LastModified(t.to_string()),
@@ -341,21 +358,27 @@ impl HttpStorage {
match status {
206 => {
let cr = header(&resp, "content-range").ok_or_else(|| {
RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url))
RemoteError::BadResponse(format!(
"{}: 206 without Content-Range",
self.redactor.shown()
))
})?;
let (a, b, total) = content_range(cr).ok_or_else(|| {
RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url))
RemoteError::BadResponse(format!(
"{}: bad Content-Range {cr:?}",
self.redactor.shown()
))
})?;
let total = total.ok_or_else(|| {
RemoteError::BadResponse(format!(
"{}: the server does not report the file's length (Content-Range {cr:?})",
self.url
self.redactor.shown()
))
})?;
if a != 0 || b >= total || b > n - 1 {
return Err(RemoteError::BadResponse(format!(
"{}: asked for bytes 0-{}, got Content-Range {cr:?}",
self.url,
self.redactor.shown(),
n - 1
)));
}
@@ -367,14 +390,15 @@ impl HttpStorage {
return Err(RemoteError::RangeNotSupported(format!(
"{} answered a range request with the whole file (status 200); set \
HttpOptions::allow_full_download to download it",
self.url
self.redactor.shown()
)));
}
let want = header(&resp, "content-length").and_then(|v| v.trim().parse().ok());
if want.is_some_and(|w: u64| w > self.options.max_full_download) {
return Err(RemoteError::Usage(format!(
"{}: the file is larger than HttpOptions::max_full_download ({} bytes)",
self.url, self.options.max_full_download
self.redactor.shown(),
self.options.max_full_download
)));
}
let bytes = self.body(resp, want, self.options.max_full_download)?;
@@ -382,11 +406,11 @@ impl HttpStorage {
}
416 => Err(RemoteError::Usage(format!(
"{}: status 416 for the first bytes (an empty file?)",
self.url
self.redactor.shown()
))),
code => Err(RemoteError::Status {
code,
what: self.url.clone(),
what: self.redactor.shown().to_string(),
}),
}
}
@@ -398,14 +422,15 @@ impl HttpStorage {
.request(Some((start, end - 1)))
.call()
.map_err(transport)?;
let changed = |why: String| RemoteError::FileChanged(format!("{}: {why}", self.url));
check_identity(&self.url, &resp)?;
let changed =
|why: String| RemoteError::FileChanged(format!("{}: {why}", self.redactor.shown()));
check_identity(self.redactor.shown(), &resp)?;
match resp.status().as_u16() {
206 => {}
200 => {
return Err(RemoteError::RangeNotSupported(format!(
"{} answered a range request with the whole file (status 200)",
self.url
self.redactor.shown()
)));
}
412 => {
@@ -417,7 +442,7 @@ impl HttpStorage {
code => {
return Err(RemoteError::Status {
code,
what: format!("{} bytes {start}-{}", self.url, end - 1),
what: format!("{} bytes {start}-{}", self.redactor.shown(), end - 1),
});
}
}
@@ -439,10 +464,16 @@ impl HttpStorage {
Validator::None => {}
}
let cr = header(&resp, "content-range").ok_or_else(|| {
RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url))
RemoteError::BadResponse(format!(
"{}: 206 without Content-Range",
self.redactor.shown()
))
})?;
let (a, b, total) = content_range(cr).ok_or_else(|| {
RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url))
RemoteError::BadResponse(format!(
"{}: bad Content-Range {cr:?}",
self.redactor.shown()
))
})?;
if let Some(total) = total
&& total != self.len
@@ -452,7 +483,7 @@ impl HttpStorage {
if a != start || b != end - 1 {
return Err(RemoteError::BadResponse(format!(
"{}: asked for bytes {start}-{}, got Content-Range {cr:?}",
self.url,
self.redactor.shown(),
end - 1
)));
}
@@ -461,6 +492,7 @@ impl HttpStorage {
fn fetch(&self, start: u64, end: u64) -> Result<Vec<u8>, RemoteError> {
self.with_retries(|| self.fetch_once(start, end))
.map_err(|e| e.scrubbed(&self.redactor))
}
/// `[offset, offset + len)` clamped to the file, or `None` if empty.
@@ -539,7 +571,7 @@ impl Storage for HttpStorage {
if out.len() != ranges.len() {
return Err(FormatError::Storage(format!(
"{}: a parallel range read failed",
self.url
self.redactor.shown()
)));
}
Ok(out)