This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs-object-store.git
The following commit(s) were added to refs/heads/main by this push:
new e28807d feat: presigned URLs with extra query params and signed
headers (SignedUrlOptions) (#771)
e28807d is described below
commit e28807de7338abebf14ecf2ab959294cb3096d83
Author: Zac Farrell <[email protected]>
AuthorDate: Wed Aug 19 14:17:35 2026 -0700
feat: presigned URLs with extra query params and signed headers
(SignedUrlOptions) (#771)
* feat(aws): presign URLs with extra query params and signed headers
* feat(gcp): presign URLs with extra query params and signed headers
* fix(signer): correct presign query encoding and validate extras
* refactor(signer): adopt SignedUrlOptions and fix canonicalization
* test(aws): verify signed headers, checksum enforcement, query encoding
against S3
* test(aws): cover multipart assembly, tamper rejection, expiry,
conditional create
* fix(docs): correct broken Signer::signed_url_opts intra-doc link
* test(aws): gate presign signature-enforcement tests behind real S3
* test(aws): delete before asserting in signed_url_expires
* test(aws): isolate presign tests in a dedicated bucket
* Mark SignedUrlOptions as non exhaustive
* docs(aws): add end-to-end presigned multipart example
* test(aws): make presign test bucket configurable via env
* test(aws): preflight signing bucket to fail fast on bad creds
* ci(aws): run presign signature-enforcement tests on MinIO
* test(aws): clarify preflight validates write access
* docs: add plain-HTTP MinIO setup for presign tests
* docs: document S3 permissions needed for presign tests
* docs: clarify presign enforcement tests run in CI on MinIO
* docs(aws): move presigned multipart example onto create_multipart
* publically re-export structs used in API
* Avoid panic when trying to sign non-utf8 values
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
.github/workflows/ci.yml | 29 ++
CONTRIBUTING.md | 70 +++++
src/aws/credential.rs | 333 ++++++++++++++++++++--
src/aws/mod.rs | 708 ++++++++++++++++++++++++++++++++++++++++++++++-
src/azure/mod.rs | 7 +
src/gcp/credential.rs | 127 ++++++++-
src/gcp/mod.rs | 39 ++-
src/lib.rs | 21 ++
src/prefix.rs | 12 +
src/signer.rs | 166 ++++++++++-
src/util.rs | 196 +++++++++++++
11 files changed, 1664 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9885130..dadfcf6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -208,12 +208,23 @@ jobs:
aws --endpoint-url=http://localhost:4566 s3 mb
s3://test-bucket-for-spawn
aws --endpoint-url=http://localhost:4566 s3 mb
s3://test-bucket-for-checksum
aws --endpoint-url=http://localhost:4566 s3 mb
s3://test-bucket-for-copy-if-not-exists
+ aws --endpoint-url=http://localhost:4566 s3 mb
s3://test-bucket-for-signing
aws --endpoint-url=http://localhost:4566 s3api create-bucket
--bucket test-object-lock --object-lock-enabled-for-bucket
KMS_KEY=$(aws --endpoint-url=http://localhost:4566 kms create-key
--description "test key")
AWS_SSE_KMS_KEY_ID=$(jq -r .KeyMetadata.KeyId <<< "$KMS_KEY")
echo "AWS_SSE_KMS_KEY_ID=$AWS_SSE_KMS_KEY_ID" >> $GITHUB_ENV
+ # MinIO validates SigV4 (LocalStack does not), so it backs the
presigned-URL
+ # signature-enforcement tests below.
+ - name: Setup MinIO (SigV4-validating S3)
+ run: |
+ MINIO_CONTAINER=$(docker run -d -p 9000:9000 -e
MINIO_ROOT_USER=minio -e MINIO_ROOT_PASSWORD=minio123 minio/minio server /data)
+ echo "MINIO_CONTAINER=$MINIO_CONTAINER" >> $GITHUB_ENV
+ for _ in $(seq 1 30); do curl -fsS
http://localhost:9000/minio/health/ready && break; sleep 1; done
+ AWS_ACCESS_KEY_ID=minio AWS_SECRET_ACCESS_KEY=minio123 \
+ aws --endpoint-url=http://localhost:9000 s3 mb
s3://test-bucket-for-signing
+
- name: Configure Azurite (Azure emulation)
# the magical connection string is from
#
https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
@@ -239,6 +250,20 @@ jobs:
AWS_CONDITIONAL_PUT: etag
AWS_COPY_IF_NOT_EXISTS: multipart
+ # These tests assert the backend REJECTS a tampered signature, an
expired URL, or an
+ # altered signed header, so they need a backend that validates SigV4 and
are gated behind
+ # TEST_S3_SIGNATURE_ENFORCEMENT (skipped against LocalStack elsewhere).
Run them against
+ # MinIO. `env -u` drops the job-wide SSE-KMS settings, whose LocalStack
key MinIO does not
+ # have; the tests only need a plain authenticated round trip.
+ - name: Run presigned-URL signature-enforcement tests (MinIO)
+ run: env -u AWS_SERVER_SIDE_ENCRYPTION -u AWS_SSE_KMS_KEY_ID cargo
test --lib --features=aws aws::tests::signed_url
+ env:
+ TEST_S3_SIGNATURE_ENFORCEMENT: 1
+ AWS_ENDPOINT: http://localhost:9000
+ AWS_ACCESS_KEY_ID: minio
+ AWS_SECRET_ACCESS_KEY: minio123
+ AWS_BUCKET: test-bucket-for-signing
+
# Exercise the `ring` crypto provider at runtime (the other jobs only
# compile it)
#
@@ -258,6 +283,10 @@ jobs:
if: ${{ !cancelled() }}
run: docker logs $LOCALSTACK_CONTAINER
+ - name: MinIO Output
+ if: ${{ !cancelled() }}
+ run: docker logs $MINIO_CONTAINER
+
- name: EC2 Metadata Output
if: ${{ !cancelled() }}
run: docker logs $EC2_METADATA_CONTAINER
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0cc703d..e15db34 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -67,6 +67,7 @@ Or directly with:
```shell
aws s3 mb s3://test-bucket --endpoint-url=http://localhost:4566
aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket-for-spawn
+aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket-for-signing
aws --endpoint-url=http://localhost:4566 dynamodb create-table --table-name
test-table --key-schema AttributeName=path,KeyType=HASH
AttributeName=etag,KeyType=RANGE --attribute-definitions
AttributeName=path,AttributeType=S AttributeName=etag,AttributeType=S
--provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5
```
@@ -148,6 +149,75 @@ export TEST_S3_SSEC_ENCRYPTION=1
cargo test --features aws --package object_store --lib
aws::tests::test_s3_ssec_encryption_with_minio -- --exact --nocapture
```
+#### Presigned URL signature-enforcement tests
+
+A handful of presigned-URL tests assert that the *storage backend* rejects an
invalid request:
+a tampered signature, an expired URL, or a signed header whose value the
client changed. These
+require a backend that actually validates SigV4. LocalStack does not (it
accepts presigned
+requests regardless of signature or expiry), so these tests are gated behind a
separate
+`TEST_S3_SIGNATURE_ENFORCEMENT` variable rather than running in the main
LocalStack integration
+pass. CI runs them against MinIO instead, which does validate SigV4 (the "Run
presigned-URL
+signature-enforcement tests (MinIO)" step in `ci.yml`), so they stay
exercised. The steps below
+reproduce that locally or point at real S3.
+
+These tests use a dedicated `test-bucket-for-signing` bucket so their writes
cannot contaminate the
+shared `test-bucket` whose exact contents `s3_test` asserts. To point at a
bucket in your own
+account instead, set `OBJECT_STORE_SIGNING_BUCKET` (no source edit required).
+
+MinIO is the recommended local backend: unlike LocalStack it validates SigV4
signatures and expiry,
+so the enforcement assertions actually exercise. Plain HTTP is enough (these
tests don't use SSE-C,
+so the self-signed-cert setup from the SSE-C section is not needed, and the
test client would
+reject that cert anyway):
+
+```shell
+docker run -d -p 9000:9000 \
+ -e MINIO_ROOT_USER=minio -e MINIO_ROOT_PASSWORD=minio123 \
+ minio/minio server /data
+
+export AWS_ENDPOINT=http://localhost:9000
+export AWS_ALLOW_HTTP=true
+export AWS_ACCESS_KEY_ID=minio
+export AWS_SECRET_ACCESS_KEY=minio123
+export AWS_REGION=us-east-1
+aws --endpoint-url=http://localhost:9000 s3 mb s3://test-bucket-for-signing
+```
+
+Then run the tests. Running against real S3 also works (unset
`AWS_ENDPOINT`/`AWS_ALLOW_HTTP` and
+use real credentials + the bucket's region); note that a `403 AccessDenied`
there means your IAM
+principal lacks permission on the bucket, not a signing bug (an incorrect
signature returns
+`SignatureDoesNotMatch`).
+
+```shell
+export TEST_INTEGRATION=1
+export TEST_S3_SIGNATURE_ENFORCEMENT=1
+# Optional: point at your own bucket instead of the default
`test-bucket-for-signing`.
+# export OBJECT_STORE_SIGNING_BUCKET=my-bucket
+cargo test --features aws --package object_store --lib aws::tests::signed_url
-- --nocapture
+```
+
+Required S3 permissions (real S3 only). The tests exercise four object-level
actions on the signing
+bucket: `s3:PutObject` (also covers CreateMultipartUpload, UploadPart, and
CompleteMultipartUpload),
+`s3:GetObject`, `s3:DeleteObject`, and `s3:AbortMultipartUpload`. A minimal
policy:
+
+```json
+{
+ "Version": "2012-10-17",
+ "Statement": [{
+ "Effect": "Allow",
+ "Action": [
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:DeleteObject",
+ "s3:AbortMultipartUpload"
+ ],
+ "Resource": "arn:aws:s3:::YOUR_BUCKET/*"
+ }]
+}
+```
+
+A `403 AccessDenied` (as opposed to `SignatureDoesNotMatch`) means the
credentials are missing one
+of these or are blocked by a bucket policy or SCP, not that the signing is
wrong.
+
### Azure
To test the Azure integration
diff --git a/src/aws/credential.rs b/src/aws/credential.rs
index 7e2245e..72017fd 100644
--- a/src/aws/credential.rs
+++ b/src/aws/credential.rs
@@ -23,7 +23,7 @@ use crate::client::{
CryptoProvider, DigestAlgorithm, HttpClient, HttpError, HttpRequest,
TokenProvider,
crypto_provider,
};
-use crate::util::{hex_digest, hex_encode};
+use crate::util::{append_strict_query_pairs, hex_digest, hex_encode};
use crate::{CredentialProvider, Result, RetryConfig};
use async_trait::async_trait;
use bytes::Buf;
@@ -299,12 +299,45 @@ impl<'a> AwsAuthorizer<'a> {
Ok(())
}
- pub(crate) fn sign(&self, method: Method, url: &mut Url, expires_in:
Duration) -> Result<()> {
+ /// Generate a presigned URL, additionally folding `extra_query`
parameters and
+ /// `signed_headers` into the SigV4 signature.
+ ///
+ /// `extra_query` lets callers sign query parameters that the recipient
must send (e.g.
+ /// `partNumber` and `uploadId` for a multipart `UploadPart`, or a
`versionId`).
+ ///
+ /// `signed_headers` lets callers bind specific request headers to the
signature (e.g.
+ /// `x-amz-checksum-sha256`, `content-type`, or SSE headers). For a
presigned URL these
+ /// values are fixed at signing time and the recipient must send exactly
these headers and
+ /// values. The mandatory `host` header is always signed.
+ pub(crate) fn sign_with(
+ &self,
+ method: Method,
+ url: &mut Url,
+ extra_query: &[(String, String)],
+ signed_headers: &HeaderMap,
+ expires_in: Duration,
+ ) -> Result<()> {
let crypto = crypto_provider(self.crypto)?;
let date = self.date.unwrap_or_else(Utc::now);
let scope = self.scope(date);
+ // Append any caller-provided query parameters before signing so they
are folded into
+ // `canonicalize_query` and become part of the signature.
+ append_strict_query_pairs(url, extra_query);
+
+ // The `host` header is always signed; callers may bind additional
headers whose values
+ // are committed to the signature at signing time.
+ let host =
&url[url::Position::BeforeHost..url::Position::AfterPort].to_string();
+ let mut headers = HeaderMap::with_capacity(1 + signed_headers.len());
+ let host_val = HeaderValue::from_str(host).unwrap();
+ headers.insert("host", host_val);
+ for (name, value) in signed_headers {
+ headers.append(name.clone(), value.clone());
+ }
+
+ let (signed_headers, canonical_headers) =
canonicalize_headers(&headers);
+
//
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
url.query_pairs_mut()
.append_pair("X-Amz-Algorithm", ALGORITHM)
@@ -314,7 +347,7 @@ impl<'a> AwsAuthorizer<'a> {
)
.append_pair("X-Amz-Date",
&date.format("%Y%m%dT%H%M%SZ").to_string())
.append_pair("X-Amz-Expires", &expires_in.as_secs().to_string())
- .append_pair("X-Amz-SignedHeaders", "host");
+ .append_pair("X-Amz-SignedHeaders", &signed_headers);
if self.request_payer {
// For signed URLs, include x-amz-request-payer=requester in the
request
@@ -333,13 +366,6 @@ impl<'a> AwsAuthorizer<'a> {
// We don't have a payload; the user is going to send the payload
directly themselves.
let digest = UNSIGNED_PAYLOAD;
- let host =
&url[url::Position::BeforeHost..url::Position::AfterPort].to_string();
- let mut headers = HeaderMap::new();
- let host_val = HeaderValue::from_str(host).unwrap();
- headers.insert("host", host_val);
-
- let (signed_headers, canonical_headers) =
canonicalize_headers(&headers);
-
let string_to_sign = self.string_to_sign(
crypto,
date,
@@ -444,6 +470,10 @@ impl CredentialExt for HttpRequestBuilder {
/// Canonicalizes query parameters into the AWS canonical form
///
+/// Parameters are sorted by encoded name, with ties broken by encoded value,
as required by the
+/// canonical request specification — sorting by the decoded name (or ignoring
the value for
+/// duplicate names) can order parameters differently from how the server
verifies them.
+///
///
<https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html>
fn canonicalize_query(url: &Url) -> String {
use std::fmt::Write;
@@ -452,23 +482,24 @@ fn canonicalize_query(url: &Url) -> String {
Some(q) if !q.is_empty() => q.len(),
_ => return String::new(),
};
- let mut encoded = String::with_capacity(capacity + 1);
- let mut headers = url.query_pairs().collect::<Vec<_>>();
- headers.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
+ let mut params = url
+ .query_pairs()
+ .map(|(k, v)| {
+ (
+ utf8_percent_encode(k.as_ref(),
&STRICT_ENCODE_SET).to_string(),
+ utf8_percent_encode(v.as_ref(),
&STRICT_ENCODE_SET).to_string(),
+ )
+ })
+ .collect::<Vec<_>>();
+ params.sort_unstable();
- let mut first = true;
- for (k, v) in headers {
- if !first {
+ let mut encoded = String::with_capacity(capacity + 1);
+ for (i, (k, v)) in params.iter().enumerate() {
+ if i > 0 {
encoded.push('&');
}
- first = false;
- let _ = write!(
- encoded,
- "{}={}",
- utf8_percent_encode(k.as_ref(), &STRICT_ENCODE_SET),
- utf8_percent_encode(v.as_ref(), &STRICT_ENCODE_SET)
- );
+ let _ = write!(encoded, "{k}={v}");
}
encoded
}
@@ -929,6 +960,7 @@ mod tests {
use http::{Method, Response};
#[cfg(feature = "reqwest")]
use reqwest::Client;
+ use std::collections::HashMap;
use std::env;
// Test generated using
https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
@@ -1142,7 +1174,13 @@ mod tests {
let mut url =
Url::parse("https://examplebucket.s3.amazonaws.com/test.txt";).unwrap();
authorizer
- .sign(Method::GET, &mut url, Duration::from_secs(86400))
+ .sign_with(
+ Method::GET,
+ &mut url,
+ &[],
+ &HeaderMap::new(),
+ Duration::from_secs(86400),
+ )
.unwrap();
assert_eq!(
@@ -1186,7 +1224,13 @@ mod tests {
let mut url =
Url::parse("https://examplebucket.s3.amazonaws.com/test.txt";).unwrap();
authorizer
- .sign(Method::GET, &mut url, Duration::from_secs(86400))
+ .sign_with(
+ Method::GET,
+ &mut url,
+ &[],
+ &HeaderMap::new(),
+ Duration::from_secs(86400),
+ )
.unwrap();
assert_eq!(
@@ -1205,6 +1249,245 @@ mod tests {
);
}
+ #[test]
+ fn signed_url_with_query_value_containing_space() {
+ // A space in a query value must be encoded as `%20` in the URL (not
`+`, as
+ // application/x-www-form-urlencoded would produce) so that the URL
bytes match the
+ // canonical query string that is signed. The expected signature was
computed with an
+ // independent SigV4 implementation.
+ let credential = AwsCredential {
+ key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
+ secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
+ token: None,
+ };
+
+ let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+
+ let authorizer = AwsAuthorizer {
+ date: Some(date),
+ crypto: None,
+ credential: &credential,
+ service: "s3",
+ region: "us-east-1",
+ token_header: None,
+ sign_payload: false,
+ request_payer: false,
+ };
+
+ let mut url =
Url::parse("https://examplebucket.s3.amazonaws.com/test.txt";).unwrap();
+ authorizer
+ .sign_with(
+ Method::PUT,
+ &mut url,
+ &[("prefix".to_string(), "a b".to_string())],
+ &HeaderMap::new(),
+ Duration::from_secs(86400),
+ )
+ .unwrap();
+
+ // The URL carries `%20`, never `+`.
+ assert!(url.query().unwrap().contains("prefix=a%20b"));
+ assert!(!url.query().unwrap().contains('+'));
+
+ let signature = url
+ .query_pairs()
+ .find(|(k, _)| k == "X-Amz-Signature")
+ .unwrap()
+ .1
+ .into_owned();
+ assert_eq!(
+ signature,
+ "e392940dfa4480f872625b35ef1224bfa1ae989c73874df01022e6d1fa4c2f00"
+ );
+ }
+
+ #[test]
+ fn signed_url_with_duplicate_query_keys_sorted_by_value() {
+ // Duplicate query parameter names must be sorted by encoded value in
the canonical
+ // request, regardless of the order they are supplied. The expected
signature was computed
+ // with an independent SigV4 implementation for the sorted form
`partNumber=1&partNumber=2`.
+ let credential = AwsCredential {
+ key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
+ secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
+ token: None,
+ };
+
+ let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+
+ let authorizer = AwsAuthorizer {
+ date: Some(date),
+ crypto: None,
+ credential: &credential,
+ service: "s3",
+ region: "us-east-1",
+ token_header: None,
+ sign_payload: false,
+ request_payer: false,
+ };
+
+ let mut url =
Url::parse("https://examplebucket.s3.amazonaws.com/test.txt";).unwrap();
+ authorizer
+ .sign_with(
+ Method::PUT,
+ &mut url,
+ // Supplied in reverse value order; canonicalization must sort
them.
+ &[
+ ("partNumber".to_string(), "2".to_string()),
+ ("partNumber".to_string(), "1".to_string()),
+ ],
+ &HeaderMap::new(),
+ Duration::from_secs(86400),
+ )
+ .unwrap();
+
+ let signature = url
+ .query_pairs()
+ .find(|(k, _)| k == "X-Amz-Signature")
+ .unwrap()
+ .1
+ .into_owned();
+ assert_eq!(
+ signature,
+ "d3d68662bef0097adce4878c453f22ce9449c7084789b03d9abe9e8c05db38e7"
+ );
+ }
+
+ #[test]
+ fn signed_url_with_query_params() {
+ // Presign a multipart `UploadPart` request, where `partNumber` and
`uploadId` must be
+ // folded into the signature.
+ let credential = AwsCredential {
+ key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
+ secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
+ token: None,
+ };
+
+ let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+
+ let authorizer = AwsAuthorizer {
+ date: Some(date),
+ crypto: None,
+ credential: &credential,
+ service: "s3",
+ region: "us-east-1",
+ token_header: None,
+ sign_payload: false,
+ request_payer: false,
+ };
+
+ let mut url =
Url::parse("https://examplebucket.s3.amazonaws.com/test.txt";).unwrap();
+ authorizer
+ .sign_with(
+ Method::PUT,
+ &mut url,
+ &[
+ ("partNumber".to_string(), "1".to_string()),
+ ("uploadId".to_string(), "abc123".to_string()),
+ ],
+ &HeaderMap::new(),
+ Duration::from_secs(86400),
+ )
+ .unwrap();
+
+ let pairs: HashMap<_, _> = url.query_pairs().into_owned().collect();
+ // Caller-provided params are present and signed (only `host` is a
signed header).
+ assert_eq!(pairs.get("partNumber").map(String::as_str), Some("1"));
+ assert_eq!(pairs.get("uploadId").map(String::as_str), Some("abc123"));
+ assert_eq!(
+ pairs.get("X-Amz-SignedHeaders").map(String::as_str),
+ Some("host")
+ );
+ assert!(pairs.contains_key("X-Amz-Signature"));
+
+ // Full golden URL locks the signature; the query params change it
relative to a plain
+ // presigned PUT, proving they were folded into the canonical request.
+ assert_eq!(
+ url,
+ Url::parse(
+ "https://examplebucket.s3.amazonaws.com/test.txt?\
+ partNumber=1&\
+ uploadId=abc123&\
+ X-Amz-Algorithm=AWS4-HMAC-SHA256&\
+
X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&\
+ X-Amz-Date=20130524T000000Z&\
+ X-Amz-Expires=86400&\
+ X-Amz-SignedHeaders=host&\
+
X-Amz-Signature=4fb562e4da520986ae14e0e0259e2a2355625743452bb3c0ab8a6b50d8f0fcd4"
+ )
+ .unwrap()
+ );
+ }
+
+ #[test]
+ fn signed_url_with_signed_headers() {
+ // Bind a `content-type` header to the signature; the recipient must
send exactly this
+ // header and value.
+ let credential = AwsCredential {
+ key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
+ secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
+ token: None,
+ };
+
+ let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+
+ let authorizer = AwsAuthorizer {
+ date: Some(date),
+ crypto: None,
+ credential: &credential,
+ service: "s3",
+ region: "us-east-1",
+ token_header: None,
+ sign_payload: false,
+ request_payer: false,
+ };
+
+ let mut signed_headers = HeaderMap::new();
+ signed_headers.insert(
+ HeaderName::from_static("content-type"),
+ HeaderValue::from_static("text/plain"),
+ );
+
+ let mut url =
Url::parse("https://examplebucket.s3.amazonaws.com/test.txt";).unwrap();
+ authorizer
+ .sign_with(
+ Method::PUT,
+ &mut url,
+ &[],
+ &signed_headers,
+ Duration::from_secs(86400),
+ )
+ .unwrap();
+
+ let pairs: HashMap<_, _> = url.query_pairs().into_owned().collect();
+ // The extra header is reflected in X-Amz-SignedHeaders, sorted
alongside `host`.
+ assert_eq!(
+ pairs.get("X-Amz-SignedHeaders").map(String::as_str),
+ Some("content-type;host")
+ );
+
+ assert_eq!(
+ url,
+ Url::parse(
+ "https://examplebucket.s3.amazonaws.com/test.txt?\
+ X-Amz-Algorithm=AWS4-HMAC-SHA256&\
+
X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&\
+ X-Amz-Date=20130524T000000Z&\
+ X-Amz-Expires=86400&\
+ X-Amz-SignedHeaders=content-type%3Bhost&\
+
X-Amz-Signature=653d5b202899172830dfe331b817de16c682b8b1e366de1b251ae709a20e20f9"
+ )
+ .unwrap()
+ );
+ }
+
#[cfg(feature = "reqwest")]
#[test]
fn test_sign_port() {
diff --git a/src/aws/mod.rs b/src/aws/mod.rs
index 5d30fbf..dd6c05e 100644
--- a/src/aws/mod.rs
+++ b/src/aws/mod.rs
@@ -44,8 +44,8 @@ use crate::client::CredentialProvider;
use crate::client::get::GetClientExt;
use crate::client::list::{ListClient, ListClientExt};
use crate::multipart::{MultipartStore, PartId};
-use crate::signer::Signer;
-use crate::util::STRICT_ENCODE_SET;
+use crate::signer::{SignedUrlOptions, Signer};
+use crate::util::{STRICT_ENCODE_SET, validate_signed_url_extras};
use crate::{
CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult,
MultipartId, MultipartUpload,
ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions, PutOptions,
PutPayload, PutResult,
@@ -141,6 +141,66 @@ impl Signer for AmazonS3 {
/// # }
/// ```
async fn signed_url(&self, method: Method, path: &Path, expires_in:
Duration) -> Result<Url> {
+ self.signed_url_opts(method, path, expires_in,
&SignedUrlOptions::default())
+ .await
+ }
+
+ /// Create a signed URL, additionally folding the query parameters and
headers in `options`
+ /// into the SigV4 signature.
+ ///
+ /// `extra_query` lets callers sign query parameters that the recipient
must send, such as
+ /// `partNumber` and `uploadId` for a multipart `UploadPart`, or a
`versionId`. `signed_headers`
+ /// binds specific request headers (e.g. `x-amz-checksum-sha256`,
`content-type`, SSE headers)
+ /// to the signature; the recipient must send exactly these headers and
values.
+ ///
+ /// # Example
+ ///
+ /// ```
+ /// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
+ /// # use object_store::{aws::AmazonS3Builder, path::Path,
signer::{Signer, SignedUrlOptions}};
+ /// # use http::Method;
+ /// # use std::time::Duration;
+ /// #
+ /// let s3 = AmazonS3Builder::new()
+ /// .with_region("us-east-1")
+ /// .with_bucket_name("my-bucket")
+ /// .with_access_key_id("my-access-key-id")
+ /// .with_secret_access_key("my-secret-access-key")
+ /// .build()?;
+ ///
+ /// // Presign a multipart UploadPart request.
+ /// let options = SignedUrlOptions::default()
+ /// .with_query([("partNumber", "1"), ("uploadId", "abc123")]);
+ /// let url = s3.signed_url_opts(
+ /// Method::PUT,
+ /// &Path::from("some-folder/some-file.txt"),
+ /// Duration::from_secs(60 * 60),
+ /// &options,
+ /// ).await?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// See [`AmazonS3::create_multipart`] for a complete end-to-end example
that presigns every
+ /// part of a multipart upload so a credential-less client can upload them.
+ ///
+ /// [`AmazonS3::create_multipart`]: crate::aws::AmazonS3::create_multipart
+ async fn signed_url_opts(
+ &self,
+ method: Method,
+ path: &Path,
+ expires_in: Duration,
+ options: &SignedUrlOptions,
+ ) -> Result<Url> {
+ // Validate the caller-provided extras up front, rejecting reserved
query parameters and
+ // headers controlled by the signer.
+ validate_signed_url_extras(
+ STORE,
+ &options.extra_query,
+ &options.signed_headers,
+ "x-amz-",
+ )?;
+
let crypto = self.client.config.crypto()?;
let credential = self.credentials().get_credential().await?;
let authorizer = AwsAuthorizer::new(&credential, "s3",
&self.client.config.region)
@@ -153,7 +213,13 @@ impl Signer for AmazonS3 {
source: format!("Unable to parse url {path_url}: {e}").into(),
})?;
- authorizer.sign(method, &mut url, expires_in)?;
+ authorizer.sign_with(
+ method,
+ &mut url,
+ &options.extra_query,
+ &options.signed_headers,
+ expires_in,
+ )?;
Ok(url)
}
@@ -534,9 +600,66 @@ impl MultipartStore for AmazonS3 {
/// To discard an upload instead of completing it, call
/// [`abort_multipart`] with the same `id`.
///
+ /// # Example: presigned multipart upload
+ ///
+ /// The same upload can be driven by a client that holds no credentials.
The credential holder
+ /// starts the upload and presigns an `UploadPart` URL for each part with
+ /// [`Signer::signed_url_opts`]; the client `PUT`s its bytes to that URL
and returns the `ETag`
+ /// from the response; the credential holder then completes the upload
with those ETags. This
+ /// is how you hand out per-part upload URLs without sharing credentials.
+ ///
+ /// ```no_run
+ /// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
+ /// # use object_store::{aws::AmazonS3Builder, multipart::{MultipartStore,
PartId}, path::Path};
+ /// # use object_store::signer::{Signer, SignedUrlOptions};
+ /// # use http::Method;
+ /// # use std::time::Duration;
+ /// #
+ /// let s3 = AmazonS3Builder::new()
+ /// .with_region("us-east-1")
+ /// .with_bucket_name("my-bucket")
+ /// .with_access_key_id("my-access-key-id")
+ /// .with_secret_access_key("my-secret-access-key")
+ /// .build()?;
+ ///
+ /// let path = Path::from("data/large_file");
+ ///
+ /// // Credential holder: start the upload.
+ /// let id = s3.create_multipart(&path).await?;
+ ///
+ /// // Credential holder: presign an `UploadPart` URL for each part.
`partNumber` and `uploadId`
+ /// // must be signed query parameters, or S3 rejects the request.
+ /// let mut part_urls = Vec::new();
+ /// for part_number in 1..=3 {
+ /// let options = SignedUrlOptions::default().with_query([
+ /// ("partNumber", part_number.to_string()),
+ /// ("uploadId", id.clone()),
+ /// ]);
+ /// let url = s3
+ /// .signed_url_opts(Method::PUT, &path, Duration::from_secs(60 *
60), &options)
+ /// .await?;
+ /// part_urls.push(url);
+ /// }
+ ///
+ /// // Client (no credentials): `PUT` each part's bytes to its URL, read
the `ETag` response
+ /// // header, and return the ETags to the credential holder in part
order, e.g. with `reqwest`:
+ /// //
+ /// // let resp =
reqwest::Client::new().put(url).body(bytes).send().await?;
+ /// // let etag =
resp.headers()[http::header::ETAG].to_str()?.to_string();
+ /// let etags: Vec<String> = // returned by the client
+ /// # vec![];
+ ///
+ /// // Credential holder: complete the upload with the collected ETags.
+ /// let parts = etags.into_iter().map(|content_id| PartId { content_id
}).collect();
+ /// s3.complete_multipart(&path, &id, parts).await?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
/// [`ObjectStoreExt::put_multipart`]: crate::ObjectStoreExt::put_multipart
/// [`complete_multipart`]: MultipartStore::complete_multipart
/// [`abort_multipart`]: MultipartStore::abort_multipart
+ /// [`Signer::signed_url_opts`]: crate::signer::Signer::signed_url_opts
/// [Amazon S3 multipart upload limits]:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
async fn create_multipart(&self, path: &Path) -> Result<MultipartId> {
self.client
@@ -610,9 +733,52 @@ mod tests {
use base64::Engine;
use base64::prelude::BASE64_STANDARD;
use http::HeaderMap;
+ use http::HeaderValue;
const NON_EXISTENT_NAME: &str = "nonexistentname";
+ /// Presigned-URL tests run against a dedicated bucket so their concurrent
writes cannot
+ /// contaminate the shared `test-bucket`, whose exact contents `s3_test`
asserts.
+ ///
+ /// Set `OBJECT_STORE_SIGNING_BUCKET` to run them against a bucket in your
own account
+ /// (e.g. real S3) without editing this source; it defaults to
`test-bucket-for-signing`.
+ const DEFAULT_SIGNING_BUCKET: &str = "test-bucket-for-signing";
+
+ fn signing_bucket() -> String {
+ std::env::var("OBJECT_STORE_SIGNING_BUCKET")
+ .unwrap_or_else(|_| DEFAULT_SIGNING_BUCKET.to_string())
+ }
+
+ /// Builds the signing store and preflights write access with an
authenticated `PutObject`
+ /// probe that fails fast, with an actionable message, if the credentials
cannot write to the
+ /// bucket. Without this a permissions problem surfaces as an opaque `403
AccessDenied` panic
+ /// deep inside whichever presign assertion happens to run first (which is
exactly what made a
+ /// reviewer's real-S3 run hard to diagnose).
+ async fn signing_store() -> AmazonS3 {
+ let store = AmazonS3Builder::from_env()
+ .with_bucket_name(signing_bucket())
+ .build()
+ .unwrap();
+
+ let bucket = signing_bucket();
+ let probe = Path::from(".object_store_presign_preflight");
+ if let Err(source) = store.put(&probe,
PutPayload::from_static(b"ok")).await {
+ panic!(
+ "presign test preflight failed: the configured credentials
cannot write to \
+ bucket `{bucket}`.\n\
+ Verify the bucket exists, is in your configured region, and
that the credentials \
+ (from the AWS_* env vars read by `from_env`) can access it.
The tests exercise \
+ s3:PutObject, s3:GetObject, and s3:DeleteObject; this
preflight checks writes. \
+ Point at a different bucket with
OBJECT_STORE_SIGNING_BUCKET.\n\
+ Underlying error: {source}"
+ );
+ }
+ // Best-effort cleanup; the probe object is harmless if it lingers.
+ let _ = store.delete(&probe).await;
+
+ store
+ }
+
#[tokio::test]
async fn write_multipart_file_with_signature() {
maybe_skip_integration!();
@@ -646,6 +812,542 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn signed_url_multipart_upload() {
+ maybe_skip_integration!();
+
+ // Exercises presigning a multipart `UploadPart` request: the
`partNumber` and `uploadId`
+ // query parameters must be folded into the signature, and the
resulting URL must be
+ // usable by a client that has no access to the credentials.
+ let integration = signing_store().await;
+
+ let path = Path::from("test_signed_multipart_upload.bin");
+ let _ = integration.delete(&path).await;
+
+ let upload_id = integration.create_multipart(&path).await.unwrap();
+
+ let part = vec![42u8; 1024];
+ let options = SignedUrlOptions::default()
+ .with_query([("partNumber", "1"), ("uploadId",
upload_id.as_str())]);
+ let url = integration
+ .signed_url_opts(Method::PUT, &path, Duration::from_secs(300),
&options)
+ .await
+ .unwrap();
+
+ // Upload the part using only the presigned URL, as a credential-less
client would.
+ let resp = reqwest::Client::new()
+ .put(url)
+ .body(part.clone())
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ resp.status().is_success(),
+ "UploadPart via presigned URL failed: {resp:?}"
+ );
+ let etag = resp
+ .headers()
+ .get(http::header::ETAG)
+ .expect("ETag in UploadPart response")
+ .to_str()
+ .unwrap()
+ .to_string();
+
+ integration
+ .complete_multipart(&path, &upload_id, vec![PartId { content_id:
etag }])
+ .await
+ .unwrap();
+
+ let got = integration.get(&path).await.unwrap().bytes().await.unwrap();
+ assert_eq!(got.as_ref(), part.as_slice());
+
+ integration.delete(&path).await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn signed_url_with_signed_checksum_header_is_enforced() {
+ maybe_skip_integration!();
+ maybe_skip_signature_enforcement!();
+
+ // Presign a PUT that binds `x-amz-checksum-sha256` to a fixed value.
This is the
+ // storage-enforced-checksum path: the server must accept a body
matching the signed
+ // checksum and reject one that does not, and the header is part of
the signature so it
+ // cannot be omitted.
+ let integration = signing_store().await;
+
+ let path = Path::from("test_signed_checksum.bin");
+ let _ = integration.delete(&path).await;
+
+ let body = b"hello world".to_vec();
+ // base64(sha256(b"hello world")), computed independently.
+ const CHECKSUM: &str = "uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=";
+
+ let options = SignedUrlOptions::default().with_signed_header(
+ HeaderName::from_static("x-amz-checksum-sha256"),
+ HeaderValue::from_static(CHECKSUM),
+ );
+ let url = integration
+ .signed_url_opts(Method::PUT, &path, Duration::from_secs(300),
&options)
+ .await
+ .unwrap();
+
+ let client = reqwest::Client::new();
+
+ // 1. Matching body + checksum header is accepted.
+ let ok = client
+ .put(url.clone())
+ .header("x-amz-checksum-sha256", CHECKSUM)
+ .body(body.clone())
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ ok.status().is_success(),
+ "matching checksum rejected: {ok:?}"
+ );
+
+ // 2. A body that does not match the signed checksum is rejected
(server enforces it).
+ let bad_body = client
+ .put(url.clone())
+ .header("x-amz-checksum-sha256", CHECKSUM)
+ .body(b"tampered content".to_vec())
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ !bad_body.status().is_success(),
+ "mismatched body was NOT rejected: {bad_body:?}"
+ );
+
+ // 3. Omitting the signed header is rejected (the header is bound to
the signature).
+ let missing_header =
client.put(url).body(body.clone()).send().await.unwrap();
+ assert!(
+ !missing_header.status().is_success(),
+ "missing signed header was NOT rejected: {missing_header:?}"
+ );
+
+ let got = integration.get(&path).await.unwrap().bytes().await.unwrap();
+ assert_eq!(got.as_ref(), body.as_slice());
+
+ integration.delete(&path).await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn signed_url_with_signed_content_type_is_enforced() {
+ maybe_skip_integration!();
+ maybe_skip_signature_enforcement!();
+
+ // Presign a PUT binding a `content-type` (with internal whitespace).
The recipient must
+ // send exactly this value; a different one breaks the signature.
+ let integration = signing_store().await;
+
+ let path = Path::from("test_signed_content_type.bin");
+ let _ = integration.delete(&path).await;
+
+ let body = b"some body".to_vec();
+ const CONTENT_TYPE_VALUE: &str = "text/plain; charset=utf-8";
+
+ let options = SignedUrlOptions::default().with_signed_header(
+ http::header::CONTENT_TYPE,
+ HeaderValue::from_static(CONTENT_TYPE_VALUE),
+ );
+ let url = integration
+ .signed_url_opts(Method::PUT, &path, Duration::from_secs(300),
&options)
+ .await
+ .unwrap();
+
+ let client = reqwest::Client::new();
+
+ // Matching content-type is accepted.
+ let ok = client
+ .put(url.clone())
+ .header(http::header::CONTENT_TYPE, CONTENT_TYPE_VALUE)
+ .body(body.clone())
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ ok.status().is_success(),
+ "matching content-type rejected: {ok:?}"
+ );
+
+ // A different content-type is rejected (the header is bound to the
signature).
+ let wrong = client
+ .put(url)
+ .header(http::header::CONTENT_TYPE, "application/octet-stream")
+ .body(body)
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ !wrong.status().is_success(),
+ "mismatched content-type was NOT rejected: {wrong:?}"
+ );
+
+ integration.delete(&path).await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn signed_url_query_value_with_space() {
+ maybe_skip_integration!();
+
+ // A signed query value containing a space must be `%20`-encoded so
the URL bytes match
+ // the canonical query string; a real server rejects the signature
otherwise. Uses a GET
+ // with a `response-content-disposition` override, whose value
contains spaces.
+ let integration = signing_store().await;
+
+ let path = Path::from("test_signed_query_space.bin");
+ let body = b"contents".to_vec();
+ integration.put(&path, body.clone().into()).await.unwrap();
+
+ let options = SignedUrlOptions::default().with_query([(
+ "response-content-disposition",
+ "attachment; filename=\"a b.txt\"",
+ )]);
+ let url = integration
+ .signed_url_opts(Method::GET, &path, Duration::from_secs(300),
&options)
+ .await
+ .unwrap();
+
+ let resp = reqwest::Client::new().get(url).send().await.unwrap();
+ assert!(
+ resp.status().is_success(),
+ "GET with space-containing signed query rejected: {resp:?}"
+ );
+ // The server honoured the override, echoing it back in the response.
+ let disposition = resp
+ .headers()
+ .get(http::header::CONTENT_DISPOSITION)
+ .and_then(|v| v.to_str().ok())
+ .unwrap_or_default()
+ .to_string();
+ let got = resp.bytes().await.unwrap();
+ assert_eq!(got.as_ref(), body.as_slice());
+ assert!(
+ disposition.contains("a b.txt"),
+ "unexpected content-disposition: {disposition:?}"
+ );
+
+ integration.delete(&path).await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn signed_url_baseline_roundtrip_without_options() {
+ maybe_skip_integration!();
+
+ // Regression: the no-options path (now routed through
`signed_url_opts`) still produces a
+ // working presigned PUT and GET.
+ let integration = signing_store().await;
+ let path = Path::from("test_signed_baseline.bin");
+ let _ = integration.delete(&path).await;
+ let body = b"baseline body".to_vec();
+ let client = reqwest::Client::new();
+
+ let put_url = integration
+ .signed_url(Method::PUT, &path, Duration::from_secs(300))
+ .await
+ .unwrap();
+ let put = client.put(put_url).body(body.clone()).send().await.unwrap();
+ assert!(put.status().is_success(), "baseline PUT failed: {put:?}");
+
+ let get_url = integration
+ .signed_url(Method::GET, &path, Duration::from_secs(300))
+ .await
+ .unwrap();
+ let got = client.get(get_url).send().await.unwrap();
+ assert!(got.status().is_success(), "baseline GET failed: {got:?}");
+ assert_eq!(got.bytes().await.unwrap().as_ref(), body.as_slice());
+
+ integration.delete(&path).await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn signed_url_multipart_multiple_parts_roundtrip() {
+ maybe_skip_integration!();
+
+ // Full multipart round trip with three parts (exercising S3's 5 MiB
minimum-part rule)
+ // and query parameters supplied in non-alphabetical order, proving
the signer sorts the
+ // canonical query string rather than signing in call order.
+ let integration = signing_store().await;
+ let path = Path::from("test_signed_multipart_large.bin");
+ let _ = integration.delete(&path).await;
+
+ const MIB: usize = 1024 * 1024;
+ let parts_data = [vec![1u8; 6 * MIB], vec![2u8; 6 * MIB], vec![3u8;
MIB]];
+
+ let upload_id = integration.create_multipart(&path).await.unwrap();
+ let client = reqwest::Client::new();
+ let mut part_ids = Vec::new();
+
+ for (idx, data) in parts_data.iter().enumerate() {
+ let part_number = (idx + 1).to_string();
+ // `uploadId` is supplied before `partNumber` — i.e. not in
canonical (sorted) order.
+ let options = SignedUrlOptions::default().with_query([
+ ("uploadId", upload_id.as_str()),
+ ("partNumber", part_number.as_str()),
+ ]);
+ let url = integration
+ .signed_url_opts(Method::PUT, &path, Duration::from_secs(600),
&options)
+ .await
+ .unwrap();
+ let resp =
client.put(url).body(data.clone()).send().await.unwrap();
+ assert!(
+ resp.status().is_success(),
+ "UploadPart {part_number} failed: {resp:?}"
+ );
+ let etag = resp
+ .headers()
+ .get(http::header::ETAG)
+ .expect("ETag")
+ .to_str()
+ .unwrap()
+ .to_string();
+ part_ids.push(PartId { content_id: etag });
+ }
+
+ integration
+ .complete_multipart(&path, &upload_id, part_ids)
+ .await
+ .unwrap();
+
+ let got = integration.get(&path).await.unwrap().bytes().await.unwrap();
+ let expected: Vec<u8> = parts_data.concat();
+ assert_eq!(got.len(), expected.len(), "assembled length mismatch");
+ assert_eq!(
+ got.as_ref(),
+ expected.as_slice(),
+ "assembled bytes mismatch"
+ );
+
+ integration.delete(&path).await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn signed_url_tampering_is_rejected() {
+ maybe_skip_integration!();
+ maybe_skip_signature_enforcement!();
+
+ // Proves the signed query parameters and headers are actually bound
to the signature:
+ // mutating them must produce SignatureDoesNotMatch, while an extra
*unsigned* header is
+ // accepted (we don't over-constrain).
+ let integration = signing_store().await;
+ let path = Path::from("test_signed_tamper.bin");
+ let _ = integration.delete(&path).await;
+ let client = reqwest::Client::new();
+
+ // --- query parameter tampering, in a multipart context ---
+ let upload_id = integration.create_multipart(&path).await.unwrap();
+ let options = SignedUrlOptions::default()
+ .with_query([("partNumber", "1"), ("uploadId",
upload_id.as_str())]);
+ let url = integration
+ .signed_url_opts(Method::PUT, &path, Duration::from_secs(300),
&options)
+ .await
+ .unwrap();
+ let url_str = url.as_str();
+
+ // Mutating partNumber breaks the signature.
+ let tampered_part = Url::parse(&url_str.replace("partNumber=1",
"partNumber=2")).unwrap();
+ let resp = client
+ .put(tampered_part)
+ .body(vec![0u8; 16])
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(
+ resp.status().as_u16(),
+ 403,
+ "tampered partNumber was accepted: {resp:?}"
+ );
+
+ // Mutating uploadId breaks the signature.
+ let bogus = format!("{}XXXX", upload_id);
+ let tampered_id = Url::parse(&url_str.replace(upload_id.as_str(),
bogus.as_str())).unwrap();
+ let resp = client
+ .put(tampered_id)
+ .body(vec![0u8; 16])
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(
+ resp.status().as_u16(),
+ 403,
+ "tampered uploadId was accepted: {resp:?}"
+ );
+
+ // An extra *unsigned* header is fine — only signed headers are bound.
This also actually
+ // uploads the part, leaving the multipart upload to be reaped by
teardown.
+ let resp = client
+ .put(url)
+ .header("x-custom-unsigned", "anything")
+ .body(vec![0u8; 16])
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ resp.status().is_success(),
+ "extra unsigned header was rejected: {resp:?}"
+ );
+
+ // --- signed header value tampering ---
+ const CHECKSUM: &str = "uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=";
+ let options = SignedUrlOptions::default().with_signed_header(
+ HeaderName::from_static("x-amz-checksum-sha256"),
+ HeaderValue::from_static(CHECKSUM),
+ );
+ let url = integration
+ .signed_url_opts(Method::PUT, &path, Duration::from_secs(300),
&options)
+ .await
+ .unwrap();
+ // Sending a *different* (well-formed) value than was signed breaks
the signature itself,
+ // i.e. 403 rather than a checksum (400) error.
+ let resp = client
+ .put(url)
+ .header(
+ "x-amz-checksum-sha256",
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ )
+ .body(b"hello world".to_vec())
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(
+ resp.status().as_u16(),
+ 403,
+ "tampered signed header value was accepted: {resp:?}"
+ );
+
+ integration.abort_multipart(&path, &upload_id).await.ok();
+ let _ = integration.delete(&path).await;
+ }
+
+ #[tokio::test]
+ async fn signed_url_special_character_key() {
+ maybe_skip_integration!();
+
+ // Proves path percent-encoding and signed query parameters coexist: a
key containing a
+ // space, `=`, and a non-ASCII character is presigned for a multipart
part and lands at the
+ // correct key. This is the exact path/query conflation class of bug
we fixed.
+ let integration = signing_store().await;
+ let path = Path::from("test signed/a b=c/π.bin");
+ let _ = integration.delete(&path).await;
+
+ let upload_id = integration.create_multipart(&path).await.unwrap();
+ let body = vec![7u8; 2048];
+ let options = SignedUrlOptions::default()
+ .with_query([("partNumber", "1"), ("uploadId",
upload_id.as_str())]);
+ let url = integration
+ .signed_url_opts(Method::PUT, &path, Duration::from_secs(300),
&options)
+ .await
+ .unwrap();
+ let resp = reqwest::Client::new()
+ .put(url)
+ .body(body.clone())
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ resp.status().is_success(),
+ "special-key UploadPart failed: {resp:?}"
+ );
+ let etag = resp
+ .headers()
+ .get(http::header::ETAG)
+ .expect("ETag")
+ .to_str()
+ .unwrap()
+ .to_string();
+ integration
+ .complete_multipart(&path, &upload_id, vec![PartId { content_id:
etag }])
+ .await
+ .unwrap();
+
+ // Read back through the normal (credentialed) path to confirm the
object landed at the
+ // intended key.
+ let got = integration.get(&path).await.unwrap().bytes().await.unwrap();
+ assert_eq!(got.as_ref(), body.as_slice());
+
+ integration.delete(&path).await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn signed_url_expires() {
+ maybe_skip_integration!();
+ maybe_skip_signature_enforcement!();
+
+ // A short-lived signed URL is rejected after it expires, confirming
the TTL is part of the
+ // signed policy.
+ let integration = signing_store().await;
+ let path = Path::from("test_signed_expiry.bin");
+ let _ = integration.delete(&path).await;
+
+ let url = integration
+ .signed_url(Method::PUT, &path, Duration::from_secs(1))
+ .await
+ .unwrap();
+ std::thread::sleep(Duration::from_secs(3));
+ let resp = reqwest::Client::new()
+ .put(url)
+ .body(b"too late".to_vec())
+ .send()
+ .await
+ .unwrap();
+ let status = resp.status();
+ // Clean up before asserting: if the PUT was wrongly accepted, the
object exists and would
+ // otherwise leak past the panic into the shared bucket.
+ let _ = integration.delete(&path).await;
+ assert_eq!(
+ status.as_u16(),
+ 403,
+ "expired signed URL was accepted: {resp:?}"
+ );
+ }
+
+ #[tokio::test]
+ async fn signed_url_conditional_create_blocks_overwrite() {
+ maybe_skip_integration!();
+
+ // A presigned PUT that signs `If-None-Match: *` succeeds when the
object is absent and is
+ // rejected (412) on replay once the object exists — the leaked-URL
replay guard.
+ let integration = signing_store().await;
+ let path = Path::from("test_signed_conditional.bin");
+ let _ = integration.delete(&path).await;
+ let client = reqwest::Client::new();
+
+ let options = SignedUrlOptions::default()
+ .with_signed_header(IF_NONE_MATCH, HeaderValue::from_static("*"));
+ let url = integration
+ .signed_url_opts(Method::PUT, &path, Duration::from_secs(300),
&options)
+ .await
+ .unwrap();
+
+ let first = client
+ .put(url.clone())
+ .header(IF_NONE_MATCH, "*")
+ .body(b"first write".to_vec())
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ first.status().is_success(),
+ "conditional create on absent object failed: {first:?}"
+ );
+
+ let replay = client
+ .put(url)
+ .header(IF_NONE_MATCH, "*")
+ .body(b"overwrite attempt".to_vec())
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(
+ replay.status().as_u16(),
+ 412,
+ "replay overwrite was not blocked: {replay:?}"
+ );
+
+ integration.delete(&path).await.unwrap();
+ }
+
#[tokio::test]
async fn copy_multipart_file_with_signature() {
maybe_skip_integration!();
diff --git a/src/azure/mod.rs b/src/azure/mod.rs
index f149078..91e4a96 100644
--- a/src/azure/mod.rs
+++ b/src/azure/mod.rs
@@ -190,6 +190,13 @@ impl Signer for MicrosoftAzure {
/// Create a URL containing the relevant [Service SAS] query parameters
that authorize a request
/// via `method` to the resource at `path` valid for the duration
specified in `expires_in`.
///
+ /// Unlike S3 and GCS, Azure does not implement
[`Signer::signed_url_opts`]: a SAS signs a
+ /// fixed set of fields rather than folding arbitrary query parameters or
request headers into
+ /// the signature, so that method returns an error if extra query
parameters or headers are
+ /// supplied. This is not needed for presigned block-blob uploads — a SAS
authorizes by
+ /// permission and resource, so a client can append the unsigned
`comp=block` and `blockid`
+ /// operation parameters to the URL returned by [`Signer::signed_url`].
+ ///
/// [Service SAS]:
https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas
///
/// # Example
diff --git a/src/gcp/credential.rs b/src/gcp/credential.rs
index 4a43929..498288c 100644
--- a/src/gcp/credential.rs
+++ b/src/gcp/credential.rs
@@ -23,7 +23,7 @@ use crate::client::{
CryptoProvider, HttpClient, HttpError, Signer, SigningAlgorithm,
TokenProvider,
};
use crate::gcp::{GcpSigningCredentialProvider, STORE};
-use crate::util::{STRICT_ENCODE_SET, hex_digest, hex_encode};
+use crate::util::{STRICT_ENCODE_SET, append_strict_query_pairs, hex_digest,
hex_encode};
use crate::{RetryConfig, StaticCredentialProvider};
use async_trait::async_trait;
use base64::Engine;
@@ -734,11 +734,13 @@ impl TokenProvider for AuthorizedUserCredentials {
}
}
-/// Trim whitespace from header values
+/// Normalize a header value for the canonical request: trim leading/trailing
whitespace and
+/// collapse runs of internal whitespace to a single space, as required by the
canonical request
+/// specification.
+///
+///
<https://cloud.google.com/storage/docs/authentication/canonical-requests#about-headers>
fn trim_header_value(value: &str) -> String {
- let mut ret = value.to_string();
- ret.retain(|c| !c.is_whitespace());
- ret
+ value.split_whitespace().collect::<Vec<_>>().join(" ")
}
/// A Google Cloud Storage Authorizer for generating signed URL using [Google
SigV4]
@@ -759,11 +761,23 @@ impl GCSAuthorizer {
}
}
- pub(crate) async fn sign(
+ /// Generate a signed URL, additionally folding `extra_query` parameters
and `signed_headers`
+ /// into the [GOOG4 signed URLs] signature.
+ ///
+ /// `extra_query` lets callers sign query parameters the recipient must
send (e.g. an upload
+ /// `partNumber`/`uploadId`), and `signed_headers` binds request headers
(e.g. `content-type`)
+ /// to the signature. For a signed URL these values are fixed at signing
time. The mandatory
+ /// `host` header is always signed.
+ ///
+ /// [GOOG4 signed URLs]:
https://cloud.google.com/storage/docs/access-control/signed-urls
+ #[allow(clippy::too_many_arguments)]
+ pub(crate) async fn sign_with(
&self,
crypto: &dyn CryptoProvider,
method: Method,
url: &mut Url,
+ extra_query: &[(String, String)],
+ signed_headers: &HeaderMap,
expires_in: Duration,
client: &GoogleCloudStorageClient,
) -> crate::Result<()> {
@@ -772,8 +786,17 @@ impl GCSAuthorizer {
let scope = self.scope(date);
let credential_with_scope = format!("{email}/{scope}");
- let mut headers = HeaderMap::new();
+ // Append any caller-provided query parameters before signing so they
are folded into
+ // the canonical query string and become part of the signature.
+ append_strict_query_pairs(url, extra_query);
+
+ // The `host` header is always signed; callers may bind additional
headers whose values
+ // are committed to the signature at signing time.
+ let mut headers = HeaderMap::with_capacity(1 + signed_headers.len());
headers.insert("host", DEFAULT_GCS_SIGN_BLOB_HOST.parse().unwrap());
+ for (name, value) in signed_headers {
+ headers.append(name.clone(), value.clone());
+ }
let (_, signed_headers) = Self::canonicalize_headers(&headers);
@@ -829,17 +852,21 @@ impl GCSAuthorizer {
/// Canonicalizes query parameters into the GCP canonical form
/// form like `max-keys=2&prefix=object`
///
+ /// Parameters are sorted by encoded name, with ties broken by encoded
value, as required by
+ /// the canonical request specification — sorting by the decoded name (or
ignoring the value
+ /// for duplicate names) can order parameters differently from how the
server verifies them.
+ ///
///
<https://cloud.google.com/storage/docs/authentication/canonical-requests#about-query-strings>
fn canonicalize_query(url: &Url) -> String {
url.query_pairs()
- .sorted_unstable_by(|a, b| a.0.cmp(&b.0))
.map(|(k, v)| {
- format!(
- "{}={}",
- utf8_percent_encode(k.as_ref(), &STRICT_ENCODE_SET),
- utf8_percent_encode(v.as_ref(), &STRICT_ENCODE_SET)
+ (
+ utf8_percent_encode(k.as_ref(),
&STRICT_ENCODE_SET).to_string(),
+ utf8_percent_encode(v.as_ref(),
&STRICT_ENCODE_SET).to_string(),
)
})
+ .sorted_unstable()
+ .map(|(k, v)| format!("{k}={v}"))
.join("&")
}
@@ -931,6 +958,7 @@ mod tests {
ClientOptions, DigestAlgorithm, DigestContext, HmacContext,
StaticCredentialProvider,
};
use crate::gcp::client::{GoogleCloudStorageClient,
GoogleCloudStorageConfig};
+ use http::{HeaderName, HeaderValue};
const SIGNATURE_BYTES: &[u8] = &[0x00, 0x01, 0x02, 0xab, 0xcd];
@@ -1038,10 +1066,12 @@ mod tests {
GoogleCloudStorageClient::new(config,
HttpClient::new(UnusedHttpService)).unwrap();
let mut url =
Url::parse("https://storage.googleapis.com/bucket/object";).unwrap();
- futures_executor::block_on(authorizer.sign(
+ futures_executor::block_on(authorizer.sign_with(
&FixedCryptoProvider,
Method::GET,
&mut url,
+ &[],
+ &HeaderMap::new(),
Duration::from_secs(60),
&client,
))
@@ -1055,6 +1085,64 @@ mod tests {
assert_eq!(signature, hex_encode(SIGNATURE_BYTES));
}
+ #[test]
+ fn signed_url_with_folds_query_and_headers() {
+ let signing_credential = Arc::new(GcpSigningCredential {
+ email: "[email protected]".into(),
+ private_key: Some(ServiceAccountKey::new(Box::new(FixedSigner))),
+ });
+ let authorizer = GCSAuthorizer::new(Arc::clone(&signing_credential));
+ let config = GoogleCloudStorageConfig {
+ base_url: DEFAULT_GCS_BASE_URL.into(),
+ credentials: Arc::new(StaticCredentialProvider::new(GcpCredential {
+ bearer: "bearer".into(),
+ })),
+ signing_credentials:
Arc::new(StaticCredentialProvider::new(GcpSigningCredential {
+ email: "[email protected]".into(),
+ private_key: None,
+ })),
+ crypto: None,
+ bucket_name: "bucket".into(),
+ retry_config: RetryConfig::default(),
+ client_options: ClientOptions::default(),
+ skip_signature: false,
+ };
+ let client =
+ GoogleCloudStorageClient::new(config,
HttpClient::new(UnusedHttpService)).unwrap();
+ let mut url =
Url::parse("https://storage.googleapis.com/bucket/object";).unwrap();
+
+ let mut signed_headers = HeaderMap::new();
+ signed_headers.insert(
+ HeaderName::from_static("content-type"),
+ HeaderValue::from_static("text/plain"),
+ );
+
+ futures_executor::block_on(authorizer.sign_with(
+ &FixedCryptoProvider,
+ Method::PUT,
+ &mut url,
+ &[
+ ("partNumber".to_string(), "1".to_string()),
+ ("uploadId".to_string(), "abc123".to_string()),
+ ],
+ &signed_headers,
+ Duration::from_secs(60),
+ &client,
+ ))
+ .unwrap();
+
+ let pairs: std::collections::HashMap<_, _> =
url.query_pairs().into_owned().collect();
+ // Caller-provided query parameters are present in the signed URL.
+ assert_eq!(pairs.get("partNumber").map(String::as_str), Some("1"));
+ assert_eq!(pairs.get("uploadId").map(String::as_str), Some("abc123"));
+ // The extra header is reflected in X-Goog-SignedHeaders, sorted
alongside `host`.
+ assert_eq!(
+ pairs.get("X-Goog-SignedHeaders").map(String::as_str),
+ Some("content-type;host")
+ );
+ assert!(pairs.contains_key("X-Goog-Signature"));
+ }
+
#[test]
fn test_canonicalize_headers() {
let mut input_header = HeaderMap::new();
@@ -1085,4 +1173,17 @@ x-goog-meta-reviewer:jane,john"
"max-keys=2&prefix=object".to_string()
);
}
+
+ #[test]
+ fn trim_header_value_collapses_internal_whitespace() {
+ // The canonical request collapses runs of whitespace to a single
space and trims the
+ // ends, rather than stripping all whitespace, so a signed
`content-type` with parameters
+ // matches the value the recipient sends.
+ assert_eq!(trim_header_value(" foo bar "), "foo bar");
+ assert_eq!(
+ trim_header_value("text/plain; charset=utf-8"),
+ "text/plain; charset=utf-8"
+ );
+ assert_eq!(trim_header_value("single"), "single");
+ }
}
diff --git a/src/gcp/mod.rs b/src/gcp/mod.rs
index 90e9129..2dbb0c7 100644
--- a/src/gcp/mod.rs
+++ b/src/gcp/mod.rs
@@ -43,7 +43,8 @@ use std::time::Duration;
use crate::CopyOptions;
use crate::client::{CredentialProvider, crypto_provider};
use crate::gcp::credential::GCSAuthorizer;
-use crate::signer::Signer;
+use crate::signer::{SignedUrlOptions, Signer};
+use crate::util::validate_signed_url_extras;
use crate::{
GetOptions, GetResult, ListResult, MultipartId, MultipartUpload,
ObjectMeta, ObjectStore,
PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
UploadPart, multipart::PartId,
@@ -323,6 +324,23 @@ impl MultipartStore for GoogleCloudStorage {
#[async_trait]
impl Signer for GoogleCloudStorage {
async fn signed_url(&self, method: Method, path: &Path, expires_in:
Duration) -> Result<Url> {
+ self.signed_url_opts(method, path, expires_in,
&SignedUrlOptions::default())
+ .await
+ }
+
+ /// Create a signed URL, additionally folding the query parameters and
headers in `options`
+ /// into the signature.
+ ///
+ /// `extra_query` lets callers sign query parameters the recipient must
send, and
+ /// `signed_headers` binds request headers (e.g. `content-type`) to the
signature; the
+ /// recipient must send exactly these headers and values.
+ async fn signed_url_opts(
+ &self,
+ method: Method,
+ path: &Path,
+ expires_in: Duration,
+ options: &SignedUrlOptions,
+ ) -> Result<Url> {
if expires_in.as_secs() > 604800 {
return Err(crate::Error::Generic {
store: STORE,
@@ -330,6 +348,15 @@ impl Signer for GoogleCloudStorage {
});
}
+ // Validate the caller-provided extras up front, rejecting reserved
query parameters and
+ // headers controlled by the signer.
+ validate_signed_url_extras(
+ STORE,
+ &options.extra_query,
+ &options.signed_headers,
+ "x-goog-",
+ )?;
+
let config = self.client.config();
let path_url = config.path_url(path);
let mut url = Url::parse(&path_url).map_err(|e| crate::Error::Generic {
@@ -342,7 +369,15 @@ impl Signer for GoogleCloudStorage {
let crypto = crypto_provider(self.client.config().crypto.as_deref())?;
authorizer
- .sign(crypto, method, &mut url, expires_in, &self.client)
+ .sign_with(
+ crypto,
+ method,
+ &mut url,
+ &options.extra_query,
+ &options.signed_headers,
+ expires_in,
+ &self.client,
+ )
.await?;
Ok(url)
diff --git a/src/lib.rs b/src/lib.rs
index ebe04d4..7138bdb 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2366,6 +2366,27 @@ mod tests {
}
pub(crate) use maybe_skip_integration;
+ /// Skip a test that asserts the *storage backend* rejects an invalid
presigned request
+ /// (tampered signature, expired URL, unmet signed header) unless
+ /// `TEST_S3_SIGNATURE_ENFORCEMENT` is set.
+ ///
+ /// These tests require a backend that actually validates SigV4 — real S3
or MinIO. The
+ /// LocalStack emulator used by the default integration suite does not
validate presigned
+ /// signatures or expiry and would return success, so they are gated
separately. See
+ /// CONTRIBUTING.md for the MinIO/S3 setup.
+ macro_rules! maybe_skip_signature_enforcement {
+ () => {
+ if std::env::var("TEST_S3_SIGNATURE_ENFORCEMENT").is_err() {
+ eprintln!(
+ "Skipping signature-enforcement test - set
TEST_S3_SIGNATURE_ENFORCEMENT \
+ and point at a backend that validates SigV4 (real S3 or
MinIO)"
+ );
+ return;
+ }
+ };
+ }
+ pub(crate) use maybe_skip_signature_enforcement;
+
/// Test that the returned stream does not borrow the lifetime of Path
fn list_store<'a>(
store: &'a dyn ObjectStore,
diff --git a/src/prefix.rs b/src/prefix.rs
index 1f53237..6c3e80b 100644
--- a/src/prefix.rs
+++ b/src/prefix.rs
@@ -260,6 +260,18 @@ impl<T: Signer> Signer for PrefixStore<T> {
.await
}
+ async fn signed_url_opts(
+ &self,
+ method: http::Method,
+ path: &Path,
+ expires_in: std::time::Duration,
+ options: &crate::signer::SignedUrlOptions,
+ ) -> Result<url::Url> {
+ self.inner
+ .signed_url_opts(method, &self.full_path(path), expires_in,
options)
+ .await
+ }
+
async fn signed_urls(
&self,
method: http::Method,
diff --git a/src/signer.rs b/src/signer.rs
index f7a74b3..a3770a0 100644
--- a/src/signer.rs
+++ b/src/signer.rs
@@ -19,10 +19,75 @@
use crate::{Result, path::Path};
use async_trait::async_trait;
-pub use http::Method;
use std::{fmt, time::Duration};
+
+// publicly re-export types from http/url used in API so downstream consumers
do
+// not have to explicitly add those crates as dependencies
+pub use http::Method;
+pub use http::{HeaderMap, HeaderName, HeaderValue};
pub use url::Url;
+/// Additional parameters to fold into a presigned URL's signature, used with
+/// [`Signer::signed_url_opts`].
+///
+/// All values are fixed at signing time: the recipient of the presigned URL
must send exactly
+/// these query parameters and headers, with these values, for the request to
be accepted.
+///
+/// Construct with [`SignedUrlOptions::default`] (or the builder methods) and
set only the fields
+/// you need, so that future additions remain backwards compatible:
+///
+/// ```
+/// # use object_store::signer::SignedUrlOptions;
+/// # use http::header::{CONTENT_TYPE, HeaderValue};
+/// let options = SignedUrlOptions::default()
+/// .with_query([("partNumber", "1"), ("uploadId", "abc123")])
+/// .with_signed_header(CONTENT_TYPE,
HeaderValue::from_static("text/plain"));
+/// ```
+#[derive(Debug, Clone, Default)]
+#[non_exhaustive]
+pub struct SignedUrlOptions {
+ /// Query parameters to sign and append to the URL.
+ ///
+ /// Used for requests that carry signed query parameters — for example a
multipart
+ /// `UploadPart` (`partNumber`, `uploadId`) or pinning a `versionId`.
+ pub extra_query: Vec<(String, String)>,
+ /// Request headers to bind to the signature.
+ ///
+ /// Used to require the recipient to send specific headers, such as a
checksum
+ /// (`x-amz-checksum-sha256`), `content-type`, or server-side-encryption
headers.
+ pub signed_headers: HeaderMap,
+}
+
+impl SignedUrlOptions {
+ /// Create an empty set of options, equivalent to
[`SignedUrlOptions::default`].
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Append query parameters to sign.
+ pub fn with_query<I, K, V>(mut self, query: I) -> Self
+ where
+ I: IntoIterator<Item = (K, V)>,
+ K: Into<String>,
+ V: Into<String>,
+ {
+ self.extra_query
+ .extend(query.into_iter().map(|(k, v)| (k.into(), v.into())));
+ self
+ }
+
+ /// Bind a request header to the signature.
+ pub fn with_signed_header(mut self, name: HeaderName, value: HeaderValue)
-> Self {
+ self.signed_headers.append(name, value);
+ self
+ }
+
+ /// Returns `true` if no extra query parameters or signed headers have
been set.
+ pub fn is_empty(&self) -> bool {
+ self.extra_query.is_empty() && self.signed_headers.is_empty()
+ }
+}
+
/// Universal API to generate presigned URLs from multiple object store
services.
#[async_trait]
pub trait Signer: Send + Sync + fmt::Debug + 'static {
@@ -32,6 +97,38 @@ pub trait Signer: Send + Sync + fmt::Debug + 'static {
/// access to the object store's credentials, to allow limited access to
the object store.
async fn signed_url(&self, method: Method, path: &Path, expires_in:
Duration) -> Result<Url>;
+ /// Like [`Signer::signed_url`], but additionally folds the query
parameters and headers in
+ /// `options` into the signature. See [`SignedUrlOptions`].
+ ///
+ /// This presigns a *single* request (for example one multipart
`UploadPart`). Orchestrating a
+ /// multipart upload — creating and completing it — is not in scope and is
typically done by
+ /// the credential-holding server via
[`MultipartStore`](crate::multipart::MultipartStore).
+ ///
+ /// The default implementation delegates to [`Signer::signed_url`] when
`options` is empty, and
+ /// otherwise returns [`crate::Error::NotSupported`]: implementations that
do not support
+ /// signing additional query parameters or headers must not silently drop
them, as that would
+ /// produce a URL that does not enforce the requested constraints.
+ ///
+ /// There is intentionally no `signed_urls_opts` batch counterpart: signed
parameters such as
+ /// `partNumber` are per-request, so a batch sharing one set of options
across many paths has
+ /// no clear meaning.
+ async fn signed_url_opts(
+ &self,
+ method: Method,
+ path: &Path,
+ expires_in: Duration,
+ options: &SignedUrlOptions,
+ ) -> Result<Url> {
+ if options.is_empty() {
+ return self.signed_url(method, path, expires_in).await;
+ }
+ Err(crate::Error::NotSupported {
+ source: "this object store does not support signing URLs with
additional \
+ query parameters or headers"
+ .into(),
+ })
+ }
+
/// Generate signed urls for multiple paths.
///
/// See [`Signer::signed_url`] for more details.
@@ -48,3 +145,70 @@ pub trait Signer: Send + Sync + fmt::Debug + 'static {
Ok(urls)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Error;
+ use http::header::CONTENT_TYPE;
+
+ /// A [`Signer`] that only implements the required `signed_url`, relying
on the default
+ /// `signed_url_opts` — mirroring providers, such as Azure, that only
implement `signed_url`.
+ #[derive(Debug)]
+ struct MinimalSigner;
+
+ #[async_trait]
+ impl Signer for MinimalSigner {
+ async fn signed_url(
+ &self,
+ _method: Method,
+ path: &Path,
+ _expires_in: Duration,
+ ) -> Result<Url> {
+ Ok(Url::parse(&format!("https://example.com/{path}";)).unwrap())
+ }
+ }
+
+ #[tokio::test]
+ async fn default_signed_url_opts_delegates_when_empty() {
+ let signer = MinimalSigner;
+ let url = signer
+ .signed_url_opts(
+ Method::GET,
+ &Path::from("file.txt"),
+ Duration::from_secs(60),
+ &SignedUrlOptions::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(url.as_str(), "https://example.com/file.txt";);
+ }
+
+ #[tokio::test]
+ async fn default_signed_url_opts_rejects_extras() {
+ let signer = MinimalSigner;
+
+ let query_err = signer
+ .signed_url_opts(
+ Method::PUT,
+ &Path::from("file.txt"),
+ Duration::from_secs(60),
+ &SignedUrlOptions::default().with_query([("partNumber", "1")]),
+ )
+ .await
+ .unwrap_err();
+ assert!(matches!(query_err, Error::NotSupported { .. }));
+
+ let header_err = signer
+ .signed_url_opts(
+ Method::PUT,
+ &Path::from("file.txt"),
+ Duration::from_secs(60),
+ &SignedUrlOptions::default()
+ .with_signed_header(CONTENT_TYPE,
HeaderValue::from_static("text/plain")),
+ )
+ .await
+ .unwrap_err();
+ assert!(matches!(header_err, Error::NotSupported { .. }));
+ }
+}
diff --git a/src/util.rs b/src/util.rs
index 764ab11..47591c2 100644
--- a/src/util.rs
+++ b/src/util.rs
@@ -301,6 +301,96 @@ pub(crate) const STRICT_ENCODE_SET:
percent_encoding::AsciiSet = percent_encodin
.remove(b'_')
.remove(b'~');
+/// Append `pairs` to the query string of `url`, percent-encoding keys and
values with
+/// [`STRICT_ENCODE_SET`].
+///
+/// [`url::Url::query_pairs_mut`]'s `append_pair` serializes using
+/// `application/x-www-form-urlencoded`, which encodes spaces as `+`. AWS
SigV4 and GCS V4 require
+/// spaces as `%20` and sign the canonicalized query string, so the bytes in
the URL must match what
+/// is signed. Encoding the pairs ourselves keeps the URL and the signature
consistent regardless of
+/// the characters a caller supplies.
+#[cfg(any(feature = "aws-base", feature = "gcp-base"))]
+pub(crate) fn append_strict_query_pairs(url: &mut url::Url, pairs: &[(String,
String)]) {
+ use percent_encoding::utf8_percent_encode;
+ use std::fmt::Write;
+
+ if pairs.is_empty() {
+ return;
+ }
+
+ let mut query = url.query().unwrap_or_default().to_owned();
+ for (key, value) in pairs {
+ if !query.is_empty() {
+ query.push('&');
+ }
+ let _ = write!(
+ query,
+ "{}={}",
+ utf8_percent_encode(key, &STRICT_ENCODE_SET),
+ utf8_percent_encode(value, &STRICT_ENCODE_SET),
+ );
+ }
+ url.set_query(Some(&query));
+}
+
+/// Headers that are controlled by the signer (`host`) or silently dropped
during canonicalization
+/// (`authorization`, `content-length`, `user-agent`). Allowing a caller to
"sign" these would
+/// either corrupt the signature or be a silent no-op, so they are rejected.
+#[cfg(any(feature = "aws-base", feature = "gcp-base"))]
+const RESERVED_SIGNED_HEADERS: [&str; 4] =
+ ["host", "authorization", "content-length", "user-agent"];
+
+/// Validate the `extra_query` and `signed_headers` from a
[`SignedUrlOptions`].
+///
+/// Query parameter names that are reserved for the signing protocol (those
starting with
+/// `reserved_query_prefix`, e.g. `x-amz-`/`x-goog-`) are rejected so a caller
cannot inject a
+/// duplicate `X-Amz-Signature`, `X-Amz-Expires`, etc. Header names in
[`RESERVED_SIGNED_HEADERS`]
+/// are likewise rejected.
+///
+/// Header values must be valid UTF-8. While [`http::HeaderValue`] permits
+/// opaque bytes (0x80–0xFF) that UTF-8 does not, the canonical request is
built
+/// as a string, so such values cannot be signed.
+///
+/// [`SignedUrlOptions`]: crate::signer::SignedUrlOptions
+#[cfg(any(feature = "aws-base", feature = "gcp-base"))]
+pub(crate) fn validate_signed_url_extras(
+ store: &'static str,
+ extra_query: &[(String, String)],
+ signed_headers: &http::HeaderMap,
+ reserved_query_prefix: &str,
+) -> Result<()> {
+ let err = |source: String| crate::Error::Generic {
+ store,
+ source: source.into(),
+ };
+
+ for (name, _) in extra_query {
+ if name.is_empty() {
+ return Err(err("query parameter name must not be
empty".to_string()));
+ }
+ if name.to_ascii_lowercase().starts_with(reserved_query_prefix) {
+ return Err(err(format!(
+ "query parameter {name:?} is reserved for request signing and
cannot be set via SignedUrlOptions"
+ )));
+ }
+ }
+
+ for (name, value) in signed_headers {
+ if RESERVED_SIGNED_HEADERS.contains(&name.as_str()) {
+ return Err(err(format!(
+ "header {name:?} is controlled by the signer and cannot be
signed via SignedUrlOptions"
+ )));
+ }
+ if std::str::from_utf8(value.as_bytes()).is_err() {
+ return Err(err(format!(
+ "value of header {name:?} is not valid UTF-8 and cannot be
signed via SignedUrlOptions"
+ )));
+ }
+ }
+
+ Ok(())
+}
+
/// Computes the SHA256 digest of `body` returned as a hex encoded string
#[cfg(any(feature = "aws-base", feature = "gcp-base"))]
pub(crate) fn hex_digest(
@@ -486,4 +576,110 @@ mod tests {
let range = GetRange::Offset(1);
assert_eq!(range.as_range(2).unwrap(), 1..2);
}
+
+ #[cfg(any(feature = "aws-base", feature = "gcp-base"))]
+ fn owned_pairs(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
+ pairs
+ .iter()
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect()
+ }
+
+ #[cfg(any(feature = "aws-base", feature = "gcp-base"))]
+ #[test]
+ fn append_strict_query_pairs_uses_percent_encoding() {
+ let mut url = url::Url::parse("https://example.com/object";).unwrap();
+ append_strict_query_pairs(
+ &mut url,
+ &owned_pairs(&[("a key", "a value"), ("plus", "a+b/c=d")]),
+ );
+ // Spaces are `%20` (not `+`), and reserved characters are
percent-encoded.
+ assert_eq!(url.query().unwrap(),
"a%20key=a%20value&plus=a%2Bb%2Fc%3Dd");
+ // Appending to an existing query preserves it.
+ append_strict_query_pairs(&mut url, &owned_pairs(&[("x", "y")]));
+ assert!(url.query().unwrap().ends_with("&x=y"));
+ // Empty input is a no-op.
+ let before = url.query().unwrap().to_owned();
+ append_strict_query_pairs(&mut url, &[]);
+ assert_eq!(url.query().unwrap(), before);
+ }
+
+ #[cfg(any(feature = "aws-base", feature = "gcp-base"))]
+ #[test]
+ fn validate_signed_url_extras_accepts_and_rejects() {
+ use http::{HeaderMap, HeaderName, HeaderValue};
+
+ let header = |name: &'static str| {
+ let mut h = HeaderMap::new();
+ h.insert(HeaderName::from_static(name),
HeaderValue::from_static("v"));
+ h
+ };
+
+ // Legitimate extras are accepted.
+ validate_signed_url_extras(
+ "S3",
+ &owned_pairs(&[("partNumber", "1"), ("uploadId", "abc")]),
+ &header("content-type"),
+ "x-amz-",
+ )
+ .unwrap();
+
+ // Reserved query parameters are rejected (case-insensitively).
+ for key in ["X-Amz-Signature", "x-amz-expires",
"X-Amz-Security-Token"] {
+ let err = validate_signed_url_extras(
+ "S3",
+ &owned_pairs(&[(key, "x")]),
+ &HeaderMap::new(),
+ "x-amz-",
+ )
+ .unwrap_err();
+ assert!(
+ matches!(err, Error::Generic { .. }),
+ "{key} should be rejected"
+ );
+ }
+ // GCS uses a different reserved prefix.
+ assert!(
+ validate_signed_url_extras(
+ "GCS",
+ &owned_pairs(&[("X-Goog-Signature", "x")]),
+ &HeaderMap::new(),
+ "x-goog-"
+ )
+ .is_err()
+ );
+
+ // Empty query parameter names are rejected.
+ assert!(
+ validate_signed_url_extras(
+ "S3",
+ &owned_pairs(&[("", "x")]),
+ &HeaderMap::new(),
+ "x-amz-"
+ )
+ .is_err()
+ );
+
+ // Headers controlled by the signer or dropped during canonicalization
are rejected.
+ for name in ["host", "authorization", "content-length", "user-agent"] {
+ let err = validate_signed_url_extras("S3", &[], &header(name),
"x-amz-").unwrap_err();
+ assert!(
+ matches!(err, Error::Generic { .. }),
+ "{name} should be rejected"
+ );
+ }
+
+ // Header values that are not valid UTF-8 (which `HeaderValue`
permits) are rejected
+ // with an error rather than panicking during canonicalization.
+ let mut non_utf8 = HeaderMap::new();
+ non_utf8.insert(
+ HeaderName::from_static("x-custom"),
+ HeaderValue::from_bytes(&[0xFF]).unwrap(),
+ );
+ let err = validate_signed_url_extras("S3", &[], &non_utf8,
"x-amz-").unwrap_err();
+ assert!(
+ matches!(err, Error::Generic { .. }),
+ "non-UTF-8 header value should be rejected"
+ );
+ }
}
