This is an automated email from the ASF dual-hosted git repository.
bnolsen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new c0da45fcda slice plugin: simplify self healing by switching to ident
header (#12408)
c0da45fcda is described below
commit c0da45fcda4b1b27bad02bdaea28b98e1ec69cc5
Author: Brian Olsen <[email protected]>
AuthorDate: Wed Aug 20 12:54:04 2025 -0600
slice plugin: simplify self healing by switching to ident header (#12408)
* cache_range_requests: remove IMS header support and expand ident header
* slice plugin initial code to remove IMS header and rely on Ident header
to simplify self healing
* get crr autest working again
* crr_ident: add fresh to stale test
* add Stale ident directive in case we're in reference slice refetch mode
and there aren't any ident headers
* update crr documentation
* refactor crr plugin identifier/validator check logic and autest, update
docs
---
.../plugins/cache_range_requests.en.rst | 59 ++--
doc/admin-guide/plugins/slice.en.rst | 54 +--
.../cache_range_requests/cache_range_requests.cc | 365 ++++++++++++---------
plugins/slice/Config.cc | 12 +-
plugins/slice/Config.h | 1 -
plugins/slice/Data.h | 15 +-
plugins/slice/server.cc | 104 ++----
plugins/slice/util.cc | 49 +--
...range_requests_cache_complete_responses.test.py | 2 -
.../cache_range_requests_ident.test.py | 254 +++++++++++---
.../cache_range_requests_ims.test.py | 133 --------
.../pluginTest/slice/gold/slice_crr_ident.gold | 8 +-
.../pluginTest/slice/gold/slice_ident.gold | 8 +-
.../pluginTest/slice/slice_ident.test.py | 6 +-
.../pluginTest/slice/slice_selfhealing.test.py | 9 +-
15 files changed, 536 insertions(+), 543 deletions(-)
diff --git a/doc/admin-guide/plugins/cache_range_requests.en.rst
b/doc/admin-guide/plugins/cache_range_requests.en.rst
index e7fd928437..416c18856d 100644
--- a/doc/admin-guide/plugins/cache_range_requests.en.rst
+++ b/doc/admin-guide/plugins/cache_range_requests.en.rst
@@ -112,37 +112,6 @@ which includes information about the partial content
range. In this mode,
all requests (include partial content) will use consistent hashing method
for parent selection.
-X-Crr-Ims header support
-------------------------
-
-.. option:: --consider-ims
-.. option:: -c
-.. option:: --ims-header=[header name] (default: X-Crr-Ims)
-.. option:: -i
-
-To support slice plugin self healing an option to force revalidation
-after cache lookup complete was added. This option is triggered by a
-special header:
-
-.. code::
-
- X-Crr-Ims: Tue, 19 Nov 2019 13:26:45 GMT
-
-When this header is provided and a `cache hit fresh` is encountered the
-``Date`` header of the object in cache is compared to this header date
-value. If the cache date is *less* than this IMS date then the object
-is marked as STALE and an appropriate If-Modified-Since or If-Match
-request along with this X-Crr-Ims header is passed up to the parent.
-
-In order for this to properly work in a CDN each cache in the
-chain *SHOULD* also contain a remap rule with the
-:program:`cache_range_requests` plugin with this option set.
-
-When used with the :program:`slice` plugin its `--crr-ims-header`
-option must have the same value (or not be defined) in order to work.
-
-Presence of the `--ims-header` automatically sets the `--consider-ims` option.
-
X-Crr-Ident header support
--------------------------
@@ -158,13 +127,29 @@ to this header.
.. code::
- X-Crr-Ident: Etag: "foo"
- X-Crr-Ident: Last-Modified: Tue, 19 Nov 2019 13:26:45 GMT
+ X-Crr-Ident: Etag "foo"
+ X-Crr-Ident: Last-Modified Tue, 19 Nov 2019 13:26:45 GMT
+ X-Crr-Ident: Stale
+
+During the cache lookup hook the identifer is used in the following ways:
+
+If a range request cache lookup is considered STALE the identifier from
+this header will be compared to the stale cache identifier. If the values
+match the response will be changed to FRESH, preventing the transaction
+from contacting a parent.
+
+If a range request cache lookup is considered FRESH the identifier from
+this header will be compared to the stale cache identifier. If the values
+do not match the transaction will be changed to STALE, resulting in an
+IMS check being sent to the parent cache.
+
+A FRESH to STALE cache lookup state may be forced by the "Stale"
+identifier tag. This is used by the slice plugin when an interior
+range request returns a 404 indicating that the asset has been removed
+at the origin.
-During the cache lookup hook if a range request is considered STALE
-the identifier from this header will be compared to the stale cache
-identifier. If the values match the response will be changed to FRESH,
-preventing the transaction from contacting a parent.
+Based on RFC7232 and ATS internals if present the ETag header is evaluated
+first, followed by the Last-Modified header.
When used with the :program:`slice` plugin its `--crr-ident-header`
option must have the same value (or not be defined) in order to work.
diff --git a/doc/admin-guide/plugins/slice.en.rst
b/doc/admin-guide/plugins/slice.en.rst
index d0dce11b20..40d606129c 100644
--- a/doc/admin-guide/plugins/slice.en.rst
+++ b/doc/admin-guide/plugins/slice.en.rst
@@ -132,21 +132,13 @@ The slice plugin supports the following options::
to indicate that the slice plugin should be skipped.
-s for short
- --crr-ims-header=<header name> (default: X-Crr-Ims)
- Header name used by the slice plugin to tell the
- `cache_range_requests` plugin that a request should
- be marked as STALE. Used for self healing.
- This must match the `--ims-header` option used by the
- `cache_range_requests` plugin.
- -i for short
-
--crr-ident-header=<header name> (default: X-Crr-Ident)
Header name used by the slice plugin to tell the
`cache_range_requests` plugin the identifier of the
first/reference slice fetched. First Etag is preferred
followed by Last-Modified. The `cache_range_requests`
- can use this identifier to flip a STALE asset back to
- FRESH in order to limit unnecessary IMS requests.
+ plugin uses this header to flip cache lookup
status
+ to STALE or FRESH depending on the header.
--prefetch-count=<int> (optional)
Default is 0
@@ -309,31 +301,39 @@ Self Healing
------------
The slice plugin uses the very first slice as a reference slice which
-uses content-length and last-modified and/or etags to ensure assembled
+uses content-length and etag or last-modified headers to ensure assembled
blocks come from the same asset. In the case where a slice from a parent
is fetched which indicates that the asset has changed, the slice plugin
will attempt to self heal the asset. The `cache_range_requests` plugin
-must be configured with the `--consider-ims` parameter in order for
+must be configured with the `--consider-ident` parameter in order for
this to work.
Example `remap.config` configuration::
map http://slice/ http://parent/ @plugin=slice.so
@pparam=--remap-host=cache_range_requests
- map http://cache_range_requests/ http://parent/
@plugin=cache_range_requests.so @pparam=--consider-ims
-
-When a request is served, the slice plugin uses reference slice 0 to
-build a response to the client. When subsequent slices are fetched they
-are checked against this reference slice. If a mismatch occurs an IMS
-request for the offending slice is made through the `cache_range_requests`
-plugin using an X-Crr-Ims header. If the refetched slice still mismatches
-then the client connection is aborted a crr IMS request is made for
-the reference slice in an attempt to refetch it.
-
-Optionally (but not recommended) the plugin may be configured to use
-the first slice in the request as the reference slice. This option
-is faster since it does not visit any slices outside those needed to
-fulfill a request. However this may still cause problems if the
-requested range was calculated from a newer version of the asset.
+ map http://cache_range_requests/ http://parent/
@plugin=cache_range_requests.so @pparam=--consider-ident
+
+When a request is served, the slice plugin uses the header from slice 0
+requested range build a response to the client. When subsequent slices
+are requested from the parent the X-Crr-Ident header is populated with
+the reference identifier (etag or last-modified) and the request is made
+through the `cache_range_requests` plugin. The `cache_range_requests`
+plugin will then decide whether to send back the current cached slice
+(if the identifier matches) or attempt to refetch or "heal" that slice
+by marking it STALE.
+
+If the slice returned by the cache_range_requests plugin still
+doesn't match the reference slice then client side of the transaction
+is aborted. The plugin will then attempt to "heal" the reference
+slice. The X-Crr-Ident header is populated with the new identifer
+and the reference slice is re-requested with the intent of having the
+`cache_range_requests` plugin "heal" the reference slice.
+
+The plugin may be configured to use the first slice of the request
+as the reference slice instead of the asset slice 0. This option is
+faster as it does not visit any slices outside those needed to fulfill
+a request. This option may cause serious out of sync issues as range
+requests may end up being served from temporally different assets.
Purge Requests
--------------
diff --git a/plugins/cache_range_requests/cache_range_requests.cc
b/plugins/cache_range_requests/cache_range_requests.cc
index c2c149ee1b..750c735b8e 100644
--- a/plugins/cache_range_requests/cache_range_requests.cc
+++ b/plugins/cache_range_requests/cache_range_requests.cc
@@ -41,6 +41,25 @@
#define DEBUG_LOG(fmt, ...) Dbg(dbg_ctl, fmt, ##__VA_ARGS__)
#define ERROR_LOG(fmt, ...) TSError("[%s:%d] %s(): " fmt, __FILE__, __LINE__,
__func__, ##__VA_ARGS__)
+// The following allows for scope based defer.
+#ifndef defer
+struct defer_dummy {
+};
+template <class F> struct deferrer {
+ F f;
+ ~deferrer() { f(); }
+};
+template <class F>
+deferrer<F>
+operator*(defer_dummy, F f)
+{
+ return {f};
+}
+#define DEFER_(LINE) zz_defer##LINE
+#define DEFER(LINE) DEFER_(LINE)
+#define defer auto DEFER(__LINE__) = defer_dummy{} *[&]()
+#endif // defer
+
namespace
{
DbgCtl dbg_ctl{PLUGIN_NAME};
@@ -50,7 +69,6 @@ enum parent_select_mode_t {
PS_CACHEKEY_URL, // Set parent selection url to cache_key url
};
-constexpr std::string_view DefaultImsHeader = {"X-Crr-Ims"};
constexpr std::string_view DefaultIdentHeader = {"X-Crr-Ident"};
constexpr std::string_view SLICE_CRR_HEADER = {"Slice-Crr-Status"};
constexpr std::string_view SLICE_CRR_VAL = "1";
@@ -58,15 +76,14 @@ constexpr std::string_view SKIP_CRR_HDR_NAME =
{"X-Skip-Crr"};
std::string_view const Etag(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
std::string_view const LastModified(TS_MIME_FIELD_LAST_MODIFIED,
TS_MIME_LEN_LAST_MODIFIED);
+std::string_view const Stale{"Stale"};
struct pluginconfig {
parent_select_mode_t ps_mode{PS_DEFAULT};
- bool consider_ims_header{false};
bool consider_ident_header{false};
bool modify_cache_key{true};
bool verify_cacheability{false};
bool cache_complete_responses{false};
- std::string ims_header;
std::string ident_header;
};
@@ -74,7 +91,6 @@ struct txndata {
pluginconfig const *config{nullptr};
std::string range_value;
TSHttpStatus origin_status{TS_HTTP_STATUS_NONE};
- time_t ims_time{0};
bool ident_check{false};
bool verify_cacheability{false};
bool cache_complete_responses{false};
@@ -118,9 +134,7 @@ create_pluginconfig(int argc, char *const argv[])
}
static const struct option longopts[] = {
- {const_cast<char *>("consider-ims"), no_argument,
nullptr, 'c'},
{const_cast<char *>("consider-ident"), no_argument,
nullptr, 'd'},
- {const_cast<char *>("ims-header"), required_argument,
nullptr, 'i'},
{const_cast<char *>("ident-header"), required_argument,
nullptr, 'j'},
{const_cast<char *>("no-modify-cachekey"), no_argument,
nullptr, 'n'},
{const_cast<char *>("ps-cachekey"), no_argument,
nullptr, 'p'},
@@ -134,25 +148,16 @@ create_pluginconfig(int argc, char *const argv[])
--argv;
for (;;) {
- int const opt = getopt_long(argc, argv, "i:", longopts, nullptr);
+ int const opt = getopt_long(argc, argv, "j:", longopts, nullptr);
if (-1 == opt) {
break;
}
switch (opt) {
- case 'c': {
- DEBUG_LOG("Plugin considers the ims header");
- pc->consider_ims_header = true;
- } break;
case 'd': {
DEBUG_LOG("Plugin considers the ident header");
pc->consider_ident_header = true;
} break;
- case 'i': {
- DEBUG_LOG("Plugin uses custom ims header: %s", optarg);
- pc->ims_header.assign(optarg);
- pc->consider_ims_header = true;
- } break;
case 'j': {
DEBUG_LOG("Plugin uses custom ident header: %s", optarg);
pc->ident_header.assign(optarg);
@@ -185,11 +190,6 @@ create_pluginconfig(int argc, char *const argv[])
pc->ps_mode = PS_CACHEKEY_URL;
}
- if (pc->consider_ims_header && pc->ims_header.empty()) {
- pc->ims_header = DefaultImsHeader;
- DEBUG_LOG("Plugin uses default ims header: %s", pc->ims_header.c_str());
- }
-
if (pc->consider_ident_header && pc->ident_header.empty()) {
pc->ident_header = DefaultIdentHeader;
DEBUG_LOG("Plugin uses default ident header: %s",
pc->ident_header.c_str());
@@ -261,8 +261,16 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
TSMLoc hdr_loc = TS_NULL_MLOC;
if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &hdr_buf, &hdr_loc)) {
+ defer
+ {
+ TSHandleMLocRelease(hdr_buf, TS_NULL_MLOC, hdr_loc);
+ };
TSMLoc const range_loc = TSMimeHdrFieldFind(hdr_buf, hdr_loc,
TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE);
if (TS_NULL_MLOC != range_loc) {
+ defer
+ {
+ TSHandleMLocRelease(hdr_buf, hdr_loc, range_loc);
+ };
int len = 0;
char const *const hdr_value = TSMimeHdrFieldValueStringGet(hdr_buf,
hdr_loc, range_loc, 0, &len);
@@ -308,7 +316,8 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
const char *start = cache_key_url;
const char *end = cache_key_url + cache_key_url_len;
if (TS_SUCCESS == TSUrlCreate(hdr_buf, &ps_loc)) {
- if (TS_PARSE_DONE == TSUrlParse(hdr_buf, ps_loc, &start, end) &&
// This should always succeed.
+ // This should always succeed.
+ if (TS_PARSE_DONE == TSUrlParse(hdr_buf, ps_loc, &start, end) &&
TS_SUCCESS == TSHttpTxnParentSelectionUrlSet(txnp, hdr_buf,
ps_loc)) {
DEBUG_LOG("Setting Parent Selection URL to '%.*s'",
cache_key_url_len, cache_key_url);
}
@@ -316,28 +325,13 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
}
}
- // optionally consider an ims header
- bool ims_active = false;
- if (pc->consider_ims_header) {
- TSMLoc const imsloc = TSMimeHdrFieldFind(hdr_buf, hdr_loc,
pc->ims_header.data(), pc->ims_header.size());
- if (TS_NULL_MLOC != imsloc) {
- time_t const itime = TSMimeHdrFieldValueDateGet(hdr_buf,
hdr_loc, imsloc);
- DEBUG_LOG("Servicing the '%s' header", pc->ims_header.c_str());
- TSHandleMLocRelease(hdr_buf, hdr_loc, imsloc);
- if (0 < itime) {
- txn_state->ims_time = itime;
- ims_active = true;
- }
- }
- }
-
// If not revalidating then consider the identity header
- if (!ims_active && pc->consider_ident_header) {
+ if (pc->consider_ident_header) {
TSMLoc const identloc = TSMimeHdrFieldFind(hdr_buf, hdr_loc,
pc->ident_header.data(), pc->ident_header.size());
if (TS_NULL_MLOC != identloc) {
DEBUG_LOG("Servicing the '%s' header", pc->ident_header.c_str());
- TSHandleMLocRelease(hdr_buf, hdr_loc, identloc);
txn_state->ident_check = true;
+ TSHandleMLocRelease(hdr_buf, hdr_loc, identloc);
};
}
@@ -358,7 +352,7 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, txn_contp);
DEBUG_LOG("Added TS_HTTP_SEND_REQUEST_HDR_HOOK,
TS_HTTP_SEND_RESPONSE_HDR_HOOK, and TS_HTTP_TXN_CLOSE_HOOK");
- if (0 < txn_state->ims_time || txn_state->ident_check) {
+ if (txn_state->ident_check) {
TSHttpTxnHookAdd(txnp, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK,
txn_contp);
DEBUG_LOG("Also Added TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK");
}
@@ -370,11 +364,9 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc)
txn_state->slice_request = true;
}
}
- TSHandleMLocRelease(hdr_buf, hdr_loc, range_loc);
} else {
DEBUG_LOG("No range request header.");
}
- TSHandleMLocRelease(hdr_buf, TS_NULL_MLOC, hdr_loc);
}
}
@@ -394,13 +386,16 @@ handle_send_origin_request(TSCont contp, TSHttpTxn txnp,
txndata *const txn_stat
return;
}
- if (TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &hdr_buf, &hdr_loc) &&
!rv.empty()) {
- if (set_header(hdr_buf, hdr_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE,
rv.data(), rv.size())) {
+ if (TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &hdr_buf, &hdr_loc)) {
+ defer
+ {
+ TSHandleMLocRelease(hdr_buf, TS_NULL_MLOC, hdr_loc);
+ };
+ if (!rv.empty() && set_header(hdr_buf, hdr_loc, TS_MIME_FIELD_RANGE,
TS_MIME_LEN_RANGE, rv.data(), rv.size())) {
DEBUG_LOG("Added range header: %s", rv.c_str());
TSHttpTxnHookAdd(txnp, TS_HTTP_READ_RESPONSE_HDR_HOOK, contp);
}
}
- TSHandleMLocRelease(hdr_buf, TS_NULL_MLOC, hdr_loc);
}
/**
@@ -416,15 +411,21 @@ handle_client_send_response(TSHttpTxn txnp, txndata
*const txn_state)
TSMBuffer resp_buf = nullptr;
TSMLoc resp_loc = TS_NULL_MLOC;
if (TS_SUCCESS == TSHttpTxnClientRespGet(txnp, &resp_buf, &resp_loc)) {
+ defer
+ {
+ TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc);
+ };
+
TSHttpStatus const status = TSHttpHdrStatusGet(resp_buf, resp_loc);
// a cached status will be 200 with expected parent response status of 206
if (TS_HTTP_STATUS_OK == status) {
if (txn_state->origin_status == TS_HTTP_STATUS_NONE ||
txn_state->origin_status == TS_HTTP_STATUS_NOT_MODIFIED) { // cache
hit or revalidation
// status is always TS_HTTP_STATUS_NONE on cache hit; its value is
only set during handle_server_read_response()
- TSMLoc content_range_loc = TSMimeHdrFieldFind(resp_buf, resp_loc,
TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE);
+ TSMLoc const content_range_loc =
+ TSMimeHdrFieldFind(resp_buf, resp_loc, TS_MIME_FIELD_CONTENT_RANGE,
TS_MIME_LEN_CONTENT_RANGE);
- if (content_range_loc) {
+ if (TS_NULL_MLOC != content_range_loc) {
DEBUG_LOG("Got TS_HTTP_STATUS_OK on cache hit or revalidation and
Content-Range header present in response");
partial_content_reason = true;
TSHandleMLocRelease(resp_buf, resp_loc, content_range_loc);
@@ -452,7 +453,6 @@ handle_client_send_response(TSHttpTxn txnp, txndata *const
txn_state)
} else {
DEBUG_LOG("Ignoring status code %d; txn_state->origin_status=%d",
status, txn_state->origin_status);
}
- TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc);
}
if (partial_content_reason) {
@@ -485,44 +485,49 @@ handle_server_read_response(TSHttpTxn txnp, txndata
*const txn_state)
TSMLoc resp_loc = TS_NULL_MLOC;
int cache_lookup;
- if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &resp_buf, &resp_loc)) {
- TSHttpStatus const status = TSHttpHdrStatusGet(resp_buf, resp_loc);
- txn_state->origin_status = status;
- if (TS_HTTP_STATUS_PARTIAL_CONTENT == status) {
- DEBUG_LOG("Got TS_HTTP_STATUS_PARTIAL_CONTENT.");
- // changing the status code from 206 to 200 forces the object into cache
- TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_OK);
- DEBUG_LOG("Set response header to TS_HTTP_STATUS_OK.");
-
- if (txn_state->verify_cacheability && !TSHttpTxnIsCacheable(txnp,
nullptr, resp_buf)) {
- DEBUG_LOG("transaction is not cacheable; resetting status code to
206");
- TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_PARTIAL_CONTENT);
- }
- } else if (TS_HTTP_STATUS_OK == status) {
- bool cacheable = txn_state->cache_complete_responses;
+ if (TS_SUCCESS != TSHttpTxnServerRespGet(txnp, &resp_buf, &resp_loc)) {
+ return;
+ }
- if (cacheable && txn_state->verify_cacheability) {
- DEBUG_LOG("Received a cacheable complete response from the origin;
verifying cacheability");
- cacheable = TSHttpTxnIsCacheable(txnp, nullptr, resp_buf);
- }
+ defer
+ {
+ TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc);
+ };
- // 200s are cached by default; only cache if configured to do so
- if (!cacheable && TS_SUCCESS == TSHttpTxnCntlSet(txnp,
TS_HTTP_CNTL_SERVER_NO_STORE, true)) {
- DEBUG_LOG("Cache write has been disabled for this transaction.");
- } else {
- DEBUG_LOG("Allowing object to be cached.");
- }
+ TSHttpStatus const status = TSHttpHdrStatusGet(resp_buf, resp_loc);
+ txn_state->origin_status = status;
+ if (TS_HTTP_STATUS_PARTIAL_CONTENT == status) {
+ DEBUG_LOG("Got TS_HTTP_STATUS_PARTIAL_CONTENT.");
+ // changing the status code from 206 to 200 forces the object into cache
+ TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_OK);
+ DEBUG_LOG("Set response header to TS_HTTP_STATUS_OK.");
+
+ if (txn_state->verify_cacheability && !TSHttpTxnIsCacheable(txnp, nullptr,
resp_buf)) {
+ DEBUG_LOG("transaction is not cacheable; resetting status code to 206");
+ TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_PARTIAL_CONTENT);
}
+ } else if (TS_HTTP_STATUS_OK == status) {
+ bool cacheable = txn_state->cache_complete_responses;
- // slice requesting cache lookup status and cacheability (only on miss or
validation)
- if ((txn_state->origin_status == TS_HTTP_STATUS_PARTIAL_CONTENT ||
txn_state->origin_status == TS_HTTP_STATUS_NOT_MODIFIED) &&
- txn_state->slice_request && TSHttpTxnIsCacheable(txnp, nullptr,
resp_buf) &&
- TSHttpTxnCacheLookupStatusGet(txnp, &cache_lookup) == TS_SUCCESS &&
- (cache_lookup == TS_CACHE_LOOKUP_MISS || cache_lookup ==
TS_CACHE_LOOKUP_HIT_STALE)) {
- txn_state->slice_response = true;
+ if (cacheable && txn_state->verify_cacheability) {
+ DEBUG_LOG("Received a cacheable complete response from the origin;
verifying cacheability");
+ cacheable = TSHttpTxnIsCacheable(txnp, nullptr, resp_buf);
}
- TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc);
+ // 200s are cached by default; only cache if configured to do so
+ if (!cacheable && TS_SUCCESS == TSHttpTxnCntlSet(txnp,
TS_HTTP_CNTL_SERVER_NO_STORE, true)) {
+ DEBUG_LOG("Cache write has been disabled for this transaction.");
+ } else {
+ DEBUG_LOG("Allowing object to be cached.");
+ }
+ }
+
+ // slice requesting cache lookup status and cacheability (only on miss or
validation)
+ if ((txn_state->origin_status == TS_HTTP_STATUS_PARTIAL_CONTENT ||
txn_state->origin_status == TS_HTTP_STATUS_NOT_MODIFIED) &&
+ txn_state->slice_request && TSHttpTxnIsCacheable(txnp, nullptr,
resp_buf) &&
+ TSHttpTxnCacheLookupStatusGet(txnp, &cache_lookup) == TS_SUCCESS &&
+ (cache_lookup == TS_CACHE_LOOKUP_MISS || cache_lookup ==
TS_CACHE_LOOKUP_HIT_STALE)) {
+ txn_state->slice_response = true;
}
}
@@ -540,7 +545,6 @@ remove_header(TSMBuffer buf, TSMLoc hdr_loc, const char
*header, int len)
while (TS_NULL_MLOC != field) {
TSMLoc const tmp = TSMimeHdrFieldNextDup(buf, hdr_loc, field);
-
++cnt;
TSMimeHdrFieldDestroy(buf, hdr_loc, field);
TSHandleMLocRelease(buf, hdr_loc, field);
@@ -598,104 +602,143 @@ set_header(TSMBuffer buf, TSMLoc hdr_loc, const char
*header, int len, const cha
return ret;
}
-time_t
-get_date_from_cached_hdr(TSHttpTxn txn)
+void
+debugTxnUrl(TSHttpTxn const txnp, std::string_view const desc, std::string
const &rangeval)
{
- TSMBuffer buf = nullptr;
- TSMLoc hdr_loc = TS_NULL_MLOC;
- time_t date = 0;
-
- if (TSHttpTxnCachedRespGet(txn, &buf, &hdr_loc) == TS_SUCCESS) {
- TSMLoc const date_loc = TSMimeHdrFieldFind(buf, hdr_loc,
TS_MIME_FIELD_DATE, TS_MIME_LEN_DATE);
- if (TS_NULL_MLOC != date_loc) {
- date = TSMimeHdrFieldValueDateGet(buf, hdr_loc, date_loc);
- TSHandleMLocRelease(buf, hdr_loc, date_loc);
- }
- TSHandleMLocRelease(buf, TS_NULL_MLOC, hdr_loc);
+ int url_len = 0;
+ char *const req_url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
+ if (nullptr != req_url) {
+ DEBUG_LOG("%.*s: %.*s-%s", (int)desc.length(), desc.data(), url_len,
req_url, rangeval.c_str());
+ TSfree(req_url);
}
-
- return date;
}
/**
- * Handle a special IMS request or identity check on stale asset
+ * Handle the identity/validator check.
+ * Based on RFC7232 and
HttpTransactCache::match_response_to_request_conditionals
+ * Compare ETAG if provided.
+ * else Compare Last-Modified if provided.
*/
void
handle_cache_lookup_complete(TSHttpTxn txnp, txndata *const txn_state)
{
+ TSAssert(txn_state->ident_check);
int cachestat;
if (TS_SUCCESS == TSHttpTxnCacheLookupStatusGet(txnp, &cachestat)) {
- if (TS_CACHE_LOOKUP_HIT_FRESH == cachestat) {
- time_t const ch_time = get_date_from_cached_hdr(txnp);
- DEBUG_LOG("IMS Cached header time %jd vs IMS %jd",
static_cast<intmax_t>(ch_time),
- static_cast<intmax_t>(txn_state->ims_time));
- if (ch_time < txn_state->ims_time) {
+ if (TS_CACHE_LOOKUP_HIT_FRESH != cachestat && TS_CACHE_LOOKUP_HIT_STALE !=
cachestat) {
+ return;
+ }
+
+ // ensure the cached response is a 200
+ TSMBuffer cbuf = nullptr;
+ TSMLoc chloc = TS_NULL_MLOC;
+ if (TSHttpTxnCachedRespGet(txnp, &cbuf, &chloc) != TS_SUCCESS) {
+ return;
+ }
+
+ defer
+ {
+ TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chloc);
+ };
+
+ if (TS_HTTP_STATUS_OK != TSHttpHdrStatusGet(cbuf, chloc)) {
+ return;
+ }
+
+ pluginconfig const *const pc = txn_state->config;
+
+ // request identifier/validator
+ TSMBuffer rbuf = nullptr;
+ TSMLoc rhloc = TS_NULL_MLOC;
+ if (TS_SUCCESS != TSHttpTxnClientReqGet(txnp, &rbuf, &rhloc)) {
+ return;
+ }
+
+ defer
+ {
+ TSHandleMLocRelease(rbuf, TS_NULL_MLOC, rhloc);
+ };
+
+ // No directive found (should not happen)
+ TSMLoc const riloc = TSMimeHdrFieldFind(rbuf, rhloc,
pc->ident_header.data(), pc->ident_header.size());
+ if (TS_NULL_MLOC == riloc) {
+ return;
+ }
+ defer
+ {
+ TSHandleMLocRelease(rbuf, rhloc, riloc);
+ };
+
+ int rilen = 0;
+ char const *const ristr = TSMimeHdrFieldValueStringGet(rbuf, rhloc, riloc,
-1, &rilen);
+ if (0 == rilen) {
+ return;
+ }
+
+ // pull the identifier from the request header
+ std::string_view rident(ristr, rilen);
+ std::string_view rtag;
+ if (rident.substr(0, Etag.length()) == Etag) {
+ DEBUG_LOG("Etag identifier provided in '%.*s'", rilen, ristr);
+ rtag = Etag;
+ rident = rident.substr(Etag.length() + 1);
+ } else if (rident.substr(0, LastModified.length()) == LastModified) {
+ DEBUG_LOG("Last-Modified indentifier provided in '%.*s'", rilen, ristr);
+ rtag = LastModified;
+ rident = rident.substr(LastModified.length() + 1);
+ } else if (rident.substr(0, Stale.length()) == Stale) {
+ // note tag isn't set here
+ if (TS_CACHE_LOOKUP_HIT_FRESH == cachestat) {
TSHttpTxnCacheLookupStatusSet(txnp, TS_CACHE_LOOKUP_HIT_STALE);
if (dbg_ctl.on()) {
- int url_len = 0;
- char *const req_url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
- if (nullptr != req_url) {
- std::string const &rv = txn_state->range_value;
- DEBUG_LOG("Forced revalidate %.*s-%s", url_len, req_url,
rv.c_str());
-
- TSfree(req_url);
- }
+ debugTxnUrl(txnp, "force to stale", txn_state->range_value);
}
}
- } else if (TS_CACHE_LOOKUP_HIT_STALE == cachestat &&
txn_state->ident_check) {
- pluginconfig const *const pc = txn_state->config;
- DEBUG_LOG("Stale asset ident check");
-
- TSMBuffer resp_buf = nullptr;
- TSMLoc resp_loc = TS_NULL_MLOC;
-
- if (TS_SUCCESS == TSHttpTxnCachedRespGet(txnp, &resp_buf, &resp_loc)) {
- if (TS_HTTP_STATUS_OK == TSHttpHdrStatusGet(resp_buf, resp_loc)) {
- // get the request identifier
- TSMBuffer req_buf = nullptr;
- TSMLoc req_loc = TS_NULL_MLOC;
- if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc)) {
- TSMLoc const ident_loc = TSMimeHdrFieldFind(req_buf, req_loc,
pc->ident_header.data(), pc->ident_header.size());
- if (TS_NULL_MLOC != ident_loc) {
- DEBUG_LOG("Checking identifier against the '%s' header",
pc->ident_header.c_str());
-
- int len = 0;
- char const *const str = TSMimeHdrFieldValueStringGet(req_buf,
req_loc, ident_loc, -1, &len);
-
- // determine which identifier has been provided
- std::string_view const svreq(str, len);
- std::string_view tag;
- if (svreq.substr(0, Etag.length()) == Etag) {
- DEBUG_LOG("Etag identifier provided in '%.*s'", len, str);
- tag = Etag;
- } else if (svreq.substr(0, LastModified.length()) ==
LastModified) {
- DEBUG_LOG("Last-Modified indentifier provided in '%.*s'", len,
str);
- tag = LastModified;
- }
+ return;
+ } else {
+ DEBUG_LOG("Unknown indentifier provided in '%.*s'", rilen, ristr);
+ return;
+ }
- if (!tag.empty()) {
- TSMLoc const id_loc = TSMimeHdrFieldFind(resp_buf, resp_loc,
tag.data(), tag.size());
- if (TS_NULL_MLOC != id_loc) {
- int len = 0;
- char const *const str =
TSMimeHdrFieldValueStringGet(resp_buf, resp_loc, id_loc, 0, &len);
- std::string_view const sv(str, len);
-
- DEBUG_LOG("Checking cached '%.*s' against request '%.*s'",
len, str, (int)svreq.size(), svreq.data());
-
- if (std::string_view::npos != svreq.rfind(sv)) {
- DEBUG_LOG("Flipping cache lookup status from STALE to
FRESH");
- TSHttpTxnCacheLookupStatusSet(txnp,
TS_CACHE_LOOKUP_HIT_FRESH);
- }
- TSHandleMLocRelease(resp_buf, resp_loc, id_loc);
- }
- }
+ // pull the identifier from the cached response
+ // first priority is etag
+ std::string_view ctag = Etag;
+ TSMLoc ciloc = TSMimeHdrFieldFind(cbuf, chloc, ctag.data(),
ctag.length());
+ if (TS_NULL_MLOC == ciloc) {
+ ctag = LastModified;
+ ciloc = TSMimeHdrFieldFind(cbuf, chloc, ctag.data(), ctag.length());
+ if (TS_NULL_MLOC == ciloc) {
+ return;
+ }
+ }
- TSHandleMLocRelease(req_buf, req_loc, ident_loc);
- }
- TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
- }
+ defer
+ {
+ TSHandleMLocRelease(cbuf, chloc, ciloc);
+ };
+
+ int cilen = 0;
+ char const *const cistr = TSMimeHdrFieldValueStringGet(cbuf, chloc, ciloc,
-1, &cilen);
+ if (0 == cilen) {
+ return;
+ }
+ std::string_view cident(cistr, cilen);
+
+ // fresh to stale with ident mismatch
+ if (TS_CACHE_LOOKUP_HIT_FRESH == cachestat) {
+ if (rtag != ctag || rident != cident) {
+ TSHttpTxnCacheLookupStatusSet(txnp, TS_CACHE_LOOKUP_HIT_STALE);
+ if (dbg_ctl.on()) {
+ debugTxnUrl(txnp, "flip to stale", txn_state->range_value);
+ }
+ }
+ // stale to fresh with ident match
+ } else if (TS_CACHE_LOOKUP_HIT_STALE == cachestat) {
+ if (rtag == ctag && rident == cident) {
+ TSHttpTxnCacheLookupStatusSet(txnp, TS_CACHE_LOOKUP_HIT_FRESH);
+ if (dbg_ctl.on()) {
+ debugTxnUrl(txnp, "flip to fresh", txn_state->range_value);
}
- TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc);
}
}
}
diff --git a/plugins/slice/Config.cc b/plugins/slice/Config.cc
index 98da5bd410..aa007cccb8 100644
--- a/plugins/slice/Config.cc
+++ b/plugins/slice/Config.cc
@@ -28,7 +28,6 @@
namespace
{
constexpr std::string_view DefaultSliceSkipHeader = {"X-Slicer-Info"};
-constexpr std::string_view DefaultCrrImsHeader = {"X-Crr-Ims"};
constexpr std::string_view DefaultCrrIdentHeader = {"X-Crr-Ident"};
} // namespace
@@ -114,7 +113,6 @@ Config::fromArgs(int const argc, char const *const argv[])
// standard parsing
constexpr struct option longopts[] = {
{const_cast<char *>("blockbytes"), required_argument, nullptr,
'b'},
- {const_cast<char *>("crr-ims-header"), required_argument, nullptr,
'c'},
{const_cast<char *>("disable-errorlog"), no_argument, nullptr,
'd'},
{const_cast<char *>("exclude-regex"), required_argument, nullptr,
'e'},
{const_cast<char *>("crr-ident-header"), required_argument, nullptr,
'g'},
@@ -135,7 +133,7 @@ Config::fromArgs(int const argc, char const *const argv[])
// getopt assumes args start at '1' so this hack is needed
char *const *argvp = (const_cast<char *const *>(argv) - 1);
for (;;) {
- int const opt = getopt_long(argc + 1, argvp, "b:dc:e:g:i:lm:p:r:s:t:x:z:",
longopts, nullptr);
+ int const opt = getopt_long(argc + 1, argvp, "b:de:g:i:lm:p:r:s:t:x:z:",
longopts, nullptr);
if (-1 == opt) {
break;
}
@@ -152,10 +150,6 @@ Config::fromArgs(int const argc, char const *const argv[])
ERROR_LOG("Invalid blockbytes: %s", optarg);
}
} break;
- case 'c': {
- m_crr_ims_header.assign(optarg);
- DEBUG_LOG("Using override crr ims header %s", optarg);
- } break;
case 'd': {
m_paceerrsecs = -1;
} break;
@@ -280,10 +274,6 @@ Config::fromArgs(int const argc, char const *const argv[])
} else {
DEBUG_LOG("Block stitching error logs at most every %d sec(s)",
m_paceerrsecs);
}
- if (m_crr_ims_header.empty()) {
- m_crr_ims_header = DefaultCrrImsHeader;
- DEBUG_LOG("Using default crr ims header %s", m_crr_ims_header.c_str());
- }
if (m_crr_ident_header.empty()) {
m_crr_ident_header = DefaultCrrIdentHeader;
DEBUG_LOG("Using default crr ident header %s", m_crr_ident_header.c_str());
diff --git a/plugins/slice/Config.h b/plugins/slice/Config.h
index 824ffddd40..27a99ccb74 100644
--- a/plugins/slice/Config.h
+++ b/plugins/slice/Config.h
@@ -51,7 +51,6 @@ struct Config {
uint64_t m_min_size_to_slice{0}; // Only strip objects larger than this
std::string m_skip_header;
- std::string m_crr_ims_header;
std::string m_crr_ident_header;
// Convert optarg to bytes
diff --git a/plugins/slice/Data.h b/plugins/slice/Data.h
index 754a2c331e..85f335d4c1 100644
--- a/plugins/slice/Data.h
+++ b/plugins/slice/Data.h
@@ -30,12 +30,10 @@
struct Config;
enum BlockState {
- Pending,
- PendingInt, // Pending internal refectch
- PendingRef, // Pending reference refetch
- Active,
- ActiveInt, // Active internal refetch
- ActiveRef, // Active reference refetch
+ Pending, // waiting for response
+ Active, // processing current block
+ PendingRef, // waiting for reference refetch response
+ ActiveRef, // processing current reference refetch
Done,
Passthru, // non 206 response passthru
Fail,
@@ -59,9 +57,7 @@ struct Data {
char m_hostname[8192];
int m_hostlen{0};
- // read from slice block 0
- char m_date[33];
- int m_datelen{0};
+ // best identifier headers, initially from reference slice
char m_etag[8192];
int m_etaglen{0};
char m_lastmodified[33];
@@ -104,7 +100,6 @@ struct Data {
explicit Data(Config *const config) : m_config(config)
{
- m_date[0] = '\0';
m_hostname[0] = '\0';
m_etag[0] = '\0';
m_lastmodified[0] = '\0';
diff --git a/plugins/slice/server.cc b/plugins/slice/server.cc
index bad9947994..f723af38d0 100644
--- a/plugins/slice/server.cc
+++ b/plugins/slice/server.cc
@@ -113,6 +113,8 @@ update_object_size(TSHttpTxn txnp, int64_t size, Config
&config)
}
}
+// This sets up the reference data with the content length and any
+// strong/weak identifiers for comparing against subsequent slices.
HeaderState
handleFirstServerHeader(Data *const data, TSCont const contp)
{
@@ -201,10 +203,6 @@ handleFirstServerHeader(Data *const data, TSCont const
contp)
return HeaderState::Fail;
}
- // save data header string
- data->m_datelen = sizeof(data->m_date);
- header.valueForKey(TS_MIME_FIELD_DATE, TS_MIME_LEN_DATE, data->m_date,
&data->m_datelen);
-
// save weak cache header identifiers (rfc7232 section 2)
data->m_etaglen = sizeof(data->m_etag);
header.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, data->m_etag,
&data->m_etaglen);
@@ -387,6 +385,7 @@ handleNextServerHeader(Data *const data)
DEBUG_LOG("Next Header:\n%s", header.toString().c_str());
}
+ // assume the next block header is from the same asset until proven false
bool same = true;
switch (header.status()) {
@@ -394,7 +393,7 @@ handleNextServerHeader(Data *const data)
if (data->onlyHeader()) {
return false;
}
- // need to reissue reference slice
+ // asset is gone, reissue the reference block
logSliceError("404 internal block response (asset gone)", data, header);
same = false;
break;
@@ -412,6 +411,7 @@ handleNextServerHeader(Data *const data)
// can't parse the content range header, abort -- might be too strict
ContentRange blockcr;
+ // check the content range (offset and size)
if (same) {
blockcr = contentRangeFrom(header);
if (!blockcr.isValid() || blockcr.m_length != data->m_contentlen) {
@@ -420,10 +420,16 @@ handleNextServerHeader(Data *const data)
}
}
+ // identifiers. Use them
+ char etag[8192];
+ int etaglen = 0;
+ char lastmodified[33];
+ int lastmodifiedlen = 0;
+
+ // check the identifiers
if (same) {
- // prefer the etag but use Last-Modified if we must.
- char etag[8192];
- int etaglen = sizeof(etag);
+ // prefer the etag
+ etaglen = sizeof(etag);
header.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, etag, &etaglen);
if (0 < data->m_etaglen || 0 < etaglen) {
@@ -432,8 +438,8 @@ handleNextServerHeader(Data *const data)
logSliceError("Mismatch block Etag", data, header);
}
} else {
- char lastmodified[33];
- int lastmodifiedlen = sizeof(lastmodified);
+ // use Last-Modified if we must
+ lastmodifiedlen = sizeof(lastmodified);
header.valueForKey(TS_MIME_FIELD_LAST_MODIFIED,
TS_MIME_LEN_LAST_MODIFIED, lastmodified, &lastmodifiedlen);
if (0 < data->m_lastmodifiedlen || 0 < lastmodifiedlen) {
same = data->m_lastmodifiedlen == lastmodifiedlen && 0 ==
strncmp(lastmodified, data->m_lastmodified, lastmodifiedlen);
@@ -445,59 +451,29 @@ handleNextServerHeader(Data *const data)
}
// Header mismatch
- if (same) {
- // If we were in reference block refetch mode and the headers
- // still match there is a problem
- if (BlockState::ActiveRef == data->m_blockstate) {
- ERROR_LOG("Reference block refetched, got the same block back again");
- return false;
- }
- } else {
- switch (data->m_blockstate) {
- case BlockState::Active: {
+ if (!same) {
+ if (data->m_blockstate == BlockState::Active) {
data->m_upstream.abort();
- // Refetch the current interior slice
- data->m_blockstate = BlockState::PendingInt;
-
- time_t date = 0;
- header.timeForKey(TS_MIME_FIELD_DATE, TS_MIME_LEN_DATE, &date);
-
- // Ask for any slice newer than the cached one
- time_t const dateims = date + 1;
-
- DEBUG_LOG("Attempting to reissue interior slice block request with IMS
header time: %jd", static_cast<intmax_t>(dateims));
-
- // add special CRR IMS header to the request
- HttpHeader headerreq(data->m_req_hdrmgr.m_buffer,
data->m_req_hdrmgr.m_lochdr);
- Config const *const conf = data->m_config;
- if (!headerreq.setKeyTime(conf->m_crr_ims_header.data(),
conf->m_crr_ims_header.size(), dateims)) {
- ERROR_LOG("Failed setting '%s'", conf->m_crr_ims_header.c_str());
- return false;
- }
-
- } break;
- case BlockState::ActiveInt: {
- data->m_upstream.abort();
+ DEBUG_LOG("Starting refetch of reference block");
- // New interior slice still mismatches, refetch the reference slice
+ // Interior slice doesn't match reference slice, refetch reference
+ // In this case we've given up but are trying to fix the reference
+ // for next time
data->m_blockstate = BlockState::PendingRef;
- // convert reference date header to time_t
- time_t const date = TSMimeParseDate(data->m_date, data->m_datelen);
-
- // Ask for any slice newer than the cached one
- time_t const dateims = date + 1;
-
- DEBUG_LOG("Attempting to reissue reference slice block request with IMS
header time: %jd", static_cast<intmax_t>(dateims));
-
- // add special CRR IMS header to the request
- HttpHeader headerreq(data->m_req_hdrmgr.m_buffer,
data->m_req_hdrmgr.m_lochdr);
- Config const *const conf = data->m_config;
- if (!headerreq.setKeyTime(conf->m_crr_ims_header.data(),
conf->m_crr_ims_header.size(), dateims)) {
- ERROR_LOG("Failed setting '%s'", conf->m_crr_ims_header.c_str());
- return false;
+ // interior headers for new identifier reference
+ data->m_etaglen = etaglen;
+ if (0 < etaglen) {
+ strncpy(data->m_etag, etag, etaglen);
}
+ data->m_lastmodifiedlen = lastmodifiedlen;
+ if (0 < lastmodifiedlen) {
+ strncpy(data->m_lastmodified, lastmodified, lastmodifiedlen);
+ }
+
+ // potentially new content length
+ data->m_contentlen = blockcr.m_length;
// Reset for first block
if (Config::RefType::First == data->m_config->m_reftype) {
@@ -507,17 +483,6 @@ handleNextServerHeader(Data *const data)
}
return true;
-
- } break;
- // Refetch the reference slice
- case BlockState::ActiveRef: {
- // In this state the reference changed otherwise the asset is toast
- // reset the content length (if content length drove the mismatch)
- data->m_contentlen = blockcr.m_length;
- return true;
- } break;
- default:
- break;
}
}
@@ -603,8 +568,7 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const
data)
// header may have been successfully parsed but with caveats
switch (data->m_blockstate) {
- // request new version of current internal slice
- case BlockState::PendingInt:
+ // request new version of reference slice
case BlockState::PendingRef: {
if (!request_block(contp, data)) {
data->m_blockstate = BlockState::Fail;
diff --git a/plugins/slice/util.cc b/plugins/slice/util.cc
index 7967185b81..579aac447a 100644
--- a/plugins/slice/util.cc
+++ b/plugins/slice/util.cc
@@ -57,7 +57,6 @@ request_block(TSCont contp, Data *const data)
switch (data->m_blockstate) {
case BlockState::Pending:
- case BlockState::PendingInt:
case BlockState::PendingRef:
break;
default:
@@ -76,7 +75,7 @@ request_block(TSCont contp, Data *const data)
DEBUG_LOG("requestBlock: %s", rangestr);
- // reuse the incoming client header, just change the range
+ // reuse the original client request header, just change the range
HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr);
// if configured, remove range header from head requests
@@ -99,22 +98,31 @@ request_block(TSCont contp, Data *const data)
}
// Attach the identifier header
+ // The cache_range_requests plugin will:
+ // mark a fresh block stale if the identifiers mismatch
+ // mark a stale block fresh if the identifiers match.
Config const *const cfg = data->m_config;
- if (!cfg->m_crr_ident_header.empty() &&
!header.hasKey(cfg->m_crr_ident_header.data(), cfg->m_crr_ident_header.size()))
{
- swoc::LocalBufferWriter<8192> idbuf;
- if (0 < data->m_etaglen) {
- idbuf.write(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
- idbuf.write(": ");
- idbuf.write(data->m_etag, data->m_etaglen);
- } else if (0 < data->m_lastmodifiedlen) {
- idbuf.write(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED);
- idbuf.write(": ");
- idbuf.write(data->m_lastmodified, data->m_lastmodifiedlen);
- }
+ if (!cfg->m_crr_ident_header.empty()) {
+ // Set ident header if header not set or if refetch reference slice
+ if (BlockState::PendingRef == data->m_blockstate ||
+ !header.hasKey(cfg->m_crr_ident_header.data(),
cfg->m_crr_ident_header.size())) {
+ swoc::LocalBufferWriter<8192> idbuf;
+ if (0 < data->m_etaglen) {
+ idbuf.write(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG);
+ idbuf.write(" ");
+ idbuf.write(data->m_etag, data->m_etaglen);
+ } else if (0 < data->m_lastmodifiedlen) {
+ idbuf.write(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED);
+ idbuf.write(" ");
+ idbuf.write(data->m_lastmodified, data->m_lastmodifiedlen);
+ } else if (BlockState::PendingRef == data->m_blockstate) {
+ idbuf.write("Stale");
+ }
- if (0 < idbuf.size()) {
- DEBUG_LOG("Adding identity '%.*s'", (int)idbuf.size(), idbuf.data());
- header.setKeyVal(cfg->m_crr_ident_header.data(),
cfg->m_crr_ident_header.size(), idbuf.data(), idbuf.size());
+ if (0 < idbuf.size()) {
+ DEBUG_LOG("Adding identity '%.*s'", (int)idbuf.size(), idbuf.data());
+ header.setKeyVal(cfg->m_crr_ident_header.data(),
cfg->m_crr_ident_header.size(), idbuf.data(), idbuf.size());
+ }
}
}
@@ -174,15 +182,8 @@ request_block(TSCont contp, Data *const data)
case BlockState::Pending:
data->m_blockstate = BlockState::Active;
break;
- case BlockState::PendingInt: {
- data->m_blockstate = BlockState::ActiveInt;
- Config const *const conf = data->m_config;
- header.removeKey(conf->m_crr_ims_header.c_str(),
conf->m_crr_ims_header.size());
- } break;
case BlockState::PendingRef: {
- data->m_blockstate = BlockState::ActiveRef;
- Config const *const conf = data->m_config;
- header.removeKey(conf->m_crr_ims_header.c_str(),
conf->m_crr_ims_header.size());
+ data->m_blockstate = BlockState::ActiveRef;
} break;
default:
ERROR_LOG("Invalid blockstate");
diff --git
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py
index e36f4f2d73..faa3be859e 100644
---
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py
+++
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cache_complete_responses.test.py
@@ -16,8 +16,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import time
-
Test.Summary = '''
cache_range_requests cache-complete-responses test
'''
diff --git
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ident.test.py
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ident.test.py
index e08ffa4d45..eb3ae9a923 100644
---
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ident.test.py
+++
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ident.test.py
@@ -16,7 +16,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import datetime
import os
+import time
Test.Summary = '''
cache_range_requests X-Crr-Ident plugin test
@@ -30,13 +32,9 @@ Test.SkipUnless(
Condition.PluginExists('cache_range_requests.so'),
Condition.PluginExists('xdebug.so'),
)
-#Test.ContinueOnFail = False
-Test.ContinueOnFail = True
+Test.ContinueOnFail = False
Test.testName = "cache_range_requests_ident"
-# Define and configure ATS
-ts = Test.MakeATSProcess("ts")
-
# Define and configure origin server
server = Test.MakeOriginServer("server")
@@ -54,16 +52,17 @@ server.addResponse("sessionlog.json", req_chk, res_chk)
body = "lets go surfin now"
bodylen = len(body)
-req_full = {
- "headers": "GET /path HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
+# baseline for testing
+last_modified = "Fri, 07 Mar 2025 18:06:58 GMT"
+etag = '"772102f4-56f4bc1e6d417"'
+
+req_both = {
+ "headers": "GET /both HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
"timestamp": "1469733493.993",
"body": ""
}
-last_modified = "Fri, 07 Mar 2025 18:06:58 GMT"
-etag = '"772102f4-56f4bc1e6d417"'
-
-res_full = {
+res_both = {
"headers":
"HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=1\r\n" +
"Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + "Connection:
close\r\n" + 'Etag: ' + etag + '\r\n' +
@@ -72,10 +71,44 @@ res_full = {
"body": body
}
-server.addResponse("sessionlog.json", req_full, res_full)
+server.addResponse("sessionlog.json", req_both, res_both)
+
+req_etag = {
+ "headers": "GET /etag HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": ""
+}
+
+res_etag = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=1\r\n" +
+ "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + "Connection:
close\r\n" + 'Etag: ' + etag + '\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": body
+}
+
+server.addResponse("sessionlog.json", req_etag, res_etag)
+
+req_lm = {
+ "headers": "GET /lm HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Accept:
*/*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": ""
+}
+res_lm = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=1\r\n" +
+ "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + "Connection:
close\r\n" + 'Last-Modified: ' + last_modified +
+ '\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": body
+}
+
+server.addResponse("sessionlog.json", req_lm, res_lm)
+
+# test for custom Ident header
req_custom = {
- "headers": "GET /pathheader HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
+ "headers": "GET /custom HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
"timestamp": "1469733493.993",
"body": ""
}
@@ -92,6 +125,29 @@ res_custom = {
server.addResponse("sessionlog.json", req_custom, res_custom)
+# Long lived asset for FRESH to STALE testing
+req_fresh = {
+ "headers": "GET /fresh HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": ""
+}
+
+etag_fresh = 'fresh'
+
+res_fresh = {
+ "headers":
+ "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=3600\r\n" +
+ "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + "Connection:
close\r\n" + 'Etag: ' + etag_fresh + '\r\n' + '\r\n' +
+ 'Last-Modified: ' + last_modified + '\r\n' + "\r\n",
+ "timestamp": "1469733493.993",
+ "body": body
+}
+
+server.addResponse("sessionlog.json", req_fresh, res_fresh)
+
+# Define and configure ATS
+ts = Test.MakeATSProcess("ts")
+
# cache range requests plugin remap
ts.Disk.remap_config.AddLines(
[
@@ -111,85 +167,181 @@ ts.Disk.records_config.update({
curl_and_args = '-s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug:
x-cache"'.format(ts.Variables.port)
-# 0 Test - Fetch asset into cache
+##
+## Stale to Fresh testing
+##
+
+## Fetch short lived assets into cache
+
+# 0 Test - Fetch both asset into cache
tr = Test.AddTestRun("0- range cache load")
ps = tr.Processes.Default
ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
ps.StartBefore(Test.Processes.ts)
-tr.MakeCurlCommand(curl_and_args + ' http://ident/path -r 0-', ts=ts)
+tr.MakeCurlCommand(curl_and_args + ' http://ident/both -r 0-', ts=ts)
ps.ReturnCode = 0
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss for load")
tr.StillRunningAfter = ts
-# 1 Test - Fetch asset into cache
-tr = Test.AddTestRun("0- range cache load")
+# 1 Test - Fetch etag asset into cache
+tr = Test.AddTestRun("0- etag cache load")
ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + ' http://identheader/pathheader -r 0-',
ts=ts)
+tr.MakeCurlCommand(curl_and_args + ' http://ident/etag -r 0-', ts=ts)
ps.ReturnCode = 0
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss for load")
tr.StillRunningAfter = ts
-# 2 Test - Ensure range is fetched
-tr = Test.AddTestRun("0- cache hit check")
+# 2 Test - Fetch lm asset into cache
+tr = Test.AddTestRun("0- lm cache load")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + ' http://ident/lm -r 0-', ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss for load")
+tr.StillRunningAfter = ts
+
+## both tests
+
+# 3 Test - Ensure Etag header match results in hit-fresh
+tr = Test.AddTestRun("0- both Etag check")
+tr.DelayStart = 2 # Time to ensure stale
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + f" http://ident/both -r 0- -H 'X-Crr-Ident:
Etag {etag}'", ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache hit-fresh")
+tr.StillRunningAfter = ts
+
+# 4 Test - Plugin expects Etag even if Last-Modified matches - hit-stale
+tr = Test.AddTestRun("0- both Last-Modified check")
ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + ' http://ident/path -r 0-', ts=ts)
+tr.MakeCurlCommand(curl_and_args + f' http://ident/both -r 0- -H "X-Crr-Ident:
Last-Modified {last_modified}"', ts=ts)
ps.ReturnCode = 0
-ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit",
"expected cache hit")
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
tr.StillRunningAfter = ts
-# 3 Test - Ensure range is fetched
-tr = Test.AddTestRun("0- cache hit check")
+# 5 Test - Bad etag stays stale
+tr = Test.AddTestRun("0- both bad Etag check")
ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + ' http://identheader/pathheader -r 0-',
ts=ts)
+tr.MakeCurlCommand(curl_and_args + f" http://ident/both -r 0- -H 'X-Crr-Ident:
Etag no_match'", ts=ts)
ps.ReturnCode = 0
-ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit",
"expected cache hit")
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
tr.StillRunningAfter = ts
-# These requests should flip from STALE to FRESH
+## etag only supplied
-# 4 Test - Ensure X-Crr-Ident Etag header results in hit-fresh
-tr = Test.AddTestRun("0- range X-Crr-Ident Etag check")
-tr.DelayStart = 2
+# 6 Test - Ensure Etag header match results in hit-fresh
+tr = Test.AddTestRun("0- etag Etag check")
ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + f" http://ident/path -r 0- -H 'X-Crr-Ident:
Etag: {etag}'", ts=ts)
+tr.MakeCurlCommand(curl_and_args + f" http://ident/etag -r 0- -H 'X-Crr-Ident:
Etag {etag}'", ts=ts)
ps.ReturnCode = 0
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache hit-fresh")
tr.StillRunningAfter = ts
-# 5 Test - Ensure X-Crr-Ident Etag header results in hit-fresh, custom header
-tr = Test.AddTestRun("0- range CrrIdent Etag check")
-tr.DelayStart = 2
+# 7 Test - Last modified will result in stale
+tr = Test.AddTestRun("0- etag lm cache stale check")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + f' http://ident/etag -r 0- -H "X-Crr-Ident:
Last-Modified {last_modified}"', ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache stale")
+tr.StillRunningAfter = ts
+
+# 8 Test - Bad etag stays stale
+tr = Test.AddTestRun("0- etag bad Etag check")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + f" http://ident/etag -r 0- -H 'X-Crr-Ident:
Etag no_match'", ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
+tr.StillRunningAfter = ts
+
+## last modified
+
+# 9 Test - Last modified will result in fresh
+tr = Test.AddTestRun("0- lm lm fresh check")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + f' http://ident/lm -r 0- -H "X-Crr-Ident:
Last-Modified {last_modified}"', ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache fresh")
+tr.StillRunningAfter = ts
+
+# 10 Test - Ensure Etag header match results in hit-fresh
+tr = Test.AddTestRun("0- lm Etag check")
ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + f" http://identheader/pathheader -r 0- -H
'CrrIdent: Etag: {etag_custom}'", ts=ts)
+tr.MakeCurlCommand(curl_and_args + f" http://ident/lm -r 0- -H 'X-Crr-Ident:
Etag {etag}'", ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
+tr.StillRunningAfter = ts
+
+## Fresh to stale testing
+
+# 11 Test - Fetch "fresh" into cache
+tr = Test.AddTestRun("0- fresh range cache load")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + ' http://ident/fresh -r 0-', ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss for load")
+tr.StillRunningAfter = ts
+
+# 12 Test - Ensure "fresh" is in cache
+tr = Test.AddTestRun("0- fresh range cache check")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + ' http://ident/fresh -r 0-', ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache fresh")
+tr.StillRunningAfter = ts
+
+# 13 request with different etag and ensure it goes stale
+tr = Test.AddTestRun("0- fresh range to stale")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + " http://ident/fresh -r 0- -H 'X-Crr-Ident:
Etag not_the_same'", ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
+tr.StillRunningAfter = ts
+
+# 14 request with Last-Modified ensure it goes stale (expected etag)
+tr = Test.AddTestRun("0- etag fresh range to stale")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + f' http://ident/fresh -r 0- -H
"X-Crr-Ident: Last-Modified {last_modified}"', ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
+tr.StillRunningAfter = ts
+
+# 15 Test - Ensure Etag header match results in hit-fresh
+tr = Test.AddTestRun("0- fresh ensure Etag check")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + f" http://ident/fresh -r 0- -H
'X-Crr-Ident: Etag {etag_fresh}'", ts=ts)
ps.ReturnCode = 0
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache hit-fresh")
tr.StillRunningAfter = ts
-# 6 Test - Ensure X-Crr-Ident Last-Modified header results in hit-fresh
-tr = Test.AddTestRun("0- range X-Crr-Ident Last-Modified check")
+# 16 hit test asset to ensure its still fresh
+tr = Test.AddTestRun("0- plain request path again")
ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + f' http://ident/path -r 0- -H "X-Crr-Ident:
Last-Modified: {last_modified}"', ts=ts)
+tr.MakeCurlCommand(curl_and_args + ' http://ident/fresh -r 0-', ts=ts)
ps.ReturnCode = 0
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache hit-fresh")
tr.StillRunningAfter = ts
-# 7 Test - Provide a mismatch Etag force IMS request
-tr = Test.AddTestRun("0- range X-Crr-Ident check")
+# 17 request with force stale
+tr = Test.AddTestRun("0- force stale")
ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + f' http://ident/path -r 0- -H "X-Crr-Ident:
Last-Modified: foo"', ts=ts)
+tr.MakeCurlCommand(curl_and_args + " http://ident/fresh -r 0- -H 'X-Crr-Ident:
Stale'", ts=ts)
ps.ReturnCode = 0
ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
tr.StillRunningAfter = ts
-# post checks for traffic.out
+## custom header
-ts.Disk.traffic_out.Content = Testers.ContainsExpression(
- """Checking cached '"772102f4-56f4bc1e6d417"' against request 'Etag:
"772102f4-56f4bc1e6d417"'""",
- "Etag is correctly considered")
-
-ts.Disk.traffic_out.Content = Testers.ContainsExpression(
- """Checking cached 'foo' against request 'Etag: foo'""", "Etag custom
header is correctly considered")
+# 18 Test - Fetch custom asset into cache
+tr = Test.AddTestRun("0- custom range cache load")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + ' http://identheader/custom -r 0-', ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss for load")
+tr.StillRunningAfter = ts
-ts.Disk.traffic_out.Content = Testers.ContainsExpression(
- """Checking cached 'Fri, 07 Mar 2025 18:06:58 GMT' against request
'Last-Modified: Fri, 07 Mar 2025 18:06:58 GMT'""",
- "Last-Modified is correctly considered")
+# 19 Test - Ensure CrrIdent Etag header results in hit-fresh, custom header
+tr = Test.AddTestRun("0- fresh CrrIdent Etag check")
+ps = tr.Processes.Default
+tr.MakeCurlCommand(curl_and_args + f" http://identheader/custom -r 0- -H
'CrrIdent: Etag {etag_custom}'", ts=ts)
+ps.ReturnCode = 0
+ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-fresh",
"expected cache hit-fresh")
+tr.StillRunningAfter = ts
diff --git
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py
b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py
deleted file mode 100644
index bb8e4cdd8b..0000000000
---
a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py
+++ /dev/null
@@ -1,133 +0,0 @@
-'''
-'''
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import time
-
-Test.Summary = '''
-cache_range_requests X-CRR-IMS plugin test
-'''
-
-# Test description:
-# Preload the cache with the entire asset to be range requested.
-# Reload remap rule with cache_range_requests plugin
-# Request content through the cache_range_requests plugin
-
-Test.SkipUnless(
- Condition.PluginExists('cache_range_requests.so'),
- Condition.PluginExists('xdebug.so'),
-)
-Test.ContinueOnFail = False
-Test.testName = "cache_range_requests_ims"
-
-# Define and configure ATS
-ts = Test.MakeATSProcess("ts")
-
-# Define and configure origin server
-server = Test.MakeOriginServer("server")
-
-# default root
-req_chk = {
- "headers": "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "uuid:
none\r\n" + "\r\n",
- "timestamp": "1469733493.993",
- "body": ""
-}
-
-res_chk = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" +
"\r\n", "timestamp": "1469733493.993", "body": ""}
-
-server.addResponse("sessionlog.json", req_chk, res_chk)
-
-body = "lets go surfin now"
-bodylen = len(body)
-
-req_full = {
- "headers": "GET /path HTTP/1.1\r\n" + "Host: www.example.com\r\n" +
"Accept: */*\r\n" + "Range: bytes=0-\r\n" + "\r\n",
- "timestamp": "1469733493.993",
- "body": ""
-}
-
-res_full = {
- "headers":
- "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" +
"Cache-Control: max-age=500\r\n" +
- "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + "Connection:
close\r\n" + 'Etag: "772102f4-56f4bc1e6d417"\r\n' +
- "\r\n",
- "timestamp": "1469733493.993",
- "body": body
-}
-
-server.addResponse("sessionlog.json", req_full, res_full)
-
-# cache range requests plugin remap
-ts.Disk.remap_config.AddLines(
- [
- f'map http://ims http://127.0.0.1:{server.Variables.Port}' + '
@plugin=cache_range_requests.so @pparam=--consider-ims',
- f'map http://imsheader http://127.0.0.1:{server.Variables.Port}' +
- ' @plugin=cache_range_requests.so @pparam=--consider-ims' + '
@pparam=--ims-header=CrrIms',
- ])
-
-# cache debug
-ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache')
-
-# minimal configuration
-ts.Disk.records_config.update({
- 'proxy.config.diags.debug.enabled': 1,
- 'proxy.config.diags.debug.tags': 'cache_range_requests',
-})
-
-curl_and_args = '-s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug:
x-cache"'.format(ts.Variables.port)
-
-# 0 Test - Fetch whole asset into cache
-tr = Test.AddTestRun("0- range cache load")
-ps = tr.Processes.Default
-ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
-ps.StartBefore(Test.Processes.ts)
-tr.MakeCurlCommand(curl_and_args + ' http://ims/path -r 0-', ts=ts)
-ps.ReturnCode = 0
-ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss",
"expected cache miss for load")
-tr.StillRunningAfter = ts
-
-# test inner range
-# 1 Test - Fetch range into cache
-tr = Test.AddTestRun("0- cache hit check")
-ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + ' http://ims/path -r 0-', ts=ts)
-ps.ReturnCode = 0
-ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit",
"expected cache hit")
-tr.StillRunningAfter = ts
-
-# set up the IMS date field (go in the future) RFC 2616
-futuretime = time.time() + 100 # seconds
-futurestr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(futuretime))
-
-# 2 Test - Ensure X-CRR-IMS header results in hit-stale
-tr = Test.AddTestRun("0- range X-CRR-IMS check")
-ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + ' http://ims/path -r 0- -H "X-CRR-IMS:
{}"'.format(futurestr), ts=ts)
-ps.ReturnCode = 0
-ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
-tr.StillRunningAfter = ts
-
-futuretime += 10 # seconds
-futurestr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(futuretime))
-
-# 3 Test - Ensure CrrIms header results in hit-stale
-tr = Test.AddTestRun("0- range CrrIms check, override header")
-ps = tr.Processes.Default
-tr.MakeCurlCommand(curl_and_args + ' http://imsheader/path -r 0- -H "CrrIms:
{}"'.format(futurestr), ts=ts)
-ps.ReturnCode = 0
-ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale",
"expected cache hit-stale")
-tr.StillRunningAfter = ts
diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold
b/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold
index f81ffa57f8..8cb833638f 100644
--- a/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold
+++ b/tests/gold_tests/pluginTest/slice/gold/slice_crr_ident.gold
@@ -1,13 +1,13 @@
cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2::
x-crr-ident=::-:: uid=::plain 0:: crc=TCP_MISS
-cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=3-5::
x-crr-ident=::Etag: "plain":: uid=::plain 1:: crc=TCP_MISS
+cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=3-5::
x-crr-ident=::Etag "plain":: uid=::plain 1:: crc=TCP_MISS
cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::plain:: crc=TCP_MISS
cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2::
x-crr-ident=::-:: uid=::plain 0:: crc=TCP_REFRESH_MISS
-cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5::
x-crr-ident=::Etag: "plain":: uid=::-:: crc=TCP_HIT
+cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5:: x-crr-ident=::Etag
"plain":: uid=::-:: crc=TCP_HIT
cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::plain:: crc=TCP_MISS
cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=0-2::
x-crr-ident=::-:: uid=::chg 0:: crc=TCP_REFRESH_MISS
-cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=3-5::
x-crr-ident=::Etag: "chg":: uid=::chg 1:: crc=TCP_REFRESH_MISS
+cpuup=/plain sssc=200 pssc=206 phr=DIRECT range=::bytes=3-5::
x-crr-ident=::Etag "chg":: uid=::chg 1:: crc=TCP_REFRESH_MISS
cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::chg:: crc=TCP_MISS
cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=0-2:: x-crr-ident=::-::
uid=::-:: crc=TCP_HIT
-cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5::
x-crr-ident=::Etag: "chg":: uid=::-:: crc=TCP_HIT
+cpuup=/plain sssc=000 pssc=206 phr=NONE range=::bytes=3-5:: x-crr-ident=::Etag
"chg":: uid=::-:: crc=TCP_HIT
cpuup=/plain sssc=200 pssc=200 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::chg:: crc=TCP_MISS
cpuup=/404.txt sssc=404 pssc=404 phr=DIRECT range=::-:: x-crr-ident=::-::
uid=::-:: crc=TCP_MISS
diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_ident.gold
b/tests/gold_tests/pluginTest/slice/gold/slice_ident.gold
index ed44f114db..c32629f0fc 100644
--- a/tests/gold_tests/pluginTest/slice/gold/slice_ident.gold
+++ b/tests/gold_tests/pluginTest/slice/gold/slice_ident.gold
@@ -1,15 +1,15 @@
/etag 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
/lm 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
/etag 000 206 range=::bytes=0-10:: x-crr-ident=::-:: crrident=::-::
-/etag 000 206 range=::bytes=11-21:: x-crr-ident=::Etag: "foo":: crrident=::-::
+/etag 000 206 range=::bytes=11-21:: x-crr-ident=::Etag "foo":: crrident=::-::
/etag 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
/etag 000 206 range=::bytes=0-10:: x-crr-ident=::-:: crrident=::-::
-/etag 000 206 range=::bytes=11-21:: x-crr-ident=::-:: crrident=::Etag: "foo"::
+/etag 000 206 range=::bytes=11-21:: x-crr-ident=::-:: crrident=::Etag "foo"::
/etag 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
/lm 000 206 range=::bytes=0-10:: x-crr-ident=::-:: crrident=::-::
-/lm 000 206 range=::bytes=11-21:: x-crr-ident=::Last-Modified: Fri, 07 Mar
2025 18:06:58 GMT:: crrident=::-::
+/lm 000 206 range=::bytes=11-21:: x-crr-ident=::Last-Modified Fri, 07 Mar 2025
18:06:58 GMT:: crrident=::-::
/lm 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
/lm 000 206 range=::bytes=0-10:: x-crr-ident=::-:: crrident=::-::
-/lm 000 206 range=::bytes=11-21:: x-crr-ident=::-:: crrident=::Last-Modified:
Fri, 07 Mar 2025 18:06:58 GMT::
+/lm 000 206 range=::bytes=11-21:: x-crr-ident=::-:: crrident=::Last-Modified
Fri, 07 Mar 2025 18:06:58 GMT::
/lm 200 200 range=::-:: x-crr-ident=::-:: crrident=::-::
/404.txt 000 404 range=::-:: x-crr-ident=::-:: crrident=::-::
diff --git a/tests/gold_tests/pluginTest/slice/slice_ident.test.py
b/tests/gold_tests/pluginTest/slice/slice_ident.test.py
index 7c7784a27d..7babe58ea3 100644
--- a/tests/gold_tests/pluginTest/slice/slice_ident.test.py
+++ b/tests/gold_tests/pluginTest/slice/slice_ident.test.py
@@ -169,19 +169,19 @@ ps.Streams.stdout.Content =
Testers.ContainsExpression("200 OK", "expected 200 O
tr.StillRunningAfter = ts
# 6 Test - add token to transaction log
-tr = Test.AddTestRun("Fetch last-modified asset")
+tr = Test.AddTestRun("Add 404 token to transaction log")
ps = tr.Processes.Default
tr.MakeCurlCommand(curl_and_args + ' http://prefetch/404.txt', ts=ts)
ps.ReturnCode = 0
ps.Streams.stdout.Content = Testers.ContainsExpression("404", "expected 404
Not Found response")
tr.StillRunningAfter = ts
+# 7 - wait for logs
condwaitpath = os.path.join(Test.Variables.AtsTestToolsDir, 'condwait')
-
tslog = os.path.join(ts.Variables.LOGDIR, 'transaction.log')
Test.AddAwaitFileContainsTestRun('Await ts transactions to finish logging.',
tslog, '404.txt')
-# 6 Check logs
+# 8 Check logs
tr = Test.AddTestRun()
tr.Processes.Default.Command = (f"cat {tslog}")
tr.Streams.stdout = "gold/slice_ident.gold"
diff --git a/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py
b/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py
index 2bced732d8..5f2ef1a70b 100644
--- a/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py
+++ b/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py
@@ -68,12 +68,11 @@ ts.Disk.remap_config.AddLines(
[
f'map http://slice/ http://127.0.0.1:{server.Variables.Port}/' +
' @plugin=slice.so @pparam=--blockbytes-test=3
@pparam=--remap-host=crr',
- f'map http://crr/ http://127.0.0.1:{server.Variables.Port}/' +
- ' @plugin=cache_range_requests.so @pparam=--consider-ims
@pparam=--consider-ident',
+ f'map http://crr/ http://127.0.0.1:{server.Variables.Port}/' + '
@plugin=cache_range_requests.so @pparam=--consider-ident',
f'map http://slicehdr/ http://127.0.0.1:{server.Variables.Port}/' + '
@plugin=slice.so @pparam=--blockbytes-test=3' +
- ' @pparam=--remap-host=crrhdr @pparam=--crr-ims-header=crr-foo',
+ ' @pparam=--remap-host=crrhdr @pparam=--crr-ident-header=crr-foo',
f'map http://crrhdr/ http://127.0.0.1:{server.Variables.Port}/'
- ' @plugin=cache_range_requests.so @pparam=--ims-header=crr-foo',
+ ' @plugin=cache_range_requests.so @pparam=--ident-header=crr-foo',
])
ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache')
@@ -239,7 +238,7 @@ ps.Streams.stderr = "gold/aaa.gold"
ps.Streams.stdout.Content = Testers.ContainsExpression("etagold", "expected
etagold")
tr.StillRunningAfter = ts
-# 5 Test - Preload reference etagnew-1
+# 5 Test - Preload slice 1 etagnew-1
tr = Test.AddTestRun("Preload slice etagnew-1")
ps = tr.Processes.Default
tr.MakeCurlCommand(curl_and_args + ' http://crr/reference -r 3-5 -H "uuid:
etagnew-1"', ts=ts)
