Fix those and I think this is a good part-1 patch that can be applied to
trunk.
Amos
found missing declaration in the flags struct.
a fixed patch included.
=== modified file 'src/ClientRequestContext.h'
--- src/ClientRequestContext.h 2012-08-14 11:53:07 +0000
+++ src/ClientRequestContext.h 2012-09-09 12:52:34 +0000
@@ -33,6 +33,8 @@
void clientAccessCheckDone(const allow_t &answer);
void clientRedirectStart();
void clientRedirectDone(char *result);
+ void clientStoreurlStart();
+ void clientStoreurlDone(char *result);
void checkNoCache();
void checkNoCacheDone(const allow_t &answer);
#if USE_ADAPTATION
@@ -53,7 +55,9 @@
ClientHttpRequest *http;
ACLChecklist *acl_checklist; /* need ptr back so we can unreg if needed */
int redirect_state;
+ int storeurl_state;
+ bool storeurl_done;
bool host_header_verify_done;
bool http_access_done;
bool adapted_http_access_done;
=== modified file 'src/client_side.cc'
--- src/client_side.cc 2012-09-06 13:12:26 +0000
+++ src/client_side.cc 2012-09-09 12:54:28 +0000
@@ -714,6 +714,7 @@
ClientHttpRequest::freeResources()
{
safe_free(uri);
+ safe_free(store_uri);
safe_free(log_uri);
safe_free(redirect.location);
range_iter.boundary.clean();
=== modified file 'src/client_side_request.cc'
--- src/client_side_request.cc 2012-08-28 19:12:13 +0000
+++ src/client_side_request.cc 2012-09-09 13:01:23 +0000
@@ -131,6 +131,7 @@
static int clientHierarchical(ClientHttpRequest * http);
static void clientInterpretRequestHeaders(ClientHttpRequest * http);
static RH clientRedirectDoneWrapper;
+static RH clientStoreurlDoneWrapper;
static void checkNoCacheDoneWrapper(allow_t, void *);
extern "C" CSR clientGetMoreData;
extern "C" CSS clientReplyStatus;
@@ -151,10 +152,11 @@
debugs(85,3, HERE << this << " ClientRequestContext destructed");
}
-ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), error(NULL), readNextRequest(false)
+ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), storeurl_state (REDIRECT_NONE), error(NULL), readNextRequest(false)
{
http_access_done = false;
redirect_done = false;
+ storeurl_done = false;
no_cache_done = false;
interpreted_req_hdrs = false;
#if USE_SSL
@@ -916,6 +918,31 @@
redirectStart(http, clientRedirectDoneWrapper, this);
}
+static void
+clientStoreurlAccessCheckDone(allow_t answer, void *data)
+{
+ ClientRequestContext *context = (ClientRequestContext *)data;
+ ClientHttpRequest *http = context->http;
+ context->acl_checklist = NULL;
+
+ if (answer == ACCESS_ALLOWED)
+ storeurlStart(http, clientStoreurlDoneWrapper, context);
+ else
+ context->clientStoreurlDone(NULL);
+}
+
+void
+ClientRequestContext::clientStoreurlStart()
+{
+ debugs(33, 5, HERE << "' ClientRequestContext::clientStoreurlStart " << http->uri << "'");
+
+ if (Config.accessList.storeurl) {
+ acl_checklist = clientAclChecklistCreate(Config.accessList.storeurl, http);
+ acl_checklist->nonBlockingCheck(clientStoreurlAccessCheckDone, this);
+ } else
+ clientStoreurlDone(NULL)
+}
+
static int
clientHierarchical(ClientHttpRequest * http)
{
@@ -1262,6 +1289,37 @@
http->doCallouts();
}
+void
+clientStoreurlDoneWrapper(void *data, char *result)
+{
+ ClientRequestContext *calloutContext = (ClientRequestContext *)data;
+
+ if (!calloutContext->httpStateIsValid())
+ return;
+
+ calloutContext->clientStoreurlDone(result);
+}
+
+void
+ClientRequestContext::clientStoreurlDone(char *result)
+{
+ debugs(85, 5, "clientStoreurlDone: '" << http->uri << "' result=" << (result ? result : "NULL"));
+ assert(storeurl_state == REDIRECT_PENDING);
+ storeurl_state = REDIRECT_DONE;
+
+ /*
+ * the next step is to implement here the actuall store_url_rewrite proccess of pushing
+ * the store_url into thr http request.
+ */
+ if (result) {
+ debugs(85, DBG_CRITICAL, "A result for storu_url accepted: " << result);
+ } else {
+ debugs(85, DBG_CRITICAL, "A result for storu_url was not accepted: " << result);
+ }
+
+ http->doCallouts();
+}
+
/** Test cache allow/deny configuration
* Sets flags.cachable=1 if caching is not denied.
*/
@@ -1689,6 +1747,18 @@
if (ih != NULL)
ih->logType = logType;
#endif
+
+ if (!calloutContext->storeurl_done) {
+ calloutContext->storeurl_done = true;
+ assert(calloutContext->storeurl_state == REDIRECT_NONE);
+
+ if (Config.Program.storeurl) {
+ debugs(83, 3, HERE << "Doing calloutContext->clientSotreurlStart()");
+ calloutContext->storeurl_state = REDIRECT_PENDING;
+ calloutContext->clientStoreurlStart();
+ return;
+ }
+ }
}
#if !_USE_INLINE_
=== modified file 'src/client_side_request.h'
--- src/client_side_request.h 2012-08-28 19:12:13 +0000
+++ src/client_side_request.h 2012-09-09 13:03:15 +0000
@@ -98,6 +98,7 @@
HttpRequest *request; /* Parsed URL ... */
char *uri;
char *log_uri;
+ char *store_uri;
struct {
int64_t offset;
@@ -209,6 +210,7 @@
/* ones that should be elsewhere */
extern void redirectStart(ClientHttpRequest *, RH *, void *);
extern void tunnelStart(ClientHttpRequest *, int64_t *, int *);
+extern void storeurlStart(ClientHttpRequest *, RH *, void *);
#if _USE_INLINE_
#include "Store.h"
=== modified file 'src/redirect.cc'
--- src/redirect.cc 2012-08-28 19:12:13 +0000
+++ src/redirect.cc 2012-09-09 13:04:48 +0000
@@ -71,8 +71,11 @@
static HLPCB redirectHandleReply;
static void redirectStateFree(redirectStateData * r);
static helper *redirectors = NULL;
+static helper *storeurl_rewriters = NULL;
static OBJH redirectStats;
+static OBJH storeurlStats;
static int n_bypassed = 0;
+static int n_storeurl_bypassed = 0;
CBDATA_TYPE(redirectStateData);
static void
@@ -81,7 +84,7 @@
redirectStateData *r = static_cast<redirectStateData *>(data);
char *t;
void *cbdata;
- debugs(61, 5, "redirectHandleRead: {" << (reply && *reply != '\0' ? reply : "<NULL>") << "}");
+ debugs(61, 5, HERE << (reply && *reply != '\0' ? reply : "<NULL>") << "}");
if (reply) {
if ((t = strchr(reply, ' ')))
@@ -119,6 +122,21 @@
"because all redirectors were busy: %d\n", n_bypassed);
}
+static void
+storeurlStats(StoreEntry * sentry)
+{
+ if (storeurl_rewriters == NULL) {
+ storeAppendPrintf(sentry, "No Store Url Rewritters defined\n");
+ return;
+ }
+
+ helperStats(sentry, storeurl_rewriters, "Store Url Rewriters Statistics");
+
+ if (Config.onoff.storeurl_bypass)
+ storeAppendPrintf(sentry, "\nNumber of requests bypassed "
+ "because all Store Url Rewriters were busy: %d\n", n_storeurl_bypassed);
+}
+
/**** PUBLIC FUNCTIONS ****/
void
@@ -237,46 +255,171 @@
redirectRegisterWithCacheManager(void)
{
Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1);
+ Mgr::RegisterAction("storeurl", "Store URL Rewritter Stats", storeurlStats, 0, 1);
}
void
redirectInit(void)
{
- static int init = 0;
-
redirectRegisterWithCacheManager();
-
- if (!Config.Program.redirect)
+ if (!Config.Program.redirect && !Config.Program.storeurl)
return;
- if (redirectors == NULL)
- redirectors = new helper("redirector");
-
- redirectors->cmdline = Config.Program.redirect;
-
- redirectors->childs.updateLimits(Config.redirectChildren);
-
- redirectors->ipc_type = IPC_STREAM;
-
- helperOpenServers(redirectors);
-
- if (!init) {
- init = 1;
- CBDATA_INIT_TYPE(redirectStateData);
- }
+ if (Config.Program.redirect){
+ if (redirectors == NULL)
+ redirectors = new helper("redirector");
+
+ redirectors->cmdline = Config.Program.redirect;
+ redirectors->childs.updateLimits(Config.redirectChildren);
+ redirectors->ipc_type = IPC_STREAM;
+ helperOpenServers(redirectors);
+ }
+
+ if (Config.Program.storeurl){
+ if (storeurl_rewriters == NULL)
+ storeurl_rewriters = new helper("storeurl");
+
+ storeurl_rewriters->cmdline = Config.Program.storeurl;
+ storeurl_rewriters->childs.updateLimits(Config.storeurlChildren);
+ storeurl_rewriters->ipc_type = IPC_STREAM;
+ helperOpenServers(storeurl_rewriters);
+ }
+ static bool init = false;
+ if (init) {
+ init = true;
+ CBDATA_INIT_TYPE(redirectStateData);
+ }
}
void
redirectShutdown(void)
{
- if (!redirectors)
+ if (!redirectors && !storeurl_rewriters)
return;
- helperShutdown(redirectors);
+ if (redirectors)
+ helperShutdown(redirectors);
+
+ if (storeurl_rewriters)
+ helperShutdown(storeurl_rewriters);
if (!shutting_down)
return;
delete redirectors;
redirectors = NULL;
+ delete storeurl_rewriters;
+ storeurl_rewriters = NULL;
+}
+
+void
+storeurlStart(ClientHttpRequest * http, RH * handler, void *data)
+{
+ ConnStateData * conn = http->getConn();
+ redirectStateData *r = NULL;
+ const char *fqdn;
+ char buf[MAX_REDIRECTOR_REQUEST_STRLEN];
+ int sz;
+ http_status status;
+ char claddr[MAX_IPSTRLEN];
+ char myaddr[MAX_IPSTRLEN];
+ assert(http);
+ assert(handler);
+ debugs(61, 5, "storeurlStart: '" << http->uri << "'");
+
+ if (Config.onoff.storeurl_bypass && storeurl_rewriters->stats.queue_size) {
+ /* Skip store_url_rewritter if there is one request queued */
+ ++n_storeurl_bypassed;
+ handler(data, NULL);
+ return;
+ }
+
+ r = cbdataAlloc(redirectStateData);
+ r->orig_url = xstrdup(http->uri);
+ if (conn != NULL)
+ r->client_addr = conn->log_addr;
+ else
+ r->client_addr.SetNoAddr();
+ r->client_ident = NULL;
+#if USE_AUTH
+ if (http->request->auth_user_request != NULL) {
+ r->client_ident = http->request->auth_user_request->username();
+ debugs(61, 5, HERE << "auth-user=" << (r->client_ident?r->client_ident:"NULL"));
+ }
+#endif
+
+ // HttpRequest initializes with null_string. So we must check both defined() and size()
+ if (!r->client_ident && http->request->extacl_user.defined() && http->request->extacl_user.size()) {
+ r->client_ident = http->request->extacl_user.termedBuf();
+ debugs(61, 5, HERE << "acl-user=" << (r->client_ident?r->client_ident:"NULL"));
+ }
+
+ if (!r->client_ident && conn != NULL && conn->clientConnection != NULL && conn->clientConnection->rfc931[0]) {
+ r->client_ident = conn->clientConnection->rfc931;
+ debugs(61, 5, HERE << "ident-user=" << (r->client_ident?r->client_ident:"NULL"));
+ }
+
+#if USE_SSL
+
+ if (!r->client_ident && conn != NULL && Comm::IsConnOpen(conn->clientConnection)) {
+ r->client_ident = sslGetUserEmail(fd_table[conn->clientConnection->fd].ssl);
+ debugs(61, 5, HERE << "ssl-user=" << (r->client_ident?r->client_ident:"NULL"));
+ }
+#endif
+
+ if (!r->client_ident)
+ r->client_ident = dash_str;
+
+ r->method_s = RequestMethodStr(http->request->method);
+
+ r->handler = handler;
+
+ r->data = cbdataReference(data);
+
+ if ((fqdn = fqdncache_gethostbyaddr(r->client_addr, 0)) == NULL)
+ fqdn = dash_str;
+
+ sz = snprintf(buf, MAX_REDIRECTOR_REQUEST_STRLEN, "%s %s/%s %s %s myip=%s myport=%d\n",
+ r->orig_url,
+ r->client_addr.NtoA(claddr,MAX_IPSTRLEN),
+ fqdn,
+ r->client_ident[0] ? rfc1738_escape(r->client_ident) : dash_str,
+ r->method_s,
+ http->request->my_addr.NtoA(myaddr,MAX_IPSTRLEN),
+ http->request->my_addr.GetPort());
+
+ if ((sz<=0) || (sz>=MAX_REDIRECTOR_REQUEST_STRLEN)) {
+ if (sz<=0) {
+ status = HTTP_INTERNAL_SERVER_ERROR;
+ debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Can not build request to be passed to storeurl rewriter. Request ABORTED.");
+ } else {
+ status = HTTP_REQUEST_URI_TOO_LARGE;
+ debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Request passed to storeurl rewriter exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN << "). Request ABORTED.");
+ }
+
+ clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
+ clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
+ assert (repContext);
+ Ip::Address tmpnoaddr;
+ tmpnoaddr.SetNoAddr();
+ repContext->setReplyToError(ERR_GATEWAY_FAILURE, status,
+ http->request->method, NULL,
+ http->getConn() != NULL && http->getConn()->clientConnection != NULL ?
+ http->getConn()->clientConnection->remote : tmpnoaddr,
+ http->request,
+ NULL,
+#if USE_AUTH
+ http->getConn() != NULL && http->getConn()->auth_user_request != NULL ?
+ http->getConn()->auth_user_request : http->request->auth_user_request);
+#else
+ NULL);
+#endif
+
+ node = (clientStreamNode *)http->client_stream.tail->data;
+ clientStreamRead(node, http, node->readBuffer);
+ return;
+ }
+
+ debugs(61,6, HERE << "sending '" << buf << "' to the helper");
+ helperSubmit(storeurl_rewriters, buf, redirectHandleReply, r);
}
=== modified file 'src/structs.h'
--- src/structs.h 2012-08-28 13:00:30 +0000
+++ src/structs.h 2012-09-09 15:29:07 +0000
@@ -306,6 +306,8 @@
#endif
wordlist *redirect;
+ wordlist *storeurl;
+
#if USE_UNLINKD
char *unlinkd;
@@ -323,6 +325,8 @@
#endif
HelperChildConfig redirectChildren;
+ HelperChildConfig storeurlChildren;
+
time_t authenticateGCInterval;
time_t authenticateTTL;
time_t authenticateIpTTL;
@@ -421,6 +425,7 @@
int nonhierarchical_direct;
int strip_query_terms;
int redirector_bypass;
+ int storeurl_bypass;
int ignore_unknown_nameservers;
int client_pconns;
int server_pconns;
@@ -484,6 +489,7 @@
acl_access *brokenPosts;
#endif
acl_access *redirector;
+ acl_access *storeurl;
acl_access *reply;
acl_address *outgoing_address;
#if USE_HTCP
@@ -962,7 +968,7 @@
};
struct request_flags {
- request_flags(): range(0),nocache(0),ims(0),auth(0),cachable(0),hierarchical(0),loopdetect(0),proxy_keepalive(0),proxying(0),refresh(0),redirected(0),need_validation(0),fail_on_validation_err(0),stale_if_hit(0),accelerated(0),ignore_cc(0),intercepted(0),hostVerified(0),spoof_client_ip(0),internal(0),internalclient(0),must_keepalive(0),pinned(0),canRePin(0),chunked_reply(0),stream_error(0),sslPeek(0),sslBumped(0),destinationIPLookedUp_(0) {
+ request_flags(): range(0),nocache(0),ims(0),auth(0),cachable(0),hierarchical(0),loopdetect(0),proxy_keepalive(0),proxying(0),refresh(0),redirected(0),storeurl_rewritted(0),need_validation(0),fail_on_validation_err(0),stale_if_hit(0),accelerated(0),ignore_cc(0),intercepted(0),hostVerified(0),spoof_client_ip(0),internal(0),internalclient(0),must_keepalive(0),pinned(0),canRePin(0),chunked_reply(0),stream_error(0),sslPeek(0),sslBumped(0),destinationIPLookedUp_(0) {
#if USE_HTTP_VIOLATIONS
nocache_hack = 0;
#endif
@@ -983,6 +989,7 @@
1; /* this should be killed, also in httpstateflags */
unsigned int refresh:1;
unsigned int redirected:1;
+ unsigned int storeurl_rewritted:1;
unsigned int need_validation:1;
unsigned int fail_on_validation_err:1; ///< whether we should fail if validation fails
unsigned int stale_if_hit:1; ///< reply is stale if it is a hit
