TCP client connections tagging is useful for faking various forms of
connection-based "authentication" when standard HTTP authentication
cannot be used. A URL rewriter or, external ACL helper may mark the
"authenticated" client connection to avoid going through
"authentication" steps during subsequent requests on the same connection
and to share connection "authentication" information with Squid ACLs,
other helpers, and logs.
After this change, Squid accepts optional clt_conn_id=ID pair from a
helper and associates the received ID with the client TCP connection.
Squid treats the received clt_conn_id=ID pair as a regular annotation,
but also keeps it across all requests on the same client connection. A
helper may update the client connection ID value during subsequent requests.
This patch documents the clt_conn_id key=value pair in cf.data.pre file
only for url rewriters. Because annotations are common to all helpers we
may want to make a special section at the beginning of cf.data.per for
all helpers. Suggestions are welcome.
I must also note that this patch adds an inconsistency. All annotation
key=values pairs received from helpers, accumulated to the existing key
notes values. The clt_conn_id=Id pair is always unique and replaces the
existing clt_conn_id=Id annotation pair.
We may want to make all annotations unique, or maybe implement a
configuration mechanism to define which annotations are overwriting
their previous values and which appending the new values.
This is a Measurement Factory project
Regards,
Christos
Support client connection annotation by helpers via clt_conn_id=ID.
TCP client connections tagging is useful for faking various forms of
connection-based "authentication" when standard HTTP authentication cannot be
used. A URL rewriter or, external ACL helper may mark the "authenticated"
client connection to avoid going through "authentication" steps during
subsequent requests on the same connection and to share connection
"authentication" information with Squid ACLs, other helpers, and logs.
After this change, Squid accepts optional clt_conn_id=ID pair from a
helper and associates the received ID with the client TCP connection.
Squid treats the received clt_conn_id=ID pair as a regular annotation, but
also keeps it across all requests on the same client connection. A helper may
update the client connection ID value during subsequent requests.
To send clt_conn_id=ID pair to a URL rewriter, use url_rewrite_extras with a
%{clt_conn_id}note macro.
This is a Measurement Factory project
=== modified file 'src/Notes.cc'
--- src/Notes.cc 2014-04-30 09:41:25 +0000
+++ src/Notes.cc 2014-06-10 15:20:30 +0000
@@ -14,40 +14,41 @@
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
*
*/
#include "squid.h"
#include "AccessLogEntry.h"
#include "acl/FilledChecklist.h"
#include "acl/Gadgets.h"
+#include "client_side.h"
#include "ConfigParser.h"
#include "globals.h"
#include "HttpReply.h"
#include "HttpRequest.h"
#include "SquidConfig.h"
#include "Store.h"
#include "StrList.h"
#include <algorithm>
#include <string>
Note::Value::~Value()
{
aclDestroyAclList(&aclList);
}
Note::Value::Pointer
Note::addValue(const String &value)
{
Value::Pointer v = new Value(value);
@@ -186,40 +187,54 @@
return value.size() ? value.termedBuf() : NULL;
}
const char *
NotePairs::findFirst(const char *noteKey) const
{
for (std::vector<NotePairs::Entry *>::const_iterator i = entries.begin(); i != entries.end(); ++i) {
if ((*i)->name.cmp(noteKey) == 0)
return (*i)->value.termedBuf();
}
return NULL;
}
void
NotePairs::add(const char *key, const char *note)
{
entries.push_back(new NotePairs::Entry(key, note));
}
void
+NotePairs::remove(const char *key)
+{
+ std::vector<NotePairs::Entry *>::iterator i = entries.begin();
+ while(i != entries.end()) {
+ if ((*i)->name.cmp(key) == 0) {
+ delete *i;
+ i = entries.erase(i);
+ } else {
+ ++i;
+ }
+ }
+}
+
+void
NotePairs::addStrList(const char *key, const char *values)
{
String strValues(values);
const char *item;
const char *pos = NULL;
int ilen = 0;
while (strListGetItem(&strValues, ',', &item, &ilen, &pos)) {
String v;
v.append(item, ilen);
entries.push_back(new NotePairs::Entry(key, v.termedBuf()));
}
}
bool
NotePairs::hasPair(const char *key, const char *value) const
{
for (std::vector<NotePairs::Entry *>::const_iterator i = entries.begin(); i != entries.end(); ++i) {
if ((*i)->name.cmp(key) == 0 && (*i)->value.cmp(value) == 0)
return true;
}
@@ -241,20 +256,36 @@
if (!hasPair((*i)->name.termedBuf(), (*i)->value.termedBuf()))
entries.push_back(new NotePairs::Entry((*i)->name.termedBuf(), (*i)->value.termedBuf()));
}
}
NotePairs &
SyncNotes(AccessLogEntry &ale, HttpRequest &request)
{
// XXX: auth code only has access to HttpRequest being authenticated
// so we must handle the case where HttpRequest is set without ALE being set.
if (!ale.notes) {
if (!request.notes)
request.notes = new NotePairs;
ale.notes = request.notes;
} else {
assert(ale.notes == request.notes);
}
return *ale.notes;
}
+
+void
+UpdateRequestNotes(ConnStateData *csd, HttpRequest &request, NotePairs const &helperNotes)
+{
+ // Tag client connection if the helper responded with clt_conn_id=ID.
+ if (const char *connId = helperNotes.findFirst("clt_conn_id")) {
+ if (csd)
+ csd->connectionId(connId);
+ // The new clt_conn_id will replace any previously set value.
+ if (request.notes != NULL)
+ request.notes->remove("clt_conn_id");
+ }
+ if (!request.notes)
+ request.notes = new NotePairs;
+ request.notes->appendNewOnly(&helperNotes);
+}
=== modified file 'src/Notes.h'
--- src/Notes.h 2014-04-30 09:41:25 +0000
+++ src/Notes.h 2014-06-10 15:15:23 +0000
@@ -143,58 +143,68 @@
/**
* Returns a comma separated list of notes with key 'noteKey'.
* Use findFirst instead when a unique kv-pair is needed.
*/
const char *find(const char *noteKey, const char *sep = ",") const;
/**
* Returns the first note value for this key or an empty string.
*/
const char *findFirst(const char *noteKey) const;
/**
* Adds a note key and value to the notes list.
* If the key name already exists in list, add the given value to its set
* of values.
*/
void add(const char *key, const char *value);
/**
+ * Remove all notes with a given key.
+ */
+ void remove(const char *key);
+
+ /**
* Adds a note key and values strList to the notes list.
* If the key name already exists in list, add the new values to its set
* of values.
*/
void addStrList(const char *key, const char *values);
/**
* Return true if the key/value pair is already stored
*/
bool hasPair(const char *key, const char *value) const;
/**
* Convert NotePairs list to a string consist of "Key: Value"
* entries separated by sep string.
*/
const char *toString(const char *sep = "\r\n") const;
/**
* True if there are not entries in the list
*/
bool empty() const {return entries.empty();}
std::vector<NotePairs::Entry *> entries; ///< The key/value pair entries
private:
NotePairs &operator = (NotePairs const &); // Not implemented
NotePairs(NotePairs const &); // Not implemented
};
MEMPROXY_CLASS_INLINE(NotePairs::Entry);
class AccessLogEntry;
/**
* Keep in sync HttpRequest and the corresponding AccessLogEntry objects
*/
NotePairs &SyncNotes(AccessLogEntry &ale, HttpRequest &request);
+class ConnStateData;
+/**
+ * Updates ConnStateData ids and HttpRequest notes from helpers received notes.
+ */
+void UpdateRequestNotes(ConnStateData *csd, HttpRequest &request, NotePairs const ¬es);
#endif
=== modified file 'src/auth/UserRequest.cc'
--- src/auth/UserRequest.cc 2014-05-22 09:12:48 +0000
+++ src/auth/UserRequest.cc 2014-06-10 15:19:29 +0000
@@ -247,44 +247,42 @@
auth_user_request->authenticate(request, conn, type);
}
static Auth::UserRequest::Pointer
authTryGetUser(Auth::UserRequest::Pointer auth_user_request, ConnStateData * conn, HttpRequest * request)
{
Auth::UserRequest::Pointer res;
if (auth_user_request != NULL)
res = auth_user_request;
else if (request != NULL && request->auth_user_request != NULL)
res = request->auth_user_request;
else if (conn != NULL)
res = conn->getAuth();
// attach the credential notes from helper to the transaction
if (request != NULL && res != NULL && res->user() != NULL) {
// XXX: we have no access to the transaction / AccessLogEntry so cant SyncNotes().
// workaround by using anything already set in HttpRequest
// OR use new and rely on a later Sync copying these to AccessLogEntry
- if (!request->notes)
- request->notes = new NotePairs;
- request->notes->appendNewOnly(&res->user()->notes);
+ UpdateRequestNotes(conn, *request, res->user()->notes);
}
return res;
}
/* returns one of
* AUTH_ACL_CHALLENGE,
* AUTH_ACL_HELPER,
* AUTH_ACL_CANNOT_AUTHENTICATE,
* AUTH_AUTHENTICATED
*
* How to use: In your proxy-auth dependent acl code, use the following
* construct:
* int rv;
* if ((rv = AuthenticateAuthenticate()) != AUTH_AUTHENTICATED)
* return rv;
*
* when this code is reached, the request/connection is authenticated.
*
* if you have non-acl code, but want to force authentication, you need a
=== modified file 'src/cf.data.pre'
--- src/cf.data.pre 2014-05-31 16:22:44 +0000
+++ src/cf.data.pre 2014-06-09 14:56:06 +0000
@@ -4561,75 +4561,83 @@
COMMENT_START
OPTIONS FOR URL REWRITING
-----------------------------------------------------------------------------
COMMENT_END
NAME: url_rewrite_program redirect_program
TYPE: wordlist
LOC: Config.Program.redirect
DEFAULT: none
DOC_START
Specify the location of the executable URL rewriter to use.
Since they can perform almost any function there isn't one included.
For each requested URL, the rewriter will receive on line with the format
[channel-ID <SP>] URL [<SP> extras]<NL>
- After processing the request the helper must reply using the following format:
+ After processing the request the helper must reply using the following
+ format:
- [channel-ID <SP>] result [<SP> kv-pairs]
+ [channel-ID <SP>] result [<SP> key=value ...]
The result code can be:
OK status=30N url="..."
Redirect the URL to the one supplied in 'url='.
'status=' is optional and contains the status code to send
the client in Squids HTTP response. It must be one of the
HTTP redirect status codes: 301, 302, 303, 307, 308.
When no status is given Squid will use 302.
OK rewrite-url="..."
Rewrite the URL to the one supplied in 'rewrite-url='.
The new URL is fetched directly by Squid and returned to
the client as the response to its request.
OK
When neither of url= and rewrite-url= are sent Squid does
not change the URL.
ERR
Do not change the URL.
BH
An internal error occurred in the helper, preventing
a result being identified. The 'message=' key name is
reserved for delivering a log message.
- In the future, the interface protocol will be extended with
- key=value pairs ("kv-pairs" shown above). Helper programs
- should be prepared to receive and possibly ignore additional
- whitespace-separated tokens on each input line.
+ Squid understands the following optional key=value pairs received from
+ URL rewriters:
+ clt_conn_id=ID
+ Associates the received ID with the client TCP connection.
+ The clt_conn_id=ID pair is treated as a regular annotation but
+ it persists across all transactions on the client connection
+ rather than disappearing after the current request. A helper
+ may update the client connection ID value during subsequent
+ requests by returning a new ID value. To send the connection
+ ID to the URL rewriter, use url_rewrite_extras:
+ url_rewrite_extras clt_conn_id=%{clt_conn_id}note ...
When using the concurrency= option the protocol is changed by
introducing a query channel tag in front of the request/response.
The query channel tag is a number between 0 and concurrency-1.
This value must be echoed back unchanged to Squid as the first part
of the response relating to its request.
WARNING: URL re-writing ability should be avoided whenever possible.
Use the URL redirect form of response instead.
Re-write creates a difference in the state held by the client
and server. Possibly causing confusion when the server response
contains snippets of its view state. Embeded URLs, response
and content Location headers, etc. are not re-written by this
interface.
By default, a URL rewriter is not used.
DOC_END
NAME: url_rewrite_children redirect_children
=== modified file 'src/client_side.h'
--- src/client_side.h 2014-06-05 14:57:58 +0000
+++ src/client_side.h 2014-06-11 14:06:45 +0000
@@ -358,40 +358,44 @@
if (!sslServerBump)
sslServerBump = srvBump;
else
assert(sslServerBump == srvBump);
}
/// Fill the certAdaptParams with the required data for certificate adaptation
/// and create the key for storing/retrieve the certificate to/from the cache
void buildSslCertGenerationParams(Ssl::CertificateProperties &certProperties);
/// Called when the client sends the first request on a bumped connection.
/// Returns false if no [delayed] error should be written to the client.
/// Otherwise, writes the error to the client and returns true. Also checks
/// for SQUID_X509_V_ERR_DOMAIN_MISMATCH on bumped requests.
bool serveDelayedError(ClientSocketContext *context);
Ssl::BumpMode sslBumpMode; ///< ssl_bump decision (Ssl::bumpEnd if n/a).
#else
bool switchedToHttps() const { return false; }
#endif
+ /* clt_conn_id=ID annotation access */
+ const SBuf &connectionId() const { return connectionId_; }
+ void connectionId(const char *anId) { connectionId_ = anId; }
+
protected:
void startDechunkingRequest();
void finishDechunkingRequest(bool withSuccess);
void abortChunkedRequestBody(const err_type error);
err_type handleChunkedRequestBody(size_t &putSize);
void startPinnedConnectionMonitoring();
void clientPinnedConnectionRead(const CommIoCbParams &io);
private:
int connFinishedWithConn(int size);
void clientAfterReadingRequests();
bool concurrentRequestQueueFilled() const;
#if USE_AUTH
/// some user details that can be used to perform authentication on this connection
Auth::UserRequest::Pointer auth_;
#endif
HttpParser parser_;
@@ -401,34 +405,36 @@
#if USE_OPENSSL
bool switchedToHttps_;
/// The SSL server host name appears in CONNECT request or the server ip address for the intercepted requests
String sslConnectHostOrIp; ///< The SSL server host name as passed in the CONNECT request
String sslCommonName; ///< CN name for SSL certificate generation
String sslBumpCertKey; ///< Key to use to store/retrieve generated certificate
/// HTTPS server cert. fetching state for bump-ssl-server-first
Ssl::ServerBump *sslServerBump;
Ssl::CertSignAlgorithm signAlgorithm; ///< The signing algorithm to use
#endif
/// the reason why we no longer write the response or nil
const char *stoppedSending_;
/// the reason why we no longer read the request or nil
const char *stoppedReceiving_;
AsyncCall::Pointer reader; ///< set when we are reading
BodyPipe::Pointer bodyPipe; // set when we are reading request body
+ SBuf connectionId_; ///< clt_conn_id=ID annotation for client connection
+
CBDATA_CLASS2(ConnStateData);
};
void setLogUri(ClientHttpRequest * http, char const *uri, bool cleanUrl = false);
const char *findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *end = NULL);
int varyEvaluateMatch(StoreEntry * entry, HttpRequest * req);
void clientOpenListenSockets(void);
void clientHttpConnectionsClose(void);
void httpRequestFree(void *);
#endif /* SQUID_CLIENTSIDE_H */
=== modified file 'src/client_side_request.cc'
--- src/client_side_request.cc 2014-06-05 14:57:58 +0000
+++ src/client_side_request.cc 2014-06-10 15:14:04 +0000
@@ -1220,46 +1220,46 @@
void
clientStoreIdDoneWrapper(void *data, const HelperReply &result)
{
ClientRequestContext *calloutContext = (ClientRequestContext *)data;
if (!calloutContext->httpStateIsValid())
return;
calloutContext->clientStoreIdDone(result);
}
void
ClientRequestContext::clientRedirectDone(const HelperReply &reply)
{
HttpRequest *old_request = http->request;
debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
assert(redirect_state == REDIRECT_PENDING);
redirect_state = REDIRECT_DONE;
+ UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
+
// Put helper response Notes into the transaction state record (ALE) eventually
// do it early to ensure that no matter what the outcome the notes are present.
- if (http->al != NULL) {
- NotePairs ¬es = SyncNotes(*http->al, *old_request);
- notes.append(&reply.notes);
- }
+ if (http->al != NULL)
+ (void)SyncNotes(*http->al, *old_request);
switch (reply.result) {
case HelperReply::Unknown:
case HelperReply::TT:
// Handler in redirect.cc should have already mapped Unknown
// IF it contained valid entry for the old URL-rewrite helper protocol
debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
break;
case HelperReply::BrokenHelper:
debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply << ", attempt #" << (redirect_fail_count+1) << " of 2");
if (redirect_fail_count < 2) { // XXX: make this configurable ?
++redirect_fail_count;
// reset state flag to try redirector again from scratch.
redirect_done = false;
}
break;
case HelperReply::Error:
// no change to be done.
@@ -1341,46 +1341,46 @@
if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
fd_note(http->getConn()->clientConnection->fd, http->uri);
assert(http->uri);
http->doCallouts();
}
/**
* This method handles the different replies from StoreID helper.
*/
void
ClientRequestContext::clientStoreIdDone(const HelperReply &reply)
{
HttpRequest *old_request = http->request;
debugs(85, 5, "'" << http->uri << "' result=" << reply);
assert(store_id_state == REDIRECT_PENDING);
store_id_state = REDIRECT_DONE;
+ UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
+
// Put helper response Notes into the transaction state record (ALE) eventually
// do it early to ensure that no matter what the outcome the notes are present.
- if (http->al != NULL) {
- NotePairs ¬es = SyncNotes(*http->al, *old_request);
- notes.append(&reply.notes);
- }
+ if (http->al != NULL)
+ (void)SyncNotes(*http->al, *old_request);
switch (reply.result) {
case HelperReply::Unknown:
case HelperReply::TT:
// Handler in redirect.cc should have already mapped Unknown
// IF it contained valid entry for the old helper protocol
debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
break;
case HelperReply::BrokenHelper:
debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply << ", attempt #" << (store_id_fail_count+1) << " of 2");
if (store_id_fail_count < 2) { // XXX: make this configurable ?
++store_id_fail_count;
// reset state flag to try StoreID again from scratch.
store_id_done = false;
}
break;
case HelperReply::Error:
// no change to be done.
@@ -1670,40 +1670,47 @@
* longer valid, it should call cbdataReferenceDone() so that
* ClientHttpRequest's reference count goes to zero and it will get
* deleted. ClientHttpRequest will then delete ClientRequestContext.
*
* Note that we set the _done flags here before actually starting
* the callout. This is strictly for convenience.
*/
tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
void
ClientHttpRequest::doCallouts()
{
assert(calloutContext);
/*Save the original request for logging purposes*/
if (!calloutContext->http->al->request) {
calloutContext->http->al->request = request;
HTTPMSGLOCK(calloutContext->http->al->request);
+
+ NotePairs ¬es = SyncNotes(*calloutContext->http->al, *calloutContext->http->request);
+ // Make the previously set client connection ID available as annotation.
+ if (ConnStateData *csd = calloutContext->http->getConn()) {
+ if (csd->connectionId().length())
+ notes.add("clt_conn_id", SBuf(csd->connectionId()).c_str());
+ }
}
if (!calloutContext->error) {
// CVE-2009-0801: verify the Host: header is consistent with other known details.
if (!calloutContext->host_header_verify_done) {
debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
calloutContext->host_header_verify_done = true;
calloutContext->hostHeaderVerify();
return;
}
if (!calloutContext->http_access_done) {
debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
calloutContext->http_access_done = true;
calloutContext->clientAccessCheck();
return;
}
#if USE_ADAPTATION
if (!calloutContext->adaptation_acl_check_done) {
=== modified file 'src/external_acl.cc'
--- src/external_acl.cc 2014-05-15 07:32:10 +0000
+++ src/external_acl.cc 2014-06-10 15:17:50 +0000
@@ -1517,44 +1517,42 @@
ACL *acl = ACL::FindByName(AclMatchedName);
assert(acl);
ACLExternal *me = dynamic_cast<ACLExternal *> (acl);
assert (me);
ACLExternal::ExternalAclLookup(checklist, me);
}
/// Called when an async lookup returns
void
ExternalACLLookup::LookupDone(void *data, void *result)
{
ACLFilledChecklist *checklist = Filled(static_cast<ACLChecklist*>(data));
checklist->extacl_entry = cbdataReference((external_acl_entry *)result);
// attach the helper kv-pair to the transaction
if (checklist->extacl_entry) {
if (HttpRequest * req = checklist->request) {
// XXX: we have no access to the transaction / AccessLogEntry so cant SyncNotes().
// workaround by using anything already set in HttpRequest
// OR use new and rely on a later Sync copying these to AccessLogEntry
- if (!req->notes)
- req->notes = new NotePairs;
- req->notes->appendNewOnly(&checklist->extacl_entry->notes);
+ UpdateRequestNotes(checklist->conn(), *req, checklist->extacl_entry->notes);
}
}
checklist->resumeNonBlockingCheck(ExternalACLLookup::Instance());
}
/* This registers "external" in the registry. To do dynamic definitions
* of external ACL's, rather than a static prototype, have a Prototype instance
* prototype in the class that defines each external acl 'class'.
* Then, then the external acl instance is created, it self registers under
* it's name.
* Be sure that clone is fully functional for that acl class though!
*/
ACL::Prototype ACLExternal::RegistryProtoype(&ACLExternal::RegistryEntry_, "external");
ACLExternal ACLExternal::RegistryEntry_("external");
ACL *
ACLExternal::clone() const
{
