This patch makes Squid deal with responses having more than 4KB of responseline + headers.
- Make the HTTP protocol handler continue reading the response until all is received. This involves growing the header buffer as needed until the header fits (or limit reached). Previously the buffer based flow control made Squid stop reading the response when reaching 4KB - makes client_side_reply clone the already parsed HTTP response and skip over the ascii HTTP header received from the store, similar to the change Adrian did earlier in Squid-2 but for other reasons.
# Bazaar merge directive format 2 (Bazaar 0.90) # revision_id: [EMAIL PROTECTED] # m06h0iqek6nq8fyb # target_branch: /data/bzr/squid3/branches/SQUID_3_0 # testament_sha1: c5c5604c9077c76cd2fc94aecdb4620126c26299 # timestamp: 2008-03-30 18:45:34 +0200 # source_branch: http://www.henriknordstrom.net/bzr/squid3/hno\ # /largeresp-3.0 # base_revision_id: [EMAIL PROTECTED] # nxa2s3jig4ce6nwb # # Begin patch === modified file 'src/HttpReply.cc' --- src/HttpReply.cc 2007-11-26 20:09:54 +0000 +++ src/HttpReply.cc 2008-03-30 14:29:57 +0000 @@ -496,3 +496,17 @@ return expectBody; } + +HttpReply * +HttpReply::clone() const +{ + HttpReply *rep = new HttpReply(); + rep->header.append(&header); + rep->hdrCacheInit(); + rep->hdr_sz = hdr_sz; + rep->http_ver = http_ver; + rep->pstate = pstate; + rep->protocol = protocol; + rep->sline = sline; + return rep; +} === modified file 'src/HttpReply.h' --- src/HttpReply.h 2007-08-13 23:20:50 +0000 +++ src/HttpReply.h 2008-03-30 14:29:57 +0000 @@ -122,6 +122,11 @@ void packHeadersInto(Packer * p) const; + /// Clone this reply. + /// Could be done as a copy-contructor but we do not want to + /// accidently copy a HttpReply.. + HttpReply *clone() const; + private: /* initialize */ void init(); === modified file 'src/client_side_reply.cc' --- src/client_side_reply.cc 2008-03-14 04:45:16 +0000 +++ src/client_side_reply.cc 2008-03-30 14:29:57 +0000 @@ -352,68 +352,47 @@ sendClientOldEntry(); } - // we have a partial reply from the origin - else if (STORE_PENDING == http->storeEntry()->store_status && 0 == status) { - // header is too large, send old entry - - if (reqsize >= HTTP_REQBUF_SZ) { - debugs(88, 3, "handleIMSReply: response from origin is too large '" << http->storeEntry()->url() << "', sending old entry to client" ); - http->logType = LOG_TCP_REFRESH_FAIL; - sendClientOldEntry(); - } - - // everything looks fine, we're just waiting for more data - else { - debugs(88, 3, "handleIMSReply: incomplete headers for '" << http->storeEntry()->url() << "', waiting for more data" ); - reqofs = reqsize; - waitForMoreData(); - } - } - - // we have a reply from the origin + HttpReply *old_rep = (HttpReply *) old_entry->getReply(); + + // origin replied 304 + + if (status == HTTP_NOT_MODIFIED) { + http->logType = LOG_TCP_REFRESH_UNMODIFIED; + + // update headers on existing entry + HttpReply *old_rep = (HttpReply *) old_entry->getReply(); + old_rep->updateOnNotModified(http->storeEntry()->getReply()); + old_entry->timestampsSet(); + + // if client sent IMS + + if (http->request->flags.ims) { + // forward the 304 from origin + debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and forwarding 304 to client"); + sendClientUpstreamResponse(); + } else { + // send existing entry, it's still valid + debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and sending " << + old_rep->sline.status << " to client"); + sendClientOldEntry(); + } + } + + // origin replied with a non-error code + else if (status > HTTP_STATUS_NONE && status < HTTP_INTERNAL_SERVER_ERROR) { + // forward response from origin + http->logType = LOG_TCP_REFRESH_MODIFIED; + debugs(88, 3, "handleIMSReply: origin replied " << status << ", replacing existing entry and forwarding to client"); + sendClientUpstreamResponse(); + } + + // origin replied with an error else { - HttpReply *old_rep = (HttpReply *) old_entry->getReply(); - - // origin replied 304 - - if (status == HTTP_NOT_MODIFIED) { - http->logType = LOG_TCP_REFRESH_UNMODIFIED; - - // update headers on existing entry - HttpReply *old_rep = (HttpReply *) old_entry->getReply(); - old_rep->updateOnNotModified(http->storeEntry()->getReply()); - old_entry->timestampsSet(); - - // if client sent IMS - - if (http->request->flags.ims) { - // forward the 304 from origin - debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and forwarding 304 to client"); - sendClientUpstreamResponse(); - } else { - // send existing entry, it's still valid - debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and sending " << - old_rep->sline.status << " to client"); - sendClientOldEntry(); - } - } - - // origin replied with a non-error code - else if (status > HTTP_STATUS_NONE && status < HTTP_INTERNAL_SERVER_ERROR) { - // forward response from origin - http->logType = LOG_TCP_REFRESH_MODIFIED; - debugs(88, 3, "handleIMSReply: origin replied " << status << ", replacing existing entry and forwarding to client"); - sendClientUpstreamResponse(); - } - - // origin replied with an error - else { - // ignore and let client have old entry - http->logType = LOG_TCP_REFRESH_FAIL; - debugs(88, 3, "handleIMSReply: origin replied with error " << - status << ", sending old entry (" << old_rep->sline.status << ") to client"); - sendClientOldEntry(); - } + // ignore and let client have old entry + http->logType = LOG_TCP_REFRESH_FAIL; + debugs(88, 3, "handleIMSReply: origin replied with error " << + status << ", sending old entry (" << old_rep->sline.status << ") to client"); + sendClientOldEntry(); } } @@ -472,33 +451,6 @@ /* update size of the request */ reqsize = result.length + reqofs; - if (e->getReply()->sline.status == 0) { - /* - * we don't have full reply headers yet; either wait for more or - * punt to clientProcessMiss. - */ - - if (e->mem_status == IN_MEMORY || e->store_status == STORE_OK) { - processMiss(); - } else if (result.length + reqofs >= HTTP_REQBUF_SZ - && http->out.offset == 0) { - processMiss(); - } else { - debugs(88, 3, "clientCacheHit: waiting for HTTP reply headers"); - reqofs += result.length; - assert(reqofs <= HTTP_REQBUF_SZ); - /* get the next users' buffer */ - StoreIOBuffer tempBuffer; - tempBuffer.offset = http->out.offset + reqofs; - tempBuffer.length = next()->readBuffer.length - reqofs; - tempBuffer.data = next()->readBuffer.data + reqofs; - storeClientCopy(sc, e, - tempBuffer, CacheHit, this); - } - - return; - } - /* * Got the headers, now grok them */ @@ -1371,32 +1323,14 @@ void -clientReplyContext::buildReply(const char *buf, size_t size) +clientReplyContext::cloneReply() { - size_t k = headersEnd(buf, size); - - if (!k) - return; - assert(reply == NULL); - HttpReply *rep = new HttpReply; + HttpReply *rep = http->storeEntry()->getReply()->clone(); reply = HTTPMSGLOCK(rep); - if (!reply->parseCharBuf(buf, k)) { - /* parsing failure, get rid of the invalid reply */ - HTTPMSGUNLOCK(reply); - - if (http->request->range) { - debugs(0,0,HERE << "look for bug here"); - /* this will fail and destroy request->range */ - // clientBuildRangeHeader(http, reply); - } - - return; - } - /* enforce 1.0 reply version */ reply->sline.version = HttpVersion(1,0); @@ -1703,32 +1637,6 @@ return getNextNode(); } -void -clientReplyContext::waitForMoreData () -{ - debugs(88, 5, "clientReplyContext::waitForMoreData: Waiting for more data to parse reply headers in client side."); - /* We don't have enough to parse the metadata yet */ - /* TODO: the store should give us out of band metadata and - * obsolete this routine - */ - /* wait for more to arrive */ - startSendProcess(); -} - -void -clientReplyContext::startSendProcess() -{ - debugs(88, 5, "clientReplyContext::startSendProcess: triggering store read to SendMoreData"); - assert(reqofs <= HTTP_REQBUF_SZ); - /* TODO: copy into the supplied buffer */ - StoreIOBuffer tempBuffer; - tempBuffer.offset = reqofs; - tempBuffer.length = next()->readBuffer.length - reqofs; - tempBuffer.data = next()->readBuffer.data + reqofs; - storeClientCopy(sc, http->storeEntry(), - tempBuffer, SendMoreData, this); -} - /* * Calculates the maximum size allowed for an HTTP response */ @@ -1841,8 +1749,10 @@ http->loggingEntry(http->storeEntry()); ssize_t body_size = reqofs - reply->hdr_sz; - - assert(body_size >= 0); + if (body_size < 0) { + reqofs = reply->hdr_sz; + body_size = 0; + } debugs(88, 3, "clientReplyContext::sendMoreData: Appending " << (int) body_size << " bytes after " << reply->hdr_sz << @@ -1872,7 +1782,7 @@ StoreIOBuffer tempBuffer; char *buf = next()->readBuffer.data; - char *body_buf = buf + reply->hdr_sz; + char *body_buf = buf + reply->hdr_sz - next()->readBuffer.offset; //Server side may disable ranges under some circumstances. @@ -1916,23 +1826,11 @@ char *body_buf = buf; - /* This is always valid until we get the headers as metadata from - * storeClientCopy. - * Then it becomes reqofs == next->readBuffer.offset() - */ - assert(reqofs == 0 || flags.storelogiccomplete); - - if (flags.headersSent && buf != result.data) { + if (buf != result.data) { /* we've got to copy some data */ assert(result.length <= next()->readBuffer.length); xmemcpy(buf, result.data, result.length); body_buf = buf; - } else if (!flags.headersSent && - buf + reqofs !=result.data) { - /* we've got to copy some data */ - assert(result.length + reqofs <= next()->readBuffer.length); - xmemcpy(buf + reqofs, result.data, result.length); - body_buf = buf; } /* We've got the final data to start pushing... */ @@ -1971,38 +1869,23 @@ return; } - buildReply(buf, reqofs); - - if (reply) { - - /* handle headers */ - - if (Config.onoff.log_mime_hdrs) { - size_t k; - - if ((k = headersEnd(buf, reqofs))) { - safe_free(http->al.headers.reply); - http->al.headers.reply = (char *)xcalloc(k + 1, 1); - xstrncpy(http->al.headers.reply, buf, k); - } - } - - holdingBuffer = result; - processReplyAccess(); - return; - - } else if (reqofs < HTTP_REQBUF_SZ && entry->store_status == STORE_PENDING) { - waitForMoreData(); - return; - } else { - debugs(88, 0, "clientReplyContext::sendMoreData: Unable to parse reply headers within a single HTTP_REQBUF_SZ length buffer"); - StoreIOBuffer tempBuffer; - tempBuffer.flags.error = 1; - /* XXX FIXME: make an html error page here */ - sendStreamError(tempBuffer); - return; + cloneReply(); + + /* handle headers */ + + if (Config.onoff.log_mime_hdrs) { + size_t k; + + if ((k = headersEnd(buf, reqofs))) { + safe_free(http->al.headers.reply); + http->al.headers.reply = (char *)xcalloc(k + 1, 1); + xstrncpy(http->al.headers.reply, buf, k); + } } - fatal ("clientReplyContext::sendMoreData: Unreachable code reached \n"); + + holdingBuffer = result; + processReplyAccess(); + return; } === modified file 'src/client_side_reply.h' --- src/client_side_reply.h 2008-02-26 06:08:50 +0000 +++ src/client_side_reply.h 2008-03-30 14:29:57 +0000 @@ -124,15 +124,13 @@ bool errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const ; void sendStreamError(StoreIOBuffer const &result); void pushStreamData(StoreIOBuffer const &result, char *source); - void waitForMoreData (); clientStreamNode * next() const; - void startSendProcess(); StoreIOBuffer holdingBuffer; HttpReply *reply; void processReplyAccess(); static PF ProcessReplyAccessResult; void processReplyAccessResult(bool accessAllowed); - void buildReply(const char *buf, size_t size); + void cloneReply(); void buildReplyHeader (); bool alwaysAllowResponse(http_status sline) const; int checkTransferDone(); === modified file 'src/http.cc' --- src/http.cc 2007-11-19 05:00:58 +0000 +++ src/http.cc 2008-03-30 16:38:00 +0000 @@ -1213,12 +1213,16 @@ * handler until we get a notification from someone that * its okay to read again. */ - if (read_sz < 2) - return; + if (read_sz < 2) { + if (flags.headers_parsed) + return; + else + read_sz = 1024; + } if (flags.do_next_read) { - flags.do_next_read = 0; - entry->delayAwareRead(fd, readBuf->space(), read_sz, ReadReplyWrapper, this); + flags.do_next_read = 0; + entry->delayAwareRead(fd, readBuf->space(read_sz), read_sz, ReadReplyWrapper, this); } } # Begin bundle IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWQoOMgwACBjfgEgwef///3/v /4C////6YA49b7lGasq+zF2ZKGqNsb03TpvZ3s7s0JWgDShdBhoYoSZMmU0yGJhlPU9Ro0xNAAAN DQA0Bkmk0xNEzEqe0iAPUDQeoAGmjQNANBoONGTIwjEAwmgwCaDQMmTRkyGEBhIiCFDTEyp6eFHk U08mo9QyAaGgBoB6gBFJJmkaDJMjVPwknpp6mnpNoTU/RTE00NqaNAA0EigQ0BCYIGgU8RomJqG1 GjTTTID1DTTSiSRRCoxC8n+nr+rq+vRWVaL+N/5VG+MrVbqQxsHlR/hNdVE8zzhUnZRfD0MarJqk Yx4/DGbntFHHfWUeYyic5zA04L0ZdRJb9og2KzBC1YY0IEEsYxhQgOKBqy1yhCT23xJ+TERtgN0j BYsq97Cr9JQtYLCtlV46crqth5N4w4sWwbq/DP1ISAy6mm0DGNibAbabYMjZ69/X1h3hT4vZ/de1 iegY2a2oq//jgj5y/T2oP8o6kMbG20gbG9McDzA5x+Tbtw0ZDIHr+HLDDHBccIUOj4D4XpKL0vMF WQREFFevPQgtFnxpGAyAmImkTyhu7N8GPLYg0yZUyygoc0UnljDk17fYGDPA289rNbI223PwtlSI PdOnZ2Obl6TTI+nodWkv1eijBErF+Mq7gHN2le6I5XDv9aAQB+CMK2ESpvGJyDjttYO8+jShRwse aB02hoiUL/VlS/PYevo38DjrYx6ZIyNZubS843S2dWkyRajoTGwStbznejG6MAQ9j1Bemyld6uGL xKRfJCc923b/PH2W5a3n2P6lcq+p+z09x5DwROyLm7wyl9NlPdTSzDS/M3AzsEjC+GS3KoBYGkh8 Xgnqah+J+bYMKMiIUOHtlhRFW3NIl6rxVlJ6Gdxxx3/t9mvTX27Rj52Gep10lE2wRY16mGWrvYLp CusMcTo5s849JDK+pvQZw8+luTqDvVefcu1MEutLFMaz6aQQS22lDDUyL4mxR7kvTdPd8ElAXBJ0 NaiqKZKZb+vXCdjSVaKTEwFpXIdzIt6RA468/O1MHLyg+yHvlAwbXBByHizYBENx3UPeSWrbreII 2MSbXS1DJ8tgmp7kiam169ikURvYFmAjeRRGGNVIJRiOwZalMj4XlhTkMJxGIDgoK9l5YzwSsr3Q mUMJKBNdQsbFEyGTGWkFWYIL60VZYnoXIoO1l+o/LWuYmWwLjdG8/Ul4CyOPl8chBr1OFD0txlWh YZs1w7hql07k5QzAgOgr6ZIGlSosFso4a3qlalhkFtJiDekkLQjMdABqECT1Ey6hkTStlfnZMU++ kA4wIE2HhE2GM5yHU2oK2wimswJHZ0GZZA3mJWdxjEnhCDcd+qhHClF6qksG0kF+hWHdVFooZFGk 6NmokiSEFEEhLIkkljILAWoaTlvzXzDMCxchlaQzuhYXkLHmMwiPHJGM08kEFN4D2gWqWaSkUYKZ YRMDAgYFTAkHJJQ80tqrnK3XnnST0zjA82OGASIlVoWDOpBLYSXFnqM0syZUTkBsGYklC7GEQScC ZUtLDWX2YXpRtJQY3mJ5kdhKka52YFnKpoYEyYUGSPLnXmti28sqZzrqx2zNhsLrVYyW652kNZaQ AoM0K7jRXm4okpwInMW33F1pb1SIk0tpEroXIMbtHvyMgJ/C4ieBmNpOYiMEyuXVRL1xXGcXPoC2 2AS7QWh+bBeB5XyvklYWYlFklkikKWUpU3FqqWJLHKhEuwF+Z3cMBETGsYhg8mgwGQlv9cnGlM6w GIYFsUlQqLlaSruJOks6ELpJQlqD6CtmxbuGKud7gDXANiDVeW26ExlTQ1FRdwtq2ReSsSTNMzfR e0e16KwgNk7oE7JaiUnZqssNRmZYnQR1FlEiJMgFStZlS8zHUlPKVSwy5L6M5SpleyV5C8q9C0Cu QCCooHLaLeYnQaNjZeEiEgSsHhwdz9kIUDPCBWtmaR/xE2FywFSgyYzEmaFgdYucrU9OSSpwtDmi a85RYYNEhmTUO4lYaFCFMIrQr6x16i21JMtwSkY2dZ1EIzMCpUl0FJoMTMmXESZyS6ygZljDiDlE eaZW0cJmZamKVAHBIpmAzoVvQDf4PW1A8AHVpGkJ5jEX7OO4xwjhlgJyA0FBruolFKgDQH0ppQL0 sF9rLLkMRNdI01mUG6YCG4GaAaGyWBftPlTa3rvUEqylS1dWhUXwAKJpLeICS7f6fP8sD512g2mN pNs+/T2PhaaqBl4LmPPKfzhJh4zdw8n9ac5jUHPMjI3mJ0N+I4/cioTJ/nwXqXzpoGb8tJKczdJH E7t2zGwpcuz8l7Ffc6mXQf7T4ZYrYuNWsrn4gQvTbgFwPpl0/yNWFVWIR0oIWnRqn7NitOc79KqK 8+xdbFnfZI0DVWcup2DaFxBEBLFDBafhtlMKIoKX9baJjAY0edBsZTqOhT5g+suAPvX3fvJmBU8T 3FD5YYxXj8o2gshpF5S8T4UBmAYHgPyPwfEawB7NyDQCdxQDQj68jVGxpGYpakT6114IiGFrqxHj xmCA1n3nvNRnPE+0+BUQ/E5DOPG4/A5jAwLCQy8obtD8TtP8fb9uahZ8XDtCH8EgM1s2+nciQoKU uAzMgddT1rd6bTefQgqfX45lQUhIHa8xzP1qDtWeymBZfYHgLQLrfgglRLhZC/qKQILaznz3Dy5t 4Z9UsuqFGRwbhQTB47LgatZ2Gw7DA8DxJl5XsLihQiXmSCZIwSO0gWE1EoGZMuBECYYEGAw3+hMu Lzh2WDJi8ege8xOmmanGZgxAWW6Ii6SaFdXFFTtGaIZ1omrBuk4rj+6IzhghWC/VIhVJgx5GvuQS Ve7GXCAhuaGzOOk5ztOws29PfWuy6h1GKw5Ejm1cUNe8DeK7cxUT4m4N8o2qdtEWgqirvLfkdZAg wwSGUCRsC38csqA1bQmV5oSR8w1DuaPP0bbbOeB097eBAoyKMUMVetBclf9SoAjcA2A0Ntg2hgNT 6LMXB8ZdS3B9xlieWlwF4LSjQYiyHxOsW5hE6+S/cwAu0CXcUQBM0Q9baFcvMvyQMfOEUorfAO82 HYSN31HaTPE8TzKnv91TIxLic+XdSrgKwOSDfvPSZwM5jCnKt7RwPkncmtN13dO8D02hIMJKWdy9 ozuhTmbbWHiL1gU1iEDiCgj6bgmvmHeviL1LwF8IeoA2dgRIRxCG66YVnnqbGI8v7NlJ4ElgJEEm RIkx/wgYLDpOxWZ4ReEpQkiiWg1rUoJRMAQjgGCTDeExgxa0oFsAhprHkxFCxCBfsGHNnYIOI6kJ 8rUheoJKgQrP6IXZ1lUL3g4Q5LCY44k6vQ0t9uBdw9x2AvSlOnzOVrAyQGPwdnSWkOXBDZjPUs1e sfYtNYgT5QZCB2rhea03uBuU8mRWrn8jj9DFfdtmeo+BJC5OaEgISuzIwuYfcRBA3BesnKTlyO2u scdrM7WtzuJDZAgXIJpRxnQsAW73zslrNfM33ZWvWBRKcl9asjLFDejDQsgxNBIDVAukDmJtNcgj 43wZDQmxWitIYHUIFN/ZF1NQuJ/PTzNM/SIHVLgOFUClYcjBEGIvhAWgMSIoJrBZ+HEWvyXdyz9c +3r9hz1L/txt71uLJ9ZYHZfNAd3uJEtdiCUhvXQ1YZIPFYHmYerJ0pgKIsRggzRpkD8XCoEitxIU rcMJIOAUs5XF+S9xyD6TyV6iDTG0xtdAyVGFoVlkHe73qU6dxWB2ucyg+ZXovY6oEHEkzcXzIHDI 5Bh9V1kpfkW7IxcusZH0juytAVFxQSZSHhAa0vA7n5zSYXgEjTkt788UcZzxUExSDcTWTUhtYZai GCDdRZcEkmwanwZJrTERDRY2DB1zCSDgkgGl1LVmwmDEBaQXYmYJdLRljAmxXIBiaaBisYoxZSPP +kN/uDsjbT2hgUU17/arUprITFOukJb5C7eYYye6fO3BeU9UAN4HyYtgJNKzeGKR0qvfqCLR3r7F h9iJAZtsOYu7R/p4ENSLgWBYenahBUspsA49mVOMJs29sAmssIQaSVz6MUpylLkDWAq3EwYFgmJo lYhMk4p8FPMRar1BXZCrFuhkl8338JSNZumLShhB95ChQfRF6EQicImGFMT9QP4IwOFxrU8IfeZ5 cT4dieIQEgzbQuJ3clBmErsV2rVaW4MyGs3UgvRpCjhe9q4Lg8cDR6VYCqp+466n+tZoZpj2q20A GN1odsKGtihQNtNyyAY4gShTzJcKIlPpZsqYMJeNA0L8mnWOWCryobVcjn0mhDhmIg7X/xdyRThQ kAoOMgw=
