Index: SPONSORS =================================================================== RCS file: /server/cvs-server/squid/squid/SPONSORS,v retrieving revision 1.6 diff -u -p -r1.6 SPONSORS --- SPONSORS 20 Dec 2001 23:37:32 -0000 1.6 +++ SPONSORS 12 Aug 2002 06:08:51 -0000 @@ -33,3 +33,10 @@ SGI - http://www.sgi.com/ SGI has provided hardware donations for Squid developers. +Zope Corporation - http://www.zope.com/ + + Zope Corporation funded the development of the ESI protocol + (http://www.esi.org) in Squid to provide greater cachability + of dynamic and personalized pages by caching common page + components. Zope engaged one of the core Squid developers + for the project. Index: doc/debug-sections.txt =================================================================== RCS file: /server/cvs-server/squid/squid/doc/debug-sections.txt,v retrieving revision 1.12 diff -u -p -r1.12 debug-sections.txt --- doc/debug-sections.txt 21 Jul 2002 14:59:54 -0000 1.12 +++ doc/debug-sections.txt 12 Aug 2002 06:08:59 -0000 @@ -88,3 +88,6 @@ section 81 Cache Store section 82 External ACL section 83 SSL accelerator support section 84 Helper process maintenance +section 85 Client side request management - after parsing, before caching +section 87 client side stream management +section 88 Client side reply management - from store to stream Index: src/Makefile.am =================================================================== RCS file: /server/cvs-server/squid/squid/src/Makefile.am,v retrieving revision 1.27 diff -u -p -r1.27 Makefile.am --- src/Makefile.am 12 Jul 2002 15:00:10 -0000 1.27 +++ src/Makefile.am 12 Aug 2002 06:09:03 -0000 @@ -114,6 +114,9 @@ squid_SOURCES = \ cbdata.c \ client_db.c \ client_side.c \ + client_side_reply.c \ + client_side_request.c \ + clientStream.c \ comm.c \ comm_select.c \ comm_poll.c \ Index: src/client_side.c =================================================================== RCS file: /server/cvs-server/squid/squid/src/client_side.c,v retrieving revision 1.584 diff -u -p -r1.584 client_side.c --- src/client_side.c 9 Aug 2002 11:13:33 -0000 1.584 +++ src/client_side.c 12 Aug 2002 06:09:05 -0000 @@ -93,57 +93,19 @@ static PF clientReadRequest; static PF connStateFree; static PF requestTimeout; static PF clientLifetimeTimeout; -static int clientCheckTransferDone(clientHttpRequest *); -static int clientGotNotEnough(clientHttpRequest *); static void checkFailureRatio(err_type, hier_code); -static void clientProcessMiss(clientHttpRequest *); -static void clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep); static clientHttpRequest *parseHttpRequestAbort(ConnStateData * conn, const char *uri); static clientHttpRequest *parseHttpRequest(ConnStateData *, method_t *, int *, char **, size_t *); -static RH clientRedirectDone; -static void clientCheckNoCache(clientHttpRequest *); -static void clientCheckNoCacheDone(int answer, void *data); -static STCB clientHandleIMSReply; -static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); -static int checkAccelOnly(clientHttpRequest *); #if USE_IDENT static IDCB clientIdentDone; #endif -static int clientOnlyIfCached(clientHttpRequest * http); -static STCB clientSendMoreData; -static STCB clientCacheHit; +static CSR clientSocketRecipient; static void clientSetKeepaliveFlag(clientHttpRequest *); -static void clientInterpretRequestHeaders(clientHttpRequest *); -static void clientProcessRequest(clientHttpRequest *); -static void clientProcessExpired(void *data); -static void clientProcessOnlyIfCachedMiss(clientHttpRequest * http); -static int clientCachable(clientHttpRequest * http); -static int clientHierarchical(clientHttpRequest * http); static int clientCheckContentLength(request_t * r); static DEFER httpAcceptDefer; -static log_type clientProcessRequest2(clientHttpRequest * http); -static int clientReplyBodyTooLarge(HttpReply *, ssize_t clen); static int clientRequestBodyTooLarge(int clen); static void clientProcessBody(ConnStateData * conn); -static int -checkAccelOnly(clientHttpRequest * http) -{ - /* return TRUE if someone makes a proxy request to us and - * we are in httpd-accel only mode */ - if (!Config2.Accel.on) - return 0; - if (Config.onoff.accel_with_proxy) - return 0; - if (http->request->protocol == PROTO_CACHEOBJ) - return 0; - if (http->flags.accel) - return 0; - if (http->request->method == METHOD_PURGE) - return 0; - return 1; -} - #if USE_IDENT static void clientIdentDone(const char *ident, void *data) @@ -154,556 +116,6 @@ clientIdentDone(const char *ident, void #endif -static aclCheck_t * -clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http) -{ - aclCheck_t *ch; - ConnStateData *conn = http->conn; - ch = aclChecklistCreate(acl, - http->request, - conn->rfc931); - - /* - * hack for ident ACL. It needs to get full addresses, and a - * place to store the ident result on persistent connections... - */ - /* connection oriented auth also needs these two lines for it's operation. */ - ch->conn = cbdataReference(conn); /* unreferenced in acl.c */ - - return ch; -} - -void -clientAccessCheck(void *data) -{ - clientHttpRequest *http = data; - if (checkAccelOnly(http)) { - /* deny proxy requests in accel_only mode */ - debug(33, 1) ("clientAccessCheck: proxy request denied in accel_only mode\n"); - clientAccessCheckDone(ACCESS_DENIED, http); - return; - } - http->acl_checklist = clientAclChecklistCreate(Config.accessList.http, http); - aclNBCheck(http->acl_checklist, clientAccessCheckDone, http); -} - -/* - * returns true if client specified that the object must come from the cache - * without contacting origin server - */ -static int -clientOnlyIfCached(clientHttpRequest * http) -{ - const request_t *r = http->request; - assert(r); - return r->cache_control && - EBIT_TEST(r->cache_control->mask, CC_ONLY_IF_CACHED); -} - -StoreEntry * -clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags) -{ - StoreEntry *e; - /* - * For erroneous requests, we might not have a h->request, - * so make a fake one. - */ - if (h->request == NULL) - h->request = requestLink(requestCreate(m, PROTO_NONE, null_string)); - e = storeCreateEntry(h->uri, h->log_uri, flags, m); - h->sc = storeClientListAdd(e, h); -#if DELAY_POOLS - delaySetStoreClient(h->sc, delayClient(h)); -#endif - h->reqofs = 0; - h->reqsize = 0; - /* I don't think this is actually needed! -- adrian */ - /* h->reqbuf = h->norm_reqbuf; */ - assert(h->reqbuf == h->norm_reqbuf); - storeClientCopy(h->sc, e, 0, HTTP_REQBUF_SZ, h->reqbuf, - clientSendMoreData, h); - return e; -} - -void -clientAccessCheckDone(int answer, void *data) -{ - clientHttpRequest *http = data; - err_type page_id; - http_status status; - ErrorState *err = NULL; - char *proxy_auth_msg = NULL; - debug(33, 2) ("The request %s %s is %s, because it matched '%s'\n", - RequestMethodStr[http->request->method], http->uri, - answer == ACCESS_ALLOWED ? "ALLOWED" : "DENIED", - AclMatchedName ? AclMatchedName : "NO ACL's"); - proxy_auth_msg = authenticateAuthUserRequestMessage(http->conn->auth_user_request ? http->conn->auth_user_request : http->request->auth_user_request); - http->acl_checklist = NULL; - if (answer == ACCESS_ALLOWED) { - safe_free(http->uri); - http->uri = xstrdup(urlCanonical(http->request)); - assert(http->redirect_state == REDIRECT_NONE); - http->redirect_state = REDIRECT_PENDING; - redirectStart(http, clientRedirectDone, http); - } else { - debug(33, 5) ("Access Denied: %s\n", http->uri); - debug(33, 5) ("AclMatchedName = %s\n", - AclMatchedName ? AclMatchedName : ""); - debug(33, 5) ("Proxy Auth Message = %s\n", - proxy_auth_msg ? proxy_auth_msg : ""); - /* - * NOTE: get page_id here, based on AclMatchedName because - * if USE_DELAY_POOLS is enabled, then AclMatchedName gets - * clobbered in the clientCreateStoreEntry() call - * just below. Pedro Ribeiro - */ - page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName); - http->log_type = LOG_TCP_DENIED; - http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); - if (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName)) { - if (!http->flags.accel) { - /* Proxy authorisation needed */ - status = HTTP_PROXY_AUTHENTICATION_REQUIRED; - } else { - /* WWW authorisation needed */ - status = HTTP_UNAUTHORIZED; - } - if (page_id == ERR_NONE) - page_id = ERR_CACHE_ACCESS_DENIED; - } else { - status = HTTP_FORBIDDEN; - if (page_id == ERR_NONE) - page_id = ERR_ACCESS_DENIED; - } - err = errorCon(page_id, status); - err->request = requestLink(http->request); - err->src_addr = http->conn->peer.sin_addr; - if (http->conn->auth_user_request) - err->auth_user_request = http->conn->auth_user_request; - else if (http->request->auth_user_request) - err->auth_user_request = http->request->auth_user_request; - /* lock for the error state */ - if (err->auth_user_request) - authenticateAuthUserRequestLock(err->auth_user_request); - err->callback_data = NULL; - errorAppendEntry(http->entry, err); - } -} - -static void -clientRedirectDone(void *data, char *result) -{ - clientHttpRequest *http = data; - request_t *new_request = NULL; - request_t *old_request = http->request; - debug(33, 5) ("clientRedirectDone: '%s' result=%s\n", http->uri, - result ? result : "NULL"); - assert(http->redirect_state == REDIRECT_PENDING); - http->redirect_state = REDIRECT_DONE; - if (result) { - http_status status = (http_status) atoi(result); - if (status == HTTP_MOVED_PERMANENTLY || status == HTTP_MOVED_TEMPORARILY) { - char *t = result; - if ((t = strchr(result, ':')) != NULL) { - http->redirect.status = status; - http->redirect.location = xstrdup(t + 1); - } else { - debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); - } - } - if (strcmp(result, http->uri)) - new_request = urlParse(old_request->method, result); - } - if (new_request) { - safe_free(http->uri); - http->uri = xstrdup(urlCanonical(new_request)); - new_request->http_ver = old_request->http_ver; - httpHeaderAppend(&new_request->header, &old_request->header); - new_request->client_addr = old_request->client_addr; - new_request->my_addr = old_request->my_addr; - new_request->my_port = old_request->my_port; - new_request->flags.redirected = 1; - if (old_request->auth_user_request) { - new_request->auth_user_request = old_request->auth_user_request; - authenticateAuthUserRequestLock(new_request->auth_user_request); - } - if (old_request->body_connection) { - new_request->body_connection = old_request->body_connection; - old_request->body_connection = NULL; - } - new_request->content_length = old_request->content_length; - new_request->flags.proxy_keepalive = old_request->flags.proxy_keepalive; - requestUnlink(old_request); - http->request = requestLink(new_request); - } - clientInterpretRequestHeaders(http); -#if HEADERS_LOG - headersLog(0, 1, request->method, request); -#endif - fd_note(http->conn->fd, http->uri); - clientCheckNoCache(http); -} - -static void -clientCheckNoCache(clientHttpRequest * http) -{ - if (Config.accessList.noCache && http->request->flags.cachable) { - http->acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http); - aclNBCheck(http->acl_checklist, clientCheckNoCacheDone, http); - } else { - clientCheckNoCacheDone(http->request->flags.cachable, http); - } -} - -void -clientCheckNoCacheDone(int answer, void *data) -{ - clientHttpRequest *http = data; - http->request->flags.cachable = answer; - http->acl_checklist = NULL; - clientProcessRequest(http); -} - -static void -clientProcessExpired(void *data) -{ - clientHttpRequest *http = data; - char *url = http->uri; - StoreEntry *entry = NULL; - debug(33, 3) ("clientProcessExpired: '%s'\n", http->uri); - assert(http->entry->lastmod >= 0); - /* - * check if we are allowed to contact other servers - * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return - * a stale entry *if* it matches client requirements - */ - if (clientOnlyIfCached(http)) { - clientProcessOnlyIfCachedMiss(http); - return; - } - http->request->flags.refresh = 1; - http->old_entry = http->entry; - http->old_sc = http->sc; - http->old_reqsize = http->reqsize; - http->old_reqofs = http->reqofs; - http->reqbuf = http->ims_reqbuf; -#if STORE_CLIENT_LIST_DEBUG - /* - * Assert that 'http' is already a client of old_entry. If - * it is not, then the beginning of the object data might get - * freed from memory before we need to access it. - */ - assert(http->sc->owner == http); -#endif - entry = storeCreateEntry(url, - http->log_uri, - http->request->flags, - http->request->method); - /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ - http->sc = storeClientListAdd(entry, http); -#if DELAY_POOLS - /* delay_id is already set on original store client */ - delaySetStoreClient(http->sc, delayClient(http)); -#endif - http->request->lastmod = http->old_entry->lastmod; - debug(33, 5) ("clientProcessExpired: lastmod %ld\n", (long int) entry->lastmod); - http->entry = entry; - http->out.offset = 0; - fwdStart(http->conn->fd, http->entry, http->request); - /* Register with storage manager to receive updates when data comes in. */ - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) - debug(33, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n"); - http->reqofs = 0; - storeClientCopy(http->sc, entry, - http->out.offset, - HTTP_REQBUF_SZ, - http->reqbuf, - clientHandleIMSReply, - http); -} - -static int -clientGetsOldEntry(StoreEntry * new_entry, StoreEntry * old_entry, request_t * request) -{ - const http_status status = new_entry->mem_obj->reply->sline.status; - if (0 == status) { - debug(33, 5) ("clientGetsOldEntry: YES, broken HTTP reply\n"); - return 1; - } - /* If the reply is a failure then send the old object as a last - * resort */ - if (status >= 500 && status < 600) { - debug(33, 3) ("clientGetsOldEntry: YES, failure reply=%d\n", status); - return 1; - } - /* If the reply is anything but "Not Modified" then - * we must forward it to the client */ - if (HTTP_NOT_MODIFIED != status) { - debug(33, 5) ("clientGetsOldEntry: NO, reply=%d\n", status); - return 0; - } - /* If the client did not send IMS in the request, then it - * must get the old object, not this "Not Modified" reply */ - if (!request->flags.ims) { - debug(33, 5) ("clientGetsOldEntry: YES, no client IMS\n"); - return 1; - } - /* If the client IMS time is prior to the entry LASTMOD time we - * need to send the old object */ - if (modifiedSince(old_entry, request)) { - debug(33, 5) ("clientGetsOldEntry: YES, modified since %ld\n", - (long int) request->ims); - return 1; - } - debug(33, 5) ("clientGetsOldEntry: NO, new one is fine\n"); - return 0; -} - - -static void -clientHandleIMSReply(void *data, char *buf, ssize_t size) -{ - clientHttpRequest *http = data; - StoreEntry *entry = http->entry; - MemObject *mem; - const char *url = storeUrl(entry); - int unlink_request = 0; - StoreEntry *oldentry; - http_status status; - debug(33, 3) ("clientHandleIMSReply: %s, %ld bytes\n", url, (long int) size); - if (entry == NULL) { - return; - } - if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - return; - } - /* update size of the request */ - http->reqsize = size + http->reqofs; - mem = entry->mem_obj; - status = mem->reply->sline.status; - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - debug(33, 3) ("clientHandleIMSReply: ABORTED '%s'\n", url); - /* We have an existing entry, but failed to validate it */ - /* Its okay to send the old one anyway */ - http->log_type = LOG_TCP_REFRESH_FAIL_HIT; - storeUnregister(http->sc, entry, http); - storeUnlockObject(entry); - entry = http->entry = http->old_entry; - http->sc = http->old_sc; - http->reqbuf = http->norm_reqbuf; - http->reqofs = http->old_reqofs; - http->reqsize = http->old_reqsize; - } else if (STORE_PENDING == entry->store_status && 0 == status) { - debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); - if (size + http->reqofs >= HTTP_REQBUF_SZ) { - /* will not get any bigger than that */ - debug(33, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url); - /* use old entry, this repeats the code abovez */ - http->log_type = LOG_TCP_REFRESH_FAIL_HIT; - storeUnregister(http->sc, entry, http); - storeUnlockObject(entry); - entry = http->entry = http->old_entry; - http->sc = http->old_sc; - http->reqbuf = http->norm_reqbuf; - http->reqofs = http->old_reqofs; - http->reqsize = http->old_reqsize; - /* continue */ - } else { - http->reqofs += size; - storeClientCopy(http->sc, entry, - http->out.offset + http->reqofs, - HTTP_REQBUF_SZ - http->reqofs, - http->reqbuf + http->reqofs, - clientHandleIMSReply, - http); - return; - } - } else if (clientGetsOldEntry(entry, http->old_entry, http->request)) { - /* We initiated the IMS request, the client is not expecting - * 304, so put the good one back. First, make sure the old entry - * headers have been loaded from disk. */ - oldentry = http->old_entry; - http->log_type = LOG_TCP_REFRESH_HIT; - if (oldentry->mem_obj->request == NULL) { - oldentry->mem_obj->request = requestLink(mem->request); - unlink_request = 1; - } - /* Don't memcpy() the whole reply structure here. For example, - * www.thegist.com (Netscape/1.13) returns a content-length for - * 304's which seems to be the length of the 304 HEADERS!!! and - * not the body they refer to. */ - httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply); - storeTimestampsSet(oldentry); - storeUnregister(http->sc, entry, http); - http->sc = http->old_sc; - storeUnlockObject(entry); - entry = http->entry = oldentry; - entry->timestamp = squid_curtime; - if (unlink_request) { - requestUnlink(entry->mem_obj->request); - entry->mem_obj->request = NULL; - } - http->reqbuf = http->norm_reqbuf; - http->reqofs = http->old_reqofs; - http->reqsize = http->old_reqsize; - } else { - /* the client can handle this reply, whatever it is */ - http->log_type = LOG_TCP_REFRESH_MISS; - if (HTTP_NOT_MODIFIED == mem->reply->sline.status) { - httpReplyUpdateOnNotModified(http->old_entry->mem_obj->reply, - mem->reply); - storeTimestampsSet(http->old_entry); - http->log_type = LOG_TCP_REFRESH_HIT; - } - storeUnregister(http->old_sc, http->old_entry, http); - storeUnlockObject(http->old_entry); - } - http->old_entry = NULL; /* done with old_entry */ - http->old_sc = NULL; - http->old_reqofs = 0; - http->old_reqsize = 0; - assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); - - clientSendMoreData(data, http->reqbuf, http->reqsize); -} - -int -modifiedSince(StoreEntry * entry, request_t * request) -{ - int object_length; - MemObject *mem = entry->mem_obj; - time_t mod_time = entry->lastmod; - debug(33, 3) ("modifiedSince: '%s'\n", storeUrl(entry)); - if (mod_time < 0) - mod_time = entry->timestamp; - debug(33, 3) ("modifiedSince: mod_time = %ld\n", (long int) mod_time); - if (mod_time < 0) - return 1; - /* Find size of the object */ - object_length = mem->reply->content_length; - if (object_length < 0) - object_length = contentLen(entry); - if (mod_time > request->ims) { - debug(33, 3) ("--> YES: entry newer than client\n"); - return 1; - } else if (mod_time < request->ims) { - debug(33, 3) ("--> NO: entry older than client\n"); - return 0; - } else if (request->imslen < 0) { - debug(33, 3) ("--> NO: same LMT, no client length\n"); - return 0; - } else if (request->imslen == object_length) { - debug(33, 3) ("--> NO: same LMT, same length\n"); - return 0; - } else { - debug(33, 3) ("--> YES: same LMT, different length\n"); - return 1; - } -} - -void -clientPurgeRequest(clientHttpRequest * http) -{ - StoreEntry *entry; - ErrorState *err = NULL; - HttpReply *r; - http_status status = HTTP_NOT_FOUND; - http_version_t version; - debug(33, 3) ("Config2.onoff.enable_purge = %d\n", Config2.onoff.enable_purge); - if (!Config2.onoff.enable_purge) { - http->log_type = LOG_TCP_DENIED; - err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); - err->request = requestLink(http->request); - err->src_addr = http->conn->peer.sin_addr; - http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); - errorAppendEntry(http->entry, err); - return; - } - /* Release both IP cache */ - ipcacheInvalidate(http->request->host); - - if (!http->flags.purging) { - /* Try to find a base entry */ - http->flags.purging = 1; - entry = storeGetPublicByRequestMethod(http->request, METHOD_GET); - if (!entry) - entry = storeGetPublicByRequestMethod(http->request, METHOD_HEAD); - if (entry) { - /* Swap in the metadata */ - http->entry = entry; - storeLockObject(http->entry); - storeCreateMemObject(http->entry, http->uri, http->log_uri); - http->entry->mem_obj->method = http->request->method; - http->sc = storeClientListAdd(http->entry, http); - http->log_type = LOG_TCP_HIT; - http->reqofs = 0; - storeClientCopy(http->sc, http->entry, - http->out.offset, - HTTP_REQBUF_SZ, - http->reqbuf, - clientCacheHit, - http); - return; - } - } - http->log_type = LOG_TCP_MISS; - /* Release the cached URI */ - entry = storeGetPublicByRequestMethod(http->request, METHOD_GET); - if (entry) { - debug(33, 4) ("clientPurgeRequest: GET '%s'\n", - storeUrl(entry)); - storeRelease(entry); - status = HTTP_OK; - } - entry = storeGetPublicByRequestMethod(http->request, METHOD_HEAD); - if (entry) { - debug(33, 4) ("clientPurgeRequest: HEAD '%s'\n", - storeUrl(entry)); - storeRelease(entry); - status = HTTP_OK; - } - /* And for Vary, release the base URI if none of the headers was included in the request */ - if (http->request->vary_headers && !strstr(http->request->vary_headers, "=")) { - entry = storeGetPublic(urlCanonical(http->request), METHOD_GET); - if (entry) { - debug(33, 4) ("clientPurgeRequest: Vary GET '%s'\n", - storeUrl(entry)); - storeRelease(entry); - status = HTTP_OK; - } - entry = storeGetPublic(urlCanonical(http->request), METHOD_HEAD); - if (entry) { - debug(33, 4) ("clientPurgeRequest: Vary HEAD '%s'\n", - storeUrl(entry)); - storeRelease(entry); - status = HTTP_OK; - } - } - /* - * Make a new entry to hold the reply to be written - * to the client. - */ - http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); - httpReplyReset(r = http->entry->mem_obj->reply); - httpBuildVersion(&version, 1, 0); - httpReplySetHeaders(r, version, status, NULL, NULL, 0, 0, -1); - httpReplySwapOut(r, http->entry); - storeComplete(http->entry); -} - -int -checkNegativeHit(StoreEntry * e) -{ - if (!EBIT_TEST(e->flags, ENTRY_NEGCACHED)) - return 0; - if (e->expires <= squid_curtime) - return 0; - if (e->store_status != STORE_OK) - return 0; - return 1; -} - static void clientUpdateCounters(clientHttpRequest * http) { @@ -851,6 +263,12 @@ httpRequestFree(void *data) storeUnlockObject(e); } requestUnlink(http->request); + /* TODO: push refcount class through to head */ + { + clientStreamNode *temp = http->stream_head; + cbdataReferenceDone (temp); + cbdataFree (http->stream_head); + } assert(http != http->next); assert(http->conn->chr != NULL); /* Unlink us from the clients request list */ @@ -897,124 +315,6 @@ connStateFree(int fd, void *data) #endif } -static void -clientInterpretRequestHeaders(clientHttpRequest * http) -{ - request_t *request = http->request; - const HttpHeader *req_hdr = &request->header; - int no_cache = 0; - const char *str; - request->imslen = -1; - request->ims = httpHeaderGetTime(req_hdr, HDR_IF_MODIFIED_SINCE); - if (request->ims > 0) - request->flags.ims = 1; - if (httpHeaderHas(req_hdr, HDR_PRAGMA)) { - String s = httpHeaderGetList(req_hdr, HDR_PRAGMA); - if (strListIsMember(&s, "no-cache", ',')) - no_cache++; - stringClean(&s); - } - request->cache_control = httpHeaderGetCc(req_hdr); - if (request->cache_control) - if (EBIT_TEST(request->cache_control->mask, CC_NO_CACHE)) - no_cache++; - /* Work around for supporting the Reload button in IE browsers - * when Squid is used as an accelerator or transparent proxy, - * by turning accelerated IMS request to no-cache requests. - * Now knows about IE 5.5 fix (is actually only fixed in SP1, - * but we can't tell whether we are talking to SP1 or not so - * all 5.5 versions are treated 'normally'). - */ - if (Config.onoff.ie_refresh) { - if (http->flags.accel && request->flags.ims) { - if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) { - if (strstr(str, "MSIE 5.01") != NULL) - no_cache++; - else if (strstr(str, "MSIE 5.0") != NULL) - no_cache++; - else if (strstr(str, "MSIE 4.") != NULL) - no_cache++; - else if (strstr(str, "MSIE 3.") != NULL) - no_cache++; - } - } - } - if (no_cache) { -#if HTTP_VIOLATIONS - if (Config.onoff.reload_into_ims) - request->flags.nocache_hack = 1; - else if (refresh_nocache_hack) - request->flags.nocache_hack = 1; - else -#endif - request->flags.nocache = 1; - } - /* ignore range header in non-GETs */ - if (request->method == METHOD_GET) { - /* - * Since we're not doing ranges atm, just set the flag if - * the header exists, and then free the range header info - * -- adrian - */ - request->range = httpHeaderGetRange(req_hdr); - if (request->range) { - request->flags.range = 1; - httpHdrRangeDestroy(request->range); - request->range = NULL; - } - } - if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION)) - request->flags.auth = 1; - if (request->login[0] != '\0') - request->flags.auth = 1; - if (httpHeaderHas(req_hdr, HDR_VIA)) { - String s = httpHeaderGetList(req_hdr, HDR_VIA); - /* - * ThisCache cannot be a member of Via header, "1.0 ThisCache" can. - * Note ThisCache2 has a space prepended to the hostname so we don't - * accidentally match super-domains. - */ - if (strListIsSubstr(&s, ThisCache2, ',')) { - debugObj(33, 1, "WARNING: Forwarding loop detected for:\n", - request, (ObjPackMethod) & httpRequestPack); - request->flags.loopdetect = 1; - } -#if FORW_VIA_DB - fvdbCountVia(strBuf(s)); -#endif - stringClean(&s); - } -#if USE_USERAGENT_LOG - if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) - logUserAgent(fqdnFromAddr(http->conn->peer.sin_addr), str); -#endif -#if USE_REFERER_LOG - if ((str = httpHeaderGetStr(req_hdr, HDR_REFERER))) - logReferer(fqdnFromAddr(http->conn->peer.sin_addr), str, - http->log_uri); -#endif -#if FORW_VIA_DB - if (httpHeaderHas(req_hdr, HDR_X_FORWARDED_FOR)) { - String s = httpHeaderGetList(req_hdr, HDR_X_FORWARDED_FOR); - fvdbCountForw(strBuf(s)); - stringClean(&s); - } -#endif - if (request->method == METHOD_TRACE) { - request->max_forwards = httpHeaderGetInt(req_hdr, HDR_MAX_FORWARDS); - } - if (clientCachable(http)) - request->flags.cachable = 1; - if (clientHierarchical(http)) - request->flags.hierarchical = 1; - debug(33, 5) ("clientInterpretRequestHeaders: REQ_NOCACHE = %s\n", - request->flags.nocache ? "SET" : "NOT SET"); - debug(33, 5) ("clientInterpretRequestHeaders: REQ_CACHABLE = %s\n", - request->flags.cachable ? "SET" : "NOT SET"); - debug(33, 5) ("clientInterpretRequestHeaders: REQ_HIERARCHICAL = %s\n", - request->flags.hierarchical ? "SET" : "NOT SET"); -} - /* * clientSetKeepaliveFlag() sets request->flags.proxy_keepalive. * This is the client-side persistent connection flag. We need @@ -1055,67 +355,6 @@ clientCheckContentLength(request_t * r) /* NOT REACHED */ } -static int -clientCachable(clientHttpRequest * http) -{ - request_t *req = http->request; - method_t method = req->method; - if (req->protocol == PROTO_HTTP) - return httpCachable(method); - /* FTP is always cachable */ - if (req->protocol == PROTO_WAIS) - return 0; - if (method == METHOD_CONNECT) - return 0; - if (method == METHOD_TRACE) - return 0; - if (method == METHOD_PUT) - return 0; - if (method == METHOD_POST) - return 0; /* XXX POST may be cached sometimes.. ignored for now */ - if (req->protocol == PROTO_GOPHER) - return gopherCachable(req); - if (req->protocol == PROTO_CACHEOBJ) - return 0; - return 1; -} - -/* Return true if we can query our neighbors for this object */ -static int -clientHierarchical(clientHttpRequest * http) -{ - const char *url = http->uri; - request_t *request = http->request; - method_t method = request->method; - const wordlist *p = NULL; - - /* IMS needs a private key, so we can use the hierarchy for IMS only - * if our neighbors support private keys */ - if (request->flags.ims && !neighbors_do_private_keys) - return 0; - if (request->flags.auth) - return 0; - if (method == METHOD_TRACE) - return 1; - if (method != METHOD_GET) - return 0; - /* scan hierarchy_stoplist */ - for (p = Config.hierarchy_stoplist; p; p = p->next) - if (strstr(url, p->key)) - return 0; - if (request->flags.loopdetect) - return 0; - if (request->protocol == PROTO_HTTP) - return httpCachable(method); - if (request->protocol == PROTO_GOPHER) - return gopherCachable(request); - if (request->protocol == PROTO_WAIS) - return 0; - if (request->protocol == PROTO_CACHEOBJ) - return 0; - return 1; -} - int isTcpHit(log_type code) { @@ -1127,345 +366,13 @@ isTcpHit(log_type code) if (code == LOG_TCP_REFRESH_FAIL_HIT) return 1; if (code == LOG_TCP_REFRESH_HIT) - return 1; - if (code == LOG_TCP_NEGATIVE_HIT) - return 1; - if (code == LOG_TCP_MEM_HIT) - return 1; - if (code == LOG_TCP_OFFLINE_HIT) - return 1; - return 0; -} - - -/* - * filters out unwanted entries from original reply header - * adds extra entries if we have more info than origin server - * adds Squid specific entries - */ -static void -clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep) -{ - HttpHeader *hdr = &rep->header; - int is_hit = isTcpHit(http->log_type); - request_t *request = http->request; -#if DONT_FILTER_THESE - /* but you might want to if you run Squid as an HTTP accelerator */ - /* httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); */ - httpHeaderDelById(hdr, HDR_ETAG); -#endif - httpHeaderDelById(hdr, HDR_PROXY_CONNECTION); - /* here: Keep-Alive is a field-name, not a connection directive! */ - httpHeaderDelByName(hdr, "Keep-Alive"); - /* remove Set-Cookie if a hit */ - if (is_hit) - httpHeaderDelById(hdr, HDR_SET_COOKIE); - /* handle Connection header */ - if (httpHeaderHas(hdr, HDR_CONNECTION)) { - /* anything that matches Connection list member will be deleted */ - String strConnection = httpHeaderGetList(hdr, HDR_CONNECTION); - const HttpHeaderEntry *e; - HttpHeaderPos pos = HttpHeaderInitPos; - /* - * think: on-average-best nesting of the two loops (hdrEntry - * and strListItem) @?@ - */ - /* - * maybe we should delete standard stuff ("keep-alive","close") - * from strConnection first? - */ - while ((e = httpHeaderGetEntry(hdr, &pos))) { - if (strListIsMember(&strConnection, strBuf(e->name), ',')) - httpHeaderDelAt(hdr, pos); - } - httpHeaderDelById(hdr, HDR_CONNECTION); - stringClean(&strConnection); - } - /* - * Add a estimated Age header on cache hits. - */ - if (is_hit) { - /* - * Remove any existing Age header sent by upstream caches - * (note that the existing header is passed along unmodified - * on cache misses) - */ - httpHeaderDelById(hdr, HDR_AGE); - /* - * This adds the calculated object age. Note that the details of the - * age calculation is performed by adjusting the timestamp in - * storeTimestampsSet(), not here. - * - * BROWSER WORKAROUND: IE sometimes hangs when receiving a 0 Age - * header, so don't use it unless there is a age to report. Please - * note that Age is only used to make a conservative estimation of - * the objects age, so a Age: 0 header does not add any useful - * information to the reply in any case. - */ - if (NULL == http->entry) - (void) 0; - else if (http->entry->timestamp < 0) - (void) 0; - else if (http->entry->timestamp < squid_curtime) - httpHeaderPutInt(hdr, HDR_AGE, - squid_curtime - http->entry->timestamp); - } - /* Handle authentication headers */ - if (request->auth_user_request) - authenticateFixHeader(rep, request->auth_user_request, request, http->flags.accel, 0); - /* Append X-Cache */ - httpHeaderPutStrf(hdr, HDR_X_CACHE, "%s from %s", - is_hit ? "HIT" : "MISS", getMyHostname()); -#if USE_CACHE_DIGESTS - /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */ - httpHeaderPutStrf(hdr, HDR_X_CACHE_LOOKUP, "%s from %s:%d", - http->lookup_type ? http->lookup_type : "NONE", - getMyHostname(), getMyPort()); -#endif - if (httpReplyBodySize(request->method, rep) < 0) { - debug(33, 3) ("clientBuildReplyHeader: can't keep-alive, unknown body size\n"); - request->flags.proxy_keepalive = 0; - } - /* Signal keep-alive if needed */ - httpHeaderPutStr(hdr, - http->flags.accel ? HDR_CONNECTION : HDR_PROXY_CONNECTION, - request->flags.proxy_keepalive ? "keep-alive" : "close"); -#if ADD_X_REQUEST_URI - /* - * Knowing the URI of the request is useful when debugging persistent - * connections in a client; we cannot guarantee the order of http headers, - * but X-Request-URI is likely to be the very last header to ease use from a - * debugger [hdr->entries.count-1]. - */ - httpHeaderPutStr(hdr, HDR_X_REQUEST_URI, - http->entry->mem_obj->url ? http->entry->mem_obj->url : http->uri); -#endif - httpHdrMangleList(hdr, request); -} - -static HttpReply * -clientBuildReply(clientHttpRequest * http, const char *buf, size_t size) -{ - HttpReply *rep = httpReplyCreate(); - size_t k = headersEnd(buf, size); - if (k && httpReplyParse(rep, buf, k)) { - /* enforce 1.0 reply version */ - httpBuildVersion(&rep->sline.version, 1, 0); - /* do header conversions */ - clientBuildReplyHeader(http, rep); - } else { - /* parsing failure, get rid of the invalid reply */ - httpReplyDestroy(rep); - rep = NULL; - } - return rep; -} - -/* - * clientCacheHit should only be called until the HTTP reply headers - * have been parsed. Normally this should be a single call, but - * it might take more than one. As soon as we have the headers, - * we hand off to clientSendMoreData, clientProcessExpired, or - * clientProcessMiss. - */ -static void -clientCacheHit(void *data, char *buf, ssize_t size) -{ - clientHttpRequest *http = data; - StoreEntry *e = http->entry; - MemObject *mem; - request_t *r = http->request; - debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); - if (http->entry == NULL) { - debug(33, 3) ("clientCacheHit: request aborted\n"); - return; - } else if (size < 0) { - /* swap in failure */ - debug(33, 3) ("clientCacheHit: swapin failure for %s\n", http->uri); - http->log_type = LOG_TCP_SWAPFAIL_MISS; - if ((e = http->entry)) { - http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; - storeUnlockObject(e); - } - clientProcessMiss(http); - return; - } - assert(size > 0); - mem = e->mem_obj; - assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); - /* update size of the request */ - http->reqsize = size + http->reqofs; - if (mem->reply->sline.status == 0) { - /* - * we don't have full reply headers yet; either wait for more or - * punt to clientProcessMiss. - */ - if (e->mem_status == IN_MEMORY || e->store_status == STORE_OK) { - clientProcessMiss(http); - } else if (size + http->reqofs >= HTTP_REQBUF_SZ && http->out.offset == 0) { - clientProcessMiss(http); - } else { - debug(33, 3) ("clientCacheHit: waiting for HTTP reply headers\n"); - http->reqofs += size; - assert(http->reqofs <= HTTP_REQBUF_SZ); - storeClientCopy(http->sc, e, - http->out.offset + http->reqofs, - HTTP_REQBUF_SZ, - http->reqbuf + http->reqofs, - clientCacheHit, - http); - } - return; - } - /* - * Got the headers, now grok them - */ - assert(http->log_type == LOG_TCP_HIT); - switch (varyEvaluateMatch(e, r)) { - case VARY_NONE: - /* No variance detected. Continue as normal */ - break; - case VARY_MATCH: - /* This is the correct entity for this request. Continue */ - debug(33, 2) ("clientProcessHit: Vary MATCH!\n"); - break; - case VARY_OTHER: - /* This is not the correct entity for this request. We need - * to requery the cache. - */ - http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; - storeUnlockObject(e); - /* Note: varyEvalyateMatch updates the request with vary information - * so we only get here once. (it also takes care of cancelling loops) - */ - debug(33, 2) ("clientProcessHit: Vary detected!\n"); - clientProcessRequest(http); - return; - case VARY_CANCEL: - /* varyEvaluateMatch found a object loop. Process as miss */ - debug(33, 1) ("clientProcessHit: Vary object loop!\n"); - clientProcessMiss(http); - return; - } - if (r->method == METHOD_PURGE) { - http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; - storeUnlockObject(e); - clientPurgeRequest(http); - return; - } - if (checkNegativeHit(e)) { - http->log_type = LOG_TCP_NEGATIVE_HIT; - clientSendMoreData(data, buf, size); - } else if (r->method == METHOD_HEAD) { - /* - * RFC 2068 seems to indicate there is no "conditional HEAD" - * request. We cannot validate a cached object for a HEAD - * request, nor can we return 304. - */ - if (e->mem_status == IN_MEMORY) - http->log_type = LOG_TCP_MEM_HIT; - clientSendMoreData(data, buf, size); - } else if (refreshCheckHTTP(e, r) && !http->flags.internal) { - debug(33, 5) ("clientCacheHit: in refreshCheck() block\n"); - /* - * We hold a stale copy; it needs to be validated - */ - /* - * The 'need_validation' flag is used to prevent forwarding - * loops between siblings. If our copy of the object is stale, - * then we should probably only use parents for the validation - * request. Otherwise two siblings could generate a loop if - * both have a stale version of the object. - */ - r->flags.need_validation = 1; - if (e->lastmod < 0) { - /* - * Previous reply didn't have a Last-Modified header, - * we cannot revalidate it. - */ - http->log_type = LOG_TCP_MISS; - clientProcessMiss(http); - } else if (r->flags.nocache) { - /* - * This did not match a refresh pattern that overrides no-cache - * we should honour the client no-cache header. - */ - http->log_type = LOG_TCP_CLIENT_REFRESH_MISS; - clientProcessMiss(http); - } else if (r->protocol == PROTO_HTTP) { - /* - * Object needs to be revalidated - * XXX This could apply to FTP as well, if Last-Modified is known. - */ - http->log_type = LOG_TCP_REFRESH_MISS; - clientProcessExpired(http); - } else { - /* - * We don't know how to re-validate other protocols. Handle - * them as if the object has expired. - */ - http->log_type = LOG_TCP_MISS; - clientProcessMiss(http); - } - } else if (r->flags.ims) { - /* - * Handle If-Modified-Since requests from the client - */ - if (mem->reply->sline.status != HTTP_OK) { - debug(33, 4) ("clientCacheHit: Reply code %d != 200\n", - mem->reply->sline.status); - http->log_type = LOG_TCP_MISS; - clientProcessMiss(http); - } else if (modifiedSince(e, http->request)) { - http->log_type = LOG_TCP_IMS_HIT; - clientSendMoreData(data, buf, size); - } else { - time_t timestamp = e->timestamp; - MemBuf mb = httpPacked304Reply(e->mem_obj->reply); - http->log_type = LOG_TCP_IMS_HIT; - storeUnregister(http->sc, e, http); - http->sc = NULL; - storeUnlockObject(e); - e = clientCreateStoreEntry(http, http->request->method, null_request_flags); - /* - * Copy timestamp from the original entry so the 304 - * reply has a meaningful Age: header. - */ - e->timestamp = timestamp; - http->entry = e; - httpReplyParse(e->mem_obj->reply, mb.buf, mb.size); - storeAppend(e, mb.buf, mb.size); - memBufClean(&mb); - storeComplete(e); - } - } else { - /* - * plain ol' cache hit - */ - if (e->mem_status == IN_MEMORY) - http->log_type = LOG_TCP_MEM_HIT; - else if (Config.onoff.offline) - http->log_type = LOG_TCP_OFFLINE_HIT; - clientSendMoreData(data, buf, size); - } -} - - -static int -clientReplyBodyTooLarge(HttpReply * rep, ssize_t clen) -{ - if (0 == rep->maxBodySize) - return 0; /* disabled */ - if (clen < 0) - return 0; /* unknown */ - if (clen > rep->maxBodySize) - return 1; /* too large */ + return 1; + if (code == LOG_TCP_NEGATIVE_HIT) + return 1; + if (code == LOG_TCP_MEM_HIT) + return 1; + if (code == LOG_TCP_OFFLINE_HIT) + return 1; return 0; } @@ -1481,199 +388,64 @@ clientRequestBodyTooLarge(int clen) return 0; } - -/* Responses with no body will not have a content-type header, - * which breaks the rep_mime_type acl, which - * coincidentally, is the most common acl for reply access lists. - * A better long term fix for this is to allow acl matchs on the various - * status codes, and then supply a default ruleset that puts these - * codes before any user defines access entries. That way the user - * can choose to block these responses where appropriate, but won't get - * mysterious breakages. - */ -static int -clientAlwaysAllowResponse(http_status sline) -{ - switch (sline) { - case HTTP_CONTINUE: - case HTTP_SWITCHING_PROTOCOLS: - case HTTP_PROCESSING: - case HTTP_NO_CONTENT: - case HTTP_NOT_MODIFIED: - return 1; - /* unreached */ - break; - default: - return 0; - } -} - - /* - * accepts chunk of a http message in buf, parses prefix, filters headers and - * such, writes processed message to the client's socket + * Write a chunk of data to a client socket. If the reply is present, send the reply headers down the wire too, + * and clean them up when finished. + * Pre-condition: + * The request is one backed by a connection, not an internal request. + * data context is null + * There are no more entries in the stream chain. */ static void -clientSendMoreData(void *data, char *retbuf, ssize_t retsize) +clientSocketRecipient (clientStreamNode *node, clientHttpRequest *http, HttpReply *rep, const char *body_data, ssize_t body_size) { - clientHttpRequest *http = data; - StoreEntry *entry = http->entry; - ConnStateData *conn = http->conn; - int fd = conn->fd; - HttpReply *rep = NULL; - char *buf = http->reqbuf; - const char *body_buf = buf; - ssize_t size = http->reqofs + retsize; - ssize_t body_size = size; - MemBuf mb; - ssize_t check_size = 0; - - debug(33, 5) ("clientSendMoreData: %s, %d bytes (%d new bytes)\n", http->uri, (int) size, retsize); - assert(size <= HTTP_REQBUF_SZ); - assert(http->request != NULL); - dlinkDelete(&http->active, &ClientActiveRequests); - dlinkAdd(http, &http->active, &ClientActiveRequests); - debug(33, 5) ("clientSendMoreData: FD %d '%s', out.offset=%ld \n", - fd, storeUrl(entry), (long int) http->out.offset); - /* update size of the request */ - http->reqsize = size; - if (conn->chr != http) { - /* there is another object in progress, defer this one */ - debug(33, 2) ("clientSendMoreData: Deferring %s\n", storeUrl(entry)); - return; - } else if (http->request->flags.reset_tcp) { - comm_reset_close(fd); - return; - } else if (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - /* call clientWriteComplete so the client socket gets closed */ - clientWriteComplete(fd, NULL, 0, COMM_OK, http); - return; - } else if (retsize < 0) { - /* call clientWriteComplete so the client socket gets closed */ - clientWriteComplete(fd, NULL, 0, COMM_OK, http); - return; - } else if (retsize == 0) { - /* call clientWriteComplete so the client socket gets closed */ + int fd; + /* Test preconditions */ + assert (node != NULL); + /* TODO: handle this rather than asserting - it should only ever happen if we cause an abort and the callback chain + * loops back to here, so we can simply return. However, that itself shouldn't happen, so it stays as an assert for now. */ + assert (cbdataReferenceValid (node)); + assert (node->data == NULL); + assert (node->next == NULL); + assert (http->conn && http->conn->fd != -1); + fd = http->conn->fd; + /* EOF / Read error / aborted entry */ + if (rep == NULL && body_data == NULL && body_size == 0) { clientWriteComplete(fd, NULL, 0, COMM_OK, http); return; } - if (http->out.offset == 0) { - if (Config.onoff.log_mime_hdrs) { - size_t k; - if ((k = headersEnd(buf, size))) { - safe_free(http->al.headers.reply); - http->al.headers.reply = xcalloc(k + 1, 1); - xstrncpy(http->al.headers.reply, buf, k); - } - } - rep = clientBuildReply(http, buf, size); - if (rep) { - aclCheck_t *ch; - int rv; - httpReplyBodyBuildSize(http->request, rep, &Config.ReplyBodySize); - if (clientReplyBodyTooLarge(rep, rep->content_length)) { - ErrorState *err = errorCon(ERR_TOO_BIG, HTTP_FORBIDDEN); - err->request = requestLink(http->request); - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; - storeUnlockObject(http->entry); - http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); - errorAppendEntry(http->entry, err); - httpReplyDestroy(rep); - return; - } - body_size = size - rep->hdr_sz; - assert(body_size >= 0); - body_buf = buf + rep->hdr_sz; - debug(33, 3) ("clientSendMoreData: Appending %d bytes after %d bytes of headers\n", - (int) body_size, rep->hdr_sz); - ch = aclChecklistCreate(Config.accessList.reply, http->request, NULL); - ch->reply = rep; - rv = aclCheckFast(Config.accessList.reply, ch); - aclChecklistFree(ch); - ch = NULL; - debug(33, 2) ("The reply for %s %s is %s, because it matched '%s'\n", - RequestMethodStr[http->request->method], http->uri, - rv ? "ALLOWED" : "DENIED", - AclMatchedName ? AclMatchedName : "NO ACL's"); - if (!rv && rep->sline.status != HTTP_FORBIDDEN - && !clientAlwaysAllowResponse(rep->sline.status)) { - /* the if above is slightly broken, but there is no way - * to tell if this is a squid generated error page, or one from - * upstream at this point. */ - ErrorState *err; - err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); - err->request = requestLink(http->request); - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; - storeUnlockObject(http->entry); - http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); - errorAppendEntry(http->entry, err); - httpReplyDestroy(rep); - return; - } - } else if (size < HTTP_REQBUF_SZ && entry->store_status == STORE_PENDING) { - /* wait for more to arrive */ - http->reqofs += retsize; - assert(http->reqofs <= HTTP_REQBUF_SZ); - storeClientCopy(http->sc, entry, - http->out.offset + http->reqofs, - HTTP_REQBUF_SZ - http->reqofs, - http->reqbuf + http->reqofs, - clientSendMoreData, - http); - return; - } - } else { + /* trivial case */ + if (http->out.offset != 0) { + assert(rep == NULL); /* Avoid copying to MemBuf if we know "rep" is NULL, and we only have a body */ http->out.offset += body_size; - assert(rep == NULL); - comm_write(fd, buf, size, clientWriteBodyComplete, http, NULL); + comm_write(fd, body_data, body_size, clientWriteBodyComplete, http, NULL); /* NULL because clientWriteBodyComplete frees it */ return; - } - if (http->request->method == METHOD_HEAD) { + } else { + MemBuf mb; + /* write headers and/or body if any */ + assert(rep || (body_data && body_size)); + /* init mb; put status line and headers if any */ if (rep) { - /* do not forward body for HEAD replies */ - body_size = 0; - http->flags.done_copying = 1; - } else { - /* - * If we are here, then store_status == STORE_OK and it - * seems we have a HEAD repsponse which is missing the - * empty end-of-headers line (home.mira.net, phttpd/0.99.72 - * does this). Because clientBuildReply() fails we just - * call this reply a body, set the done_copying flag and - * continue... - */ - http->flags.done_copying = 1; - } - } - /* write headers and/or body if any */ - assert(rep || (body_buf && body_size)); - /* init mb; put status line and headers if any */ - if (rep) { - mb = httpReplyPack(rep); - http->out.offset += rep->hdr_sz; - check_size += rep->hdr_sz; + mb = httpReplyPack(rep); + http->out.offset += rep->hdr_sz; #if HEADERS_LOG - headersLog(0, 0, http->request->method, rep); + headersLog(0, 0, http->request->method, rep); #endif - httpReplyDestroy(rep); - rep = NULL; - } else { - memBufDefInit(&mb); - } - if (body_buf && body_size) { - http->out.offset += body_size; - check_size += body_size; - memBufAppend(&mb, body_buf, body_size); + httpReplyDestroy(rep); + rep = NULL; + } else { + memBufDefInit(&mb); + } + if (body_data && body_size) { + http->out.offset += body_size; + memBufAppend(&mb, body_data, body_size); + } + /* write */ + comm_write_mbuf(fd, mb, clientWriteComplete, http); + /* if we don't do it, who will? */ } - /* write */ - comm_write_mbuf(fd, mb, clientWriteComplete, http); - /* if we don't do it, who will? */ } /* @@ -1735,332 +507,81 @@ clientKeepaliveNextRequest(clientHttpReq if (0 == storeClientCopyPending(http->sc, entry, http)) { if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n"); - /* If we have any data in our reqbuf, use it */ + /* If we have any data in our reqbuf, use it */ if (http->reqsize > 0) { /* - * We can pass in reqbuf/size here, since clientSendMoreData ignores what - * is passed and uses them itself.. :-) - * -- adrian - */ - clientSendMoreData(http, http->reqbuf, http->reqsize); + * We can pass in reqbuf/size here, since clientSendMoreData ignores what + * is passed and uses them itself.. :-) + * -- adrian + */ + clientSendMoreData(http, http->reqbuf, http->reqsize); } else { assert(http->out.offset == 0); - /* - * here - have no data (don't ever think we get here..) - * so lets start copying.. - * -- adrian - */ - storeClientCopy(http->sc, entry, - http->out.offset, + /* + * here - have no data (don't ever think we get here..) + * so lets start copying.. + * -- adrian + */ + storeClientCopy(http->sc, entry, + http->out.offset, HTTP_REQBUF_SZ, http->reqbuf, clientSendMoreData, http); - } + } } } } +/* A write has just completed to the client, or we have just realised there is + * no more data to send. + * + */ static void clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) { clientHttpRequest *http = data; StoreEntry *entry = http->entry; - int done; http->out.size += size; debug(33, 5) ("clientWriteComplete: FD %d, sz %ld, err %d, off %ld, len %d\n", fd, (long int) size, errflag, (long int) http->out.offset, entry ? objectLen(entry) : 0); - if (size > 0) { + if (size > 0 && fd > -1) { kb_incr(&statCounter.client_http.kbytes_out, size); if (isTcpHit(http->log_type)) kb_incr(&statCounter.client_http.hit_kbytes_out, size); } -#if SIZEOF_SIZE_T == 4 - if (http->out.size > 0x7FFF0000) { - debug(33, 1) ("WARNING: closing FD %d to prevent counter overflow\n", fd); - debug(33, 1) ("\tclient %s\n", inet_ntoa(http->conn->peer.sin_addr)); - debug(33, 1) ("\treceived %d bytes\n", (int) http->out.size); - debug(33, 1) ("\tURI %s\n", http->log_uri); - comm_close(fd); - } else -#endif -#if SIZEOF_OFF_T == 4 - if (http->out.offset > 0x7FFF0000) { - debug(33, 1) ("WARNING: closing FD %d to prevent counter overflow\n", fd); - debug(33, 1) ("\tclient %s\n", inet_ntoa(http->conn->peer.sin_addr)); - debug(33, 1) ("\treceived %d bytes (offset %d)\n", (int) http->out.size, - (int) http->out.offset); - debug(33, 1) ("\tURI %s\n", http->log_uri); - comm_close(fd); - } else -#endif if (errflag) { /* - * just close the socket, httpRequestFree will abort if needed + * just close the socket, httpRequestFree will abort if needed. + * errflag is only EVER set by the comms callbacks */ + assert (fd != -1); comm_close(fd); - } else if (NULL == entry) { - comm_close(fd); /* yuk */ - } else if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - comm_close(fd); - } else if ((done = clientCheckTransferDone(http)) != 0 || size == 0) { - debug(33, 5) ("clientWriteComplete: FD %d transfer is DONE\n", fd); - /* We're finished case */ - if (httpReplyBodySize(http->request->method, entry->mem_obj->reply) < 0) { - debug(33, 5) ("clientWriteComplete: closing, content_length < 0\n"); - comm_close(fd); - } else if (!done) { - debug(33, 5) ("clientWriteComplete: closing, !done\n"); - comm_close(fd); - } else if (clientGotNotEnough(http)) { - debug(33, 5) ("clientWriteComplete: client didn't get all it expected\n"); - comm_close(fd); - } else if (http->request->flags.proxy_keepalive) { + return; + } + switch (clientDetermineActionAfterWrite(fd, http, entry, size)){ + case 0: debug(33, 5) ("clientWriteComplete: FD %d Keeping Alive\n", fd); clientKeepaliveNextRequest(http); - } else { - comm_close(fd); - } - } else if (clientReplyBodyTooLarge(entry->mem_obj->reply, http->out.offset)) { - comm_close(fd); - } else { - /* More data will be coming from primary server; register with - * storage manager. */ - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) - debug(33, 0) ("clientWriteComplete 2: ENTRY_ABORTED\n"); - http->reqofs = 0; - storeClientCopy(http->sc, entry, - http->out.offset, - HTTP_REQBUF_SZ, - http->reqbuf, - clientSendMoreData, - http); - } -} - -/* - * client issued a request with an only-if-cached cache-control directive; - * we did not find a cached object that can be returned without - * contacting other servers; - * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068] - */ -static void -clientProcessOnlyIfCachedMiss(clientHttpRequest * http) -{ - char *url = http->uri; - request_t *r = http->request; - ErrorState *err = NULL; - debug(33, 4) ("clientProcessOnlyIfCachedMiss: '%s %s'\n", - RequestMethodStr[r->method], url); - http->al.http.code = HTTP_GATEWAY_TIMEOUT; - err = errorCon(ERR_ONLY_IF_CACHED_MISS, HTTP_GATEWAY_TIMEOUT); - err->request = requestLink(r); - err->src_addr = http->conn->peer.sin_addr; - if (http->entry) { - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; - storeUnlockObject(http->entry); - } - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); - errorAppendEntry(http->entry, err); -} - -static log_type -clientProcessRequest2(clientHttpRequest * http) -{ - request_t *r = http->request; - StoreEntry *e; - if (r->flags.cachable || r->flags.internal) - e = http->entry = storeGetPublicByRequest(r); - else - e = http->entry = NULL; - /* Release negatively cached IP-cache entries on reload */ - if (r->flags.nocache) - ipcacheInvalidate(r->host); -#if HTTP_VIOLATIONS - else if (r->flags.nocache_hack) - ipcacheInvalidate(r->host); -#endif -#if USE_CACHE_DIGESTS - http->lookup_type = e ? "HIT" : "MISS"; -#endif - if (NULL == e) { - /* this object isn't in the cache */ - debug(33, 3) ("clientProcessRequest2: storeGet() MISS\n"); - return LOG_TCP_MISS; - } - if (Config.onoff.offline) { - debug(33, 3) ("clientProcessRequest2: offline HIT\n"); - http->entry = e; - return LOG_TCP_HIT; - } - if (http->redirect.status) { - /* force this to be a miss */ - http->entry = NULL; - return LOG_TCP_MISS; - } - if (!storeEntryValidToSend(e)) { - debug(33, 3) ("clientProcessRequest2: !storeEntryValidToSend MISS\n"); - http->entry = NULL; - return LOG_TCP_MISS; - } - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - /* Special entries are always hits, no matter what the client says */ - debug(33, 3) ("clientProcessRequest2: ENTRY_SPECIAL HIT\n"); - http->entry = e; - return LOG_TCP_HIT; - } -#if HTTP_VIOLATIONS - if (e->store_status == STORE_PENDING) { - if (r->flags.nocache || r->flags.nocache_hack) { - debug(33, 3) ("Clearing no-cache for STORE_PENDING request\n\t%s\n", - storeUrl(e)); - r->flags.nocache = 0; - r->flags.nocache_hack = 0; - } - } -#endif - if (r->flags.nocache) { - debug(33, 3) ("clientProcessRequest2: no-cache REFRESH MISS\n"); - http->entry = NULL; - return LOG_TCP_CLIENT_REFRESH_MISS; - } - /* We don't cache any range requests (for now!) -- adrian */ - if (r->flags.range) { - http->entry = NULL; - return LOG_TCP_MISS; - } - debug(33, 3) ("clientProcessRequest2: default HIT\n"); - http->entry = e; - return LOG_TCP_HIT; -} - -static void -clientProcessRequest(clientHttpRequest * http) -{ - char *url = http->uri; - request_t *r = http->request; - HttpReply *rep; - http_version_t version; - debug(33, 4) ("clientProcessRequest: %s '%s'\n", - RequestMethodStr[r->method], - url); - if (r->method == METHOD_CONNECT) { - http->log_type = LOG_TCP_MISS; - sslStart(http, &http->out.size, &http->al.http.code); - return; - } else if (r->method == METHOD_PURGE) { - clientPurgeRequest(http); - return; - } else if (r->method == METHOD_TRACE) { - if (r->max_forwards == 0) { - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); - storeReleaseRequest(http->entry); - storeBuffer(http->entry); - rep = httpReplyCreate(); - httpBuildVersion(&version, 1, 0); - httpReplySetHeaders(rep, version, HTTP_OK, NULL, "text/plain", - httpRequestPrefixLen(r), 0, squid_curtime); - httpReplySwapOut(rep, http->entry); - httpReplyDestroy(rep); - httpRequestSwapOut(r, http->entry); - storeComplete(http->entry); return; - } - /* yes, continue */ - http->log_type = LOG_TCP_MISS; - } else { - http->log_type = clientProcessRequest2(http); - } - debug(33, 4) ("clientProcessRequest: %s for '%s'\n", - log_tags[http->log_type], - http->uri); - http->out.offset = 0; - if (NULL != http->entry) { - storeLockObject(http->entry); - storeCreateMemObject(http->entry, http->uri, http->log_uri); - http->entry->mem_obj->method = r->method; - http->sc = storeClientListAdd(http->entry, http); -#if DELAY_POOLS - delaySetStoreClient(http->sc, delayClient(http)); -#endif - assert(http->log_type == LOG_TCP_HIT); - http->reqofs = 0; - storeClientCopy(http->sc, http->entry, - http->out.offset, - HTTP_REQBUF_SZ, - http->reqbuf, - clientCacheHit, - http); - } else { - /* MISS CASE, http->log_type is already set! */ - clientProcessMiss(http); - } -} - -/* - * Prepare to fetch the object as it's a cache miss of some kind. - */ -static void -clientProcessMiss(clientHttpRequest * http) -{ - char *url = http->uri; - request_t *r = http->request; - ErrorState *err = NULL; - debug(33, 4) ("clientProcessMiss: '%s %s'\n", - RequestMethodStr[r->method], url); - /* - * We might have a left-over StoreEntry from a failed cache hit - * or IMS request. - */ - if (http->entry) { - if (EBIT_TEST(http->entry->flags, ENTRY_SPECIAL)) { - debug(33, 0) ("clientProcessMiss: miss on a special object (%s).\n", url); - debug(33, 0) ("\tlog_type = %s\n", log_tags[http->log_type]); - storeEntryDump(http->entry, 1); - } - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; - storeUnlockObject(http->entry); - http->entry = NULL; - } - if (r->method == METHOD_PURGE) { - clientPurgeRequest(http); - return; - } - if (clientOnlyIfCached(http)) { - clientProcessOnlyIfCachedMiss(http); - return; - } - /* - * Deny loops when running in accelerator/transproxy mode. - */ - if (http->flags.accel && r->flags.loopdetect) { - http->al.http.code = HTTP_FORBIDDEN; - err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); - err->request = requestLink(r); - err->src_addr = http->conn->peer.sin_addr; - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); - errorAppendEntry(http->entry, err); - return; - } - assert(http->out.offset == 0); - http->entry = clientCreateStoreEntry(http, r->method, r->flags); - if (http->redirect.status) { - HttpReply *rep = httpReplyCreate(); -#if LOG_TCP_REDIRECTS - http->log_type = LOG_TCP_REDIRECT; -#endif - storeReleaseRequest(http->entry); - httpRedirectReply(rep, http->redirect.status, http->redirect.location); - httpReplySwapOut(rep, http->entry); - httpReplyDestroy(rep); - storeComplete(http->entry); - return; + case 1: /* fallthru */ + case 2: + if (fd != -1) + comm_close (fd); + return; + case 3: + /* More data will be coming from primary server; register with + * storage manager. */ + http->reqofs = 0; + storeClientCopy(http->sc, entry, + http->out.offset, + HTTP_REQBUF_SZ, + http->reqbuf, + clientSendMoreData, + http); + break; + default:fatal ("Hit unreachable code in clientWriteComplete\n"); } - if (http->flags.internal) - r->protocol = PROTO_INTERNAL; - fwdStart(http->conn->fd, http->entry, r); } static clientHttpRequest * @@ -2074,10 +595,78 @@ parseHttpRequestAbort(ConnStateData * co http->uri = xstrdup(uri); http->log_uri = xstrndup(uri, MAX_URL); http->reqbuf = http->norm_reqbuf; + /* default socket stream terminator */ + http->stream_head = NULL; + clientStreamInsertHead (&http->stream_head, clientSocketRecipient, NULL); dlinkAdd(http, &http->active, &ClientActiveRequests); return http; } +/* Utility function to perform part of request parsing */ +static clientHttpRequest * +clientParseHttpRequestLine (char *inbuf, size_t req_sz, ConnStateData * conn, method_t *method_p, char **url_p, http_version_t *http_ver_p) +{ + char *mstr = NULL; + char *url = NULL; + char *token = NULL; + char *t; + /* Barf on NULL characters in the headers */ + if (strlen(inbuf) != req_sz) { + debug(33, 1) ("parseHttpRequest: Requestheader contains NULL characters\n"); + return parseHttpRequestAbort(conn, "error:invalid-request"); + } + + /* Look for request method */ + if ((mstr = strtok(inbuf, "\t ")) == NULL) { + debug(33, 1) ("parseHttpRequest: Can't get request method\n"); + return parseHttpRequestAbort(conn, "error:invalid-request-method"); + } + *method_p = urlParseMethod(mstr); + if (*method_p == METHOD_NONE) { + debug(33, 1) ("parseHttpRequest: Unsupported method '%s'\n", mstr); + return parseHttpRequestAbort(conn, "error:unsupported-request-method"); + } + debug(33, 5) ("parseHttpRequest: Method is '%s'\n", mstr); + + /* look for URL+HTTP/x.x */ + if ((url = strtok(NULL, "\n")) == NULL) { + debug(33, 1) ("parseHttpRequest: Missing URL\n"); + return parseHttpRequestAbort(conn, "error:missing-url"); + } + while (xisspace(*url)) + url++; + t = url + strlen(url); + assert(*t == '\0'); + while (t > url) { + t--; + if (xisspace(*t) && !strncmp(t + 1, "HTTP/", 5)) { + token = t + 1; + break; + } + } + while (t > url && xisspace(*t)) + *(t--) = '\0'; + debug(33, 5) ("parseHttpRequest: URI is '%s'\n", url); + *url_p = url; + if (token == NULL) { + debug(33, 3) ("parseHttpRequest: Missing HTTP identifier\n"); +#if RELAXED_HTTP_PARSER + httpBuildVersion(http_ver_p, 0, 9); /* wild guess */ +#else + return parseHttpRequestAbort(conn, "error:missing-http-ident"); +#endif + } else { + if (sscanf(token + 5, "%d.%d", &http_ver_p->major, &http_ver_p->minor) != 2) { + debug(33, 3) ("parseHttpRequest: Invalid HTTP identifier.\n"); + return parseHttpRequestAbort(conn, "error: invalid HTTP-ident"); + } + debug(33, 6) ("parseHttpRequest: Client HTTP version %d.%d.\n", http_ver_p->major, http_ver_p->minor); + } + + /* everything was ok */ + return NULL; +} + /* * parseHttpRequest() * @@ -2090,19 +679,16 @@ parseHttpRequest(ConnStateData * conn, m char **prefix_p, size_t * req_line_sz_p) { char *inbuf = NULL; - char *mstr = NULL; char *url = NULL; char *req_hdr = NULL; http_version_t http_ver; - char *token = NULL; char *t = NULL; char *end; size_t header_sz; /* size of headers, not including first line */ size_t prefix_sz; /* size of whole request (req-line + headers) */ size_t url_sz; size_t req_sz; - method_t method; - clientHttpRequest *http = NULL; + clientHttpRequest *http; #if IPF_TRANSPARENT struct natlookup natLookup; static int natfd = -1; @@ -2133,63 +719,11 @@ parseHttpRequest(ConnStateData * conn, m xmemcpy(inbuf, conn->in.buf, req_sz); *(inbuf + req_sz) = '\0'; - /* Barf on NULL characters in the headers */ - if (strlen(inbuf) != req_sz) { - debug(33, 1) ("parseHttpRequest: Requestheader contains NULL characters\n"); - xfree(inbuf); - return parseHttpRequestAbort(conn, "error:invalid-request"); - } - /* Look for request method */ - if ((mstr = strtok(inbuf, "\t ")) == NULL) { - debug(33, 1) ("parseHttpRequest: Can't get request method\n"); - xfree(inbuf); - return parseHttpRequestAbort(conn, "error:invalid-request-method"); - } - method = urlParseMethod(mstr); - if (method == METHOD_NONE) { - debug(33, 1) ("parseHttpRequest: Unsupported method '%s'\n", mstr); - xfree(inbuf); - return parseHttpRequestAbort(conn, "error:unsupported-request-method"); - } - debug(33, 5) ("parseHttpRequest: Method is '%s'\n", mstr); - *method_p = method; - - /* look for URL+HTTP/x.x */ - if ((url = strtok(NULL, "\n")) == NULL) { - debug(33, 1) ("parseHttpRequest: Missing URL\n"); - xfree(inbuf); - return parseHttpRequestAbort(conn, "error:missing-url"); - } - while (xisspace(*url)) - url++; - t = url + strlen(url); - assert(*t == '\0'); - token = NULL; - while (t > url) { - t--; - if (xisspace(*t) && !strncmp(t + 1, "HTTP/", 5)) { - token = t + 1; - break; - } - } - while (t > url && xisspace(*t)) - *(t--) = '\0'; - debug(33, 5) ("parseHttpRequest: URI is '%s'\n", url); - if (token == NULL) { - debug(33, 3) ("parseHttpRequest: Missing HTTP identifier\n"); -#if RELAXED_HTTP_PARSER - httpBuildVersion(&http_ver, 0, 9); /* wild guess */ -#else + /* Is there a legitimate first line to the headers ? */ + if ((http = clientParseHttpRequestLine (inbuf, req_sz, conn, method_p, &url, &http_ver))) { + /* something wrong, abort */ xfree(inbuf); - return parseHttpRequestAbort(conn, "error:missing-http-ident"); -#endif - } else { - if (sscanf(token + 5, "%d.%d", &http_ver.major, &http_ver.minor) != 2) { - debug(33, 3) ("parseHttpRequest: Invalid HTTP identifier.\n"); - xfree(inbuf); - return parseHttpRequestAbort(conn, "error: invalid HTTP-ident"); - } - debug(33, 6) ("parseHttpRequest: Client HTTP version %d.%d.\n", http_ver.major, http_ver.minor); + return http; } /* @@ -2221,11 +755,16 @@ parseHttpRequest(ConnStateData * conn, m http->start = current_time; http->req_sz = prefix_sz; http->reqbuf = http->norm_reqbuf; + /* default socket based stream terminator */ + http->stream_head = NULL; + clientStreamInsertHead (&http->stream_head, clientSocketRecipient, NULL); *prefix_p = xmalloc(prefix_sz + 1); xmemcpy(*prefix_p, conn->in.buf, prefix_sz); *(*prefix_p + prefix_sz) = '\0'; dlinkAdd(http, &http->active, &ClientActiveRequests); + /* XXX this function is still way to long. here is a natural point for further simplification */ + debug(33, 5) ("parseHttpRequest: Request Header is\n%s\n", (*prefix_p) + *req_line_sz_p); #if THIS_VIOLATES_HTTP_SPECS_ON_URL_TRANSFORMATION if ((t = strchr(url, '#'))) /* remove HTML anchors */ @@ -2545,7 +1084,8 @@ clientReadRequest(int fd, void *data) commSetTimeout(fd, Config.Timeout.lifetime, clientLifetimeTimeout, http); if (parser_return_code < 0) { debug(33, 1) ("clientReadRequest: FD %d Invalid Request\n", fd); - err = errorCon(ERR_INVALID_REQ, HTTP_BAD_REQUEST); + err = clientBuildError(ERR_INVALID_REQ, HTTP_BAD_REQUEST, NULL, &conn->peer.sin_addr, + NULL); err->request_hdrs = xstrdup(conn->in.buf); http->entry = clientCreateStoreEntry(http, method, null_request_flags); errorAppendEntry(http->entry, err); @@ -2554,9 +1094,8 @@ clientReadRequest(int fd, void *data) } if ((request = urlParse(method, http->uri)) == NULL) { debug(33, 5) ("Invalid URL: %s\n", http->uri); - err = errorCon(ERR_INVALID_URL, HTTP_BAD_REQUEST); - err->src_addr = conn->peer.sin_addr; - err->url = xstrdup(http->uri); + err = clientBuildError(ERR_INVALID_URL, HTTP_BAD_REQUEST, http->uri, + &conn->peer.sin_addr, NULL); http->al.http.code = err->http_status; http->entry = clientCreateStoreEntry(http, method, null_request_flags); errorAppendEntry(http->entry, err); @@ -2598,9 +1137,8 @@ clientReadRequest(int fd, void *data) request->http_ver = http->http_ver; if (!urlCheckRequest(request) || httpHeaderHas(&request->header, HDR_TRANSFER_ENCODING)) { - err = errorCon(ERR_UNSUP_REQ, HTTP_NOT_IMPLEMENTED); - err->src_addr = conn->peer.sin_addr; - err->request = requestLink(request); + err = clientBuildError(ERR_UNSUP_REQ, HTTP_NOT_IMPLEMENTED, NULL, &conn->peer.sin_addr, + request); request->flags.proxy_keepalive = 0; http->al.http.code = err->http_status; http->entry = clientCreateStoreEntry(http, request->method, null_request_flags); @@ -2608,9 +1146,8 @@ clientReadRequest(int fd, void *data) break; } if (!clientCheckContentLength(request)) { - err = errorCon(ERR_INVALID_REQ, HTTP_LENGTH_REQUIRED); - err->src_addr = conn->peer.sin_addr; - err->request = requestLink(request); + err = clientBuildError(ERR_INVALID_REQ, HTTP_LENGTH_REQUIRED, NULL, + &conn->peer.sin_addr, request); http->al.http.code = err->http_status; http->entry = clientCreateStoreEntry(http, request->method, null_request_flags); errorAppendEntry(http->entry, err); @@ -2624,8 +1161,8 @@ clientReadRequest(int fd, void *data) request->body_connection = conn; /* Is it too large? */ if (clientRequestBodyTooLarge(request->content_length)) { - err = errorCon(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE); - err->request = requestLink(request); + err = clientBuildError(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE, NULL, + &conn->peer.sin_addr, http->request); http->entry = clientCreateStoreEntry(http, METHOD_NONE, null_request_flags); errorAppendEntry(http->entry, err); @@ -2645,7 +1182,8 @@ clientReadRequest(int fd, void *data) (int) conn->in.offset); debug(33, 1) ("Config 'request_header_max_size'= %ld bytes.\n", (long int) Config.maxRequestHeaderSize); - err = errorCon(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE); + err = clientBuildError(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE, NULL, + &conn->peer.sin_addr, NULL); http = parseHttpRequestAbort(conn, "error:request-too-large"); /* add to the client request queue */ for (H = &conn->chr; *H; H = &(*H)->next); @@ -2802,8 +1340,7 @@ requestTimeout(int fd, void *data) /* * Generate an error */ - err = errorCon(ERR_LIFETIME_EXP, HTTP_REQUEST_TIMEOUT); - err->url = xstrdup("N/A"); + err = clientBuildError(ERR_LIFETIME_EXP, HTTP_REQUEST_TIMEOUT, "N/A", &conn->peer.sin_addr, NULL); /* * Normally we shouldn't call errorSend() in client_side.c, but * it should be okay in this case. Presumably if we get here @@ -3023,86 +1560,6 @@ httpsAccept(int sock, void *data) } #endif /* USE_SSL */ - -#define SENDING_BODY 0 -#define SENDING_HDRSONLY 1 -static int -clientCheckTransferDone(clientHttpRequest * http) -{ - int sending = SENDING_BODY; - StoreEntry *entry = http->entry; - MemObject *mem; - http_reply *reply; - int sendlen; - if (entry == NULL) - return 0; - /* - * For now, 'done_copying' is used for special cases like - * Range and HEAD requests. - */ - if (http->flags.done_copying) - return 1; - /* - * Handle STORE_OK objects. - * objectLen(entry) will be set proprely. - */ - if (entry->store_status == STORE_OK) { - if (http->out.offset >= objectLen(entry)) - return 1; - else - return 0; - } - /* - * Now, handle STORE_PENDING objects - */ - mem = entry->mem_obj; - assert(mem != NULL); - assert(http->request != NULL); - reply = mem->reply; - if (reply->hdr_sz == 0) - return 0; /* haven't found end of headers yet */ - else if (reply->sline.status == HTTP_OK) - sending = SENDING_BODY; - else if (reply->sline.status == HTTP_NO_CONTENT) - sending = SENDING_HDRSONLY; - else if (reply->sline.status == HTTP_NOT_MODIFIED) - sending = SENDING_HDRSONLY; - else if (reply->sline.status < HTTP_OK) - sending = SENDING_HDRSONLY; - else if (http->request->method == METHOD_HEAD) - sending = SENDING_HDRSONLY; - else - sending = SENDING_BODY; - /* - * Figure out how much data we are supposed to send. - * If we are sending a body and we don't have a content-length, - * then we must wait for the object to become STORE_OK. - */ - if (sending == SENDING_HDRSONLY) - sendlen = reply->hdr_sz; - else if (reply->content_length < 0) - return 0; - else - sendlen = reply->content_length + reply->hdr_sz; - /* - * Now that we have the expected length, did we send it all? - */ - if (http->out.offset < sendlen) - return 0; - else - return 1; -} - -static int -clientGotNotEnough(clientHttpRequest * http) -{ - int cl = httpReplyBodySize(http->request->method, http->entry->mem_obj->reply); - int hs = http->entry->mem_obj->reply->hdr_sz; - assert(cl >= 0); - if (http->out.offset < cl + hs) - return 1; - return 0; -} /* * This function is designed to serve a fairly specific purpose. Index: src/protos.h =================================================================== RCS file: /server/cvs-server/squid/squid/src/protos.h,v retrieving revision 1.442 diff -u -p -r1.442 protos.h --- src/protos.h 20 Jul 2002 12:30:04 -0000 1.442 +++ src/protos.h 12 Aug 2002 06:09:07 -0000 @@ -125,6 +125,8 @@ extern int cbdataInternalReferenceDoneVa extern int cbdataReferenceValid(const void *p); extern cbdata_type cbdataInternalAddType(cbdata_type type, const char *label, int size, FREE * free_func); +/* client_side.c - FD related client side routines */ + extern void clientdbInit(void); extern void clientdbUpdate(struct in_addr, log_type, protocol_t, size_t); extern int clientdbCutoffDenied(struct in_addr); @@ -134,10 +136,8 @@ extern int clientdbEstablished(struct in extern void clientAccessCheck(void *); extern void clientAccessCheckDone(int, void *); -extern int modifiedSince(StoreEntry *, request_t *); extern char *clientConstructTraceEcho(clientHttpRequest *); extern void clientPurgeRequest(clientHttpRequest *); -extern int checkNegativeHit(StoreEntry *); extern void clientOpenListenSockets(void); extern void clientHttpConnectionsClose(void); extern StoreEntry *clientCreateStoreEntry(clientHttpRequest *, method_t, request_flags); @@ -145,6 +145,19 @@ extern int isTcpHit(log_type); extern void clientReadBody(request_t * req, char *buf, size_t size, CBCB * callback, void *data); extern int clientAbortBody(request_t * req); +/* client_side_request.c - client side request related routines (pure logic) */ +extern int clientDetermineActionAfterWrite(int fd, clientHttpRequest const *http, StoreEntry *entry, size_t size); +extern STCB clientSendMoreData; +extern ErrorState *clientBuildError (err_type, http_status, char const *, struct in_addr *, request_t *); +extern int clientBeginRequest (method_t, char const *, CSR *, void *, HttpHeader const *); + +/* client_side_reply.c - client side reply related routines (pure logic, no comms) */ +extern int clientCheckTransferDone(clientHttpRequest const *); + +/* clientStream.c */ +extern void clientStreamInsertHead (clientStreamNode **, CSR *, void *); +extern clientStreamNode *clientStreamNew (CSR *, void *); + extern int commSetNonBlocking(int fd); extern int commUnsetNonBlocking(int fd); extern void commSetCloseOnExec(int fd); @@ -899,6 +912,7 @@ extern EVH storeMaintainSwapSpace; extern void storeExpireNow(StoreEntry *); extern void storeReleaseRequest(StoreEntry *); extern void storeConfigure(void); +extern int storeCheckNegativeHit(StoreEntry *); extern void storeNegativeCache(StoreEntry *); extern void storeFreeMemory(void); extern int expiresMoreThan(time_t, time_t); Index: src/store.c =================================================================== RCS file: /server/cvs-server/squid/squid/src/store.c,v retrieving revision 1.544 diff -u -p -r1.544 store.c --- src/store.c 24 Oct 2001 08:19:08 -0000 1.544 +++ src/store.c 12 Aug 2002 06:09:08 -0000 @@ -1036,6 +1036,18 @@ storeKeepInMemory(const StoreEntry * e) return mem->inmem_lo == 0; } +int +storeCheckNegativeHit(StoreEntry * e) +{ + if (!EBIT_TEST(e->flags, ENTRY_NEGCACHED)) + return 0; + if (e->expires <= squid_curtime) + return 0; + if (e->store_status != STORE_OK) + return 0; + return 1; +} + void storeNegativeCache(StoreEntry * e) { Index: src/structs.h =================================================================== RCS file: /server/cvs-server/squid/squid/src/structs.h,v retrieving revision 1.423 diff -u -p -r1.423 structs.h --- src/structs.h 9 Aug 2002 10:57:43 -0000 1.423 +++ src/structs.h 12 Aug 2002 06:09:10 -0000 @@ -1046,6 +1046,12 @@ struct _AccessLogEntry { HierarchyLogEntry hier; }; +struct _clientStreamNode { + CSR *func; + void *data; + clientStreamNode *next; +}; + struct _clientHttpRequest { ConnStateData *conn; request_t *request; /* Parsed URL ... */ @@ -1089,6 +1095,7 @@ struct _clientHttpRequest { char *reqbuf; int reqofs; int reqsize; + clientStreamNode *stream_head; }; struct _ConnStateData { Index: src/typedefs.h =================================================================== RCS file: /server/cvs-server/squid/squid/src/typedefs.h,v retrieving revision 1.134 diff -u -p -r1.134 typedefs.h --- src/typedefs.h 23 Jun 2002 13:32:25 -0000 1.134 +++ src/typedefs.h 12 Aug 2002 06:09:10 -0000 @@ -106,6 +106,7 @@ typedef struct _HttpBody HttpBody; typedef struct _HttpReply HttpReply; typedef struct _HttpStateData HttpStateData; typedef struct _icpUdpData icpUdpData; +typedef struct _clientStreamNode clientStreamNode; typedef struct _clientHttpRequest clientHttpRequest; typedef struct _ConnStateData ConnStateData; typedef struct _ConnCloseHelperData ConnCloseHelperData; @@ -196,6 +197,10 @@ typedef struct _delayConfig delayConfig; typedef struct _delaySpecSet delaySpecSet; typedef struct _delaySpec delaySpec; #endif + +/* client_side.c callbacks and callforwards */ +/* client stream recipient */ +typedef void CSR (clientStreamNode *, clientHttpRequest *, HttpReply *, const char *, ssize_t); typedef void CWCB(int fd, char *, size_t size, int flag, void *data); typedef void CNCB(int fd, int status, void *); Index: src/clientStream.c =================================================================== RCS file: src/clientStream.c diff -N src/clientStream.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/clientStream.c 12 Aug 2002 06:13:19 -0000 @@ -0,0 +1,83 @@ + +/* + * $Id$ + * + * DEBUG: section 87 Client-side Strean routines. + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +CBDATA_TYPE(clientStreamNode); + + +/* Local functions */ +static FREE clientStreamFree; + +clientStreamNode * +clientStreamNew (CSR *func, void *data) +{ + clientStreamNode *temp; + CBDATA_INIT_TYPE_FREECB(clientStreamNode, clientStreamFree); + temp = cbdataAlloc (clientStreamNode); + temp->func = func; + temp->data = data; + return temp; +} + +void +clientStreamInsertHead (clientStreamNode **head, CSR *func, void *data) +{ + clientStreamNode *temp; + /* test preconditions */ + assert (head != NULL); + temp = clientStreamNew(func, data); + temp->next = *head; + *head = cbdataReference (temp); +} + + +/* Local function bodies */ +void +clientStreamFree (void *foo) +{ + clientStreamNode *node = foo; + debug (87, 9) ("Freeing clientStreamNode %p\n", node); + /* TODO: push refcount class through to head for cleanness*/ + if (node->data) { + cbdataFree (node->data); + } + if (node->next) { + clientStreamNode *temp = node->next; + cbdataReferenceDone (temp); + cbdataFree (node->next); + } + +} Index: src/client_side_reply.c =================================================================== RCS file: src/client_side_reply.c diff -N src/client_side_reply.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/client_side_reply.c 12 Aug 2002 06:13:19 -0000 @@ -0,0 +1,1165 @@ + +/* + * $Id$ + * + * DEBUG: section 88 Client-side Reply Routines + * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c) + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +static const char *const crlf = "\r\n"; + +/* Local functions */ +extern aclCheck_t * clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http); /* used in all client_* source files */ +static int clientGotNotEnough(clientHttpRequest const *); +/* static */ int clientReplyBodyTooLarge(HttpReply *, ssize_t); +/* This will go away when clientProcessRequest gets split */ +extern void clientProcessRequest(clientHttpRequest *); +static int clientOnlyIfCached(clientHttpRequest * http); +static void clientProcessExpired(void *data); +/* used from the request side */ void clientProcessMiss(clientHttpRequest *); + STCB clientCacheHit; +static void clientProcessOnlyIfCachedMiss(clientHttpRequest *); +static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); +static STCB clientHandleIMSReply; +static int modifiedSince(StoreEntry *, request_t *); + + +static void +clientProcessExpired(void *data) +{ + clientHttpRequest *http = data; + char *url = http->uri; + StoreEntry *entry = NULL; + debug(88, 3) ("clientProcessExpired: '%s'\n", http->uri); + assert(http->entry->lastmod >= 0); + /* + * check if we are allowed to contact other servers + * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return + * a stale entry *if* it matches client requirements + */ + if (clientOnlyIfCached(http)) { + clientProcessOnlyIfCachedMiss(http); + return; + } + http->request->flags.refresh = 1; + http->old_entry = http->entry; + http->old_sc = http->sc; + http->old_reqsize = http->reqsize; + http->old_reqofs = http->reqofs; + http->reqbuf = http->ims_reqbuf; +#if STORE_CLIENT_LIST_DEBUG + /* + * Assert that 'http' is already a client of old_entry. If + * it is not, then the beginning of the object data might get + * freed from memory before we need to access it. + */ + assert(http->sc->owner == http); +#endif + entry = storeCreateEntry(url, + http->log_uri, + http->request->flags, + http->request->method); + /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ + http->sc = storeClientListAdd(entry, http); +#if DELAY_POOLS + /* delay_id is already set on original store client */ + delaySetStoreClient(http->sc, delayClient(http)); +#endif + http->request->lastmod = http->old_entry->lastmod; + debug(88, 5) ("clientProcessExpired: lastmod %ld\n", (long int) entry->lastmod); + http->entry = entry; + http->out.offset = 0; + fwdStart(http->conn ? http->conn->fd : -1, http->entry, http->request); + /* Register with storage manager to receive updates when data comes in. */ + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) + debug(88, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n"); + http->reqofs = 0; + storeClientCopy(http->sc, entry, + http->out.offset, + HTTP_REQBUF_SZ, + http->reqbuf, + clientHandleIMSReply, + http); +} + +int +modifiedSince(StoreEntry * entry, request_t * request) +{ + int object_length; + MemObject *mem = entry->mem_obj; + time_t mod_time = entry->lastmod; + debug(88, 3) ("modifiedSince: '%s'\n", storeUrl(entry)); + if (mod_time < 0) + mod_time = entry->timestamp; + debug(88, 3) ("modifiedSince: mod_time = %ld\n", (long int) mod_time); + if (mod_time < 0) + return 1; + /* Find size of the object */ + object_length = mem->reply->content_length; + if (object_length < 0) + object_length = contentLen(entry); + if (mod_time > request->ims) { + debug(88, 3) ("--> YES: entry newer than client\n"); + return 1; + } else if (mod_time < request->ims) { + debug(88, 3) ("--> NO: entry older than client\n"); + return 0; + } else if (request->imslen < 0) { + debug(88, 3) ("--> NO: same LMT, no client length\n"); + return 0; + } else if (request->imslen == object_length) { + debug(88, 3) ("--> NO: same LMT, same length\n"); + return 0; + } else { + debug(88, 3) ("--> YES: same LMT, different length\n"); + return 1; + } +} + +static int +clientGetsOldEntry(StoreEntry * new_entry, StoreEntry * old_entry, request_t * request){ + const http_status status = new_entry->mem_obj->reply->sline.status; + if (0 == status) { + debug(88, 5) ("clientGetsOldEntry: YES, broken HTTP reply\n"); + return 1; + } + /* If the reply is a failure then send the old object as a last + * resort */ + if (status >= 500 && status < 600) { + debug(88, 3) ("clientGetsOldEntry: YES, failure reply=%d\n", status); + return 1; + } + /* If the reply is anything but "Not Modified" then + * we must forward it to the client */ + if (HTTP_NOT_MODIFIED != status) { + debug(88, 5) ("clientGetsOldEntry: NO, reply=%d\n", status); + return 0; + } + /* If the client did not send IMS in the request, then it + * must get the old object, not this "Not Modified" reply */ + if (!request->flags.ims) { + debug(88, 5) ("clientGetsOldEntry: YES, no client IMS\n"); + return 1; + } + /* If the client IMS time is prior to the entry LASTMOD time we + * need to send the old object */ + if (modifiedSince(old_entry, request)) { + debug(88, 5) ("clientGetsOldEntry: YES, modified since %ld\n", + (long int) request->ims); + return 1; + } + debug(88, 5) ("clientGetsOldEntry: NO, new one is fine\n"); + return 0; +} + +void +clientHandleIMSReply(void *data, char *buf, ssize_t size) +{ + clientHttpRequest *http = data; + StoreEntry *entry = http->entry; + MemObject *mem; + const char *url = storeUrl(entry); + int unlink_request = 0; + StoreEntry *oldentry; + http_status status; + debug(88, 3) ("clientHandleIMSReply: %s, %ld bytes\n", url, (long int) size); + if (entry == NULL) { + return; + } + if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + return; + } + /* update size of the request */ + http->reqsize = size + http->reqofs; + mem = entry->mem_obj; + status = mem->reply->sline.status; + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + debug(88, 3) ("clientHandleIMSReply: ABORTED '%s'\n", url); + /* We have an existing entry, but failed to validate it */ + /* Its okay to send the old one anyway */ + http->log_type = LOG_TCP_REFRESH_FAIL_HIT; + storeUnregister(http->sc, entry, http); + storeUnlockObject(entry); + entry = http->entry = http->old_entry; + http->sc = http->old_sc; + http->reqbuf = http->norm_reqbuf; + http->reqofs = http->old_reqofs; + http->reqsize = http->old_reqsize; + } else if (STORE_PENDING == entry->store_status && 0 == status) { + debug(88, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); + if (size + http->reqofs >= HTTP_REQBUF_SZ) { + /* will not get any bigger than that */ + debug(88, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url); + /* use old entry, this repeats the code abovez */ + http->log_type = LOG_TCP_REFRESH_FAIL_HIT; + storeUnregister(http->sc, entry, http); + storeUnlockObject(entry); + entry = http->entry = http->old_entry; + http->sc = http->old_sc; + http->reqbuf = http->norm_reqbuf; + http->reqofs = http->old_reqofs; + http->reqsize = http->old_reqsize; + /* continue */ + } else { + http->reqofs += size; + storeClientCopy(http->sc, entry, + http->out.offset + http->reqofs, + HTTP_REQBUF_SZ - http->reqofs, + http->reqbuf + http->reqofs, + clientHandleIMSReply, + http); + return; + } + } else if (clientGetsOldEntry(entry, http->old_entry, http->request)) { + /* We initiated the IMS request, the client is not expecting + * 304, so put the good one back. First, make sure the old entry + * headers have been loaded from disk. */ + oldentry = http->old_entry; + http->log_type = LOG_TCP_REFRESH_HIT; + if (oldentry->mem_obj->request == NULL) { + oldentry->mem_obj->request = requestLink(mem->request); + unlink_request = 1; + } + /* Don't memcpy() the whole reply structure here. For example, + * www.thegist.com (Netscape/1.13) returns a content-length for + * 304's which seems to be the length of the 304 HEADERS!!! and + * not the body they refer to. */ + httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply); + storeTimestampsSet(oldentry); + storeUnregister(http->sc, entry, http); + http->sc = http->old_sc; + storeUnlockObject(entry); + entry = http->entry = oldentry; + entry->timestamp = squid_curtime; + if (unlink_request) { + requestUnlink(entry->mem_obj->request); + entry->mem_obj->request = NULL; + } + http->reqbuf = http->norm_reqbuf; + http->reqofs = http->old_reqofs; + http->reqsize = http->old_reqsize; + } else { + /* the client can handle this reply, whatever it is */ + http->log_type = LOG_TCP_REFRESH_MISS; + if (HTTP_NOT_MODIFIED == mem->reply->sline.status) { + httpReplyUpdateOnNotModified(http->old_entry->mem_obj->reply, + mem->reply); + storeTimestampsSet(http->old_entry); + http->log_type = LOG_TCP_REFRESH_HIT; + } + storeUnregister(http->old_sc, http->old_entry, http); + storeUnlockObject(http->old_entry); + } + http->old_entry = NULL; /* done with old_entry */ + http->old_sc = NULL; + http->old_reqofs = 0; + http->old_reqsize = 0; + assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); + + clientSendMoreData(data, http->reqbuf, http->reqsize); +} + + +/* + * clientCacheHit should only be called until the HTTP reply headers + * have been parsed. Normally this should be a single call, but + * it might take more than one. As soon as we have the headers, + * we hand off to clientSendMoreData, clientProcessExpired, or + * clientProcessMiss. + */ +void +clientCacheHit(void *data, char *buf, ssize_t size) +{ + clientHttpRequest *http = data; + StoreEntry *e = http->entry; + MemObject *mem; + request_t *r = http->request; + debug(88, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); + if (http->entry == NULL) { + debug(88, 3) ("clientCacheHit: request aborted\n"); + return; + } else if (size < 0) { + /* swap in failure */ + debug(88, 3) ("clientCacheHit: swapin failure for %s\n", http->uri); + http->log_type = LOG_TCP_SWAPFAIL_MISS; + if ((e = http->entry)) { + http->entry = NULL; + storeUnregister(http->sc, e, http); + http->sc = NULL; + storeUnlockObject(e); + } + clientProcessMiss(http); + return; + } + assert(size > 0); + mem = e->mem_obj; + assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); + /* update size of the request */ + http->reqsize = size + http->reqofs; + if (mem->reply->sline.status == 0) { + /* + * we don't have full reply headers yet; either wait for more or + * punt to clientProcessMiss. + */ + if (e->mem_status == IN_MEMORY || e->store_status == STORE_OK) { + clientProcessMiss(http); + } else if (size + http->reqofs >= HTTP_REQBUF_SZ && http->out.offset == 0) { + clientProcessMiss(http); + } else { + debug(88, 3) ("clientCacheHit: waiting for HTTP reply headers\n"); + http->reqofs += size; + assert(http->reqofs <= HTTP_REQBUF_SZ); + storeClientCopy(http->sc, e, + http->out.offset + http->reqofs, + HTTP_REQBUF_SZ, + http->reqbuf + http->reqofs, + clientCacheHit, + http); + } + return; + } + /* + * Got the headers, now grok them + */ + assert(http->log_type == LOG_TCP_HIT); + switch (varyEvaluateMatch(e, r)) { + case VARY_NONE: + /* No variance detected. Continue as normal */ + break; + case VARY_MATCH: + /* This is the correct entity for this request. Continue */ + debug(88, 2) ("clientProcessHit: Vary MATCH!\n"); + break; + case VARY_OTHER: + /* This is not the correct entity for this request. We need + * to requery the cache. + */ + http->entry = NULL; + storeUnregister(http->sc, e, http); + http->sc = NULL; + storeUnlockObject(e); + /* Note: varyEvalyateMatch updates the request with vary information + * so we only get here once. (it also takes care of cancelling loops) + */ + debug(88, 2) ("clientProcessHit: Vary detected!\n"); + clientProcessRequest(http); + return; + case VARY_CANCEL: + /* varyEvaluateMatch found a object loop. Process as miss */ + debug(88, 1) ("clientProcessHit: Vary object loop!\n"); + clientProcessMiss(http); + return; + } + if (r->method == METHOD_PURGE) { + http->entry = NULL; + storeUnregister(http->sc, e, http); + http->sc = NULL; + storeUnlockObject(e); + clientPurgeRequest(http); + return; + } + if (storeCheckNegativeHit(e)) { + http->log_type = LOG_TCP_NEGATIVE_HIT; + clientSendMoreData(data, buf, size); + } else if (r->method == METHOD_HEAD) { + /* + * RFC 2068 seems to indicate there is no "conditional HEAD" + * request. We cannot validate a cached object for a HEAD + * request, nor can we return 304. + */ + if (e->mem_status == IN_MEMORY) + http->log_type = LOG_TCP_MEM_HIT; + clientSendMoreData(data, buf, size); + } else if (refreshCheckHTTP(e, r) && !http->flags.internal) { + debug(88, 5) ("clientCacheHit: in refreshCheck() block\n"); + /* + * We hold a stale copy; it needs to be validated + */ + /* + * The 'need_validation' flag is used to prevent forwarding + * loops between siblings. If our copy of the object is stale, + * then we should probably only use parents for the validation + * request. Otherwise two siblings could generate a loop if + * both have a stale version of the object. + */ + r->flags.need_validation = 1; + if (e->lastmod < 0) { + /* + * Previous reply didn't have a Last-Modified header, + * we cannot revalidate it. + */ + http->log_type = LOG_TCP_MISS; + clientProcessMiss(http); + } else if (r->flags.nocache) { + /* + * This did not match a refresh pattern that overrides no-cache + * we should honour the client no-cache header. + */ + http->log_type = LOG_TCP_CLIENT_REFRESH_MISS; + clientProcessMiss(http); + } else if (r->protocol == PROTO_HTTP) { + /* + * Object needs to be revalidated + * XXX This could apply to FTP as well, if Last-Modified is known. + */ + http->log_type = LOG_TCP_REFRESH_MISS; + clientProcessExpired(http); + } else { + /* + * We don't know how to re-validate other protocols. Handle + * them as if the object has expired. + */ + http->log_type = LOG_TCP_MISS; + clientProcessMiss(http); + } + } else if (r->flags.ims) { + /* + * Handle If-Modified-Since requests from the client + */ + if (mem->reply->sline.status != HTTP_OK) { + debug(88, 4) ("clientCacheHit: Reply code %d != 200\n", + mem->reply->sline.status); + http->log_type = LOG_TCP_MISS; + clientProcessMiss(http); + } else if (modifiedSince(e, http->request)) { + http->log_type = LOG_TCP_IMS_HIT; + clientSendMoreData(data, buf, size); + } else { + time_t timestamp = e->timestamp; + MemBuf mb = httpPacked304Reply(e->mem_obj->reply); + http->log_type = LOG_TCP_IMS_HIT; + storeUnregister(http->sc, e, http); + http->sc = NULL; + storeUnlockObject(e); + e = clientCreateStoreEntry(http, http->request->method, null_request_flags); + /* + * Copy timestamp from the original entry so the 304 + * reply has a meaningful Age: header. + */ + e->timestamp = timestamp; + http->entry = e; + httpReplyParse(e->mem_obj->reply, mb.buf, mb.size); + storeAppend(e, mb.buf, mb.size); + memBufClean(&mb); + storeComplete(e); + } + } else { + /* + * plain ol' cache hit + */ + if (e->mem_status == IN_MEMORY) + http->log_type = LOG_TCP_MEM_HIT; + else if (Config.onoff.offline) + http->log_type = LOG_TCP_OFFLINE_HIT; + clientSendMoreData(data, buf, size); + } +} + +/* + * Prepare to fetch the object as it's a cache miss of some kind. + */ +void +clientProcessMiss(clientHttpRequest * http) +{ + char *url = http->uri; + request_t *r = http->request; + ErrorState *err = NULL; + debug(88, 4) ("clientProcessMiss: '%s %s'\n", + RequestMethodStr[r->method], url); + /* + * We might have a left-over StoreEntry from a failed cache hit + * or IMS request. + */ + if (http->entry) { + if (EBIT_TEST(http->entry->flags, ENTRY_SPECIAL)) { + debug(88, 0) ("clientProcessMiss: miss on a special object (%s).\n", url); + debug(88, 0) ("\tlog_type = %s\n", log_tags[http->log_type]); + storeEntryDump(http->entry, 1); + } + storeUnregister(http->sc, http->entry, http); + http->sc = NULL; + storeUnlockObject(http->entry); + http->entry = NULL; + } + if (r->method == METHOD_PURGE) { + clientPurgeRequest(http); + return; + } + if (clientOnlyIfCached(http)) { + clientProcessOnlyIfCachedMiss(http); + return; + } + /* + * Deny loops when running in accelerator/transproxy mode. + */ + if (http->flags.accel && r->flags.loopdetect) { + http->al.http.code = HTTP_FORBIDDEN; + err = clientBuildError(ERR_ACCESS_DENIED, HTTP_FORBIDDEN, NULL, &http->conn->peer.sin_addr, + http->request); + http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + errorAppendEntry(http->entry, err); + return; + } + assert(http->out.offset == 0); + http->entry = clientCreateStoreEntry(http, r->method, r->flags); + if (http->redirect.status) { + HttpReply *rep = httpReplyCreate(); +#if LOG_TCP_REDIRECTS + http->log_type = LOG_TCP_REDIRECT; +#endif + storeReleaseRequest(http->entry); + httpRedirectReply(rep, http->redirect.status, http->redirect.location); + httpReplySwapOut(rep, http->entry); + httpReplyDestroy(rep); + storeComplete(http->entry); + return; + } + if (http->flags.internal) + r->protocol = PROTO_INTERNAL; + fwdStart(http->conn ? http->conn->fd : -1, http->entry, r); +} + +/* + * client issued a request with an only-if-cached cache-control directive; + * we did not find a cached object that can be returned without + * contacting other servers; + * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068] + */ +static void +clientProcessOnlyIfCachedMiss(clientHttpRequest * http) +{ + char *url = http->uri; + request_t *r = http->request; + ErrorState *err = NULL; + debug(88, 4) ("clientProcessOnlyIfCachedMiss: '%s %s'\n", + RequestMethodStr[r->method], url); + http->al.http.code = HTTP_GATEWAY_TIMEOUT; + err = clientBuildError (ERR_ONLY_IF_CACHED_MISS, HTTP_GATEWAY_TIMEOUT, NULL, + &http->conn->peer.sin_addr, http->request); + if (http->entry) { + storeUnregister(http->sc, http->entry, http); + http->sc = NULL; + storeUnlockObject(http->entry); + } + http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + errorAppendEntry(http->entry, err); +} + +void +clientPurgeRequest(clientHttpRequest * http) +{ + StoreEntry *entry; + ErrorState *err = NULL; + HttpReply *r; + http_status status = HTTP_NOT_FOUND; + http_version_t version; + debug(88, 3) ("Config2.onoff.enable_purge = %d\n", Config2.onoff.enable_purge); + if (!Config2.onoff.enable_purge) { + http->log_type = LOG_TCP_DENIED; + err = clientBuildError(ERR_ACCESS_DENIED, HTTP_FORBIDDEN, NULL, &http->conn->peer.sin_addr, + http->request); + http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); + errorAppendEntry(http->entry, err); + return; + } + /* Release both IP cache */ + ipcacheInvalidate(http->request->host); + + /* RC FIXME: This doesn't make sense - we should only ever be here on PURGE + * requests! + */ + if (!http->flags.purging) { + /* Try to find a base entry */ + http->flags.purging = 1; + entry = storeGetPublicByRequestMethod(http->request, METHOD_GET); + if (!entry) + entry = storeGetPublicByRequestMethod(http->request, METHOD_HEAD); + if (entry) { + /* Swap in the metadata */ + http->entry = entry; + storeLockObject(http->entry); + storeCreateMemObject(http->entry, http->uri, http->log_uri); + http->entry->mem_obj->method = http->request->method; + http->sc = storeClientListAdd(http->entry, http); + http->log_type = LOG_TCP_HIT; + http->reqofs = 0; + storeClientCopy(http->sc, http->entry, + http->out.offset, + HTTP_REQBUF_SZ, + http->reqbuf, + clientCacheHit, + http); + return; + } + } + http->log_type = LOG_TCP_MISS; + /* Release the cached URI */ + entry = storeGetPublicByRequestMethod(http->request, METHOD_GET); + if (entry) { + debug(88, 4) ("clientPurgeRequest: GET '%s'\n", + storeUrl(entry)); + storeRelease(entry); + status = HTTP_OK; + } + entry = storeGetPublicByRequestMethod(http->request, METHOD_HEAD); + if (entry) { + debug(88, 4) ("clientPurgeRequest: HEAD '%s'\n", + storeUrl(entry)); + storeRelease(entry); + status = HTTP_OK; + } + /* And for Vary, release the base URI if none of the headers was included in the request */ + if (http->request->vary_headers && !strstr(http->request->vary_headers, "=")) { + entry = storeGetPublic(urlCanonical(http->request), METHOD_GET); + if (entry) { + debug(88, 4) ("clientPurgeRequest: Vary GET '%s'\n", + storeUrl(entry)); + storeRelease(entry); + status = HTTP_OK; + } + entry = storeGetPublic(urlCanonical(http->request), METHOD_HEAD); + if (entry) { + debug(88, 4) ("clientPurgeRequest: Vary HEAD '%s'\n", + storeUrl(entry)); + storeRelease(entry); + status = HTTP_OK; + } + } + /* + * Make a new entry to hold the reply to be written + * to the client. + */ + http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); + httpReplyReset(r = http->entry->mem_obj->reply); + httpBuildVersion(&version, 1, 0); + httpReplySetHeaders(r, version, status, NULL, NULL, 0, 0, -1); + httpReplySwapOut(r, http->entry); + storeComplete(http->entry); +} + +#define SENDING_BODY 0 +#define SENDING_HDRSONLY 1 +int +clientCheckTransferDone(clientHttpRequest const * http) +{ + int sending = SENDING_BODY; + StoreEntry *entry = http->entry; + MemObject *mem; + http_reply *reply; + int sendlen; + if (entry == NULL) + return 0; + /* + * For now, 'done_copying' is used for special cases like + * Range and HEAD requests. + */ + if (http->flags.done_copying) + return 1; + /* + * Handle STORE_OK objects. + * objectLen(entry) will be set proprely. + */ + if (entry->store_status == STORE_OK) { + if (http->out.offset >= objectLen(entry)) + return 1; + else + return 0; + } + /* + * Now, handle STORE_PENDING objects + */ + mem = entry->mem_obj; + assert(mem != NULL); + assert(http->request != NULL); + reply = mem->reply; + if (reply->hdr_sz == 0) + return 0; /* haven't found end of headers yet */ + else if (reply->sline.status == HTTP_OK) + sending = SENDING_BODY; + else if (reply->sline.status == HTTP_NO_CONTENT) + sending = SENDING_HDRSONLY; + else if (reply->sline.status == HTTP_NOT_MODIFIED) + sending = SENDING_HDRSONLY; + else if (reply->sline.status < HTTP_OK) + sending = SENDING_HDRSONLY; + else if (http->request->method == METHOD_HEAD) + sending = SENDING_HDRSONLY; + else + sending = SENDING_BODY; + /* + * Figure out how much data we are supposed to send. + * If we are sending a body and we don't have a content-length, + * then we must wait for the object to become STORE_OK. + */ + if (sending == SENDING_HDRSONLY) + sendlen = reply->hdr_sz; + else if (reply->content_length < 0) + return 0; + else + sendlen = reply->content_length + reply->hdr_sz; + /* + * Now that we have the expected length, did we send it all? + */ + if (http->out.offset < sendlen) + return 0; + else + return 1; +} + + + +int +clientGotNotEnough(clientHttpRequest const * http) +{ + int cl = httpReplyBodySize(http->request->method, http->entry->mem_obj->reply); + int hs = http->entry->mem_obj->reply->hdr_sz; + assert(cl >= 0); + if (http->out.offset < cl + hs) + return 1; + return 0; +} + + +/* A write has completed, what is the next status: + * 0 transfer complete ok, persistent connections allowed + * 1 transfer complete ok, persistent connections not allowed + * 2 transfer failed + * 3 transfer incomplete + * + * Preconditions: + * *http is a valid structure. + * fd is either -1, or an open fd. + * + * TODO: enumify this + */ +int +clientDetermineActionAfterWrite(int fd, clientHttpRequest const *http, StoreEntry *entry, size_t size) +{ + int done; +#if SIZEOF_SIZE_T == 4 + if (http->out.size > 0x7FFF0000) { + debug(88, 1) ("WARNING: closing FD %d to prevent counter overflow\n", fd); + debug(88, 1) ("\tclient %s\n", inet_ntoa(http->conn ? http->conn->peer.sin_addr : no_addr)); + debug(88, 1) ("\treceived %d bytes\n", (int) http->out.size); + debug(88, 1) ("\tURI %s\n", http->log_uri); + return 2; + } +#endif +#if SIZEOF_OFF_T == 4 + if (http->out.offset > 0x7FFF0000) { + debug(88, 1) ("WARNING: closing FD %d to prevent counter overflow\n", fd); + debug(88, 1) ("\tclient %s\n", inet_ntoa(http->conn ? http->conn->peer.sin_addr : no_addr)); + debug(88, 1) ("\treceived %d bytes (offset %d)\n", (int) http->out.size, (int) http->out.offset); + debug(88, 1) ("\tURI %s\n", http->log_uri); + return 2; + } +#endif + if (entry == NULL) + return 2; /* yuk */ + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) + /* TODO: Could upstream read errors (retsize < 0) be + * lost, and result in undersize requests being considered + * complete. Should we tcp reset such connections ? + */ + return 2; + if ((done = clientCheckTransferDone(http)) != 0 || size == 0) { + debug(88, 5) ("clientWriteComplete: FD %d transfer is DONE\n", fd); + /* Ok we're finished, but how? */ + if (httpReplyBodySize(http->request->method, entry->mem_obj->reply) < 0) { + debug(88, 5) ("clientWriteComplete: closing, content_length < 0\n"); + return 2; + } else if (!done) { + debug(88, 5) ("clientWriteComplete: closing, !done, but read 0 bytes\n"); + return 2; + } else if (clientGotNotEnough(http)) { + debug(88, 5) ("clientWriteComplete: client didn't get all it expected\n"); + return 2; + } else if (http->request->flags.proxy_keepalive) { + return 0; /* allow keepalive */ + } + return 1; /* don't allow keepalive */ + } + if (clientReplyBodyTooLarge(entry->mem_obj->reply, http->out.offset)) + return 2; + /* More data will be coming from primary server; register with + * storage manager. */ + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) + fatal ("entry aborted DURING clientWriteComplete \n"); + return 3; +} + +/* Responses with no body will not have a content-type header, + * which breaks the rep_mime_type acl, which + * coincidentally, is the most common acl for reply access lists. + * A better long term fix for this is to allow acl matchs on the various + * status codes, and then supply a default ruleset that puts these + * codes before any user defines access entries. That way the user + * can choose to block these responses where appropriate, but won't get + * mysterious breakages. + */ +static int +clientAlwaysAllowResponse(http_status sline) +{ + switch (sline) { + case HTTP_CONTINUE: + case HTTP_SWITCHING_PROTOCOLS: + case HTTP_PROCESSING: + case HTTP_NO_CONTENT: + case HTTP_NOT_MODIFIED: + return 1; + /* unreached */ + break; + default: + return 0; + } +} + +/* + * filters out unwanted entries from original reply header + * adds extra entries if we have more info than origin server + * adds Squid specific entries + */ +static void +clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep) +{ + HttpHeader *hdr = &rep->header; + int is_hit = isTcpHit(http->log_type); + request_t *request = http->request; +#if DONT_FILTER_THESE + /* but you might want to if you run Squid as an HTTP accelerator */ + /* httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); */ + httpHeaderDelById(hdr, HDR_ETAG); +#endif + httpHeaderDelById(hdr, HDR_PROXY_CONNECTION); + /* here: Keep-Alive is a field-name, not a connection directive! */ + httpHeaderDelByName(hdr, "Keep-Alive"); + /* remove Set-Cookie if a hit */ + if (is_hit) + httpHeaderDelById(hdr, HDR_SET_COOKIE); + /* handle Connection header */ + if (httpHeaderHas(hdr, HDR_CONNECTION)) { + /* anything that matches Connection list member will be deleted */ + String strConnection = httpHeaderGetList(hdr, HDR_CONNECTION); + const HttpHeaderEntry *e; + HttpHeaderPos pos = HttpHeaderInitPos; + /* + * think: on-average-best nesting of the two loops (hdrEntry + * and strListItem) @?@ + */ + /* + * maybe we should delete standard stuff ("keep-alive","close") + * from strConnection first? + */ + while ((e = httpHeaderGetEntry(hdr, &pos))) { + if (strListIsMember(&strConnection, strBuf(e->name), ',')) + httpHeaderDelAt(hdr, pos); + } + httpHeaderDelById(hdr, HDR_CONNECTION); + stringClean(&strConnection); + } + /* + * Add a estimated Age header on cache hits. + */ + if (is_hit) { + /* + * Remove any existing Age header sent by upstream caches + * (note that the existing header is passed along unmodified + * on cache misses) + */ + httpHeaderDelById(hdr, HDR_AGE); + /* + * This adds the calculated object age. Note that the details of the + * age calculation is performed by adjusting the timestamp in + * storeTimestampsSet(), not here. + * + * BROWSER WORKAROUND: IE sometimes hangs when receiving a 0 Age + * header, so don't use it unless there is a age to report. Please + * note that Age is only used to make a conservative estimation of + * the objects age, so a Age: 0 header does not add any useful + * information to the reply in any case. + */ + if (NULL == http->entry) + (void) 0; + else if (http->entry->timestamp < 0) + (void) 0; + else if (http->entry->timestamp < squid_curtime) + httpHeaderPutInt(hdr, HDR_AGE, + squid_curtime - http->entry->timestamp); + } + /* Handle authentication headers */ + if (request->auth_user_request) + authenticateFixHeader(rep, request->auth_user_request, request, http->flags.accel, 0); + /* Append X-Cache */ + httpHeaderPutStrf(hdr, HDR_X_CACHE, "%s from %s", + is_hit ? "HIT" : "MISS", getMyHostname()); +#if USE_CACHE_DIGESTS + /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */ + httpHeaderPutStrf(hdr, HDR_X_CACHE_LOOKUP, "%s from %s:%d", + http->lookup_type ? http->lookup_type : "NONE", + getMyHostname(), getMyPort()); +#endif + if (httpReplyBodySize(request->method, rep) < 0) { + debug(88, 3) ("clientBuildReplyHeader: can't keep-alive, unknown body size\n"); request->flags.proxy_keepalive = 0; + } + /* Signal keep-alive if needed */ + httpHeaderPutStr(hdr, + http->flags.accel ? HDR_CONNECTION : HDR_PROXY_CONNECTION, + request->flags.proxy_keepalive ? "keep-alive" : "close"); +#if ADD_X_REQUEST_URI + /* + * Knowing the URI of the request is useful when debugging persistent + * connections in a client; we cannot guarantee the order of http headers, + * but X-Request-URI is likely to be the very last header to ease use from a + * debugger [hdr->entries.count-1]. + */ + httpHeaderPutStr(hdr, HDR_X_REQUEST_URI, + http->entry->mem_obj->url ? http->entry->mem_obj->url : http->uri); +#endif + httpHdrMangleList(hdr, request); +} + + +static HttpReply * +clientBuildReply(clientHttpRequest * http, const char *buf, size_t size) +{ + HttpReply *rep = httpReplyCreate(); + size_t k = headersEnd(buf, size); + if (k && httpReplyParse(rep, buf, k)) { + /* enforce 1.0 reply version */ + httpBuildVersion(&rep->sline.version, 1, 0); + /* do header conversions */ + clientBuildReplyHeader(http, rep); + } else { + /* parsing failure, get rid of the invalid reply */ + httpReplyDestroy(rep); + rep = NULL; + } + return rep; +} + + +/* + * accepts chunk of a http message in buf, parses prefix, filters headers and + * such, writes processed message to the message recipient + */ +void +clientSendMoreData(void *data, char *retbuf, ssize_t retsize) +{ + clientHttpRequest *http = data; + StoreEntry *entry = http->entry; + ConnStateData *conn = http->conn; + int fd = conn ? conn->fd : -1; + HttpReply *rep = NULL; + char *buf = http->reqbuf; + const char *body_buf = buf; + ssize_t size = http->reqofs + retsize; + ssize_t body_size = size; + + debug(88, 5) ("clientSendMoreData: %s, %d bytes (%d new bytes)\n", http->uri, (int) size, retsize); + assert(size <= HTTP_REQBUF_SZ); + assert(http->request != NULL); + /* TODO: remove this assert once everything is stable ? */ + assert(http->stream_head && cbdataReferenceValid(http->stream_head)); + dlinkDelete(&http->active, &ClientActiveRequests); + dlinkAdd(http, &http->active, &ClientActiveRequests); + debug(88, 5) ("clientSendMoreData: FD %d '%s', out.offset=%ld \n", + fd, storeUrl(entry), (long int) http->out.offset); + /* update size of the request */ + http->reqsize = size; + /* Internal requests go through regardless */ + if (conn && conn->chr != http) { + /* there is another object in progress, defer this one */ + debug(88, 2) ("clientSendMoreData: Deferring %s\n", storeUrl(entry)); + return; + } else if (http->request->flags.reset_tcp) { + if (fd != -1) + comm_reset_close(fd); + return; + } else if (/* aborted request */ + (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) || + /* Upstream read error */ (retsize < 0) || + /* Upstream EOF */ (body_size == 0)) { + /* call clientWriteComplete so the client socket gets closed */ + /* We call into the stream, because we don't know that there is a + * client socket! + */ + http->stream_head->func (http->stream_head, http, NULL, NULL, 0); + /* clientWriteComplete(fd, NULL, 0, COMM_OK, http); */ + return; + } + /* FIXME: Adrian says this is a dodgy artifact from the rearrangement of + * HEAD and may not be true for pipelining. + * */ + if (http->out.offset != 0) { + http->stream_head->func (http->stream_head, http, NULL, buf, size); + return; + } + /* handle headers */ + if (Config.onoff.log_mime_hdrs) { + size_t k; + if ((k = headersEnd(buf, size))) { + safe_free(http->al.headers.reply); + http->al.headers.reply = xcalloc(k + 1, 1); + xstrncpy(http->al.headers.reply, buf, k); + } + } + rep = clientBuildReply(http, buf, size); + if (rep) { + aclCheck_t *ch; + int rv; + httpReplyBodyBuildSize(http->request, rep, &Config.ReplyBodySize); + if (clientReplyBodyTooLarge(rep, rep->content_length)) { + ErrorState *err = clientBuildError(ERR_TOO_BIG, HTTP_FORBIDDEN, NULL, + http->conn ? &http->conn->peer.sin_addr : &no_addr, http->request); + storeUnregister(http->sc, http->entry, http); + http->sc = NULL; + storeUnlockObject(http->entry); + http->entry = clientCreateStoreEntry(http, http->request->method, + null_request_flags); + errorAppendEntry(http->entry, err); + httpReplyDestroy(rep); + return; + } + body_size = size - rep->hdr_sz; + assert(body_size >= 0); + body_buf = buf + rep->hdr_sz; + debug(88, 3) ("clientSendMoreData: Appending %d bytes after %d bytes of headers\n", + (int) body_size, rep->hdr_sz); + ch = aclChecklistCreate(Config.accessList.reply, http->request, NULL); + ch->reply = rep; + rv = aclCheckFast(Config.accessList.reply, ch); + aclChecklistFree(ch); + ch = NULL; + debug(88, 2) ("The reply for %s %s is %s, because it matched '%s'\n", + RequestMethodStr[http->request->method], http->uri, + rv ? "ALLOWED" : "DENIED", + AclMatchedName ? AclMatchedName : "NO ACL's"); + if (!rv && rep->sline.status != HTTP_FORBIDDEN + && !clientAlwaysAllowResponse(rep->sline.status)) { + /* the if above is slightly broken, but there is no way + * to tell if this is a squid generated error page, or one from + * upstream at this point. */ + ErrorState *err; + err = clientBuildError(ERR_ACCESS_DENIED, HTTP_FORBIDDEN, NULL, http->conn ? &http->conn->peer.sin_addr : &no_addr, + http->request); + storeUnregister(http->sc, http->entry, http); + http->sc = NULL; + storeUnlockObject(http->entry); + http->entry = clientCreateStoreEntry(http, http->request->method, + null_request_flags); + errorAppendEntry(http->entry, err); + httpReplyDestroy(rep); + return; + } + } else if (size < HTTP_REQBUF_SZ && entry->store_status == STORE_PENDING) { + /* wait for more to arrive */ + http->reqofs += retsize; + assert(http->reqofs <= HTTP_REQBUF_SZ); + storeClientCopy(http->sc, entry, + http->out.offset + http->reqofs, + HTTP_REQBUF_SZ - http->reqofs, + http->reqbuf + http->reqofs, + clientSendMoreData, + http); + return; + } + if (http->request->method == METHOD_HEAD) { + if (rep) { + /* do not forward body for HEAD replies */ + body_size = 0; + http->flags.done_copying = 1; + } else { + /* + * If we are here, then store_status == STORE_OK and it + * seems we have a HEAD repsponse which is missing the + * empty end-of-headers line (home.mira.net, phttpd/0.99.72 + * does this). Because clientBuildReply() fails we just + * call this reply a body, set the done_copying flag and + * continue... + */ + http->flags.done_copying = 1; + } + } + assert(rep || (body_buf && body_size)); + http->stream_head->func (http->stream_head, http, rep, body_buf, body_size); +} + +int +clientReplyBodyTooLarge(HttpReply * rep, ssize_t clen) +{ + if (0 == rep->maxBodySize) + return 0; /* disabled */ + if (clen < 0) + return 0; /* unknown */ + if (clen > rep->maxBodySize) + return 1; /* too large */ + return 0; +} + +/* + * returns true if client specified that the object must come from the cache + * without contacting origin server + */ +static int +clientOnlyIfCached(clientHttpRequest * http) +{ + const request_t *r = http->request; + assert(r); + return r->cache_control && + EBIT_TEST(r->cache_control->mask, CC_ONLY_IF_CACHED); +} + +StoreEntry * +clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags) +{ + StoreEntry *e; + /* + * For erroneous requests, we might not have a h->request, + * so make a fake one. + */ + if (h->request == NULL) + h->request = requestLink(requestCreate(m, PROTO_NONE, null_string)); + e = storeCreateEntry(h->uri, h->log_uri, flags, m); + h->sc = storeClientListAdd(e, h); +#if DELAY_POOLS + delaySetStoreClient(h->sc, delayClient(h)); +#endif + h->reqofs = 0; + h->reqsize = 0; + /* I don't think this is actually needed! -- adrian */ + /* h->reqbuf = h->norm_reqbuf; */ + assert(h->reqbuf == h->norm_reqbuf); + storeClientCopy(h->sc, e, 0, HTTP_REQBUF_SZ, h->reqbuf, + clientSendMoreData, h); + return e; +} + Index: src/client_side_request.c =================================================================== RCS file: src/client_side_request.c diff -N src/client_side_request.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/client_side_request.c 12 Aug 2002 06:13:20 -0000 @@ -0,0 +1,644 @@ + +/* + * $Id$ + * + * DEBUG: section 85 Client-side Request Routines + * AUTHOR: Robert Collins (Originall Duane Wessels in client_side.c) + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +#if LINGERING_CLOSE +#define comm_close comm_lingering_close +#endif + +static const char *const crlf = "\r\n"; + +/* Local functions */ +static int checkAccelOnly(clientHttpRequest *); +/*static*/ aclCheck_t * clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http); +static int clientCachable(clientHttpRequest * http); +static int clientHierarchical(clientHttpRequest * http); +static void clientInterpretRequestHeaders(clientHttpRequest * http); +static RH clientRedirectDone; +static void clientCheckNoCache(clientHttpRequest *); +static void clientCheckNoCacheDone(int answer, void *data); +static log_type clientProcessRequest2(clientHttpRequest * http); +void clientProcessRequest(clientHttpRequest *); + +/* client reply functions - TODO: put in protos.h? */ +extern STCB clientCacheHit; +extern void clientProcessMiss(clientHttpRequest *); + +/* Create a request and kick it off */ +int /* returns nonzero on failure */ +clientBeginRequest (method_t method, char const *url, CSR *streamfunc, void *streamdata, HttpHeader const *header) +{ + size_t url_sz; + http_version_t http_ver = {1,0}; + clientHttpRequest *http = cbdataAlloc(clientHttpRequest); + request_t *request; + http->http_ver = http_ver; + http->conn = NULL; + http->start = current_time; + /* this is only used to adjust the connection offset in client_side.c */ + http->req_sz = 0; + http->reqbuf = http->norm_reqbuf; + /* stream terminator */ + http->stream_head = NULL; + clientStreamInsertHead (&http->stream_head, streamfunc, streamdata); + /* make it visible in the 'current acctive requests list' */ + dlinkAdd(http, &http->active, &ClientActiveRequests); + /* Set flags */ + http->flags.accel = 1; /* internal requests only makes sense in an + accelerator today. TODO: accept flags ? */ + /* allow size for url rewriting */ + url_sz = strlen(url) + Config.appendDomainLen + 5; + http->uri = xcalloc(url_sz, 1); + strcpy(http->uri, url); + + if ((request = urlParse(method, http->uri)) == NULL) { + debug(85, 5) ("Invalid URL: %s\n", http->uri); + return -1; + } + /* now update the headers in request with our supplied headers. + * urLParse should return a blank header set, but we use Update to be sure + * of correctness. + */ + if (header) + httpHeaderUpdate(&request->header, header, NULL); + http->log_uri = xstrdup(urlCanonicalClean(request)); + /* http struct now ready */ + + /* build new header list *? + * TODO + */ + request->flags.accelerated = http->flags.accel; + /* FIXME? Do we want to detect and handle internal requests of internal + * objects ? */ + + /* Internally created requests cannot have bodies today */ + request->content_length =0; + request->client_addr = no_addr; + request->my_addr = no_addr; /* undefined for internal requests */ + request->my_port = 0; + request->http_ver = http_ver; + http->request = requestLink(request); + + /* optional - skip the access check ? */ + clientAccessCheck (http); + return 0; +} + +static int +checkAccelOnly(clientHttpRequest * http) +{ + /* return TRUE if someone makes a proxy request to us and + * we are in httpd-accel only mode */ + if (!Config2.Accel.on) + return 0; + if (Config.onoff.accel_with_proxy) + return 0; + if (http->request->protocol == PROTO_CACHEOBJ) + return 0; + if (http->flags.accel) + return 0; + if (http->request->method == METHOD_PURGE) + return 0; + return 1; +} + +aclCheck_t * +clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http) +{ + aclCheck_t *ch; + ConnStateData *conn = http->conn; + ch = aclChecklistCreate(acl, + http->request, + conn ? conn->rfc931 : dash_str); + + /* + * hack for ident ACL. It needs to get full addresses, and a + * place to store the ident result on persistent connections... + */ + /* connection oriented auth also needs these two lines for it's operation. */ + /* Internal requests do not have a connection reference, because: + * A) their byte count may be transformed before being applied to an outbound + * connection + * B) they are internal - any limiting on them should be done on the server end. + */ + if (conn) + ch->conn = cbdataReference(conn); /* unreferenced in acl.c */ + + return ch; +} + +/* This is the entry point for external users of the client_side routines */ +void +clientAccessCheck(void *data) +{ + clientHttpRequest *http = data; + if (checkAccelOnly(http)) { + /* deny proxy requests in accel_only mode */ + debug(85, 1) ("clientAccessCheck: proxy request denied in accel_only mode\n"); + clientAccessCheckDone(ACCESS_DENIED, http); + return; + } + http->acl_checklist = clientAclChecklistCreate(Config.accessList.http, http); + aclNBCheck(http->acl_checklist, clientAccessCheckDone, http); +} + +void +clientAccessCheckDone(int answer, void *data) +{ + clientHttpRequest *http = data; + err_type page_id; + http_status status; + ErrorState *err = NULL; + char *proxy_auth_msg = NULL; + debug(85, 2) ("The request %s %s is %s, because it matched '%s'\n", + RequestMethodStr[http->request->method], http->uri, + answer == ACCESS_ALLOWED ? "ALLOWED" : "DENIED", + AclMatchedName ? AclMatchedName : "NO ACL's"); + proxy_auth_msg = authenticateAuthUserRequestMessage((http->conn && http->conn->auth_user_request) ? http->conn->auth_user_request : http->request->auth_user_request); + http->acl_checklist = NULL; + if (answer == ACCESS_ALLOWED) { + safe_free(http->uri); + http->uri = xstrdup(urlCanonical(http->request)); + assert(http->redirect_state == REDIRECT_NONE); + http->redirect_state = REDIRECT_PENDING; + redirectStart(http, clientRedirectDone, http); + } else { + debug(85, 5) ("Access Denied: %s\n", http->uri); + debug(85, 5) ("AclMatchedName = %s\n", + AclMatchedName ? AclMatchedName : ""); + debug(85, 5) ("Proxy Auth Message = %s\n", + proxy_auth_msg ? proxy_auth_msg : ""); + /* + * NOTE: get page_id here, based on AclMatchedName because + * if USE_DELAY_POOLS is enabled, then AclMatchedName gets + * clobbered in the clientCreateStoreEntry() call + * just below. Pedro Ribeiro + */ + page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName); + http->log_type = LOG_TCP_DENIED; + http->entry = clientCreateStoreEntry(http, http->request->method, + null_request_flags); + if (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName)) { + if (!http->flags.accel) { + /* Proxy authorisation needed */ + status = HTTP_PROXY_AUTHENTICATION_REQUIRED; + } else { + /* WWW authorisation needed */ + status = HTTP_UNAUTHORIZED; + } + if (page_id == ERR_NONE) + page_id = ERR_CACHE_ACCESS_DENIED; + } else { + status = HTTP_FORBIDDEN; + if (page_id == ERR_NONE) + page_id = ERR_ACCESS_DENIED; + } + err = clientBuildError (page_id, status, NULL, http->conn ? &http->conn->peer.sin_addr : &no_addr, http->request); + if (http->conn && http->conn->auth_user_request) + err->auth_user_request = http->conn->auth_user_request; + else if (http->request->auth_user_request) + err->auth_user_request = http->request->auth_user_request; + /* lock for the error state */ + if (err->auth_user_request) + authenticateAuthUserRequestLock(err->auth_user_request); + err->callback_data = NULL; + errorAppendEntry(http->entry, err); + } +} + +static int +clientCachable(clientHttpRequest * http) +{ + request_t *req = http->request; + method_t method = req->method; + if (req->protocol == PROTO_HTTP) + return httpCachable(method); + /* FTP is always cachable */ + if (req->protocol == PROTO_WAIS) + return 0; + if (method == METHOD_CONNECT) + return 0; + if (method == METHOD_TRACE) + return 0; + if (method == METHOD_PUT) + return 0; + if (method == METHOD_POST) + return 0; /* XXX POST may be cached sometimes.. ignored for now */ + if (req->protocol == PROTO_GOPHER) + return gopherCachable(req); + if (req->protocol == PROTO_CACHEOBJ) + return 0; + return 1; +} + +static int +clientHierarchical(clientHttpRequest * http) +{ + const char *url = http->uri; + request_t *request = http->request; + method_t method = request->method; + const wordlist *p = NULL; + + /* IMS needs a private key, so we can use the hierarchy for IMS only + * if our neighbors support private keys */ + if (request->flags.ims && !neighbors_do_private_keys) + return 0; + if (request->flags.auth) + return 0; + if (method == METHOD_TRACE) + return 1; + if (method != METHOD_GET) + return 0; + /* scan hierarchy_stoplist */ + for (p = Config.hierarchy_stoplist; p; p = p->next) + if (strstr(url, p->key)) + return 0; + if (request->flags.loopdetect) + return 0; + if (request->protocol == PROTO_HTTP) + return httpCachable(method); + if (request->protocol == PROTO_GOPHER) + return gopherCachable(request); + if (request->protocol == PROTO_WAIS) + return 0; + if (request->protocol == PROTO_CACHEOBJ) + return 0; + return 1; +} + + +static void +clientInterpretRequestHeaders(clientHttpRequest * http) +{ + request_t *request = http->request; + const HttpHeader *req_hdr = &request->header; + int no_cache = 0; + const char *str; + request->imslen = -1; + request->ims = httpHeaderGetTime(req_hdr, HDR_IF_MODIFIED_SINCE); + if (request->ims > 0) + request->flags.ims = 1; + if (httpHeaderHas(req_hdr, HDR_PRAGMA)) { + String s = httpHeaderGetList(req_hdr, HDR_PRAGMA); + if (strListIsMember(&s, "no-cache", ',')) + no_cache++; + stringClean(&s); + } + request->cache_control = httpHeaderGetCc(req_hdr); + if (request->cache_control) + if (EBIT_TEST(request->cache_control->mask, CC_NO_CACHE)) + no_cache++; + /* Work around for supporting the Reload button in IE browsers + * when Squid is used as an accelerator or transparent proxy, + * by turning accelerated IMS request to no-cache requests. + * Now knows about IE 5.5 fix (is actually only fixed in SP1, + * but we can't tell whether we are talking to SP1 or not so + * all 5.5 versions are treated 'normally'). + */ + if (Config.onoff.ie_refresh) { + if (http->flags.accel && request->flags.ims) { + if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) { + if (strstr(str, "MSIE 5.01") != NULL) + no_cache++; + else if (strstr(str, "MSIE 5.0") != NULL) + no_cache++; + else if (strstr(str, "MSIE 4.") != NULL) + no_cache++; + else if (strstr(str, "MSIE 3.") != NULL) + no_cache++; + } + } + } + if (no_cache) { +#if HTTP_VIOLATIONS + if (Config.onoff.reload_into_ims) + request->flags.nocache_hack = 1; + else if (refresh_nocache_hack) + request->flags.nocache_hack = 1; + else +#endif + request->flags.nocache = 1; + } + /* ignore range header in non-GETs */ + if (request->method == METHOD_GET) { + /* + * Since we're not doing ranges atm, just set the flag if + * the header exists, and then free the range header info + * -- adrian + */ + request->range = httpHeaderGetRange(req_hdr); + if (request->range) { + request->flags.range = 1; + httpHdrRangeDestroy(request->range); + request->range = NULL; + } + } + if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION)) + request->flags.auth = 1; + if (request->login[0] != '\0') + request->flags.auth = 1; + if (httpHeaderHas(req_hdr, HDR_VIA)) { + String s = httpHeaderGetList(req_hdr, HDR_VIA); + /* + * ThisCache cannot be a member of Via header, "1.0 ThisCache" can. + * Note ThisCache2 has a space prepended to the hostname so we don't + * accidentally match super-domains. + */ + if (strListIsSubstr(&s, ThisCache2, ',')) { + debugObj(33, 1, "WARNING: Forwarding loop detected for:\n", + request, (ObjPackMethod) & httpRequestPack); + request->flags.loopdetect = 1; + } +#if FORW_VIA_DB + fvdbCountVia(strBuf(s)); +#endif + stringClean(&s); + } +#if USE_USERAGENT_LOG + if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) + logUserAgent(fqdnFromAddr(http->conn->peer.sin_addr), str); +#endif +#if USE_REFERER_LOG + if ((str = httpHeaderGetStr(req_hdr, HDR_REFERER))) + logReferer(fqdnFromAddr(http->conn->peer.sin_addr), str, + http->log_uri); +#endif +#if FORW_VIA_DB + if (httpHeaderHas(req_hdr, HDR_X_FORWARDED_FOR)) { + String s = httpHeaderGetList(req_hdr, HDR_X_FORWARDED_FOR); + fvdbCountForw(strBuf(s)); + stringClean(&s); + } +#endif + if (request->method == METHOD_TRACE) { + request->max_forwards = httpHeaderGetInt(req_hdr, HDR_MAX_FORWARDS); + } + if (clientCachable(http)) + request->flags.cachable = 1; + if (clientHierarchical(http)) + request->flags.hierarchical = 1; + debug(85, 5) ("clientInterpretRequestHeaders: REQ_NOCACHE = %s\n", + request->flags.nocache ? "SET" : "NOT SET"); + debug(85, 5) ("clientInterpretRequestHeaders: REQ_CACHABLE = %s\n", + request->flags.cachable ? "SET" : "NOT SET"); + debug(85, 5) ("clientInterpretRequestHeaders: REQ_HIERARCHICAL = %s\n", + request->flags.hierarchical ? "SET" : "NOT SET"); +} + +void +clientRedirectDone(void *data, char *result) +{ + clientHttpRequest *http = data; + request_t *new_request = NULL; + request_t *old_request = http->request; + debug(85, 5) ("clientRedirectDone: '%s' result=%s\n", http->uri, + result ? result : "NULL"); + assert(http->redirect_state == REDIRECT_PENDING); + http->redirect_state = REDIRECT_DONE; + if (result) { + http_status status = (http_status) atoi(result); + if (status == HTTP_MOVED_PERMANENTLY || status == HTTP_MOVED_TEMPORARILY) { + char *t = result; + if ((t = strchr(result, ':')) != NULL) { + http->redirect.status = status; + http->redirect.location = xstrdup(t + 1); + } else { + debug(85, 1) ("clientRedirectDone: bad input: %s\n", result); + } + } + if (strcmp(result, http->uri)) + new_request = urlParse(old_request->method, result); + } + if (new_request) { + safe_free(http->uri); + http->uri = xstrdup(urlCanonical(new_request)); + new_request->http_ver = old_request->http_ver; + httpHeaderAppend(&new_request->header, &old_request->header); + new_request->client_addr = old_request->client_addr; + new_request->my_addr = old_request->my_addr; + new_request->my_port = old_request->my_port; + new_request->flags.redirected = 1; + if (old_request->auth_user_request) { + new_request->auth_user_request = old_request->auth_user_request; + authenticateAuthUserRequestLock(new_request->auth_user_request); + } + if (old_request->body_connection) { + new_request->body_connection = old_request->body_connection; + old_request->body_connection = NULL; + } + new_request->content_length = old_request->content_length; + new_request->flags.proxy_keepalive = old_request->flags.proxy_keepalive; + requestUnlink(old_request); + http->request = requestLink(new_request); + } + clientInterpretRequestHeaders(http); +#if HEADERS_LOG + headersLog(0, 1, request->method, request); +#endif + /* FIXME PIPELINE: This is innacurate during pipelining */ + if (http->conn) + fd_note(http->conn->fd, http->uri); + clientCheckNoCache(http); +} + +void +clientCheckNoCache(clientHttpRequest * http) +{ + if (Config.accessList.noCache && http->request->flags.cachable) { + http->acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http); + aclNBCheck(http->acl_checklist, clientCheckNoCacheDone, http); + } else { + clientCheckNoCacheDone(http->request->flags.cachable, http); + } +} + +void +clientCheckNoCacheDone(int answer, void *data) +{ + clientHttpRequest *http = data; + http->request->flags.cachable = answer; + http->acl_checklist = NULL; + clientProcessRequest(http); +} + +static log_type +clientProcessRequest2(clientHttpRequest * http) +{ + request_t *r = http->request; + StoreEntry *e; + if (r->flags.cachable || r->flags.internal) + e = http->entry = storeGetPublicByRequest(r); + else + e = http->entry = NULL; + /* Release negatively cached IP-cache entries on reload */ + if (r->flags.nocache) + ipcacheInvalidate(r->host); +#if HTTP_VIOLATIONS + else if (r->flags.nocache_hack) + ipcacheInvalidate(r->host); +#endif +#if USE_CACHE_DIGESTS + http->lookup_type = e ? "HIT" : "MISS"; +#endif + if (NULL == e) { + /* this object isn't in the cache */ + debug(85, 3) ("clientProcessRequest2: storeGet() MISS\n"); + return LOG_TCP_MISS; + } + if (Config.onoff.offline) { + debug(85, 3) ("clientProcessRequest2: offline HIT\n"); + http->entry = e; + return LOG_TCP_HIT; + } + if (http->redirect.status) { + /* force this to be a miss */ + http->entry = NULL; + return LOG_TCP_MISS; + } + if (!storeEntryValidToSend(e)) { + debug(85, 3) ("clientProcessRequest2: !storeEntryValidToSend MISS\n"); + http->entry = NULL; + return LOG_TCP_MISS; + } + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { + /* Special entries are always hits, no matter what the client says */ + debug(85, 3) ("clientProcessRequest2: ENTRY_SPECIAL HIT\n"); + http->entry = e; + return LOG_TCP_HIT; + } +#if HTTP_VIOLATIONS + if (e->store_status == STORE_PENDING) { + if (r->flags.nocache || r->flags.nocache_hack) { + debug(85, 3) ("Clearing no-cache for STORE_PENDING request\n\t%s\n", + storeUrl(e)); + r->flags.nocache = 0; + r->flags.nocache_hack = 0; + } + } +#endif + if (r->flags.nocache) { + debug(85, 3) ("clientProcessRequest2: no-cache REFRESH MISS\n"); + http->entry = NULL; + return LOG_TCP_CLIENT_REFRESH_MISS; + } + /* We don't cache any range requests (for now!) -- adrian */ + if (r->flags.range) { + http->entry = NULL; + return LOG_TCP_MISS; + } + debug(85, 3) ("clientProcessRequest2: default HIT\n"); + http->entry = e; + return LOG_TCP_HIT; +} + +void +clientProcessRequest(clientHttpRequest * http) +{ + char *url = http->uri; + request_t *r = http->request; + HttpReply *rep; + http_version_t version; + debug(85, 4) ("clientProcessRequest: %s '%s'\n", + RequestMethodStr[r->method], + url); + if (r->method == METHOD_CONNECT) { + http->log_type = LOG_TCP_MISS; + sslStart(http, &http->out.size, &http->al.http.code); + return; + } else if (r->method == METHOD_PURGE) { + clientPurgeRequest(http); + return; + } else if (r->method == METHOD_TRACE) { + if (r->max_forwards == 0) { + http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + storeReleaseRequest(http->entry); + storeBuffer(http->entry); + rep = httpReplyCreate(); + httpBuildVersion(&version, 1, 0); + httpReplySetHeaders(rep, version, HTTP_OK, NULL, "text/plain", + httpRequestPrefixLen(r), 0, squid_curtime); + httpReplySwapOut(rep, http->entry); + httpReplyDestroy(rep); + httpRequestSwapOut(r, http->entry); + storeComplete(http->entry); + return; + } + /* yes, continue */ + http->log_type = LOG_TCP_MISS; + } else { + http->log_type = clientProcessRequest2(http); + } + debug(85, 4) ("clientProcessRequest: %s for '%s'\n", + log_tags[http->log_type], + http->uri); + /* Here is where we start requesting data - from the cache or from the network + * TODO: Split the function here, and this point on goes to client_side_reply.c + */ + http->out.offset = 0; + if (NULL != http->entry) { + storeLockObject(http->entry); + storeCreateMemObject(http->entry, http->uri, http->log_uri); + http->entry->mem_obj->method = r->method; + http->sc = storeClientListAdd(http->entry, http); +#if DELAY_POOLS + delaySetStoreClient(http->sc, delayClient(http)); +#endif + assert(http->log_type == LOG_TCP_HIT); + http->reqofs = 0; + storeClientCopy(http->sc, http->entry, + http->out.offset, + HTTP_REQBUF_SZ, + http->reqbuf, + clientCacheHit, + http); + } else { + /* MISS CASE, http->log_type is already set! */ + clientProcessMiss(http); + } +} + +/* Widely used utility functions */ +ErrorState * +clientBuildError (err_type page_id, http_status status, char const *url, struct in_addr *src_addr, request_t *request) +{ + ErrorState *err = errorCon(page_id, status); + err->src_addr = *src_addr; + if (url) + err->url = xstrdup (url); + if (request) + err->request = requestLink (request); + return err; +}