HTTP Compliance: Support If-Match and If-None-Match requests. Add support for If-Match and If-None-Match headers as described in RFC 2616 (sections 14.24 and 14.26 in particular). Moved IMS handling from clientReplyContext::cacheHit() to clientReplyContext::processConditional() while preserving the original IMS logic, except for the case when a request has both IMS and If-None-Match. Co-Advisors test cases: test_clause/rfc2616/ifMatch-mismatch-strong test_clause/rfc2616/ifMatch-mismatch-weak test_clause/rfc2616/ifNoneMatch-match-imsNone and many more === modified file 'errors/Makefile.am' --- errors/Makefile.am 2010-08-23 02:21:19 +0000 +++ errors/Makefile.am 2010-10-01 16:49:56 +0000 @@ -19,40 +19,41 @@ ERROR_TEMPLATES = \ templates/ERR_CONNECT_FAIL \ templates/ERR_DIR_LISTING \ templates/ERR_DNS_FAIL \ templates/ERR_ESI \ templates/ERR_FORWARDING_DENIED \ templates/ERR_FTP_DISABLED \ templates/ERR_FTP_FAILURE \ templates/ERR_FTP_FORBIDDEN \ templates/ERR_FTP_NOT_FOUND \ templates/ERR_FTP_PUT_CREATED \ templates/ERR_FTP_PUT_ERROR \ templates/ERR_FTP_PUT_MODIFIED \ templates/ERR_FTP_UNAVAILABLE \ templates/ERR_ICAP_FAILURE \ templates/ERR_INVALID_REQ \ templates/ERR_INVALID_RESP \ templates/ERR_INVALID_URL \ templates/ERR_LIFETIME_EXP \ templates/ERR_NO_RELAY \ templates/ERR_ONLY_IF_CACHED_MISS \ + templates/ERR_PRECONDITION_FAILED \ templates/ERR_READ_ERROR \ templates/ERR_READ_TIMEOUT \ templates/ERR_SECURE_CONNECT_FAIL \ templates/ERR_SHUTTING_DOWN \ templates/ERR_SOCKET_FAILURE \ templates/ERR_TOO_BIG \ templates/ERR_UNSUP_HTTPVERSION \ templates/ERR_UNSUP_REQ \ templates/ERR_URN_RESOLVE \ templates/ERR_WRITE_ERROR \ templates/ERR_ZERO_SIZE_OBJECT TRANSLATE_LANGUAGES = \ af.lang \ ar.lang \ az.lang \ bg.lang \ ca.lang \ cs.lang \ da.lang \ === modified file 'errors/list' --- errors/list 2009-09-19 09:15:45 +0000 +++ errors/list 2010-10-01 16:49:56 +0000 @@ -5,31 +5,32 @@ ERR_CANNOT_FORWARD ERR_CONNECT_FAIL ERR_DIR_LISTING ERR_DNS_FAIL ERR_ESI ERR_FORWARDING_DENIED ERR_FTP_DISABLED ERR_FTP_FAILURE ERR_FTP_FORBIDDEN ERR_FTP_NOT_FOUND ERR_FTP_PUT_CREATED ERR_FTP_PUT_ERROR ERR_FTP_PUT_MODIFIED ERR_FTP_UNAVAILABLE ERR_ICAP_FAILURE ERR_INVALID_REQ ERR_INVALID_RESP ERR_INVALID_URL ERR_LIFETIME_EXP ERR_NO_RELAY ERR_ONLY_IF_CACHED_MISS +ERR_PRECONDITION_FAILED ERR_READ_ERROR ERR_READ_TIMEOUT ERR_SECURE_CONNECT_FAIL ERR_SHUTTING_DOWN ERR_SOCKET_FAILURE ERR_TOO_BIG ERR_UNSUP_HTTPVERSION ERR_UNSUP_REQ ERR_URN_RESOLVE ERR_WRITE_ERROR ERR_ZERO_SIZE_OBJECT === added file 'errors/templates/ERR_PRECONDITION_FAILED' --- errors/templates/ERR_PRECONDITION_FAILED 1970-01-01 00:00:00 +0000 +++ errors/templates/ERR_PRECONDITION_FAILED 2010-10-01 17:04:18 +0000 @@ -0,0 +1,39 @@ + + + +ERROR: The requested URL could not be retrieved + + +
+

ERROR

+

The requested URL could not be retrieved

+
+
+ +
+

The following error was encountered while trying to retrieve the URL: %U

+ +
+

Precondition Failed.

+
+ +

This means:

+
+

At least one precondition specified by the HTTP client in the request header has failed.

+
+ +
+
+ +
+ + === modified file 'src/HttpRequest.cc' --- src/HttpRequest.cc 2010-09-12 00:10:47 +0000 +++ src/HttpRequest.cc 2010-10-01 16:49:56 +0000 @@ -557,40 +557,48 @@ HttpRequest::cacheable() const * The below looks questionable: what non HTTP protocols use connect, * trace, put and post? RC */ if (!method.isCacheble()) return false; /* * XXX POST may be cached sometimes.. ignored * for now */ if (protocol == PROTO_GOPHER) return gopherCachable(this); if (protocol == PROTO_CACHEOBJ) return false; return true; } +bool +HttpRequest::conditional() const +{ + return flags.ims || + header.has(HDR_IF_MATCH) || + header.has(HDR_IF_NONE_MATCH); +} + bool HttpRequest::inheritProperties(const HttpMsg *aMsg) { const HttpRequest* aReq = dynamic_cast(aMsg); if (!aReq) return false; client_addr = aReq->client_addr; #if FOLLOW_X_FORWARDED_FOR indirect_client_addr = aReq->indirect_client_addr; #endif #if USE_SQUID_EUI client_eui48 = aReq->client_eui48; client_eui64 = aReq->client_eui64; #endif my_addr = aReq->my_addr; dnsWait = aReq->dnsWait; #if USE_ADAPTATION adaptHistory_ = aReq->adaptHistory(); === modified file 'src/HttpRequest.h' --- src/HttpRequest.h 2010-09-11 23:58:15 +0000 +++ src/HttpRequest.h 2010-10-01 17:05:03 +0000 @@ -74,40 +74,42 @@ public: typedef HttpMsgPointerT Pointer; MEMPROXY_CLASS(HttpRequest); HttpRequest(); HttpRequest(const HttpRequestMethod& aMethod, protocol_t aProtocol, const char *aUrlpath); ~HttpRequest(); virtual void reset(); // use HTTPMSGLOCK() instead of calling this directly virtual HttpRequest *_lock() { return static_cast(HttpMsg::_lock()); }; void initHTTP(const HttpRequestMethod& aMethod, protocol_t aProtocol, const char *aUrlpath); virtual HttpRequest *clone() const; /* are responses to this request potentially cachable */ bool cacheable() const; + bool conditional() const; ///< has at least one recognized If-* header + /// whether the client is likely to be able to handle a 1xx reply bool canHandle1xx() const; /* Now that we care what host contains it is better off being protected. */ /* HACK: These two methods are only inline to get around Makefile dependancies */ /* caused by HttpRequest being used in places it really shouldn't. */ /* ideally they would be methods of URL instead. */ inline void SetHost(const char *src) { host_addr.SetEmpty(); host_addr = src; if ( host_addr.IsAnyAddr() ) { xstrncpy(host, src, SQUIDHOSTNAMELEN); host_is_numeric = 0; } else { host_addr.ToHostname(host, SQUIDHOSTNAMELEN); debugs(23, 3, "HttpRequest::SetHost() given IP: " << host_addr); host_is_numeric = 1; } }; inline const char* GetHost(void) const { return host; }; === modified file 'src/Store.h' --- src/Store.h 2009-12-26 00:25:57 +0000 +++ src/Store.h 2010-10-01 17:33:09 +0000 @@ -106,40 +106,44 @@ public: int checkNegativeHit() const; int locked() const; int validToSend() const; int keepInMemory() const; void createMemObject(const char *, const char *); void dump(int debug_lvl) const; void hashDelete(); void hashInsert(const cache_key *); void registerAbort(STABH * cb, void *); void reset(); void setMemStatus(mem_status_t); void timestampsSet(); void unregisterAbort(); void destroyMemObject(); int checkTooSmall(); void delayAwareRead(int fd, char *buf, int len, AsyncCall::Pointer callback); void setNoDelay (bool const); bool modifiedSince(HttpRequest * request) const; + /// has ETag matching at least one of the If-Match etags + bool hasIfMatchEtag(const HttpRequest &request) const; + /// has ETag matching at least one of the If-None-Match etags + bool hasIfNoneMatchEtag(const HttpRequest &request) const; /** What store does this entry belong too ? */ virtual RefCount store() const; MemObject *mem_obj; RemovalPolicyNode repl; /* START OF ON-DISK STORE_META_STD TLV field */ time_t timestamp; time_t lastref; time_t expires; time_t lastmod; uint64_t swap_file_sz; u_short refcount; u_short flags; /* END OF ON-DISK STORE_META_STD */ sfileno swap_filen:25; sdirno swap_dirn:7; @@ -172,40 +176,41 @@ public: #endif /** append bytes to the buffer */ virtual void append(char const *, int len); /** disable sending content to the clients */ virtual void buffer(); /** flush any buffered content */ virtual void flush(); /** reduce the memory lock count on the entry */ virtual int unlock(); /** increate the memory lock count on the entry */ virtual int64_t objectLen() const; virtual int64_t contentLen() const; virtual void lock(); virtual void release(); private: static MemAllocator *pool; bool validLength() const; + bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const; }; /// \ingroup StoreAPI class NullStoreEntry:public StoreEntry { public: static NullStoreEntry *getInstance(); bool isNull() { return true; } const char *getMD5Text() const; _SQUID_INLINE_ HttpReply const *getReply() const; void write (StoreIOBuffer) {} bool isEmpty () const {return true;} virtual size_t bytesWanted(Range const aRange) const { assert (aRange.size()); return aRange.end - 1;} === modified file 'src/client_side_reply.cc' --- src/client_side_reply.cc 2010-09-28 15:20:36 +0000 +++ src/client_side_reply.cc 2010-10-01 19:05:13 +0000 @@ -540,75 +540,43 @@ clientReplyContext::cacheHit(StoreIOBuff /* * This did not match a refresh pattern that overrides no-cache * we should honour the client no-cache header. */ http->logType = LOG_TCP_CLIENT_REFRESH_MISS; processMiss(); } else if (r->protocol == PROTO_HTTP) { /* * Object needs to be revalidated * XXX This could apply to FTP as well, if Last-Modified is known. */ processExpired(); } else { /* * We don't know how to re-validate other protocols. Handle * them as if the object has expired. */ http->logType = LOG_TCP_MISS; processMiss(); } - } else if (r->flags.ims) { - /* - * Handle If-Modified-Since requests from the client - */ - - if (e->getReply()->sline.status != HTTP_OK) { - debugs(88, 4, "clientCacheHit: Reply code " << - e->getReply()->sline.status << " != 200"); - http->logType = LOG_TCP_MISS; - processMiss(); - } else if (e->modifiedSince(http->request)) { - http->logType = LOG_TCP_IMS_HIT; - sendMoreData(result); - } else { - time_t const timestamp = e->timestamp; - HttpReply *temprep = e->getReply()->make304(); - http->logType = LOG_TCP_IMS_HIT; - removeClientStoreReference(&sc, http); - createStoreEntry(http->request->method, - request_flags()); - e = http->storeEntry(); - /* - * Copy timestamp from the original entry so the 304 - * reply has a meaningful Age: header. - */ - e->timestamp = timestamp; - e->replaceHttpReply(temprep); - e->complete(); - /* TODO: why put this in the store and then serialise it and then parse it again. - * Simply mark the request complete in our context and - * write the reply struct to the client side - */ - triggerInitialStoreRead(); - } - } else { + } else if (r->conditional()) + processConditional(result); + else { /* * plain ol' cache hit */ #if DELAY_POOLS if (e->store_status != STORE_OK) http->logType = LOG_TCP_MISS; else #endif if (e->mem_status == IN_MEMORY) http->logType = LOG_TCP_MEM_HIT; else if (Config.onoff.offline) http->logType = LOG_TCP_OFFLINE_HIT; sendMoreData(result); } } /** * Prepare to fetch the object as it's a cache miss of some kind. @@ -694,40 +662,121 @@ clientReplyContext::processMiss() } /** * client issued a request with an only-if-cached cache-control directive; * we did not find a cached object that can be returned without * contacting other servers; * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068] */ void clientReplyContext::processOnlyIfCachedMiss() { ErrorState *err = NULL; debugs(88, 4, "clientProcessOnlyIfCachedMiss: '" << RequestMethodStr(http->request->method) << " " << http->uri << "'"); http->al.http.code = HTTP_GATEWAY_TIMEOUT; err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, HTTP_GATEWAY_TIMEOUT, NULL, http->getConn()->peer, http->request); removeClientStoreReference(&sc, http); startError(err); } +/// process conditional request from client +void +clientReplyContext::processConditional(StoreIOBuffer &result) +{ + StoreEntry *e = http->storeEntry(); + + if (e->getReply()->sline.status != HTTP_OK) { + debugs(88, 4, "clientReplyContext::processConditional: Reply code " << + e->getReply()->sline.status << " != 200"); + http->logType = LOG_TCP_MISS; + processMiss(); + return; + } + + HttpRequest &r = *http->request; + + if (r.header.has(HDR_IF_MATCH) && !e->hasIfMatchEtag(r)) { + // RFC 2616: reply with 412 Precondition Failed if If-Match did not match + sendPreconditionFailedError(); + return; + } + + bool matchedIfNoneMatch = false; + if (r.header.has(HDR_IF_NONE_MATCH)) { + if (!e->hasIfNoneMatchEtag(r)) { + // RFC 2616: ignore IMS if If-None-Match did not match + r.flags.ims = 0; + r.ims = -1; + r.imslen = 0; + r.header.delById(HDR_IF_MODIFIED_SINCE); + http->logType = LOG_TCP_MISS; + sendMoreData(result); + return; + } + + if (!r.flags.ims) { + // RFC 2616: if If-None-Match matched and there is no IMS, + // reply with 412 Precondition Failed + sendPreconditionFailedError(); + return; + } + + // otherwise check IMS below to decide if we reply with 412 + matchedIfNoneMatch = true; + } + + if (r.flags.ims) { + // handle If-Modified-Since requests from the client + if (e->modifiedSince(&r)) { + http->logType = LOG_TCP_IMS_HIT; + sendMoreData(result); + return; + } + + if (matchedIfNoneMatch) { + // If-None-Match matched, reply with 412 Precondition Failed + sendPreconditionFailedError(); + return; + } + + // otherwise reply with 304 Not Modified + const time_t timestamp = e->timestamp; + HttpReply *const temprep = e->getReply()->make304(); + http->logType = LOG_TCP_IMS_HIT; + removeClientStoreReference(&sc, http); + createStoreEntry(http->request->method, request_flags()); + e = http->storeEntry(); + // Copy timestamp from the original entry so the 304 + // reply has a meaningful Age: header. + e->timestamp = timestamp; + e->replaceHttpReply(temprep); + e->complete(); + /* + * TODO: why put this in the store and then serialise it and + * then parse it again. Simply mark the request complete in + * our context and write the reply struct to the client side. + */ + triggerInitialStoreRead(); + } +} + void clientReplyContext::purgeRequestFindObjectToPurge() { /* Try to find a base entry */ http->flags.purging = 1; lookingforstore = 1; // TODO: can we use purgeAllCached() here instead of doing the // getPublicByRequestMethod() dance? StoreEntry::getPublicByRequestMethod(this, http->request, METHOD_GET); } // Purges all entries with a given url // TODO: move to SideAgent parent, when we have one /* * We probably cannot purge Vary-affected responses because their MD5 * keys depend on vary headers. */ void purgeEntriesByUrl(HttpRequest * req, const char *url) @@ -1783,40 +1832,53 @@ clientReplyContext::next() const { assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode()); return getNextNode(); } void clientReplyContext::sendBodyTooLargeError() { Ip::Address tmp_noaddr; tmp_noaddr.SetNoAddr(); // TODO: make a global const http->logType = LOG_TCP_DENIED_REPLY; ErrorState *err = clientBuildError(ERR_TOO_BIG, HTTP_FORBIDDEN, NULL, http->getConn() != NULL ? http->getConn()->peer : tmp_noaddr, http->request); removeClientStoreReference(&(sc), http); HTTPMSGUNLOCK(reply); startError(err); } +/// send 412 (Precondition Failed) to client +void +clientReplyContext::sendPreconditionFailedError() +{ + http->logType = LOG_TCP_HIT; + ErrorState *const err = + clientBuildError(ERR_PRECONDITION_FAILED, HTTP_PRECONDITION_FAILED, + NULL, http->getConn()->peer, http->request); + removeClientStoreReference(&sc, http); + HTTPMSGUNLOCK(reply); + startError(err); +} + void clientReplyContext::processReplyAccess () { /* NP: this should probably soft-fail to a zero-sized-reply error ?? */ assert(reply); /** Don't block our own responses or HTTP status messages */ if (http->logType == LOG_TCP_DENIED || http->logType == LOG_TCP_DENIED_REPLY || alwaysAllowResponse(reply->sline.status)) { headers_sz = reply->hdr_sz; processReplyAccessResult(1); return; } /** Check for reply to big error */ if (reply->expectedBodyTooLarge(*http->request)) { sendBodyTooLargeError(); return; } === modified file 'src/client_side_reply.h' --- src/client_side_reply.h 2010-05-13 06:20:23 +0000 +++ src/client_side_reply.h 2010-10-01 17:09:31 +0000 @@ -111,35 +111,37 @@ public: clientStreamNode *ourNode; /* This will go away if/when this file gets refactored some more */ private: CBDATA_CLASS(clientReplyContext); clientStreamNode *getNextNode() const; void makeThisHead(); bool errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const ; void sendStreamError(StoreIOBuffer const &result); void pushStreamData(StoreIOBuffer const &result, char *source); clientStreamNode * next() const; StoreIOBuffer holdingBuffer; HttpReply *reply; void processReplyAccess(); static PF ProcessReplyAccessResult; void processReplyAccessResult(bool accessAllowed); void cloneReply(); void buildReplyHeader (); bool alwaysAllowResponse(http_status sline) const; int checkTransferDone(); void processOnlyIfCachedMiss(); + void processConditional(StoreIOBuffer &result); void cacheHit(StoreIOBuffer result); void handleIMSReply(StoreIOBuffer result); void sendMoreData(StoreIOBuffer result); void triggerInitialStoreRead(); void sendClientOldEntry(); void purgeAllCached(); void sendBodyTooLargeError(); + void sendPreconditionFailedError(); StoreEntry *old_entry; store_client *old_sc; /* ... for entry to be validated */ bool deleting; }; #endif /* SQUID_CLIENTSIDEREPLY_H */ === modified file 'src/err_type.h' --- src/err_type.h 2009-10-31 11:53:09 +0000 +++ src/err_type.h 2010-10-01 16:49:56 +0000 @@ -17,40 +17,41 @@ typedef enum { ERR_LIFETIME_EXP, ERR_READ_ERROR, ERR_WRITE_ERROR, ERR_CONNECT_FAIL, ERR_SECURE_CONNECT_FAIL, ERR_SOCKET_FAILURE, /* DNS Errors */ ERR_DNS_FAIL, ERR_URN_RESOLVE, /* HTTP Errors */ ERR_ONLY_IF_CACHED_MISS, /* failure to satisfy only-if-cached request */ ERR_TOO_BIG, ERR_INVALID_RESP, ERR_UNSUP_HTTPVERSION, /* HTTP version is not supported */ ERR_INVALID_REQ, ERR_UNSUP_REQ, ERR_INVALID_URL, ERR_ZERO_SIZE_OBJECT, + ERR_PRECONDITION_FAILED, /* FTP Errors */ ERR_FTP_DISABLED, ERR_FTP_UNAVAILABLE, ERR_FTP_FAILURE, ERR_FTP_PUT_ERROR, ERR_FTP_NOT_FOUND, ERR_FTP_FORBIDDEN, ERR_FTP_PUT_CREATED, /* !error,a note that the file was created */ ERR_FTP_PUT_MODIFIED, /* modified, !created */ /* ESI Errors */ ERR_ESI, /* Failure to perform ESI processing */ /* ICAP Errors */ ERR_ICAP_FAILURE, /* Special Cases */ ERR_DIR_LISTING, /* Display of remote directory (FTP, Gopher) */ ERR_SQUID_SIGNATURE, /* not really an error */ === modified file 'src/store.cc' --- src/store.cc 2010-09-28 15:40:07 +0000 +++ src/store.cc 2010-10-01 18:04:27 +0000 @@ -1887,40 +1887,85 @@ StoreEntry::modifiedSince(HttpRequest * object_length = contentLen(); if (mod_time > request->ims) { debugs(88, 3, "--> YES: entry newer than client"); return true; } else if (mod_time < request->ims) { debugs(88, 3, "--> NO: entry older than client"); return false; } else if (request->imslen < 0) { debugs(88, 3, "--> NO: same LMT, no client length"); return false; } else if (request->imslen == object_length) { debugs(88, 3, "--> NO: same LMT, same length"); return false; } else { debugs(88, 3, "--> YES: same LMT, different length"); return true; } } +bool +StoreEntry::hasIfMatchEtag(const HttpRequest &request) const +{ + const String reqETags = request.header.getList(HDR_IF_MATCH); + return hasOneOfEtags(reqETags, false); +} + +bool +StoreEntry::hasIfNoneMatchEtag(const HttpRequest &request) const +{ + const String reqETags = request.header.getList(HDR_IF_NONE_MATCH); + // weak comparison is allowed only for HEAD or full-body GET requests + const bool allowWeakMatch = !request.flags.range && + (request.method == METHOD_GET || request.method == METHOD_HEAD); + return hasOneOfEtags(reqETags, allowWeakMatch); +} + +/// whether at least one of the request ETags matches entity ETag +bool +StoreEntry::hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const +{ + const ETag repETag = getReply()->header.getETag(HDR_ETAG); + if (!repETag.str) + return strListIsMember(&reqETags, "*", ','); + + bool matched = false; + const char *pos = NULL; + const char *item; + int ilen; + while (!matched && strListGetItem(&reqETags, ',', &item, &ilen, &pos)) { + if (!strncmp(item, "*", ilen)) + matched = true; + else { + String str; + str.append(item, ilen); + ETag reqETag; + if (etagParseInit(&reqETag, str.termedBuf())) { + matched = allowWeakMatch ? etagIsWeakEqual(repETag, reqETag) : + etagIsStrongEqual(repETag, reqETag); + } + } + } + return matched; +} + StorePointer StoreEntry::store() const { assert(0 <= swap_dirn && swap_dirn < Config.cacheSwap.n_configured); return INDEXSD(swap_dirn); } void StoreEntry::unlink() { store()->unlink(*this); } /* * return true if the entry is in a state where * it can accept more data (ie with write() method) */ bool StoreEntry::isAccepting() const {