private boolean explicitFreshnessRequest(final HttpRequestWrapper request, final HttpCacheEntry entry, final Date now) { for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) { for(final HeaderElement elt : h.getElements()) { if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) { try { final int maxstale = Integer.parseInt(elt.getValue()); final long age = validityPolicy.getCurrentAgeSecs(entry, now); final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry); if (age - lifetime > maxstale) { return true; } } catch (final NumberFormatException nfe) { return true; } } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName()) || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) { return true; } } } return false; }
boolean clientRequestsOurOptions(final HttpRequest request) { final RequestLine line = request.getRequestLine(); if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) { return false; } if (!"*".equals(line.getUri())) { return false; } if (!"0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue())) { return false; } return true; }
private boolean explicitFreshnessRequest( final HttpRequestWrapper request, final HttpCacheEntry entry, final Date now) { for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) { for(final HeaderElement elt : h.getElements()) { if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) { try { final int maxstale = Integer.parseInt(elt.getValue()); final long age = validityPolicy.getCurrentAgeSecs(entry, now); final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry); if (age - lifetime > maxstale) { return true; } } catch (final NumberFormatException nfe) { return true; } } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName()) || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) { return true; } } } return false; }
public boolean mayReturnStaleWhileRevalidating(final HttpCacheEntry entry, final Date now) { for (final Header h : entry.getHeaders(HeaderConstants.CACHE_CONTROL)) { for(final HeaderElement elt : h.getElements()) { if (HeaderConstants.STALE_WHILE_REVALIDATE.equalsIgnoreCase(elt.getName())) { try { final int allowedStalenessLifetime = Integer.parseInt(elt.getValue()); if (getStalenessSecs(entry, now) <= allowedStalenessLifetime) { return true; } } catch (final NumberFormatException nfe) { // skip malformed directive } } } } return false; }
private boolean mayReturnStaleIfError(final Header[] headers, final long stalenessSecs) { boolean result = false; for(final Header h : headers) { for(final HeaderElement elt : h.getElements()) { if (HeaderConstants.STALE_IF_ERROR.equals(elt.getName())) { try { final int staleIfErrorSecs = Integer.parseInt(elt.getValue()); if (stalenessSecs <= staleIfErrorSecs) { result = true; break; } } catch (final NumberFormatException nfe) { // skip malformed directive } } } } return result; }
protected long getAgeValue(final HttpCacheEntry entry) { long ageValue = 0; for (final Header hdr : entry.getHeaders(HeaderConstants.AGE)) { long hdrAge; try { hdrAge = Long.parseLong(hdr.getValue()); if (hdrAge < 0) { hdrAge = MAX_AGE; } } catch (final NumberFormatException nfe) { hdrAge = MAX_AGE; } ageValue = (hdrAge > ageValue) ? hdrAge : ageValue; } return ageValue; }
protected long getMaxAge(final HttpCacheEntry entry) { long maxage = -1; for (final Header hdr : entry.getHeaders(HeaderConstants.CACHE_CONTROL)) { for (final HeaderElement elt : hdr.getElements()) { if (HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName()) || "s-maxage".equals(elt.getName())) { try { final long currMaxAge = Long.parseLong(elt.getValue()); if (maxage == -1 || currMaxAge < maxage) { maxage = currMaxAge; } } catch (final NumberFormatException nfe) { // be conservative if can't parse maxage = 0; } } } } return maxage; }
private void stripOtherFreshnessDirectivesWithNoCache(final HttpRequest request) { final List<HeaderElement> outElts = new ArrayList<HeaderElement>(); boolean shouldStrip = false; for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) { for(final HeaderElement elt : h.getElements()) { if (!disallowedWithNoCache.contains(elt.getName())) { outElts.add(elt); } if (HeaderConstants.CACHE_CONTROL_NO_CACHE.equals(elt.getName())) { shouldStrip = true; } } } if (!shouldStrip) { return; } request.removeHeaders(HeaderConstants.CACHE_CONTROL); request.setHeader(HeaderConstants.CACHE_CONTROL, buildHeaderFromElements(outElts)); }
private RequestProtocolError requestHasWeakETagAndRange(final HttpRequest request) { // TODO: Should these be looking at all the headers marked as Range? final String method = request.getRequestLine().getMethod(); if (!(HeaderConstants.GET_METHOD.equals(method))) { return null; } final Header range = request.getFirstHeader(HeaderConstants.RANGE); if (range == null) { return null; } final Header ifRange = request.getFirstHeader(HeaderConstants.IF_RANGE); if (ifRange == null) { return null; } final String val = ifRange.getValue(); if (val.startsWith("W/")) { return RequestProtocolError.WEAK_ETAG_AND_RANGE_ERROR; } return null; }
private boolean expiresHeaderLessOrEqualToDateHeaderAndNoCacheControl( final HttpResponse response) { if (response.getFirstHeader(HeaderConstants.CACHE_CONTROL) != null) { return false; } final Header expiresHdr = response.getFirstHeader(HeaderConstants.EXPIRES); final Header dateHdr = response.getFirstHeader(HTTP.DATE_HEADER); if (expiresHdr == null || dateHdr == null) { return false; } final Date expires = DateUtils.parseDate(expiresHdr.getValue()); final Date date = DateUtils.parseDate(dateHdr.getValue()); if (expires == null || date == null) { return false; } return expires.equals(date) || expires.before(date); }
/** * If I was able to use a {@link CacheEntity} to response to the {@link org.apache.http.HttpRequest} then * generate an {@link HttpResponse} based on the cache entry. * @param request {@link HttpRequestWrapper} to generate the response for * @param entry {@link CacheEntity} to transform into an {@link HttpResponse} * @return {@link HttpResponse} that was constructed */ CloseableHttpResponse generateResponse(final HttpRequestWrapper request, final HttpCacheEntry entry) { final Date now = new Date(); final HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, entry .getStatusCode(), entry.getReasonPhrase()); response.setHeaders(entry.getAllHeaders()); if (responseShouldContainEntity(request, entry)) { final HttpEntity entity = new CacheEntity(entry); addMissingContentLengthHeader(response, entity); response.setEntity(entity); } final long age = this.validityStrategy.getCurrentAgeSecs(entry, now); if (age > 0) { if (age >= Integer.MAX_VALUE) { response.setHeader(HeaderConstants.AGE, "2147483648"); } else { response.setHeader(HeaderConstants.AGE, "" + ((int) age)); } } return Proxies.enhanceResponse(response); }
/** * Check entry against If-None-Match * @param request The current httpRequest being made * @param entry the cache entry * @return boolean does the etag validator match */ private boolean etagValidatorMatches(final HttpRequest request, final HttpCacheEntry entry) { final Header etagHeader = entry.getFirstHeader(HeaderConstants.ETAG); final String etag = (etagHeader != null) ? etagHeader.getValue() : null; final Header[] ifNoneMatch = request.getHeaders(HeaderConstants.IF_NONE_MATCH); if (ifNoneMatch != null) { for (final Header h : ifNoneMatch) { for (final HeaderElement elt : h.getElements()) { final String reqEtag = elt.toString(); if (("*".equals(reqEtag) && etag != null) || reqEtag.equals(etag)) { return true; } } } } return false; }
/** * Check entry against If-Modified-Since, if If-Modified-Since is in the future it is invalid as per * http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html * @param request The current httpRequest being made * @param entry the cache entry * @param now right NOW in time * @return boolean Does the last modified header match */ private boolean lastModifiedValidatorMatches(final HttpRequest request, final HttpCacheEntry entry, final Date now) { final Header lastModifiedHeader = entry.getFirstHeader(HeaderConstants.LAST_MODIFIED); Date lastModified = null; if (lastModifiedHeader != null) { lastModified = DateUtils.parseDate(lastModifiedHeader.getValue()); } if (lastModified == null) { return false; } for (final Header h : request.getHeaders(HeaderConstants.IF_MODIFIED_SINCE)) { final Date ifModifiedSince = DateUtils.parseDate(h.getValue()); if (ifModifiedSince != null) { if (ifModifiedSince.after(now) || lastModified.after(ifModifiedSince)) { return false; } } } return true; }
/** * Try to detect if the returned response is generated from a stale cache entry. * @param httpResponse the response to be checked * @return whether the response is stale or not */ private boolean isNotStale(final HttpResponse httpResponse) { final Header[] warnings = httpResponse.getHeaders(HeaderConstants.WARNING); if (warnings != null) { for (final Header warning : warnings) { /** * warn-codes * 110 = Response is stale * 111 = Revalidation failed */ final String warningValue = warning.getValue(); if (warningValue.startsWith("110") || warningValue.startsWith("111")) { return false; } } } return true; }
/** * When a {@link HttpCacheEntry} does not exist for a specific * {@link org.apache.http.HttpRequest} we attempt to see if an existing * {@link HttpCacheEntry} is appropriate by building a conditional * {@link org.apache.http.HttpRequest} using the variants' ETag values. * If no such values exist, the request is unmodified * * @param request the original request from the caller * @param variants * @return the wrapped request */ public HttpRequestWrapper buildConditionalRequestFromVariants(final HttpRequestWrapper request, final Map<String, Variant> variants) { final HttpRequestWrapper newRequest = HttpRequestWrapper.wrap(request.getOriginal()); newRequest.setHeaders(request.getAllHeaders()); // we do not support partial content so all etags are used final StringBuilder etags = new StringBuilder(); boolean first = true; for(final String etag : variants.keySet()) { if (!first) { etags.append(","); } first = false; etags.append(etag); } newRequest.setHeader(HeaderConstants.IF_NONE_MATCH, etags.toString()); return newRequest; }
@Test public void testBuildConditionalRequestFromVariants() throws Exception { final String etag1 = "\"123\""; final String etag2 = "\"456\""; final String etag3 = "\"789\""; final Map<String,Variant> variantEntries = new HashMap<String,Variant>(); variantEntries.put(etag1, new Variant("A","B",HttpTestUtils.makeCacheEntry(new Header[] { new BasicHeader("ETag", etag1) }))); variantEntries.put(etag2, new Variant("C","D",HttpTestUtils.makeCacheEntry(new Header[] { new BasicHeader("ETag", etag2) }))); variantEntries.put(etag3, new Variant("E","F",HttpTestUtils.makeCacheEntry(new Header[] { new BasicHeader("ETag", etag3) }))); final HttpRequest conditional = impl.buildConditionalRequestFromVariants(request, variantEntries); // seems like a lot of work, but necessary, check for existence and exclusiveness String ifNoneMatch = conditional.getFirstHeader(HeaderConstants.IF_NONE_MATCH).getValue(); Assert.assertTrue(ifNoneMatch.contains(etag1)); Assert.assertTrue(ifNoneMatch.contains(etag2)); Assert.assertTrue(ifNoneMatch.contains(etag3)); ifNoneMatch = ifNoneMatch.replace(etag1, ""); ifNoneMatch = ifNoneMatch.replace(etag2, ""); ifNoneMatch = ifNoneMatch.replace(etag3, ""); ifNoneMatch = ifNoneMatch.replace(",",""); ifNoneMatch = ifNoneMatch.replace(" ", ""); Assert.assertEquals(ifNoneMatch, ""); }
@Test public void testRunReportsJobFailedForStaleResponse() throws Exception { final String identifier = "foo"; final Header[] warning = new Header[] {new BasicHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"")}; final AsynchronousValidationRequest impl = new AsynchronousValidationRequest( mockParent, mockClient, route, request, context, mockExecAware, mockCacheEntry, identifier, 0); when( mockClient.revalidateCacheEntry( route, request, context, mockExecAware, mockCacheEntry)).thenReturn(mockResponse); when(mockResponse.getStatusLine()).thenReturn(mockStatusLine); when(mockStatusLine.getStatusCode()).thenReturn(200); when(mockResponse.getHeaders(HeaderConstants.WARNING)).thenReturn(warning); impl.run(); verify(mockClient).revalidateCacheEntry( route, request, context, mockExecAware, mockCacheEntry); verify(mockResponse).getStatusLine(); verify(mockStatusLine).getStatusCode(); verify(mockResponse).getHeaders(HeaderConstants.WARNING); verify(mockParent).markComplete(identifier); verify(mockParent).jobFailed(identifier); }
private HttpCacheEntry makeCacheEntryWithVariantMap() { final Header[] headers = new Header[5]; for (int i = 0; i < headers.length; i++) { headers[i] = new BasicHeader("header" + i, "value" + i); } final String body = "Lorem ipsum dolor sit amet"; final ProtocolVersion pvObj = new ProtocolVersion("HTTP", 1, 1); final StatusLine slObj = new BasicStatusLine(pvObj, 200, "ok"); final Map<String,String> variantMap = new HashMap<String,String>(); variantMap.put("test variant 1","true"); variantMap.put("test variant 2","true"); final HttpCacheEntry cacheEntry = new HttpCacheEntry(new Date(), new Date(), slObj, headers, new HeapResource(Base64.decodeBase64(body .getBytes(UTF8))), variantMap, HeaderConstants.GET_METHOD); return cacheEntry; }
@Test public void testFlushContentLocationEntryIfUnSafeRequest() throws Exception { final HttpHost host = new HttpHost("foo.example.com"); final HttpRequest req = new HttpPost("/foo"); final HttpResponse resp = HttpTestUtils.make200Response(); resp.setHeader("Content-Location", "/bar"); resp.setHeader(HeaderConstants.ETAG, "\"etag\""); final String key = (new CacheKeyGenerator()).getURI(host, new HttpGet("/bar")); final HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(new Header[] { new BasicHeader("Date", DateUtils.formatDate(new Date())), new BasicHeader("ETag", "\"old-etag\"") }); backing.map.put(key, entry); impl.flushInvalidatedCacheEntriesFor(host, req, resp); assertNull(backing.map.get(key)); }
@Test public void executeWithEtag() throws ClientProtocolException, IOException { EtagAnswer answer = new EtagAnswer(); Mockito.when(httpClient.execute(Matchers.any(HttpUriRequest.class), Matchers.any(HttpContext.class))).then(answer); EtagCachingHttpClient client = new EtagCachingHttpClient(httpClient); HttpResponse response = client.execute(prepareRequest(ETAG, null)); Header etag = response.getFirstHeader(HeaderConstants.ETAG); Assert.assertNotNull(etag, "Etag should not be null"); Assert.assertEquals(etag.getValue(), ETAG); Assert.assertEquals(response.getStatusLine().getStatusCode(), HttpStatus.SC_NOT_MODIFIED, "Http status must be 304 Not Modified"); Assert.assertNull(response.getEntity()); Assert.assertFalse(answer.isModified()); }
@Test public void executeWithModifiedSince() throws ClientProtocolException, IOException { EtagAnswer answer = new EtagAnswer(); Mockito.when(httpClient.execute(Matchers.any(HttpUriRequest.class), Matchers.any(HttpContext.class))).then(answer); EtagCachingHttpClient client = new EtagCachingHttpClient(httpClient); HttpResponse response = client.execute(prepareRequest(null, null)); checkResponse(response, DateUtils.formatDate(answer.getLastModified())); Assert.assertTrue(answer.isModified()); Header lastModifiedHeader = response.getFirstHeader(HeaderConstants.LAST_MODIFIED); String lastModified = lastModifiedHeader.getValue(); response = client.execute(prepareRequest(null, lastModified)); checkResponse(response, lastModified); Assert.assertFalse(answer.isModified()); }
private HttpResponse generateCachedResponse(final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry, final Date now) { final HttpResponse cachedResponse; if (request.containsHeader(HeaderConstants.IF_NONE_MATCH) || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) { cachedResponse = responseGenerator.generateNotModifiedResponse(entry); } else { cachedResponse = responseGenerator.generateResponse(request, entry); } setResponseStatus(context, CacheResponseStatus.CACHE_HIT); if (validityPolicy.getStalenessSecs(entry, now) > 0L) { cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\""); } return cachedResponse; }
private HttpResponse unvalidatedCacheHit( final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry) { final HttpResponse cachedResponse = responseGenerator.generateResponse(request, entry); setResponseStatus(context, CacheResponseStatus.CACHE_HIT); cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\""); return cachedResponse; }
private boolean mayCallBackend(final HttpRequestWrapper request) { for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) { for (final HeaderElement elt : h.getElements()) { if ("only-if-cached".equals(elt.getName())) { log.trace("Request marked only-if-cached"); return false; } } } return true; }
private CloseableHttpResponse generateCachedResponse(final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry, final Date now) { final CloseableHttpResponse cachedResponse; if (request.containsHeader(HeaderConstants.IF_NONE_MATCH) || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) { cachedResponse = responseGenerator.generateNotModifiedResponse(entry); } else { cachedResponse = responseGenerator.generateResponse(request, entry); } setResponseStatus(context, CacheResponseStatus.CACHE_HIT); if (validityPolicy.getStalenessSecs(entry, now) > 0L) { cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\""); } return cachedResponse; }
private CloseableHttpResponse unvalidatedCacheHit( final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry) { final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(request, entry); setResponseStatus(context, CacheResponseStatus.CACHE_HIT); cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\""); return cachedResponse; }
public boolean mayReturnStaleIfError(final HttpRequest request, final HttpCacheEntry entry, final Date now) { final long stalenessSecs = getStalenessSecs(entry, now); return mayReturnStaleIfError(request.getHeaders(HeaderConstants.CACHE_CONTROL), stalenessSecs) || mayReturnStaleIfError(entry.getHeaders(HeaderConstants.CACHE_CONTROL), stalenessSecs); }
protected Date getLastModifiedValue(final HttpCacheEntry entry) { final Header dateHdr = entry.getFirstHeader(HeaderConstants.LAST_MODIFIED); if (dateHdr == null) { return null; } return DateUtils.parseDate(dateHdr.getValue()); }
protected Date getExpirationDate(final HttpCacheEntry entry) { final Header expiresHeader = entry.getFirstHeader(HeaderConstants.EXPIRES); if (expiresHeader == null) { return null; } return DateUtils.parseDate(expiresHeader.getValue()); }
public boolean hasCacheControlDirective(final HttpCacheEntry entry, final String directive) { for (final Header h : entry.getHeaders(HeaderConstants.CACHE_CONTROL)) { for(final HeaderElement elt : h.getElements()) { if (directive.equalsIgnoreCase(elt.getName())) { return true; } } } return false; }
private void addVariantWithEtag(final String variantKey, final String variantCacheKey, final Map<String, Variant> variants) throws IOException { final HttpCacheEntry entry = storage.getEntry(variantCacheKey); if (entry == null) { return; } final Header etagHeader = entry.getFirstHeader(HeaderConstants.ETAG); if (etagHeader == null) { return; } variants.put(etagHeader.getValue(), new Variant(variantKey, variantCacheKey, entry)); }
private void decrementOPTIONSMaxForwardsIfGreaterThen0(final HttpRequest request) { if (!HeaderConstants.OPTIONS_METHOD.equals(request.getRequestLine().getMethod())) { return; } final Header maxForwards = request.getFirstHeader(HeaderConstants.MAX_FORWARDS); if (maxForwards == null) { return; } request.removeHeaders(HeaderConstants.MAX_FORWARDS); final int currentMaxForwards = Integer.parseInt(maxForwards.getValue()); request.setHeader(HeaderConstants.MAX_FORWARDS, Integer.toString(currentMaxForwards - 1)); }
private void verifyOPTIONSRequestWithBodyHasContentType(final HttpRequest request) { if (!HeaderConstants.OPTIONS_METHOD.equals(request.getRequestLine().getMethod())) { return; } if (!(request instanceof HttpEntityEnclosingRequest)) { return; } addContentTypeHeaderIfMissing((HttpEntityEnclosingRequest) request); }
private RequestProtocolError requestHasWeekETagForPUTOrDELETEIfMatch(final HttpRequest request) { // TODO: Should these be looking at all the headers marked as If-Match/If-None-Match? final String method = request.getRequestLine().getMethod(); if (!(HeaderConstants.PUT_METHOD.equals(method) || HeaderConstants.DELETE_METHOD .equals(method))) { return null; } final Header ifMatch = request.getFirstHeader(HeaderConstants.IF_MATCH); if (ifMatch != null) { final String val = ifMatch.getValue(); if (val.startsWith("W/")) { return RequestProtocolError.WEAK_ETAG_ON_PUTDELETE_METHOD_ERROR; } } else { final Header ifNoneMatch = request.getFirstHeader(HeaderConstants.IF_NONE_MATCH); if (ifNoneMatch == null) { return null; } final String val2 = ifNoneMatch.getValue(); if (val2.startsWith("W/")) { return RequestProtocolError.WEAK_ETAG_ON_PUTDELETE_METHOD_ERROR; } } return null; }
private RequestProtocolError requestContainsNoCacheDirectiveWithFieldName(final HttpRequest request) { for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) { for(final HeaderElement elt : h.getElements()) { if (HeaderConstants.CACHE_CONTROL_NO_CACHE.equalsIgnoreCase(elt.getName()) && elt.getValue() != null) { return RequestProtocolError.NO_CACHE_DIRECTIVE_WITH_FIELD_NAME; } } } return null; }
protected boolean isExplicitlyNonCacheable(final HttpResponse response) { final Header[] cacheControlHeaders = response.getHeaders(HeaderConstants.CACHE_CONTROL); for (final Header header : cacheControlHeaders) { for (final HeaderElement elem : header.getElements()) { if (HeaderConstants.CACHE_CONTROL_NO_STORE.equals(elem.getName()) || HeaderConstants.CACHE_CONTROL_NO_CACHE.equals(elem.getName()) || (sharedCache && HeaderConstants.PRIVATE.equals(elem.getName()))) { return true; } } } return false; }
protected boolean hasCacheControlParameterFrom(final HttpMessage msg, final String[] params) { final Header[] cacheControlHeaders = msg.getHeaders(HeaderConstants.CACHE_CONTROL); for (final Header header : cacheControlHeaders) { for (final HeaderElement elem : header.getElements()) { for (final String param : params) { if (param.equalsIgnoreCase(elem.getName())) { return true; } } } } return false; }
protected boolean isExplicitlyCacheable(final HttpResponse response) { if (response.getFirstHeader(HeaderConstants.EXPIRES) != null) { return true; } final String[] cacheableParams = { HeaderConstants.CACHE_CONTROL_MAX_AGE, "s-maxage", HeaderConstants.CACHE_CONTROL_MUST_REVALIDATE, HeaderConstants.CACHE_CONTROL_PROXY_REVALIDATE, HeaderConstants.PUBLIC }; return hasCacheControlParameterFrom(response, cacheableParams); }