diff --git a/addOns/pscanrules/CHANGELOG.md b/addOns/pscanrules/CHANGELOG.md index 138a9dae3c7..37f932013cc 100644 --- a/addOns/pscanrules/CHANGELOG.md +++ b/addOns/pscanrules/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Add alert references to HTTP Server Response Header scan rule alerts (Issue 7100, 9050). - Update alert references to latest locations to fix 404s and resolve redirections. +- Reduced usage of error level logging. ## [66] - 2025-07-25 ### Added diff --git a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/CrossDomainMisconfigurationScanRule.java b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/CrossDomainMisconfigurationScanRule.java index c75ee2ea4b4..2000311d8b1 100644 --- a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/CrossDomainMisconfigurationScanRule.java +++ b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/CrossDomainMisconfigurationScanRule.java @@ -85,66 +85,60 @@ public String getName() { @Override public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) { - try { + LOGGER.debug( + "Checking message {} for Cross-Domain misconfigurations", + msg.getRequestHeader().getURI()); + + String corsAllowOriginValue = + msg.getResponseHeader().getHeader(HttpHeader.ACCESS_CONTROL_ALLOW_ORIGIN); + // String corsAllowHeadersValue = + // msg.getResponseHeader().getHeader(HttpHeader.ACCESS_CONTROL_ALLOW_HEADERS); + // String corsAllowMethodsValue = + // msg.getResponseHeader().getHeader(HttpHeader.ACCESS_CONTROL_ALLOW_METHODS); + // String corsExposeHeadersValue = + // msg.getResponseHeader().getHeader(HttpHeader.ACCESS_CONTROL_EXPOSE_HEADERS); + + if (corsAllowOriginValue != null && corsAllowOriginValue.equals("*")) { LOGGER.debug( - "Checking message {} for Cross-Domain misconfigurations", - msg.getRequestHeader().getURI()); - - String corsAllowOriginValue = - msg.getResponseHeader().getHeader(HttpHeader.ACCESS_CONTROL_ALLOW_ORIGIN); - // String corsAllowHeadersValue = - // msg.getResponseHeader().getHeader(HttpHeader.ACCESS_CONTROL_ALLOW_HEADERS); - // String corsAllowMethodsValue = - // msg.getResponseHeader().getHeader(HttpHeader.ACCESS_CONTROL_ALLOW_METHODS); - // String corsExposeHeadersValue = - // msg.getResponseHeader().getHeader(HttpHeader.ACCESS_CONTROL_EXPOSE_HEADERS); - - if (corsAllowOriginValue != null && corsAllowOriginValue.equals("*")) { - LOGGER.debug( - "Raising a Medium risk Cross Domain alert on {}: {}", - HttpHeader.ACCESS_CONTROL_ALLOW_ORIGIN, - corsAllowOriginValue); - // Its a Medium, rather than a High (as originally thought), for the following - // reasons: - // Assumption: if an API is accessible in an unauthenticated manner, it doesn't need - // to be protected - // (if it should be protected, its a Missing Function Level Access Control issue, - // not a Cross Domain Misconfiguration) - // - // Case 1) Request sent using XHR - // - cookies will not be sent with the request at all unless withCredentials = true - // on the XHR request; - // - If a cookie was sent with the request, the browser will not give access to the - // response body via JavaScript unless the response headers say - // "Access-Control-Allow-Credentials: true" - // - If "Access-Control-Allow-Credentials: true" and "Access-Control-Allow-Origin: - // *" in the response, the browser will not give access to the response body. - // (this is an edge case, but is actually really important, because it blocks all - // the useful attacks, and is well supported by modern browsers) - // Case 2) Request sent using HTML Form POST with an iframe, for instance, and - // attempting to access the iframe body (ie, the Cross Domain response) using - // JavaScript - // - the cookie will be sent by the web browser (possibly leading to CSRF, but with - // no impact from the point of view of the Same Origin Policy / Cross Domain - // Misconfiguration - // - the HTML response is not accessible in JavaScript, regardless of the CORS - // headers sent in the response (in all my trials, at least) - // (this is even more restrictive than the equivalent request sent by XHR) - - // The CORS misconfig could still allow an attacker to access the data returned from - // an unauthenticated API, which is protected by some other form of security, such - // as IP address white-listing, for instance. - - buildAlert( - extractEvidence( - msg.getResponseHeader().toString(), - HttpHeader.ACCESS_CONTROL_ALLOW_ORIGIN)) - .raise(); - } - - } catch (Exception e) { - LOGGER.error( - "An error occurred trying to passively scan a message for Cross Domain Misconfigurations"); + "Raising a Medium risk Cross Domain alert on {}: {}", + HttpHeader.ACCESS_CONTROL_ALLOW_ORIGIN, + corsAllowOriginValue); + // Its a Medium, rather than a High (as originally thought), for the following + // reasons: + // Assumption: if an API is accessible in an unauthenticated manner, it doesn't need + // to be protected + // (if it should be protected, its a Missing Function Level Access Control issue, + // not a Cross Domain Misconfiguration) + // + // Case 1) Request sent using XHR + // - cookies will not be sent with the request at all unless withCredentials = true + // on the XHR request; + // - If a cookie was sent with the request, the browser will not give access to the + // response body via JavaScript unless the response headers say + // "Access-Control-Allow-Credentials: true" + // - If "Access-Control-Allow-Credentials: true" and "Access-Control-Allow-Origin: + // *" in the response, the browser will not give access to the response body. + // (this is an edge case, but is actually really important, because it blocks all + // the useful attacks, and is well supported by modern browsers) + // Case 2) Request sent using HTML Form POST with an iframe, for instance, and + // attempting to access the iframe body (ie, the Cross Domain response) using + // JavaScript + // - the cookie will be sent by the web browser (possibly leading to CSRF, but with + // no impact from the point of view of the Same Origin Policy / Cross Domain + // Misconfiguration + // - the HTML response is not accessible in JavaScript, regardless of the CORS + // headers sent in the response (in all my trials, at least) + // (this is even more restrictive than the equivalent request sent by XHR) + + // The CORS misconfig could still allow an attacker to access the data returned from + // an unauthenticated API, which is protected by some other form of security, such + // as IP address white-listing, for instance. + + buildAlert( + extractEvidence( + msg.getResponseHeader().toString(), + HttpHeader.ACCESS_CONTROL_ALLOW_ORIGIN)) + .raise(); } } diff --git a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureDebugErrorsScanRule.java b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureDebugErrorsScanRule.java index db4bc9cde05..eda55d67dde 100644 --- a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureDebugErrorsScanRule.java +++ b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureDebugErrorsScanRule.java @@ -114,7 +114,7 @@ private static List loadFile(Path path) { BufferedReader reader = null; File f = path.toFile(); if (!f.exists()) { - LOGGER.error("No such file: {}", f.getAbsolutePath()); + LOGGER.warn("No such file: {}", f.getAbsolutePath()); return strings; } try { diff --git a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureInUrlScanRule.java b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureInUrlScanRule.java index aa7fa84abca..1c3587e5a4c 100644 --- a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureInUrlScanRule.java +++ b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureInUrlScanRule.java @@ -132,7 +132,7 @@ private static List loadFile(String file) { List strings = new ArrayList<>(); File f = new File(Constant.getZapHome() + File.separator + file); if (!f.exists()) { - LOGGER.error("No such file: {}", f.getAbsolutePath()); + LOGGER.warn("No such file: {}", f.getAbsolutePath()); return strings; } diff --git a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureReferrerScanRule.java b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureReferrerScanRule.java index e703deabfde..d2e1b8c5efd 100644 --- a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureReferrerScanRule.java +++ b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InformationDisclosureReferrerScanRule.java @@ -184,7 +184,7 @@ private static List loadFile(String file) { List strings = new ArrayList<>(); File f = new File(Constant.getZapHome() + File.separator + file); if (!f.exists()) { - LOGGER.error("No such file: {}", f.getAbsolutePath()); + LOGGER.warn("No such file: {}", f.getAbsolutePath()); return strings; } diff --git a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InsecureAuthenticationScanRule.java b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InsecureAuthenticationScanRule.java index 89009088c13..256f617b23c 100644 --- a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InsecureAuthenticationScanRule.java +++ b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/InsecureAuthenticationScanRule.java @@ -124,7 +124,7 @@ public void scanHttpRequestSend(HttpMessage msg, int id) { alertRisk = Alert.RISK_HIGH; } } catch (IllegalArgumentException e) { - LOGGER.error( + LOGGER.warn( "Invalid Base64 value for {} Authentication: {}", authMechanism, authValues[1]); diff --git a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/RetrievedFromCacheScanRule.java b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/RetrievedFromCacheScanRule.java index 80fa9aea25d..d39ffd06db1 100644 --- a/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/RetrievedFromCacheScanRule.java +++ b/addOns/pscanrules/src/main/java/org/zaproxy/zap/extension/pscanrules/RetrievedFromCacheScanRule.java @@ -57,88 +57,82 @@ public class RetrievedFromCacheScanRule extends PluginPassiveScanner @Override public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) { - try { - LOGGER.debug( - "Checking URL {} to see if was served from a shared cache", - msg.getRequestHeader().getURI()); - - // X-Cache: HIT - // X-Cache: HIT from cache.kolich.local <-- was the data actually served from the - // cache (subject to no-cache, expiry, etc.)? - // (if X-Cache: HIT, it implies X-Cache-Lookup: HIT) - // (and if X-Cache-Lookup: MISS, it implies X-Cache: MISS) - // X-Cache-Lookup: HIT from cache.kolich.local:80 <-- was the data *available* in the - // cache? (not whether it was actually served) - - // X-Cache: MISS - // X-Cache: MISS from cache.kolich.local - // X-Cache-Lookup: MISS from cache.kolich.local:80 - - // X-Cache HIT from proxy.domain.tld, MISS from proxy.local - // X-Cache-Lookup HIT from proxy.domain.tld:3128, MISS from proxy.local:3128 - - List xcacheHeaders = msg.getResponseHeader().getHeaderValues("X-Cache"); - if (!xcacheHeaders.isEmpty()) { - for (String xcacheHeader : xcacheHeaders) { - for (String proxyServerDetails : xcacheHeader.split(",")) { - // strip off any leading space for the second and subsequent proxies - if (proxyServerDetails.startsWith(" ")) - proxyServerDetails = proxyServerDetails.substring(1); - LOGGER.trace("Proxy HIT/MISS details [{}]", proxyServerDetails); - String[] proxyServerDetailsArray = proxyServerDetails.split(" ", 3); - if (proxyServerDetailsArray.length >= 1) { - String hitormiss = - proxyServerDetailsArray[0].toUpperCase(); // HIT or MISS - if (hitormiss.equals("HIT")) { - // the response was served from cache, so raise it.. - String evidence = proxyServerDetails; - LOGGER.debug( - "{} was served from a cache, due to presence of a 'HIT' in the 'X-Cache' response header", - msg.getRequestHeader().getURI()); - // could be from HTTP/1.0 or HTTP/1.1. We don't know which. - buildAlert(evidence, false).raise(); - return; - } + LOGGER.debug( + "Checking URL {} to see if was served from a shared cache", + msg.getRequestHeader().getURI()); + + // X-Cache: HIT + // X-Cache: HIT from cache.kolich.local <-- was the data actually served from the + // cache (subject to no-cache, expiry, etc.)? + // (if X-Cache: HIT, it implies X-Cache-Lookup: HIT) + // (and if X-Cache-Lookup: MISS, it implies X-Cache: MISS) + // X-Cache-Lookup: HIT from cache.kolich.local:80 <-- was the data *available* in the + // cache? (not whether it was actually served) + + // X-Cache: MISS + // X-Cache: MISS from cache.kolich.local + // X-Cache-Lookup: MISS from cache.kolich.local:80 + + // X-Cache HIT from proxy.domain.tld, MISS from proxy.local + // X-Cache-Lookup HIT from proxy.domain.tld:3128, MISS from proxy.local:3128 + + List xcacheHeaders = msg.getResponseHeader().getHeaderValues("X-Cache"); + if (!xcacheHeaders.isEmpty()) { + for (String xcacheHeader : xcacheHeaders) { + for (String proxyServerDetails : xcacheHeader.split(",")) { + // strip off any leading space for the second and subsequent proxies + if (proxyServerDetails.startsWith(" ")) + proxyServerDetails = proxyServerDetails.substring(1); + LOGGER.trace("Proxy HIT/MISS details [{}]", proxyServerDetails); + String[] proxyServerDetailsArray = proxyServerDetails.split(" ", 3); + if (proxyServerDetailsArray.length >= 1) { + String hitormiss = proxyServerDetailsArray[0].toUpperCase(); // HIT or MISS + if (hitormiss.equals("HIT")) { + // the response was served from cache, so raise it.. + String evidence = proxyServerDetails; + LOGGER.debug( + "{} was served from a cache, due to presence of a 'HIT' in the 'X-Cache' response header", + msg.getRequestHeader().getURI()); + // could be from HTTP/1.0 or HTTP/1.1. We don't know which. + buildAlert(evidence, false).raise(); + return; } } } } + } - // The "Age" header (defined in RFC 7234) conveys the sender's estimate of the amount of - // time since the response (or its revalidation) was generated at the origin server. - // An HTTP/1.1 server that includes a cache MUST include an Age header field in every - // response generated from its own cache. - // i.e.: a valid "Age" header implies that the response was served from a cache - // lets validate that it is actually a non-negative decimal integer, as mandated by RFC - // 7234, however. - // if there are multiple "Age" headers, just look for one valid value in the multiple - // "Age" headers.. Not sure if this case is strictly valid with the spec, however. - // Note: HTTP/1.0 caches do not implement "Age", so the absence of an "Age" header does - // *not* imply that the response was served from the origin server, rather than a - // cache.. - List ageHeaders = msg.getResponseHeader().getHeaderValues("Age"); - if (!ageHeaders.isEmpty()) { - for (String ageHeader : ageHeaders) { - LOGGER.trace("Validating Age header value [{}]", ageHeader); - Long ageAsLong = null; - try { - ageAsLong = Long.parseLong(ageHeader); - } catch (NumberFormatException nfe) { - // Ignore - } - if (ageAsLong != null && ageAsLong >= 0) { - String evidence = "Age: " + ageHeader; - LOGGER.debug( - "{} was served from a HTTP/1.1 cache, due to presence of a valid (non-negative decimal integer) 'Age' response header value", - msg.getRequestHeader().getURI()); - buildAlert(evidence, true).raise(); - return; - } + // The "Age" header (defined in RFC 7234) conveys the sender's estimate of the amount of + // time since the response (or its revalidation) was generated at the origin server. + // An HTTP/1.1 server that includes a cache MUST include an Age header field in every + // response generated from its own cache. + // i.e.: a valid "Age" header implies that the response was served from a cache + // lets validate that it is actually a non-negative decimal integer, as mandated by RFC + // 7234, however. + // if there are multiple "Age" headers, just look for one valid value in the multiple + // "Age" headers.. Not sure if this case is strictly valid with the spec, however. + // Note: HTTP/1.0 caches do not implement "Age", so the absence of an "Age" header does + // *not* imply that the response was served from the origin server, rather than a + // cache.. + List ageHeaders = msg.getResponseHeader().getHeaderValues("Age"); + if (!ageHeaders.isEmpty()) { + for (String ageHeader : ageHeaders) { + LOGGER.trace("Validating Age header value [{}]", ageHeader); + Long ageAsLong = null; + try { + ageAsLong = Long.parseLong(ageHeader); + } catch (NumberFormatException nfe) { + // Ignore + } + if (ageAsLong != null && ageAsLong >= 0) { + String evidence = "Age: " + ageHeader; + LOGGER.debug( + "{} was served from a HTTP/1.1 cache, due to presence of a valid (non-negative decimal integer) 'Age' response header value", + msg.getRequestHeader().getURI()); + buildAlert(evidence, true).raise(); + return; } } - - } catch (Exception e) { - LOGGER.error("An error occurred while checking if a URL was served from a cache", e); } } diff --git a/addOns/pscanrulesAlpha/CHANGELOG.md b/addOns/pscanrulesAlpha/CHANGELOG.md index 6c5b6996a11..aeefb303cb8 100644 --- a/addOns/pscanrulesAlpha/CHANGELOG.md +++ b/addOns/pscanrulesAlpha/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased ### Removed - The two example passive scan rules were removed from this add-on and are now part of: https://github.com/zaproxy/addon-java +- Reduced usage of error level logging. ## [46] - 2025-09-18 ### Changed diff --git a/addOns/pscanrulesBeta/CHANGELOG.md b/addOns/pscanrulesBeta/CHANGELOG.md index a3514a11dff..6da8a7d522a 100644 --- a/addOns/pscanrulesBeta/CHANGELOG.md +++ b/addOns/pscanrulesBeta/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [46] - 2025-09-18 ### Changed - Update alert references to latest locations to fix 404s and resolve redirections. +- Reduced usage of error level logging. ## [45] - 2025-09-10 ### Changed diff --git a/addOns/pscanrulesBeta/src/main/java/org/zaproxy/zap/extension/pscanrulesBeta/CacheableScanRule.java b/addOns/pscanrulesBeta/src/main/java/org/zaproxy/zap/extension/pscanrulesBeta/CacheableScanRule.java index c5683df0def..ed8d6eada98 100644 --- a/addOns/pscanrulesBeta/src/main/java/org/zaproxy/zap/extension/pscanrulesBeta/CacheableScanRule.java +++ b/addOns/pscanrulesBeta/src/main/java/org/zaproxy/zap/extension/pscanrulesBeta/CacheableScanRule.java @@ -103,340 +103,377 @@ public class CacheableScanRule extends PluginPassiveScanner implements CommonPas public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) { // TODO: standardise the logic in the case of duplicate / conflicting headers. - try { - LOGGER.debug("Checking URL {} for storability", msg.getRequestHeader().getURI()); - - // storability: is the request method understood by the cache and defined as being - // cacheable? - String method = msg.getRequestHeader().getMethod(); - String methodUpper = method.toUpperCase(); - if (!(methodUpper.equals(HttpRequestHeader.GET) - || methodUpper.equals(HttpRequestHeader.HEAD) - || methodUpper.equals(HttpRequestHeader.POST))) { - // non-cacheable method ==> non-storable - LOGGER.debug( - "{} is not storable due to the use of the non-cacheable request method '{}'", - msg.getRequestHeader().getURI(), - method); - alertNonStorable(method + " ").raise(); - return; - } + LOGGER.debug("Checking URL {} for storability", msg.getRequestHeader().getURI()); + + // storability: is the request method understood by the cache and defined as being + // cacheable? + String method = msg.getRequestHeader().getMethod(); + String methodUpper = method.toUpperCase(); + if (!(methodUpper.equals(HttpRequestHeader.GET) + || methodUpper.equals(HttpRequestHeader.HEAD) + || methodUpper.equals(HttpRequestHeader.POST))) { + // non-cacheable method ==> non-storable + LOGGER.debug( + "{} is not storable due to the use of the non-cacheable request method '{}'", + msg.getRequestHeader().getURI(), + method); + alertNonStorable(method + " ").raise(); + return; + } - // is the response status code "understood" by the cache? - // this is somewhat implementation specific, so lets assume that a cache "understands" - // all 1XX, 2XX, 3XX, 4XX, and 5XX response classes for now. - // this logic will allow us to detect if the response is storable by "some" compliant - // caching server - int responseClass = msg.getResponseHeader().getStatusCode() / 100; - if ((responseClass != 1) - && (responseClass != 2) - && (responseClass != 3) - && (responseClass != 4) - && (responseClass != 5)) { - LOGGER.debug( - "{} is not storable due to the use of a HTTP response class [{}] that we do not 'understand' (we 'understand' 1XX, 2XX, 3XX, 4XX, and 5XX response classes)", - msg.getRequestHeader().getURI(), - responseClass); - alertNonStorable(String.valueOf(msg.getResponseHeader().getStatusCode())).raise(); - return; - } + // is the response status code "understood" by the cache? + // this is somewhat implementation specific, so lets assume that a cache "understands" + // all 1XX, 2XX, 3XX, 4XX, and 5XX response classes for now. + // this logic will allow us to detect if the response is storable by "some" compliant + // caching server + int responseClass = msg.getResponseHeader().getStatusCode() / 100; + if ((responseClass != 1) + && (responseClass != 2) + && (responseClass != 3) + && (responseClass != 4) + && (responseClass != 5)) { + LOGGER.debug( + "{} is not storable due to the use of a HTTP response class [{}] that we do not 'understand' (we 'understand' 1XX, 2XX, 3XX, 4XX, and 5XX response classes)", + msg.getRequestHeader().getURI(), + responseClass); + alertNonStorable(String.valueOf(msg.getResponseHeader().getStatusCode())).raise(); + return; + } - // does the "no-store" cache directive appear in request or response header fields? - // 1: check the Pragma request header (for HTTP 1.0 caches) - // 2: check the Pragma response header (for HTTP 1.0 caches) - // 3: check the Cache-Control request header (for HTTP 1.1 caches) - // 4: check the Cache-Control response header (for HTTP 1.1 caches) - List headers = new ArrayList<>(); - headers.addAll(msg.getRequestHeader().getHeaderValues(HttpHeader.PRAGMA)); - headers.addAll(msg.getResponseHeader().getHeaderValues(HttpHeader.PRAGMA)); - headers.addAll(msg.getRequestHeader().getHeaderValues(HttpHeader.CACHE_CONTROL)); - headers.addAll(msg.getResponseHeader().getHeaderValues(HttpHeader.CACHE_CONTROL)); + // does the "no-store" cache directive appear in request or response header fields? + // 1: check the Pragma request header (for HTTP 1.0 caches) + // 2: check the Pragma response header (for HTTP 1.0 caches) + // 3: check the Cache-Control request header (for HTTP 1.1 caches) + // 4: check the Cache-Control response header (for HTTP 1.1 caches) + List headers = new ArrayList<>(); + headers.addAll(msg.getRequestHeader().getHeaderValues(HttpHeader.PRAGMA)); + headers.addAll(msg.getResponseHeader().getHeaderValues(HttpHeader.PRAGMA)); + headers.addAll(msg.getRequestHeader().getHeaderValues(HttpHeader.CACHE_CONTROL)); + headers.addAll(msg.getResponseHeader().getHeaderValues(HttpHeader.CACHE_CONTROL)); + + for (String directive : headers) { + for (String directiveToken : directive.split(" ")) { + // strip off any trailing comma + if (directiveToken.endsWith(",")) + directiveToken = directiveToken.substring(0, directiveToken.length() - 1); + LOGGER.trace("Looking for 'no-store' in [{}]", directiveToken); + if (directiveToken.toLowerCase().equals("no-store")) { + LOGGER.debug( + "{} is not storable due to the use of HTTP caching directive 'no-store' in the request or response", + msg.getRequestHeader().getURI()); + alertNonStorable(directiveToken).raise(); + return; + } + } + } - for (String directive : headers) { + // does the "private" response directive appear in the response, if the cache is shared + // check the Cache-Control response header only (for HTTP 1.1 caches) + List responseHeadersCacheControl = + msg.getResponseHeader().getHeaderValues(HttpHeader.CACHE_CONTROL); + if (!responseHeadersCacheControl.isEmpty()) { + for (String directive : responseHeadersCacheControl) { for (String directiveToken : directive.split(" ")) { // strip off any trailing comma if (directiveToken.endsWith(",")) directiveToken = directiveToken.substring(0, directiveToken.length() - 1); - LOGGER.trace("Looking for 'no-store' in [{}]", directiveToken); - if (directiveToken.toLowerCase().equals("no-store")) { + LOGGER.trace("Looking for 'private' in [{}]", directiveToken); + if (directiveToken.toLowerCase().equals("private")) { LOGGER.debug( - "{} is not storable due to the use of HTTP caching directive 'no-store' in the request or response", + "{} is not storable due to the use of HTTP caching directive 'private' in the response", msg.getRequestHeader().getURI()); alertNonStorable(directiveToken).raise(); return; } } } + } - // does the "private" response directive appear in the response, if the cache is shared - // check the Cache-Control response header only (for HTTP 1.1 caches) - List responseHeadersCacheControl = - msg.getResponseHeader().getHeaderValues(HttpHeader.CACHE_CONTROL); + // does the Authorization header field appear in the request, if the cache is shared + // (which we assume it is for now) + // if so, does the response explicitly allow it to be cached? (see rfc7234 section 3.2) + // Note: this logic defines if an initial request is storable. A second request for the + // same URL + // may or may not be actually served from the cache, depending on other criteria, such + // as whether the cached response is + // considered stale (based on the values of s-maxage and other values). This is in + // accordance with rfc7234 section 3.2. + List authHeaders = msg.getRequestHeader().getHeaderValues(HttpHeader.AUTHORIZATION); + if (!authHeaders.isEmpty()) { + // there is an authorization header + // look for "must-revalidate", "public", and "s-maxage", in the response, since + // these permit + // a request with an "Authorization" request header to be cached if (!responseHeadersCacheControl.isEmpty()) { + boolean authorizedIsStorable = false; for (String directive : responseHeadersCacheControl) { for (String directiveToken : directive.split(" ")) { // strip off any trailing comma if (directiveToken.endsWith(",")) directiveToken = directiveToken.substring(0, directiveToken.length() - 1); - LOGGER.trace("Looking for 'private' in [{}]", directiveToken); - if (directiveToken.toLowerCase().equals("private")) { - LOGGER.debug( - "{} is not storable due to the use of HTTP caching directive 'private' in the response", - msg.getRequestHeader().getURI()); - alertNonStorable(directiveToken).raise(); - return; + LOGGER.trace( + "Looking for 'must-revalidate', 'public', 's-maxage' in [{}]", + directiveToken); + if ((directiveToken.toLowerCase().equals("must-revalidate")) + || (directiveToken.toLowerCase().equals("public")) + || (directiveToken.toLowerCase().startsWith("s-maxage="))) { + authorizedIsStorable = true; + break; } } } - } - - // does the Authorization header field appear in the request, if the cache is shared - // (which we assume it is for now) - // if so, does the response explicitly allow it to be cached? (see rfc7234 section 3.2) - // Note: this logic defines if an initial request is storable. A second request for the - // same URL - // may or may not be actually served from the cache, depending on other criteria, such - // as whether the cached response is - // considered stale (based on the values of s-maxage and other values). This is in - // accordance with rfc7234 section 3.2. - List authHeaders = - msg.getRequestHeader().getHeaderValues(HttpHeader.AUTHORIZATION); - if (!authHeaders.isEmpty()) { - // there is an authorization header - // look for "must-revalidate", "public", and "s-maxage", in the response, since - // these permit - // a request with an "Authorization" request header to be cached - if (!responseHeadersCacheControl.isEmpty()) { - boolean authorizedIsStorable = false; - for (String directive : responseHeadersCacheControl) { - for (String directiveToken : directive.split(" ")) { - // strip off any trailing comma - if (directiveToken.endsWith(",")) - directiveToken = - directiveToken.substring(0, directiveToken.length() - 1); - LOGGER.trace( - "Looking for 'must-revalidate', 'public', 's-maxage' in [{}]", - directiveToken); - if ((directiveToken.toLowerCase().equals("must-revalidate")) - || (directiveToken.toLowerCase().equals("public")) - || (directiveToken.toLowerCase().startsWith("s-maxage="))) { - authorizedIsStorable = true; - break; - } - } - } - // is the request with an authorisation header allowed, based on the response - // headers? - if (!authorizedIsStorable) { - LOGGER.debug( - "{} is not storable due to the use of the 'Authorisation' request header, without a compensatory 'must-revalidate', 'public', or 's-maxage' directive in the response", - msg.getRequestHeader().getURI()); - alertNonStorable(HttpHeader.AUTHORIZATION + ":").raise(); - return; - } - } else { + // is the request with an authorisation header allowed, based on the response + // headers? + if (!authorizedIsStorable) { LOGGER.debug( - "{} is not storable due to the use of the 'Authorisation' request header, without a compensatory 'must-revalidate', 'public', or 's-maxage' directive in the response (no 'Cache-Control' directive was noted)", + "{} is not storable due to the use of the 'Authorisation' request header, without a compensatory 'must-revalidate', 'public', or 's-maxage' directive in the response", msg.getRequestHeader().getURI()); alertNonStorable(HttpHeader.AUTHORIZATION + ":").raise(); return; } - } - - // in addition to the checks above, just one of the following needs to be true for the - // response to be storable - /* - * the response - * contains an Expires header field (see Section 5.3), or - * contains a max-age response directive (see Section 5.2.2.8), or - * contains a s-maxage response directive (see Section 5.2.2.9) - and the cache is shared, or - * contains a Cache Control Extension (see Section 5.2.3) that - allows it to be cached, or - * has a status code that is defined as cacheable by default (see - Section 4.2.2), or - * contains a public response directive (see Section 5.2.2.5). - */ - // TODO: replace "Expires" with some defined constant. Can't find one right now though. - // Ho Hum. - List expires = msg.getResponseHeader().getHeaderValues("Expires"); - if (!expires.isEmpty()) + } else { LOGGER.debug( - "{} *is* storable due to the basic checks, and the presence of the 'Expires' header in the response", + "{} is not storable due to the use of the 'Authorisation' request header, without a compensatory 'must-revalidate', 'public', or 's-maxage' directive in the response (no 'Cache-Control' directive was noted)", msg.getRequestHeader().getURI()); - // grab this for later. Not needed for "storability" checks. - List dates = msg.getResponseHeader().getHeaderValues("Date"); + alertNonStorable(HttpHeader.AUTHORIZATION + ":").raise(); + return; + } + } - String maxAge = null, sMaxAge = null, publicDirective = null; - if (!responseHeadersCacheControl.isEmpty()) { - for (String directive : responseHeadersCacheControl) { - for (String directiveToken : directive.split(" ")) { - // strip off any trailing comma - if (directiveToken.endsWith(",")) - directiveToken = - directiveToken.substring(0, directiveToken.length() - 1); - LOGGER.trace( - "Looking for 'max-age', 's-maxage', 'public' in [{}]", - directiveToken); - if (directiveToken.toLowerCase().startsWith("max-age=")) { - LOGGER.debug( - "{} *is* storable due to the basic checks, and the presence of the 'max-age' caching directive in the response", - msg.getRequestHeader().getURI()); - maxAge = directiveToken; - } - if (directiveToken - .toLowerCase() - .startsWith("s-maxage=")) { // for a shared cache.. - LOGGER.debug( - "{} *is* storable due to the basic checks, and the presence of the 's-maxage' caching directive in the response", - msg.getRequestHeader().getURI()); - sMaxAge = directiveToken; - } - if (directiveToken.toLowerCase().equals("public")) { - LOGGER.debug( - "{} *is* storable due to the basic checks, and the presence of the 'public' caching directive in the response", - msg.getRequestHeader().getURI()); - publicDirective = directiveToken; - } + // in addition to the checks above, just one of the following needs to be true for the + // response to be storable + /* + * the response + * contains an Expires header field (see Section 5.3), or + * contains a max-age response directive (see Section 5.2.2.8), or + * contains a s-maxage response directive (see Section 5.2.2.9) + and the cache is shared, or + * contains a Cache Control Extension (see Section 5.2.3) that + allows it to be cached, or + * has a status code that is defined as cacheable by default (see + Section 4.2.2), or + * contains a public response directive (see Section 5.2.2.5). + */ + // TODO: replace "Expires" with some defined constant. Can't find one right now though. + // Ho Hum. + List expires = msg.getResponseHeader().getHeaderValues("Expires"); + if (!expires.isEmpty()) + LOGGER.debug( + "{} *is* storable due to the basic checks, and the presence of the 'Expires' header in the response", + msg.getRequestHeader().getURI()); + // grab this for later. Not needed for "storability" checks. + List dates = msg.getResponseHeader().getHeaderValues("Date"); + + String maxAge = null, sMaxAge = null, publicDirective = null; + if (!responseHeadersCacheControl.isEmpty()) { + for (String directive : responseHeadersCacheControl) { + for (String directiveToken : directive.split(" ")) { + // strip off any trailing comma + if (directiveToken.endsWith(",")) + directiveToken = directiveToken.substring(0, directiveToken.length() - 1); + LOGGER.trace( + "Looking for 'max-age', 's-maxage', 'public' in [{}]", directiveToken); + if (directiveToken.toLowerCase().startsWith("max-age=")) { + LOGGER.debug( + "{} *is* storable due to the basic checks, and the presence of the 'max-age' caching directive in the response", + msg.getRequestHeader().getURI()); + maxAge = directiveToken; + } + if (directiveToken + .toLowerCase() + .startsWith("s-maxage=")) { // for a shared cache.. + LOGGER.debug( + "{} *is* storable due to the basic checks, and the presence of the 's-maxage' caching directive in the response", + msg.getRequestHeader().getURI()); + sMaxAge = directiveToken; + } + if (directiveToken.toLowerCase().equals("public")) { + LOGGER.debug( + "{} *is* storable due to the basic checks, and the presence of the 'public' caching directive in the response", + msg.getRequestHeader().getURI()); + publicDirective = directiveToken; } } } - // TODO: implement checks here for known (implementation specific) Cache Control - // Extensions that would - // allow the response to be cached. - - // rfc7231 defines the following response codes as cacheable by default - boolean statusCodeCacheable = false; - int response = msg.getResponseHeader().getStatusCode(); - if ((response == 200) - || (response == 203) - || (response == 204) - || (response == 206) - || (response == 300) - || (response == 301) - || (response == 404) - || (response == 405) - || (response == 410) - || (response == 414) - || (response == 501)) { - statusCodeCacheable = true; - LOGGER.debug( - "{} *is* storable due to the basic checks, and the presence of a cacheable response status code (200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501)", - msg.getRequestHeader().getURI()); - } + } + // TODO: implement checks here for known (implementation specific) Cache Control + // Extensions that would + // allow the response to be cached. + + // rfc7231 defines the following response codes as cacheable by default + boolean statusCodeCacheable = false; + int response = msg.getResponseHeader().getStatusCode(); + if ((response == 200) + || (response == 203) + || (response == 204) + || (response == 206) + || (response == 300) + || (response == 301) + || (response == 404) + || (response == 405) + || (response == 410) + || (response == 414) + || (response == 501)) { + statusCodeCacheable = true; + LOGGER.debug( + "{} *is* storable due to the basic checks, and the presence of a cacheable response status code (200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501)", + msg.getRequestHeader().getURI()); + } - if (expires.isEmpty() - && maxAge == null - && sMaxAge == null - && statusCodeCacheable == false - && publicDirective == null) { - LOGGER.debug( - "{} is not storable due to the absence of any of an 'Expires' header, 'max-age' directive, 's-maxage' directive, 'public' directive, or cacheable response status code (200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501) in the response", - msg.getRequestHeader().getURI()); - // we raise the alert with the status code as evidence, because all the other - // conditions are "absent", rather "present" (ie, it is the only possible evidence - // we can show in this case). - alertNonStorable(String.valueOf(response)).raise(); - return; + if (expires.isEmpty() + && maxAge == null + && sMaxAge == null + && statusCodeCacheable == false + && publicDirective == null) { + LOGGER.debug( + "{} is not storable due to the absence of any of an 'Expires' header, 'max-age' directive, 's-maxage' directive, 'public' directive, or cacheable response status code (200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501) in the response", + msg.getRequestHeader().getURI()); + // we raise the alert with the status code as evidence, because all the other + // conditions are "absent", rather "present" (ie, it is the only possible evidence + // we can show in this case). + alertNonStorable(String.valueOf(response)).raise(); + return; + } + + // at this point, we *know* that the response is storable. + // so check if the content is retrievable from the cache (i.e. "cacheable") + /* + * When presented with a request, a cache MUST NOT reuse a stored + * response, unless: + * o The presented effective request URI (Section 5.5 of [RFC7230]) and + * that of the stored response match, and + * o the request method associated with the stored response allows it + * to be used for the presented request, and + * o selecting header fields nominated by the stored response (if any) + * match those presented (see Section 4.1), and + * o the presented request does not contain the no-cache pragma + * (Section 5.4), nor the no-cache cache directive (Section 5.2.1), + * unless the stored response is successfully validated + * (Section 4.3), and + * o the stored response does not contain the no-cache cache directive + * (Section 5.2.2.2), unless it is successfully validated + * (Section 4.3), and + * o the stored response is either: + * * fresh (see Section 4.2), or + * * allowed to be served stale (see Section 4.2.4), or + * * successfully validated (see Section 4.3). + * Note that any of the requirements listed above can be overridden by a + * cache-control extension; see Section 5.2.3. + */ + + // 1: we assume that the presented effective request URI matches that of the stored + // response in the cache + // 2: we assume that the presented request method is compatible with the request method + // of the stored response + // 3: we assume that the presented selecting header fields match the selecting header + // fields nominated by the stored response (if any) + // 4: we assume that the presented request does not contain the no-cache pragma, nor the + // no-cache cache directive + + // check if the stored response does not contain the no-cache cache directive, unless it + // is successfully validated + // note: we cannot (passively or actively) check the re-validation process, and can only + // assume that it will properly + // respond with details of whether the cache server can serve the cached contents or + // not. In any event, this decision is made by the origin + // server, and is not at the discretion of the cache server, so we do not concern + // ourselves with it here. + headers = msg.getResponseHeader().getHeaderValues(HttpHeader.CACHE_CONTROL); + if (!headers.isEmpty()) { + for (String directive : headers) { + for (String directiveToken : directive.split(" ")) { + // strip off any trailing comma + if (directiveToken.endsWith(",")) + directiveToken = directiveToken.substring(0, directiveToken.length() - 1); + LOGGER.trace("Looking for 'no-cache' in [{}]", directiveToken); + // Note: if the directive looked like "Cache-Control: no-cache #field-name" + // (with the optional field name argument, with no comma separating them), + // then the "no-cache" directive only applies to the field name (response + // header) in question, and not the entire contents. + // In this case, the remainder of the contents may be served without + // validation. The logic below is consistent with this requirement. + if (directiveToken.toLowerCase().equals("no-cache")) { + LOGGER.debug( + "{} is not retrievable from the cache (cacheable) due to the use of the unqualified HTTP caching directive 'no-cache' in the response", + msg.getRequestHeader().getURI()); + alertStorableNonCacheable(directiveToken).raise(); + return; + } + } } + } - // at this point, we *know* that the response is storable. - // so check if the content is retrievable from the cache (i.e. "cacheable") - /* - * When presented with a request, a cache MUST NOT reuse a stored - * response, unless: - * o The presented effective request URI (Section 5.5 of [RFC7230]) and - * that of the stored response match, and - * o the request method associated with the stored response allows it - * to be used for the presented request, and - * o selecting header fields nominated by the stored response (if any) - * match those presented (see Section 4.1), and - * o the presented request does not contain the no-cache pragma - * (Section 5.4), nor the no-cache cache directive (Section 5.2.1), - * unless the stored response is successfully validated - * (Section 4.3), and - * o the stored response does not contain the no-cache cache directive - * (Section 5.2.2.2), unless it is successfully validated - * (Section 4.3), and - * o the stored response is either: - * * fresh (see Section 4.2), or - * * allowed to be served stale (see Section 4.2.4), or - * * successfully validated (see Section 4.3). - * Note that any of the requirements listed above can be overridden by a - * cache-control extension; see Section 5.2.3. - */ - - // 1: we assume that the presented effective request URI matches that of the stored - // response in the cache - // 2: we assume that the presented request method is compatible with the request method - // of the stored response - // 3: we assume that the presented selecting header fields match the selecting header - // fields nominated by the stored response (if any) - // 4: we assume that the presented request does not contain the no-cache pragma, nor the - // no-cache cache directive - - // check if the stored response does not contain the no-cache cache directive, unless it - // is successfully validated - // note: we cannot (passively or actively) check the re-validation process, and can only - // assume that it will properly - // respond with details of whether the cache server can serve the cached contents or - // not. In any event, this decision is made by the origin - // server, and is not at the discretion of the cache server, so we do not concern - // ourselves with it here. - headers = msg.getResponseHeader().getHeaderValues(HttpHeader.CACHE_CONTROL); - if (!headers.isEmpty()) { - for (String directive : headers) { - for (String directiveToken : directive.split(" ")) { - // strip off any trailing comma - if (directiveToken.endsWith(",")) - directiveToken = - directiveToken.substring(0, directiveToken.length() - 1); - LOGGER.trace("Looking for 'no-cache' in [{}]", directiveToken); - // Note: if the directive looked like "Cache-Control: no-cache #field-name" - // (with the optional field name argument, with no comma separating them), - // then the "no-cache" directive only applies to the field name (response - // header) in question, and not the entire contents. - // In this case, the remainder of the contents may be served without - // validation. The logic below is consistent with this requirement. - if (directiveToken.toLowerCase().equals("no-cache")) { - LOGGER.debug( - "{} is not retrievable from the cache (cacheable) due to the use of the unqualified HTTP caching directive 'no-cache' in the response", - msg.getRequestHeader().getURI()); - alertStorableNonCacheable(directiveToken).raise(); - return; - } + // is the stored response fresh? + // Note that fresh = freshness lifetime > current age + long lifetime = -1; + boolean lifetimeFound = false; + String freshEvidence = null; + String otherInfo = null; + + // 1: calculate the freshness lifetime of the request, using the following checks, with + // the following priority, as specified by rfc7234. + // 1a:Get the "s-maxage" response directive value (if duplicates exist, the values are + // invalid) + if (!responseHeadersCacheControl.isEmpty()) { + int lifetimesFound = 0; + for (String directive : responseHeadersCacheControl) { + for (String directiveToken : directive.split(" ")) { + // strip off any trailing comma + if (directiveToken.endsWith(",")) + directiveToken = directiveToken.substring(0, directiveToken.length() - 1); + LOGGER.trace("Looking for 's-maxage' in [{}]", directiveToken); + if (directiveToken.toLowerCase().startsWith("s-maxage=")) { + LOGGER.debug( + "{} has a caching lifetime defined by an HTTP caching directive 's-maxage' ", + msg.getRequestHeader().getURI()); + lifetimeFound = true; + lifetimesFound++; + lifetime = extractAgeValue(directiveToken, "s-maxage=".length()); + freshEvidence = directiveToken; } } } + // if duplicates exist, the values are invalid. as per rfc7234. + if (lifetimesFound > 1) { + lifetimeFound = false; + lifetime = -1; + freshEvidence = null; + LOGGER.debug( + "{} had multiple caching lifetimes defined by an HTTP caching directive 's-maxage'. Invalidating all of these!", + msg.getRequestHeader().getURI()); + } + } - // is the stored response fresh? - // Note that fresh = freshness lifetime > current age - long lifetime = -1; - boolean lifetimeFound = false; - String freshEvidence = null; - String otherInfo = null; - - // 1: calculate the freshness lifetime of the request, using the following checks, with - // the following priority, as specified by rfc7234. - // 1a:Get the "s-maxage" response directive value (if duplicates exist, the values are - // invalid) + // 1b:Get the "max-age" response directive value (if duplicates exist, the values are + // invalid) + if (!lifetimeFound) { if (!responseHeadersCacheControl.isEmpty()) { int lifetimesFound = 0; for (String directive : responseHeadersCacheControl) { - for (String directiveToken : directive.split(" ")) { + for (String directiveToken : directive.replaceAll("[ ,]+", ",").split(",")) { // strip off any trailing comma if (directiveToken.endsWith(",")) directiveToken = directiveToken.substring(0, directiveToken.length() - 1); - LOGGER.trace("Looking for 's-maxage' in [{}]", directiveToken); - if (directiveToken.toLowerCase().startsWith("s-maxage=")) { + LOGGER.trace("Looking for 'max-age' in [{}]", directiveToken); + if (directiveToken.toLowerCase().startsWith("max-age=")) { LOGGER.debug( - "{} has a caching lifetime defined by an HTTP caching directive 's-maxage' ", + "{} has a caching lifetime defined by an HTTP caching directive 'max-age' ", msg.getRequestHeader().getURI()); lifetimeFound = true; lifetimesFound++; - lifetime = extractAgeValue(directiveToken, "s-maxage=".length()); + // get the portion of the string after "maxage=" + // Cache-Control: max-age=7776000,private + try { + lifetime = extractAgeValue(directiveToken, "max-age=".length()); + } catch (NumberFormatException nfe) { + lifetimeFound = false; + lifetimesFound--; + LOGGER.debug( + "Could not parse max-age to establish lifetime. Perhaps the value exceeds Long.MAX_VALUE or contains non-number characters:{}", + directiveToken); + } freshEvidence = directiveToken; } } @@ -447,258 +484,205 @@ public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) { lifetime = -1; freshEvidence = null; LOGGER.debug( - "{} had multiple caching lifetimes defined by an HTTP caching directive 's-maxage'. Invalidating all of these!", + "{} had multiple caching lifetimes defined by an HTTP caching directive 'max-age'. Invalidating all of these!", msg.getRequestHeader().getURI()); } } + } - // 1b:Get the "max-age" response directive value (if duplicates exist, the values are - // invalid) - if (!lifetimeFound) { - if (!responseHeadersCacheControl.isEmpty()) { - int lifetimesFound = 0; - for (String directive : responseHeadersCacheControl) { - for (String directiveToken : - directive.replaceAll("[ ,]+", ",").split(",")) { - // strip off any trailing comma - if (directiveToken.endsWith(",")) - directiveToken = - directiveToken.substring(0, directiveToken.length() - 1); - LOGGER.trace("Looking for 'max-age' in [{}]", directiveToken); - if (directiveToken.toLowerCase().startsWith("max-age=")) { - LOGGER.debug( - "{} has a caching lifetime defined by an HTTP caching directive 'max-age' ", - msg.getRequestHeader().getURI()); - lifetimeFound = true; - lifetimesFound++; - // get the portion of the string after "maxage=" - // Cache-Control: max-age=7776000,private - try { - lifetime = extractAgeValue(directiveToken, "max-age=".length()); - } catch (NumberFormatException nfe) { - lifetimeFound = false; - lifetimesFound--; - LOGGER.debug( - "Could not parse max-age to establish lifetime. Perhaps the value exceeds Long.MAX_VALUE or contains non-number characters:{}", - directiveToken); - } - freshEvidence = directiveToken; - } - } - } - // if duplicates exist, the values are invalid. as per rfc7234. - if (lifetimesFound > 1) { - lifetimeFound = false; - lifetime = -1; - freshEvidence = null; - LOGGER.debug( - "{} had multiple caching lifetimes defined by an HTTP caching directive 'max-age'. Invalidating all of these!", - msg.getRequestHeader().getURI()); - } + // 1c: Get the "Expires" response header value - "Date" response header field. ("Date" + // is optional if the origin has no clock, or returned a 1XX or 5XX response, else + // mandatory) + if (!lifetimeFound) { + String expiresHeader = null; + String dateHeader = null; + if (!expires.isEmpty()) { + // Expires can be absent, or take the form of "Thu, 27 Nov 2014 12:21:57 GMT", + // "-1", "0", etc. + // Invalid dates are treated as "expired" + int expiresHeadersFound = 0; + for (String directive : expires) { + LOGGER.debug( + "{} has a caching lifetime expiry defined by an HTTP response header 'Expires'", + msg.getRequestHeader().getURI()); + expiresHeadersFound++; + expiresHeader = directive; + freshEvidence = directive; } - } - - // 1c: Get the "Expires" response header value - "Date" response header field. ("Date" - // is optional if the origin has no clock, or returned a 1XX or 5XX response, else - // mandatory) - if (!lifetimeFound) { - String expiresHeader = null; - String dateHeader = null; - if (!expires.isEmpty()) { - // Expires can be absent, or take the form of "Thu, 27 Nov 2014 12:21:57 GMT", - // "-1", "0", etc. - // Invalid dates are treated as "expired" - int expiresHeadersFound = 0; - for (String directive : expires) { - LOGGER.debug( - "{} has a caching lifetime expiry defined by an HTTP response header 'Expires'", - msg.getRequestHeader().getURI()); - expiresHeadersFound++; - expiresHeader = directive; - freshEvidence = directive; - } - // if duplicates exist, the values are invalid. as per rfc7234. - if (expiresHeadersFound > 1) { - expiresHeader = null; - LOGGER.debug( - "{} had multiple caching lifetime expiries defined by an HTTP response header 'Expires'. Invalidating all of these!", - msg.getRequestHeader().getURI()); - } else { - // we now have a single "expiry". - // Now it is time to get the "date" for the request, so we can subtract the - // "date" from the "expiry" to get the "lifetime". - if (!dates.isEmpty()) { - int dateHeadersFound = 0; - for (String directive : dates) { - LOGGER.debug( - "{} has a caching lifetime date defined by an HTTP response header 'Date'", - msg.getRequestHeader().getURI()); - dateHeadersFound++; - dateHeader = directive; - } - // if duplicates exist, the values are invalid. as per rfc7234. - if (dateHeadersFound > 1) { - dateHeader = null; - LOGGER.debug( - "{} had multiple caching lifetime dates defined by an HTTP response header 'Date'. Invalidating all of these!", - msg.getRequestHeader().getURI()); - } else { - // we have one expiry, and one date. Yippee.. Are they valid tough?? - // both dates can be invalid, or have one of 3 formats, all of which - // MUST be supported! - Date expiresDate = parseDate(expiresHeader); - - if (expiresDate != null) { - Date dateDate = parseDate(dateHeader); - if (dateDate != null) { - // calculate the lifetime = Expires - Date - lifetimeFound = true; - lifetime = - (expiresDate.getTime() - dateDate.getTime()) / 1000; - // there is multiple parts to the evidence in this case (the - // Expiry, and the Date, but lets show the Expiry) - freshEvidence = expiresHeader; - LOGGER.debug( - "{} had an 'Expires' date and a 'Date' date, which were used to calculate the lifetime of the request", - msg.getRequestHeader().getURI()); - } else { - // the "Date" date is not valid. Treat it as "expired" - LOGGER.debug( - "{} had an invalid caching lifetime date defined by an HTTP response header 'Date'. Ignoring the 'Expires' header for the purposes of lifetime calculation.", - msg.getRequestHeader().getURI()); - lifetime = -1; - } + // if duplicates exist, the values are invalid. as per rfc7234. + if (expiresHeadersFound > 1) { + expiresHeader = null; + LOGGER.debug( + "{} had multiple caching lifetime expiries defined by an HTTP response header 'Expires'. Invalidating all of these!", + msg.getRequestHeader().getURI()); + } else { + // we now have a single "expiry". + // Now it is time to get the "date" for the request, so we can subtract the + // "date" from the "expiry" to get the "lifetime". + if (!dates.isEmpty()) { + int dateHeadersFound = 0; + for (String directive : dates) { + LOGGER.debug( + "{} has a caching lifetime date defined by an HTTP response header 'Date'", + msg.getRequestHeader().getURI()); + dateHeadersFound++; + dateHeader = directive; + } + // if duplicates exist, the values are invalid. as per rfc7234. + if (dateHeadersFound > 1) { + dateHeader = null; + LOGGER.debug( + "{} had multiple caching lifetime dates defined by an HTTP response header 'Date'. Invalidating all of these!", + msg.getRequestHeader().getURI()); + } else { + // we have one expiry, and one date. Yippee.. Are they valid tough?? + // both dates can be invalid, or have one of 3 formats, all of which + // MUST be supported! + Date expiresDate = parseDate(expiresHeader); + + if (expiresDate != null) { + Date dateDate = parseDate(dateHeader); + if (dateDate != null) { + // calculate the lifetime = Expires - Date + lifetimeFound = true; + lifetime = (expiresDate.getTime() - dateDate.getTime()) / 1000; + // there is multiple parts to the evidence in this case (the + // Expiry, and the Date, but lets show the Expiry) + freshEvidence = expiresHeader; + LOGGER.debug( + "{} had an 'Expires' date and a 'Date' date, which were used to calculate the lifetime of the request", + msg.getRequestHeader().getURI()); } else { - // the expires date is not valid. Treat it as "expired" - // (will not result in a "cacheable" alert, so the evidence is - // not needed, in fact + // the "Date" date is not valid. Treat it as "expired" LOGGER.debug( - "{} had an invalid caching lifetime expiry date defined by an HTTP response header 'Expiry'. Assuming an historic/ expired lifetime.", + "{} had an invalid caching lifetime date defined by an HTTP response header 'Date'. Ignoring the 'Expires' header for the purposes of lifetime calculation.", msg.getRequestHeader().getURI()); - lifetimeFound = true; - lifetime = 0; - freshEvidence = expiresHeader; + lifetime = -1; } + } else { + // the expires date is not valid. Treat it as "expired" + // (will not result in a "cacheable" alert, so the evidence is + // not needed, in fact + LOGGER.debug( + "{} had an invalid caching lifetime expiry date defined by an HTTP response header 'Expiry'. Assuming an historic/ expired lifetime.", + msg.getRequestHeader().getURI()); + lifetimeFound = true; + lifetime = 0; + freshEvidence = expiresHeader; } - } else { - // "Dates" is not defined. Nothing to do! - LOGGER.debug( - "{} has a caching lifetime expiry defined by an HTTP response header 'Expires', but no 'Date' header to subtract from it", - msg.getRequestHeader().getURI()); } + } else { + // "Dates" is not defined. Nothing to do! + LOGGER.debug( + "{} has a caching lifetime expiry defined by an HTTP response header 'Expires', but no 'Date' header to subtract from it", + msg.getRequestHeader().getURI()); } - } else { - // "Expires" is not defined. Nothing to do! - LOGGER.debug( - "{} has no caching lifetime expiry defined by an HTTP response header 'Expires'", - msg.getRequestHeader().getURI()); } - } - - // 1d: Use a heuristic to determine a "plausible" expiration time. This is - // implementation specific, and the implementation is permitted to be liberal. - // for the purposes of this exercise, lets assume the cache chooses a "plausible" - // expiration of 1 year (expressed in seconds) - if (!lifetimeFound) { + } else { + // "Expires" is not defined. Nothing to do! LOGGER.debug( - "{} has no caching lifetime expiry of any form, so assuming that it is set 'heuristically' to 1 year (as a form of worst case)", + "{} has no caching lifetime expiry defined by an HTTP response header 'Expires'", msg.getRequestHeader().getURI()); - lifetimeFound = true; - lifetime = SECONDS_IN_YEAR; - // a liberal heuristic was assumed, for which no actual evidence exists - freshEvidence = null; - otherInfo = - Constant.messages.getString( - MESSAGE_PREFIX_STORABLE_CACHEABLE - + "otherinfo.liberallifetimeheuristic"); } + } + // 1d: Use a heuristic to determine a "plausible" expiration time. This is + // implementation specific, and the implementation is permitted to be liberal. + // for the purposes of this exercise, lets assume the cache chooses a "plausible" + // expiration of 1 year (expressed in seconds) + if (!lifetimeFound) { LOGGER.debug( - "{} has a caching lifetime of {}", msg.getRequestHeader().getURI(), lifetime); - - // 2: calculate the current age of the request - // Note that since we are not necessarily testing via a cache, the "Age" header may - // not be set (this is set by the caching server, not by the web server) - // so we can only possibly get the "apparent_age", and not the "corrected_age_value" - // documented in rfc7234. - // In any event, this is not an issue, because in the worst case, the user could be - // sending the first request for a given URL, placing - // the response in the cache, with an age approaching 0 (depending on network delay). - // By this logic, let's not even try to check the "apparent_age" (since it depends on - // our network, and could be completely different for other users) - // and let's assume that in at least some cases, the "age" can be 0 (the most extreme - // case, from the point of view of "freshness"). - // so "freshness" depends purely on the defined lifetime, in practice. - long age = 0; - - // so after all that, is the response fresh or not? - if (lifetime > age) { - // fresh, so it can be retrieved from the cache - LOGGER.debug( - "{} is retrievable from the cache (cacheable), since it is fresh", - msg.getRequestHeader().getURI()); - alertStorableCacheable(freshEvidence, otherInfo).raise(); - return; - } else { - // stale! - // is the stored response allowed to be served stale? - // if the following are not present, the response *can* be served stale.. - // Note: this area of the RFC is vague at best (and somewhat contradictory), so this - // area may need to be reviewed once the RFC has been updated - // (the version used is rfc7234 from June 2014) - /* - "must-revalidate" - OK (fairly explicit) - "proxy-revalidate" - OK (fairly explicit) - "s-maxage" - see rfc7234, section 3.2 - "max-age" - inferred, based on the case for "s-maxage" - */ - - boolean staleRetrieveAllowed = true; - String doNotRetrieveStaleEvidence = null; - if (!responseHeadersCacheControl.isEmpty()) { - for (String directive : responseHeadersCacheControl) { - for (String directiveToken : directive.split(" ")) { - // strip off any trailing comma - if (directiveToken.endsWith(",")) - directiveToken = - directiveToken.substring(0, directiveToken.length() - 1); - LOGGER.trace( - "Looking for 'must-revalidate', 'proxy-revalidate', 's-maxage', 'max-age' in [{}]", - directiveToken); - if ((directiveToken.toLowerCase().equals("must-revalidate")) - || (directiveToken.toLowerCase().equals("proxy-revalidate")) - || (directiveToken.toLowerCase().startsWith("s-maxage=")) - || (directiveToken.toLowerCase().startsWith("max-age="))) { - staleRetrieveAllowed = false; - doNotRetrieveStaleEvidence = directiveToken; - break; - } + "{} has no caching lifetime expiry of any form, so assuming that it is set 'heuristically' to 1 year (as a form of worst case)", + msg.getRequestHeader().getURI()); + lifetimeFound = true; + lifetime = SECONDS_IN_YEAR; + // a liberal heuristic was assumed, for which no actual evidence exists + freshEvidence = null; + otherInfo = + Constant.messages.getString( + MESSAGE_PREFIX_STORABLE_CACHEABLE + + "otherinfo.liberallifetimeheuristic"); + } + + LOGGER.debug("{} has a caching lifetime of {}", msg.getRequestHeader().getURI(), lifetime); + + // 2: calculate the current age of the request + // Note that since we are not necessarily testing via a cache, the "Age" header may + // not be set (this is set by the caching server, not by the web server) + // so we can only possibly get the "apparent_age", and not the "corrected_age_value" + // documented in rfc7234. + // In any event, this is not an issue, because in the worst case, the user could be + // sending the first request for a given URL, placing + // the response in the cache, with an age approaching 0 (depending on network delay). + // By this logic, let's not even try to check the "apparent_age" (since it depends on + // our network, and could be completely different for other users) + // and let's assume that in at least some cases, the "age" can be 0 (the most extreme + // case, from the point of view of "freshness"). + // so "freshness" depends purely on the defined lifetime, in practice. + long age = 0; + + // so after all that, is the response fresh or not? + if (lifetime > age) { + // fresh, so it can be retrieved from the cache + LOGGER.debug( + "{} is retrievable from the cache (cacheable), since it is fresh", + msg.getRequestHeader().getURI()); + alertStorableCacheable(freshEvidence, otherInfo).raise(); + return; + } else { + // stale! + // is the stored response allowed to be served stale? + // if the following are not present, the response *can* be served stale.. + // Note: this area of the RFC is vague at best (and somewhat contradictory), so this + // area may need to be reviewed once the RFC has been updated + // (the version used is rfc7234 from June 2014) + /* + "must-revalidate" - OK (fairly explicit) + "proxy-revalidate" - OK (fairly explicit) + "s-maxage" - see rfc7234, section 3.2 + "max-age" - inferred, based on the case for "s-maxage" + */ + + boolean staleRetrieveAllowed = true; + String doNotRetrieveStaleEvidence = null; + if (!responseHeadersCacheControl.isEmpty()) { + for (String directive : responseHeadersCacheControl) { + for (String directiveToken : directive.split(" ")) { + // strip off any trailing comma + if (directiveToken.endsWith(",")) + directiveToken = + directiveToken.substring(0, directiveToken.length() - 1); + LOGGER.trace( + "Looking for 'must-revalidate', 'proxy-revalidate', 's-maxage', 'max-age' in [{}]", + directiveToken); + if ((directiveToken.toLowerCase().equals("must-revalidate")) + || (directiveToken.toLowerCase().equals("proxy-revalidate")) + || (directiveToken.toLowerCase().startsWith("s-maxage=")) + || (directiveToken.toLowerCase().startsWith("max-age="))) { + staleRetrieveAllowed = false; + doNotRetrieveStaleEvidence = directiveToken; + break; } } } - // TODO: check for any known Cache Control Extensions here, before making a final - // call on the retrievability of the cached data. - if (staleRetrieveAllowed) { - // no directives were configured to prevent stale responses being retrieved - // (without validation) - alertStorableCacheable( - "", - Constant.messages.getString( - MESSAGE_PREFIX_STORABLE_CACHEABLE - + "otherinfo.staleretrievenotblocked")) - .raise(); - } else { - // the directives do not allow stale responses to be retrieved - // we saw just one other scenario where this could happen: where the response - // was cached, but the "no-cache" response directive was specified - alertStorableNonCacheable(doNotRetrieveStaleEvidence).raise(); - } } - } catch (Exception e) { - LOGGER.error( - "An error occurred while checking a URI [{}] for cacheability", - msg.getRequestHeader().getURI(), - e); + // TODO: check for any known Cache Control Extensions here, before making a final + // call on the retrievability of the cached data. + if (staleRetrieveAllowed) { + // no directives were configured to prevent stale responses being retrieved + // (without validation) + alertStorableCacheable( + "", + Constant.messages.getString( + MESSAGE_PREFIX_STORABLE_CACHEABLE + + "otherinfo.staleretrievenotblocked")) + .raise(); + } else { + // the directives do not allow stale responses to be retrieved + // we saw just one other scenario where this could happen: where the response + // was cached, but the "no-cache" response directive was specified + alertStorableNonCacheable(doNotRetrieveStaleEvidence).raise(); + } } } diff --git a/addOns/pscanrulesBeta/src/main/java/org/zaproxy/zap/extension/pscanrulesBeta/JsFunctionScanRule.java b/addOns/pscanrulesBeta/src/main/java/org/zaproxy/zap/extension/pscanrulesBeta/JsFunctionScanRule.java index a0389feceba..2a50cf480bf 100644 --- a/addOns/pscanrulesBeta/src/main/java/org/zaproxy/zap/extension/pscanrulesBeta/JsFunctionScanRule.java +++ b/addOns/pscanrulesBeta/src/main/java/org/zaproxy/zap/extension/pscanrulesBeta/JsFunctionScanRule.java @@ -99,7 +99,7 @@ public class JsFunctionScanRule extends PluginPassiveScanner implements CommonPa } } } catch (IOException e) { - LOGGER.error( + LOGGER.warn( "Error on opening/reading js functions file: {}{}{}{} Error: {}", File.separator, FUNC_LIST_DIR, diff --git a/addOns/retire/CHANGELOG.md b/addOns/retire/CHANGELOG.md index d148b375138..ac6759db5b0 100644 --- a/addOns/retire/CHANGELOG.md +++ b/addOns/retire/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased ### Changed - Updated with upstream retire.js pattern changes. +- Reduced usage of error level logging. ## [0.49.0] - 2025-09-18 ### Changed diff --git a/addOns/retire/src/main/java/org/zaproxy/addon/retire/RetireScanRule.java b/addOns/retire/src/main/java/org/zaproxy/addon/retire/RetireScanRule.java index 2186634e33f..cb177a44935 100644 --- a/addOns/retire/src/main/java/org/zaproxy/addon/retire/RetireScanRule.java +++ b/addOns/retire/src/main/java/org/zaproxy/addon/retire/RetireScanRule.java @@ -78,7 +78,7 @@ public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) { Repo scanRepo = getRepo(); if (!getHelper().isPage200(msg) || scanRepo == null) { if (scanRepo == null) { - LOGGER.error("\tThe Retire.js repository was null."); + LOGGER.warn("\tThe Retire.js repository was null."); } return; } diff --git a/addOns/wappalyzer/CHANGELOG.md b/addOns/wappalyzer/CHANGELOG.md index e40d67ce721..ebe8cb664f4 100644 --- a/addOns/wappalyzer/CHANGELOG.md +++ b/addOns/wappalyzer/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Icon sizing in the Technology table when a transparent placeholder needs to be used. +- Reduced usage of error level logging. ## [21.48.0] - 2025-09-02 ### Changed diff --git a/addOns/wappalyzer/src/main/java/org/zaproxy/zap/extension/wappalyzer/ExtensionWappalyzer.java b/addOns/wappalyzer/src/main/java/org/zaproxy/zap/extension/wappalyzer/ExtensionWappalyzer.java index 1e6115a345f..f8c6ac687e1 100644 --- a/addOns/wappalyzer/src/main/java/org/zaproxy/zap/extension/wappalyzer/ExtensionWappalyzer.java +++ b/addOns/wappalyzer/src/main/java/org/zaproxy/zap/extension/wappalyzer/ExtensionWappalyzer.java @@ -144,7 +144,7 @@ public void init() { .map(ExtensionWappalyzer::techToResourcePath) .forEach(technologyFiles::add); } catch (IOException e) { - LOGGER.error("Failed to enumerate Tech Detection technologies:", e); + LOGGER.warn("Failed to enumerate Tech Detection technologies:", e); } TechData result = @@ -393,7 +393,7 @@ public void sessionChanged(final Session session) { try { EventQueue.invokeAndWait(() -> sessionChangedEventHandler(session)); } catch (Exception e) { - LOGGER.error(e.getMessage(), e); + LOGGER.warn(e.getMessage(), e); } } } diff --git a/addOns/wappalyzer/src/main/java/org/zaproxy/zap/extension/wappalyzer/TechsJsonParser.java b/addOns/wappalyzer/src/main/java/org/zaproxy/zap/extension/wappalyzer/TechsJsonParser.java index 5297f36bbd6..665ae8cb579 100644 --- a/addOns/wappalyzer/src/main/java/org/zaproxy/zap/extension/wappalyzer/TechsJsonParser.java +++ b/addOns/wappalyzer/src/main/java/org/zaproxy/zap/extension/wappalyzer/TechsJsonParser.java @@ -65,8 +65,8 @@ public class TechsJsonParser { public TechsJsonParser() { this( - (pattern, e) -> LOGGER.error("Invalid pattern syntax {}", pattern, e), - e -> LOGGER.error(e.getMessage(), e)); + (pattern, e) -> LOGGER.warn("Invalid pattern syntax {}", pattern, e), + e -> LOGGER.warn(e.getMessage(), e)); } TechsJsonParser(PatternErrorHandler peh, ParsingExceptionHandler parsingExceptionHandler) { @@ -297,7 +297,7 @@ private static List jsonToCategoryList(Map categories, O if (category != null) { list.add(category); } else { - LOGGER.error("Failed to find category for {}", obj); + LOGGER.warn("Failed to find category for {}", obj); } } } @@ -327,7 +327,7 @@ private List> jsonToAppPatternMapList(String type, Objec } } } else if (json != null) { - LOGGER.error( + LOGGER.warn( "Unexpected JSON type for {} pattern: {} {}", type, json, @@ -473,10 +473,10 @@ private static AppPattern strToAppPattern(String type, String str) { } else if (values[i].startsWith(FIELD_VERSION)) { ap.setVersion(values[i].substring(FIELD_VERSION.length())); } else { - LOGGER.error("Unexpected field: {}", values[i]); + LOGGER.warn("Unexpected field: {}", values[i]); } } catch (Exception e) { - LOGGER.error("Invalid field syntax {}", values[i], e); + LOGGER.warn("Invalid field syntax {}", values[i], e); } } if (pattern.indexOf(FIELD_CONFIDENCE) > -1) { @@ -496,7 +496,7 @@ private static int parseConfidence(String confidence) { } return Integer.parseInt(confidence); } catch (NumberFormatException nfe) { - LOGGER.error("Invalid field value: {}", confidence); + LOGGER.debug("Invalid field value: {}", confidence); return 0; } }