Since Varnish did not work out on Solaris yet again. I have decided to bite the bullet and write a headers normalization patch for Squid 2.7. This patch should produce much better cache hit rates with Squid. Efficiency++

What the patch does:

1. Removes Cache-Control request headers, don’t let clients by-pass cache if it is primed.
2. Normalize Accept-Encoding Headers for a higher cache hit rate.
3. Clear Accept-Encoding Headers for content that should not be compressed such as image,video and audio.

and the patch: squid-headers-normalization.patch

Update: Fixed a minor memory leak, all good now.
Update 2: Added audio exception to strip accept-encoding.


--- src/client_side.c.og	2010-01-20 12:00:56.000000000 -0800
+++ src/client_side.c	2010-01-19 20:35:31.000000000 -0800
@@ -3983,6 +3983,7 @@
 	    errorAppendEntry(http->entry, err);
 	    return -1;
 	}
+
 	/* compile headers */
 	/* we should skip request line! */
 	if ((http->http_ver.major >= 1) && !httpMsgParseRequestHeader(request, &msg)) {
@@ -3992,10 +3993,59 @@
 	    err->url = xstrdup(http->uri);
 	    http->al.http.code = err->http_status;
 	    http->log_type = LOG_TCP_DENIED;
+
 	    http->entry = clientCreateStoreEntry(http, method, null_request_flags);
 	    errorAppendEntry(http->entry, err);
 	    return -1;
 	}
+
+	/*
+	 *  Normalize Request Cache-Control / If-Modified-Since Headers
+	 *  Don't let client by-pass the cache if there is cached content.
+	 */
+	if(httpHeaderHas(&request->header,HDR_CACHE_CONTROL)) {
+		httpHeaderDelByName(&request->header,"cache-control");
+	}
+
+	/*
+	 * Un-comment this if you want Squid to always respond with the request
+	 * instead of returning back with a 304 if the cache has not changed.
+	 */
+	/*
+	if(httpHeaderHas(&request->header,HDR_IF_MODIFIED_SINCE)) {
+		httpHeaderDelByName(&request->header,"if-modified-since");
+	}*/
+
+	/*
+	 * Normalize Accept-Encoding Headers sent from client
+	 */
+	if(httpHeaderHas(&request->header,HDR_ACCEPT_ENCODING)) {
+		String val = httpHeaderGetByName(&request->header,"accept-encoding");
+		if(val.buf) {
+			if(strstr(val.buf,"gzip") != NULL) {
+				httpHeaderDelByName(&request->header,"accept-encoding");
+				httpHeaderPutStr(&request->header,HDR_ACCEPT_ENCODING,"gzip");
+			} else if(strstr(val.buf,"deflate") != NULL) {
+				httpHeaderDelByName(&request->header,"accept-encoding");
+				httpHeaderPutStr(&request->header,HDR_ACCEPT_ENCODING,"deflate");
+			} else {
+				httpHeaderDelByName(&request->header,"accept-encoding");
+			}
+		}
+		stringClean(&val);
+	}
+
+	/*
+	 * Normalize Accept-Encoding Headers for video/image content
+	 */
+	char *mime_type = mimeGetContentType(http->uri);
+	if(mime_type) {
+		if(strstr(mime_type,"image") != NULL || strstr(mime_type,"video") != NULL || strstr(mime_type,"audio") != NULL) {
+			httpHeaderDelByName(&request->header,"accept-encoding");
+		}
+	}
+
+
 	/*
 	 * If we read past the end of this request, move the remaining
 	 * data to the beginning