php.htaccessvarnishexpires-header

How can I leverage browser cache and .htaccess are not cleared by my varnish configuration?

I want to leverage browser caching for the following cacheable resources: JS/TTF/ICO/jpeg/png/css/...

I'm testing using GMETRIX, but I keep failing on LEVERAGE browser cache.

I think some of the rules in my .htaccess are being voided in my Varnish configuration. So I've included the config below.

I've included as much info as I could (and was allowed to post here...)

How can I make sure the rules in my .htaccess are not nulled by my varnish configuration and static files are cached... ?

Here's a HTTP request for a Javascript file:

 Request URL: https://www.domain1.com/media/extendware/ewminify/files/js/0747a032c3b39cac.js
 Request Method: GET
 Status Code: 200  (from memory cache)
 Remote Address: 77.243.239.145:443
 Referrer Policy: no-referrer-when-downgrade
 accept-ranges: bytes
 age: 20
 cache-control: max-age=28800
 content-encoding: gzip
 content-length: 200303
 content-type: application/x-javascript; charset=utf-8
 date: Fri, 04 Jan 2019 12:05:17 GMT
 etag: "b929d-57e158e64a31f-gzip"
 last-modified: Fri, 28 Dec 2018 14:05:31 GMT
 server: Apache
 status: 200
 strict-transport-security: max-age=15768000
 vary: Accept-Encoding
 via: 1.1 varnish-v4
 x-magento-cache-debug: HIT
 x-varnish: 4101659 2173998
 x-varnish-currency: EUR
 x-varnish-esi-access: 
 x-varnish-esi-method: 
 x-varnish-hits: 1
 x-varnish-host: www.domain1.com
 x-varnish-store: 
 x-varnish-url: 
 /media/extendware/ewminify/files/js/0747a032c3b39cac.js
 :authority: www.domain1.com
 :method: GET
 :path: /media/extendware/ewminify/files/js/0747a032c3b39cac.js
 :scheme: https
 accept: */*
 accept-encoding: gzip, deflate, br
 accept-language: en-GB,en;q=0.9,en-US;q=0.8,nl;q=0.7
 cache-control: no-cache
 cookie: __utmz=151xxxxx.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=         
 (none); frontend=xxxxxx; epc-initiated=1; _fbp=fb.1.xxxx.xx; 
  __utmc=xxxx; adminhtml=xxxxx; currency=EUR; 
  __utma=xxxx.xx.xxx.xxxx.xxxx.4; __utmt=1; __utmb=xxx.7.10.xxx
 pragma: no-cache
 referer: https://www.domain1.com/
 user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) 
 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 
 Safari/537.36

.htacces

## default index file

DirectoryIndex index.php

<IfModule mod_php5.c>

## adjust memory limit

#    php_value memory_limit 64M
php_value memory_limit 512M
php_value max_execution_time 18000

## disable magic quotes for php request vars

php_flag magic_quotes_gpc off

## disable automatic session start
## before autoload was initialized

php_flag session.auto_start off

## enable resulting html compression

php_flag zlib.output_compression off

# disable user agent verification to not break multiple image upload

php_flag suhosin.session.cryptua off

# turn off compatibility with PHP4 when dealing with objects

php_flag zend.ze1_compatibility_mode Off

</IfModule>

<IfModule mod_security.c>
# disable POST processing to not break multiple image upload

SecFilterEngine Off
SecFilterScanPOST Off
 </IfModule>

# GZip compresssion

<ifModule mod_gzip.c>
mod_gzip_on Yes
mod_gzip_dechunk Yes
mod_gzip_item_include file .(html?|txt|css|js|php|pl)$
mod_gzip_item_include handler ^cgi-script$
mod_gzip_item_include mime ^text/.*
mod_gzip_item_include mime ^application/x-javascript.*
mod_gzip_item_exclude mime ^image/.*
mod_gzip_item_exclude rspheader ^Content-Encoding:.*gzip.*
</ifModule>

<IfModule mod_deflate.c>

## enable apache served files compression
## http://developer.yahoo.com/performance/rules.html#gzip

# Insert filter on all content
###SetOutputFilter DEFLATE
# Insert filter on selected content types only
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css 
text/javascript

# Netscape 4.x has some problems...
BrowserMatch ^Mozilla/4 gzip-only-text/html

# Netscape 4.06-4.08 have some more problems
BrowserMatch ^Mozilla/4\.0[678] no-gzip

# MSIE masquerades as Netscape, but it is fine
BrowserMatch \bMSIE !no-gzip !gzip-only-text/html

# Don't compress images
#SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png)$ no-gzip dont-vary

# Make sure proxies don't deliver the wrong content
Header append Vary User-Agent env=!dont-vary

</IfModule>

<IfModule mod_headers.c>
Header set Connection keep-alive
</IfModule>

<IfModule mod_ssl.c>

## make HTTPS env vars available for CGI mode

SSLOptions StdEnvVars

</IfModule>

RewriteEngine on
RewriteCond %{HTTPS} off
RewriteCond %{HTTP_HOST} !^www\.(.*)$ [NC]
RewriteRule ^(.*)$ http://www.%{HTTP_HOST}/$1 [R=301,L]

RewriteCond %{HTTPS} on
 RewriteCond %{HTTP_HOST} !^www\.(.*)$ [NC]
RewriteRule ^(.*)$ https://www.%{HTTP_HOST}/$1 [R=301,L]

 <IfModule mod_rewrite.c>

## enable rewrites

Options +FollowSymLinks
RewriteEngine on

## you can put here your magento root folder
## path relative to web root

#RewriteBase /magento/

## uncomment next line to enable light API calls processing

#    RewriteRule ^api/([a-z][0-9a-z_]+)/?$ api.php?type=$1 [QSA,L]

## rewrite API2 calls to api.php (by now it is REST only)

RewriteRule ^api/rest api.php?type=rest [QSA,L]

## workaround for HTTP authorization
   ## in CGI environment

RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization}]

## TRACE and TRACK HTTP methods disabled to prevent XSS attacks

RewriteCond %{REQUEST_METHOD} ^TRAC[EK]
RewriteRule .* - [L,R=405]

## redirect for mobile user agents

#RewriteCond %{REQUEST_URI} !^/mobiledirectoryhere/.*$
#RewriteCond %{HTTP_USER_AGENT} 
"android|blackberry|ipad|iphone|ipod|iemobile|opera 
mobile|palmos|webos|googlebot-mobile" [NC]
#RewriteRule ^(.*)$ /mobiledirectoryhere/ [L,R=302]

## always send 404 on missing files in these folders

RewriteCond %{REQUEST_URI} !^/(media|skin|js)/

## never rewrite for existing files, directories and links

RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_FILENAME} !-l

  ## rewrite everything else to index.php

 RewriteRule .* index.php [L]

 </IfModule>


## Prevent character encoding issues from server overrides
## If you still have problems, use the second line instead

AddDefaultCharset Off
#AddDefaultCharset UTF-8

<IfModule mod_expires.c>

## Add default Expires header
## http://developer.yahoo.com/performance/rules.html#expires

ExpiresDefault "access plus 1 year"
ExpiresActive On
ExpiresDefault A600
ExpiresByType image/x-icon A2592000
ExpiresByType application/x-javascript A604800
ExpiresByType text/css A604800
ExpiresByType image/gif A2592000
ExpiresByType image/png A2592000
ExpiresByType image/jpeg A2592000
ExpiresByType text/plain A86400
ExpiresByType application/x-shockwave-flash A2592000
ExpiresByType video/x-flv A2592000
ExpiresByType application/pdf A2592000
ExpiresByType text/html A600
ExpiresByType text/js "access plus 1 years"
ExpiresByType text/javascript "access plus 1 years"
ExpiresByType application/javascript "access plus 1 years"
ExpiresByType application/x-javascript "access plus 1 years" 
### Expire everything else 1 day from when it's last modified
### (this uses the Alternative syntax)
ExpiresDefault "modification plus 1 day"

# Add correct content-type for fonts
AddType application/vnd.ms-fontobject .eot
AddType application/x-font-ttf .ttf
AddType application/x-font-opentype .otf
AddType application/x-font-woff .woff
AddType image/svg+xml .svg



 # Images
 ExpiresByType image/jpeg "access plus 1 year"
 ExpiresByType image/gif "access plus 1 year"
 ExpiresByType image/png "access plus 1 year"
 ExpiresByType image/webp "access plus 1 year"
 ExpiresByType image/svg+xml "access plus 1 year"
 ExpiresByType image/x-icon "access plus 1 year"

 # Video
 ExpiresByType video/mp4 "access plus 1 year"
 ExpiresByType video/mpeg "access plus 1 year"

 # CSS, JavaScript
 ExpiresByType text/css "access plus 1 month"
 ExpiresByType text/javascript "access plus 1 month"
 ExpiresByType application/javascript "access plus 1 month"

 # Others
ExpiresByType application/pdf "access plus 1 month"
ExpiresByType application/x-shockwave-flash "access plus 1 month"



# Compress compressible fonts
AddOutputFilterByType DEFLATE application/x-font-ttf application/x- 
font-opentype image/svg+xml


# Add a far future Expires header for fonts
 ExpiresByType application/vnd.ms-fontobject "access plus 1 year"
 ExpiresByType application/x-font-ttf "access plus 1 year"
ExpiresByType application/x-font-opentype "access plus 1 year"
   ExpiresByType application/x-font-woff "access plus 1 year"
ExpiresByType image/svg+xml "access plus 1 year"



</IfModule>

 ## By default allow all access

Order allow,deny
Allow from all

    ## Deny access to release notes to prevent disclosure of the 
 installed Magento version

  <Files RELEASE_NOTES.txt>
    order allow,deny
    deny from all
  </Files>

  ## If running in cluster environment, uncomment this
  ## http://developer.yahoo.com/performance/rules.html#etags

  #FileETag none

/etc/varnish/default.vcl

         if (bereq.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?") {
        unset beresp.http.Vary;
        set beresp.do_gzip = true;
        if (beresp.status != 200 && beresp.status != 404) {
            set beresp.ttl = 15s;
            set beresp.uncacheable = true;
            return (deliver);
        } else {
            if (beresp.http.Set-Cookie) {
                set beresp.http.X-Varnish-Set-Cookie = beresp.http.Set-Cookie;
                unset beresp.http.Set-Cookie;
            }
            //unset beresp.http.Cache-Control;
            //unset beresp.http.Expires;
           // unset beresp.http.Pragma;
           // unset beresp.http.Cache;
           // unset beresp.http.Age;
           // if (beresp.http.X-Turpentine-Esi == "1") {
                set beresp.do_esi = true;
            }
            if (beresp.http.X-Turpentine-Cache == "0") {
                set beresp.ttl = 15s;
                set beresp.uncacheable = true;
                return (deliver);
            } else {
                if (true &&
                        bereq.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf|mp4)(?=\?|&|$)") {
                    //unset beresp.http.set-cookie;
                    set beresp.ttl = 28800s;
                    set beresp.http.Cache-Control = "max-age=28800";
                } elseif (bereq.http.X-Varnish-Esi-Method) {
                    if (bereq.http.X-Varnish-Esi-Access == "private" &&
                            bereq.http.Cookie ~ "frontend=") {
                        set beresp.http.X-Varnish-Session = regsub(bereq.http.Cookie,
                                "^.*?frontend=([^;]*);*.*$", "\1");
                    }
                    if (bereq.http.X-Varnish-Esi-Method == "ajax" &&
                            bereq.http.X-Varnish-Esi-Access == "public") 
                    {
                        // Public ajax requests
                        set beresp.http.Cache-Control = "max-age=" + regsub(
                                bereq.url, ".*/ttl/(\d+)/.*", "\1");

                    }
                    set beresp.ttl = std.duration(
                            regsub(
                            bereq.url, ".*/ttl/(\d+)/.*", "\1s"),
                            300s);
                    if (beresp.ttl == 0s) {
                        set beresp.ttl = 15s;
                        set beresp.uncacheable = true;
                        return (deliver);
                    }
                } else {
                    set beresp.ttl = 3600s;
                }
            }
        }
Solution

  • set beresp.http.Cache-Control = "max-age=28800";

    This line is overriding the 1 year expiry date and setting it to 8 hours. The documentation for GTmetrix says resources must be cached for at least 1 month to pass the browser cache test.

    set beresp.http.Cache-Control = "max-age=31556952"