Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Varnish beyond basic web acceleration - Symfony...

Thijs Feryn
September 26, 2019

Varnish beyond basic web acceleration - Symfony Live Berlin 2019

Advanced VCL examples where Varnish is used for web and API acceleration, but also for OTT video platforms and DIY CDNs.

See https://feryn.eu/speaking/varnish-beyond-basic-web-acceleration-symfony-live-berlin-2019/ for more details

Thijs Feryn

September 26, 2019
Tweet

More Decks by Thijs Feryn

Other Decks in Technology

Transcript

  1. WHY IS VARNISH SO POWERFUL? ✓ EXTREMELY LOW RESOURCE ✓

    EXTREMELY STABLE ✓ 100 GBIT PER SERVER ✓ NO DISK ACCESS AT RUNTIME ✓ REQUEST COALESCING ✓ VARNISH CONFIGURATION LANGUAGE ✓ VMODS ✓ COMPLIES TO HTTP BEST PRACTICES ✓ ENTERPRISE FEATURES*
  2. HTTP CACHING MECHANISMS Expires: Sat, 09 Sep 2017 14:30:00 GMT

    Cache-control: public, max-age=3600, s-maxage=86400 Cache-control: private, no-cache, no-store
  3. CONDITIONAL REQUESTS HTTP/1.1 200 OK Host: localhost Etag: 7c9d70604c6061da9bb9377d3f00eb27 Content-type:

    text/html; charset=UTF-8 Hello world output GET / HTTP/1.1 Host: localhost
  4. CONDITIONAL REQUESTS HTTP/1.1 304 Not Modified Host: localhost Etag: 7c9d70604c6061da9bb9377d3f00eb27

    GET / HTTP/1.1 Host: localhost If-None-Match: 7c9d70604c6061da9bb9377d3f00eb27
  5. ✓STATELESS ✓WELL-DEFINED TTL ✓CACHE / NO-CACHE PER RESOURCE ✓CACHE VARIATIONS

    ✓CONDITIONAL REQUESTS ✓PLACEHOLDERS FOR NON-CACHEABLE CONTENT IN AN IDEAL WORLD
  6. sub vcl_recv { if (req.url ~ "^/status\.php$" || req.url ~

    "^/update\.php$" || req.url ~ "^/admin$" || req.url ~ "^/admin/.*$" || req.url ~ "^/flag/.*$" || req.url ~ "^.*/ajax/.*$" || req.url ~ "^.*/ahah/.*$") { return (pass); } } URL blacklist example
  7. vcl 4.0; sub vcl_recv { if (req.http.Cookie) { set req.http.Cookie

    = ";" + req.http.Cookie; set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); set req.http.Cookie = regsuball(req.http.Cookie, ";(SESS[a-z0-9]+|SSESS[a-z0-9]+|NO_CACHE)=", "; \1="); set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", ""); if (req.http.Cookie == "") { unset req.http.Cookie; } else { return (pass); } } } Only keep certain cookies
  8. sub vcl_recv { if (req.http.Cookie) { set req.http.Cookie = ";"

    + req.http.Cookie; set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); set req.http.Cookie = regsuball(req.http.Cookie, ";(language)=", "; \1="); set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", ""); if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; return(pass); } return(hash); } } sub vcl_hash { hash_data(regsub( req.http.Cookie, "^.*language=([^;]*);*.*$", "\1" )); } Cookie cache variation
  9. import accept; sub vcl_init { new format = accept.rule("text/plain"); format.add("text/html");

    format.add("application/json"); new lang = accept.rule("en"); lang.add("fr"); } sub vcl_recv { set req.http.accept = format.filter(req.http.accept); set req.http.accept-language = lang.filter(req.http.accept-language); } Sanitize accept headers
  10. import crypto; import edgestash; import xbody; sub vcl_backend_response{ if (beresp.http.Content-Type

    ~ "html") { xbody.regsub("<script>", "<script nonce={{nonce}}>"); edgestash.parse_response(); } } sub vcl_deliver { if (edgestash.is_edgestash()) { set req.http.X-nonce = crypto.hex_encode(crypto.urandom(16)); set resp.http.Content-Security-Policy = "script-src 'nonce-" + req.http.X-nonce + "'"; edgestash.add_json({" { "nonce":""} + req.http.X-nonce + {"" } "}); edgestash.execute(); } } Personalized Javascript CSP nonce
  11. import cookieplus; import edgestash; import kvstore; import xbody; sub vcl_init

    { new session_json = kvstore.init(); } sub vcl_recv { unset req.http.X-do-capture; unset req.http.X-session; if (req.method != "HEAD" && req.method != "GET") { return (pass); } if (!session_json.get(cookieplus.get("session"))) { return (pass); } return (hash); } CACHE PERSONALIZATION
  12. sub vcl_pass { set req.http.X-do-capture = "true"; } sub vcl_backend_response

    { if (beresp.http.Content-Type ~ "text") { if (bereq.http.X-do-capture) { xbody.capture("name", "Hello (\S+)", "\1"); } else { xbody.regsub("Hello \S+", "Hello {{name}}"); edgestash.parse_response(); } } } sub vcl_deliver { if (cookieplus.get("session")) { set req.http.X-session = cookieplus.get("session"); } else if (cookieplus.setcookie_get("session")) { set req.http.X-session = cookieplus.setcookie_get("session"); } if (req.http.X-session && xbody.get("name")) { session_json.set(req.http.X-session, xbody.get_all(), 24h); } if (edgestash.is_edgestash()) { edgestash.add_json(session_json.get(req.http.X-session)); edgestash.execute(); } } Capture value Turn value into placeholder Store value in K/V store Parse value into placeholder
  13. import deviceatlas; sub vcl_recv { set req.http.MobilePhone = "no"; if

    (deviceatlas.lookup_int(req.http.User-Agent, "isMobilePhone")) { set req.http.MobilePhone = "yes"; } } Device detection
  14. vcl 4.0; import mmdb; backend default { .host = "192.0.2.11";

    .port = "8080"; } # create a database object sub vcl_init { new geodb = mmdb.init("/path/to/db"); } sub vcl_recv { # retrieve the name of the request's origin set req.http.Country-Name = geodb.country_name(client.ip); # if the country doesn't come from Germany or Belgium, deny access if (req.http.Country-Name != "Germany" || req.http.Country-Name != "Belgium") { return (synth(403, "Sorry, only available in Germany and Belgium")); } } Geo blocking
  15. vcl 4.0; import http; backend default { .host = "origin";

    .port = "80"; } sub vcl_recv { set req.http.X-prefetch = http.varnish_url("/"); } sub vcl_backend_response { if (beresp.http.Link ~ "<.+>.*(prefetch|next)") { set bereq.http.X-link = regsub(beresp.http.Link, "^.*<([^>]*)>.*$", "\1"); set bereq.http.X-prefetch = regsub(bereq.http.X-prefetch, "/$", bereq.http.X-link); http.init(0); http.req_copy_headers(0); http.req_set_url(0, bereq.http.X-prefetch); http.req_send_and_finish(0); } } Pre-fetching
  16. #EXTM3U #EXT-X-VERSION:3 #EXT-X-TARGETDURATION:6 #EXT-X-MEDIA-SEQUENCE:0 #EXT-X-PLAYLIST-TYPE:VOD #EXTINF:6.000000, stream_00.ts #EXTINF:6.000000, stream_01.ts #EXTINF:6.000000,

    stream_02.ts #EXTINF:6.000000, stream_03.ts #EXTINF:6.000000, stream_04.ts #EXTINF:6.000000, stream_05.ts #EXTINF:6.000000, stream_06.ts #EXTINF:6.000000, stream_07.ts #EXTINF:6.000000, stream_08.ts #EXTINF:6.000000, stream_09.ts #EXTINF:5.280000, stream_010.ts #EXT-X-ENDLIST STREAM.M3U8
  17. <html> <head> <link href="https://vjs.zencdn.net/7.5.4/video-js.css" rel="stylesheet"> </head> <body> <video id='my-video' class='video-js'

    controls preload='auto' height='640' data- setup='{}'> <source src="/live/stream.m3u8" type="application/vnd.apple.mpegurl"> </video> <script src='https://vjs.zencdn.net/7.5.4/video.js'></script> </body> </html>
  18. MORE VIDEO FEATURES ✓ GEOIP ✓ DIGITAL RIGHTS MANAGEMENT ✓

    AUTHENTICATION ✓ THROTTLING & RATE LIMITING ✓ ADAPTIVE BITRATE FILTERING
  19. vcl 4.1; import http; import std; backend default { .host

    = "origin"; .port = "80"; } sub vcl_recv { if (req.url ~ "^/vod/.+\.ts$") { http.init(0); http.req_set_max_loops(0,1); http.req_copy_headers(0); http.req_set_method(0, "HEAD"); set req.http.x-next-url = http.prefetch_next_url(); std.log("Prefetching " + req.http.x-next-url); http.req_set_url(0, req.http.x-next-url); http.req_send_and_finish(0); } }
  20. vcl 4.0; import cookieplus; import redis; import synthbackend; sub vcl_init

    { new db = redis.db( location="redis:6379", type=master, connection_timeout=500, shared_connections=false, max_connections=1); } sub vcl_recv { set req.http.X-Login = "false"; set req.http.x-session = cookieplus.get("PHPSESSID","guest"); if(req.http.x-session != "guest") { db.command("EXISTS"); db.push("sf_s"+cookieplus.get("PHPSESSID")); db.execute(); if(db.get_integer_reply() == 1) { set req.http.X-Login = "true"; } } } Synthetic HTTP
  21. sub vcl_backend_fetch { if(bereq.url == "/session") { if(bereq.http.X-Login != "true")

    { set bereq.backend = synthbackend.from_string("{}"); return(fetch); } db.command("EVAL"); db.push({" local session = redis.call('GET', KEYS[1]) if session == nil then return '{}' end local result = string.gsub(session, '[%c]', '') local username = string.gsub(result,'.+Userusername\";s:[0-9]+:\"([^\"]+)\";.+','%1') if username == nil then return '{}' end return '{"username":"'.. username ..'"}' "}); db.push(1); db.push("sf_s"+cookieplus.get("PHPSESSID")); db.execute(); set bereq.backend = synthbackend.from_string(db.get_string_reply()); } } sub vcl_backend_response { if(bereq.url == "/session") { set beresp.http.Content-Type = "application/json; charset=utf-8"; set beresp.ttl = 3600s; set beresp.http.vary = "x-session"; } } Synthetic HTTP
  22. VARNISH ENTERPRISE ✓ CLIENT SSL TERMINATION ✓ BACKEND SSL CONNECTIONS

    ✓ PARALLEL ESI ✓ MASSIVE STORAGE ENGINE ✓ ENCRYPTION ✓ THROTTLING ✓ RATE LIMITING ✓ PREFETCHING ✓ GEOLOCATION ✓ AUTHENTICATION ✓ EDGESTASH ✓ CUSTOM STATISTICS ✓ ADMIN MODULE ✓ SUPPORT ✓ HIGH AVAILABILITY ✓ MOBILE APP