Merge pull request #2348 from asannikov/varnish

Varnish
This commit is contained in:
Shao Yu-Lung (Allen) 2020-02-03 16:14:44 +08:00 committed by GitHub
commit 10c9decf78
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 451 additions and 14 deletions

View File

@ -693,6 +693,7 @@ You may wanna change the default security configuration, so go to `http://localh
<br> <br>
<a name="Use-Redis"></a> <a name="Use-Redis"></a>
## Use Redis ## Use Redis
1 - First make sure you run the Redis Container (`redis`) with the `docker-compose up` command. 1 - First make sure you run the Redis Container (`redis`) with the `docker-compose up` command.
@ -780,12 +781,81 @@ Read the [Laravel official documentation](https://laravel.com/docs/5.7/redis#con
``` ```
<br>
<a name="Use-Varnish"></a>
## Use Varnish
The goal was to proxy request to varnish server using nginx. So only nginx has been configured for Varnish proxy.
Nginx is on port 80 or 443. Nginx sends request through varnish server and varnish server sends request back to nginx on port 81 (external port is defined in `VARNISH_BACKEND_PORT`).
The idea was taken from this [post](https://www.linode.com/docs/websites/varnish/use-varnish-and-nginx-to-serve-wordpress-over-ssl-and-http-on-debian-8/)
The Varnish configuration was developed and tested for Wordpress only. Probably it works with other systems.
#### Steps to configure varnish proxy server:
1. You have to set domain name for VARNISH_PROXY1_BACKEND_HOST variable.
2. If you want to use varnish for different domains, you have to add new configuration section in your env file.
```
VARNISH_PROXY1_CACHE_SIZE=128m
VARNISH_PROXY1_BACKEND_HOST=replace_with_your_domain.name
VARNISH_PROXY1_SERVER=SERVER1
```
3. Then you have to add new config section into docker-compose.yml with related variables:
```
custom_proxy_name:
container_name: custom_proxy_name
build: ./varnish
expose:
- ${VARNISH_PORT}
environment:
- VARNISH_CONFIG=${VARNISH_CONFIG}
- CACHE_SIZE=${VARNISH_PROXY2_CACHE_SIZE}
- VARNISHD_PARAMS=${VARNISHD_PARAMS}
- VARNISH_PORT=${VARNISH_PORT}
- BACKEND_HOST=${VARNISH_PROXY2_BACKEND_HOST}
- BACKEND_PORT=${VARNISH_BACKEND_PORT}
- VARNISH_SERVER=${VARNISH_PROXY2_SERVER}
ports:
- "${VARNISH_PORT}:${VARNISH_PORT}"
links:
- workspace
networks:
- frontend
```
4. change your varnish config and add nginx configuration. Example Nginx configuration is here: `nginx/sites/laravel_varnish.conf.example`.
5. `varnish/default.vcl` is old varnish configuration, which was used in the previous version. Use `default_wordpress.vcl` instead.
#### How to run:
1. Rename `default_wordpress.vcl` to `default.vcl`
2. `docker-compose up -d nginx`
3. `docker-compose up -d proxy`
Keep in mind that varnish server must be built after Nginx cause varnish checks domain affordability.
#### FAQ:
1. How to purge cache? <br>
run from any cli: <br>`curl -X PURGE https://yourwebsite.com/`.
2. How to reload varnish?<br>
`docker container exec proxy varnishreload`
3. Which varnish commands are allowed?
- varnishadm
- varnishd
- varnishhist
- varnishlog
- varnishncsa
- varnishreload
- varnishstat
- varnishtest
- varnishtop
4. How to reload Nginx?<br>
`docker exec Nginx nginx -t`<br>
`docker exec Nginx nginx -s reload`
<br> <br>
<a name="Use-Mongo"></a> <a name="Use-Mongo"></a>
## Use Mongo ## Use Mongo
1 - First install `mongo` in the Workspace and the PHP-FPM Containers: 1 - First install `mongo` in the Workspace and the PHP-FPM Containers:

View File

@ -322,6 +322,7 @@ services:
ports: ports:
- "${NGINX_HOST_HTTP_PORT}:80" - "${NGINX_HOST_HTTP_PORT}:80"
- "${NGINX_HOST_HTTPS_PORT}:443" - "${NGINX_HOST_HTTPS_PORT}:443"
- "${VARNISH_BACKEND_PORT}:81"
depends_on: depends_on:
- php-fpm - php-fpm
networks: networks:
@ -869,6 +870,7 @@ services:
### Varnish ########################################## ### Varnish ##########################################
proxy: proxy:
container_name: proxy
build: ./varnish build: ./varnish
expose: expose:
- ${VARNISH_PORT} - ${VARNISH_PORT}
@ -880,12 +882,15 @@ services:
- BACKEND_HOST=${VARNISH_PROXY1_BACKEND_HOST} - BACKEND_HOST=${VARNISH_PROXY1_BACKEND_HOST}
- BACKEND_PORT=${VARNISH_BACKEND_PORT} - BACKEND_PORT=${VARNISH_BACKEND_PORT}
- VARNISH_SERVER=${VARNISH_PROXY1_SERVER} - VARNISH_SERVER=${VARNISH_PROXY1_SERVER}
ports:
- "${VARNISH_PORT}:${VARNISH_PORT}"
links: links:
- workspace - workspace
networks: networks:
- frontend - frontend
proxy2: proxy2:
container_name: proxy2
build: ./varnish build: ./varnish
expose: expose:
- ${VARNISH_PORT} - ${VARNISH_PORT}
@ -897,6 +902,8 @@ services:
- BACKEND_HOST=${VARNISH_PROXY2_BACKEND_HOST} - BACKEND_HOST=${VARNISH_PROXY2_BACKEND_HOST}
- BACKEND_PORT=${VARNISH_BACKEND_PORT} - BACKEND_PORT=${VARNISH_BACKEND_PORT}
- VARNISH_SERVER=${VARNISH_PROXY2_SERVER} - VARNISH_SERVER=${VARNISH_PROXY2_SERVER}
ports:
- "${VARNISH_PORT}:${VARNISH_PORT}"
links: links:
- workspace - workspace
networks: networks:

View File

@ -386,8 +386,8 @@ MAILDEV_SMTP_PORT=25
### VARNISH ############################################### ### VARNISH ###############################################
VARNISH_CONFIG=/etc/varnish/default.vcl VARNISH_CONFIG=/etc/varnish/default.vcl
VARNISH_PORT=8080 VARNISH_PORT=6081
VARNISH_BACKEND_PORT=8888 VARNISH_BACKEND_PORT=81
VARNISHD_PARAMS="-p default_ttl=3600 -p default_grace=3600" VARNISHD_PARAMS="-p default_ttl=3600 -p default_grace=3600"
### Varnish ############################################### ### Varnish ###############################################

View File

@ -18,6 +18,8 @@ RUN apk update \
&& apk add --no-cache openssl \ && apk add --no-cache openssl \
&& apk add --no-cache bash && apk add --no-cache bash
RUN apk add --no-cache curl
RUN set -x ; \ RUN set -x ; \
addgroup -g 82 -S www-data ; \ addgroup -g 82 -S www-data ; \
adduser -u 82 -D -S -G www-data www-data && exit 0 ; exit 1 adduser -u 82 -D -S -G www-data www-data && exit 0 ; exit 1
@ -39,4 +41,4 @@ ADD ./startup.sh /opt/startup.sh
RUN sed -i 's/\r//g' /opt/startup.sh RUN sed -i 's/\r//g' /opt/startup.sh
CMD ["/bin/bash", "/opt/startup.sh"] CMD ["/bin/bash", "/opt/startup.sh"]
EXPOSE 80 443 EXPOSE 80 81 443

View File

@ -0,0 +1,110 @@
server {
listen 80;
listen [::]:80;
server_name www.laravel.test;
rewrite ^(.*) https://laravel.test$1/ permanent;
}
server {
listen 80;
listen [::]:80;
server_name laravel.test;
rewrite ^(.*) https://laravel.test$1/ permanent;
}
server {
listen 443 ssl ;
listen [::]:443 ssl;
ssl_certificate /etc/nginx/ssl/laravel.test.crt;
ssl_certificate_key /etc/nginx/ssl/laravel.test.key;
server_name www.laravel.test;
rewrite ^(.*) https://laravel.test$1/ permanent;
}
server {
server_name laravel.test;
# For https
listen 443 ssl ;
listen [::]:443 ssl;
ssl_certificate /etc/nginx/ssl/laravel.test.crt;
ssl_certificate_key /etc/nginx/ssl/laravel.test.key;
port_in_redirect off;
add_header Strict-Transport-Security "max-age=31536000";
add_header X-Content-Type-Options nosniff;
location / {
proxy_pass http://proxy:6081;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header HTTPS "on";
proxy_redirect off;
}
}
server {
server_name laravel.test;
listen 81;
listen [::]:81;
root /var/www/laravel.test/www;
index index.php index.html index.htm;
location / {
try_files $uri $uri/ /index.php$is_args$args;
}
location ~ \.php$ {
fastcgi_max_temp_file_size 4m;
fastcgi_pass php-upstream;
# Additional configs
fastcgi_pass_header Set-Cookie;
fastcgi_pass_header Cookie;
fastcgi_ignore_headers Cache-Control Expires Set-Cookie;
try_files $uri /index.php =404;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_param PATH_TRANSLATED $document_root$fastcgi_path_info;
fastcgi_param HTTPS on;
fastcgi_buffers 16 16k;
fastcgi_buffer_size 32k;
fastcgi_intercept_errors on;
#fixes timeouts
fastcgi_read_timeout 600;
include fastcgi_params;
}
# Caching
location ~* \.(ico|jpg|webp|jpeg|gif|css|png|js|ico|bmp|zip|woff)$ {
access_log off;
log_not_found off;
add_header Pragma public;
add_header Cache-Control "public";
expires 14d;
}
location ~* \.(php|html)$ {
access_log on;
log_not_found on;
add_header Pragma public;
add_header Cache-Control "public";
expires 14d;
}
location ~ /\.ht {
deny all;
}
}

View File

@ -1,16 +1,8 @@
FROM debian:latest FROM varnish:6.3
LABEL maintainer="ZeroC0D3 Team<zeroc0d3.team@gmail.com>"
# Set Environment Variables # Set Environment Variables
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
# Install Dependencies
RUN apt-get update && apt-get install -y apt-utils && apt-get upgrade -y
RUN mkdir /home/site && mkdir /home/site/cache
RUN apt-get install -y varnish
RUN rm -rf /var/lib/apt/lists/*
# Setting Configurations # Setting Configurations
ENV VARNISH_CONFIG /etc/varnish/default.vcl ENV VARNISH_CONFIG /etc/varnish/default.vcl
ENV CACHE_SIZE 128m ENV CACHE_SIZE 128m

View File

@ -52,6 +52,7 @@ sub vcl_init {
# vdir.add_backend(servern); # vdir.add_backend(servern);
} }
# This function is used when a request is send by a HTTP client (Browser)
sub vcl_recv { sub vcl_recv {
# Called at the beginning of a request, after the complete request has been received and parsed. # Called at the beginning of a request, after the complete request has been received and parsed.
# Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable, # Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable,
@ -75,8 +76,12 @@ sub vcl_recv {
# Not from an allowed IP? Then die with an error. # Not from an allowed IP? Then die with an error.
return (synth(405, "This IP is not allowed to send PURGE requests.")); return (synth(405, "This IP is not allowed to send PURGE requests."));
} }
# If you got this stage (and didn't error out above), purge the cached result
return (purge); ban("req.http.host == " + req.http.host);
# Throw a synthetic page so the request won't go to the backend.
return(synth(200, "Ban added"));
# If allowed, do a cache_lookup -> vlc_hit() or vlc_miss()
#return (purge);
} }
# Only deal with "normal" types # Only deal with "normal" types

View File

@ -0,0 +1,243 @@
vcl 4.1;
# Based on: https://github.com/mattiasgeniar/varnish-6.0-configuration-templates/blob/master/default.vcl
import std;
import directors;
backend everpracticalsolutionsServer { # Define one backend
.host = "${BACKEND_HOST}"; # IP or Hostname of backend
.port = "${BACKEND_PORT}"; # Port Apache or whatever is listening
.max_connections = 300; # That's it
.probe = {
#.url = "/"; # short easy way (GET /)
# We prefer to only do a HEAD /
.request =
"HEAD /health_check.php HTTP/1.1"
"Host: ${BACKEND_HOST}"
"Connection: close"
"User-Agent: Varnish Health Probe";
.interval = 5s; # check the health of each backend every 5 seconds
.timeout = 1s; # timing out after 1 second.
.window = 5; # If 3 out of the last 5 polls succeeded the backend is considered healthy, otherwise it will be marked as sick
.threshold = 3;
}
.first_byte_timeout = 300s; # How long to wait before we receive a first byte from our backend?
.connect_timeout = 5s; # How long to wait for a backend connection?
.between_bytes_timeout = 2s; # How long to wait between bytes received from our backend?
}
# Only allow purging from specific IPs
acl purge {
"localhost";
"127.0.0.1";
"192.168.16.5";
"192.168.16.6";
"185.228.234.203";
}
# This function is used when a request is send by a HTTP client (Browser)
sub vcl_recv {
# Normalize the header, remove the port (in case you're testing this on various TCP ports)
set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
# Allow purging from ACL
if (req.method == "PURGE") {
# If not allowed then a error 405 is returned
if (!client.ip ~ purge) {
return(synth(405, "This IP is not allowed to send PURGE requests."));
}
ban("req.http.host == " + req.http.host);
# Throw a synthetic page so the request won't go to the backend.
return(synth(200, "Ban added"));
# If allowed, do a cache_lookup -> vlc_hit() or vlc_miss()
#return (purge);
}
# Post requests will not be cached
if (req.http.Authorization || req.method == "POST") {
return (pass);
}
# --- WordPress specific configuration
# Did not cache the RSS feed
if (req.url ~ "/feed") {
return (pass);
}
# Blitz hack
if (req.url ~ "/mu-.*") {
return (pass);
}
# Did not cache the admin and login pages
if (req.url ~ "/wp-(login|admin)") {
return (pass);
}
# Remove the "has_js" cookie
set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
# Remove any Google Analytics based cookies
set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
# Remove the Quant Capital cookies (added by some plugin, all __qca)
set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");
# Remove the wp-settings-1 cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-1=[^;]+(; )?", "");
# Remove the wp-settings-time-1 cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-time-1=[^;]+(; )?", "");
# Remove the wp test cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wordpress_test_cookie=[^;]+(; )?", "");
# Are there cookies left with only spaces or that are empty?
if (req.http.cookie ~ "^ *$") {
unset req.http.cookie;
}
# Cache the following files extensions
if (req.url ~ "\.(css|js|png|gif|jp(e)?g|swf|ico)") {
unset req.http.cookie;
}
# Normalize Accept-Encoding header and compression
# https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
if (req.http.Accept-Encoding) {
# Do no compress compressed files...
if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
unset req.http.Accept-Encoding;
}
}
# Check the cookies for wordpress-specific items
if (req.http.Cookie ~ "wordpress_" || req.http.Cookie ~ "comment_") {
return (pass);
}
if (!req.http.cookie) {
unset req.http.cookie;
}
# --- End of WordPress specific configuration
# Do not cache HTTP authentication and HTTP Cookie
if (req.http.Authorization || req.http.Cookie) {
# Not cacheable by default
return (pass);
}
# Cache all others requests
return (hash);
}
sub vcl_pipe {
return (pipe);
}
sub vcl_pass {
return (fetch);
}
# The data on which the hashing will take place
sub vcl_hash {
hash_data(req.url);
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
# If the client supports compression, keep that in a different cache
if (req.http.Accept-Encoding) {
hash_data(req.http.Accept-Encoding);
}
return (lookup);
}
# This function is used when a request is sent by our backend (Nginx server)
sub vcl_backend_response {
# Remove some headers we never want to see
unset beresp.http.Server;
unset beresp.http.X-Powered-By;
# For static content strip all backend cookies
if (bereq.url ~ "\.(css|js|png|gif|jp(e?)g)|swf|ico") {
unset beresp.http.cookie;
}
# Only allow cookies to be set if we're in admin area
if (beresp.http.Set-Cookie && bereq.url !~ "^/wp-(login|admin)") {
unset beresp.http.Set-Cookie;
}
# don't cache response to posted requests or those with basic auth
if ( bereq.method == "POST" || bereq.http.Authorization ) {
set beresp.uncacheable = true;
set beresp.ttl = 120s;
return (deliver);
}
# don't cache search results
if ( bereq.url ~ "\?s=" ){
set beresp.uncacheable = true;
set beresp.ttl = 120s;
return (deliver);
}
# only cache status ok
if ( beresp.status != 200 ) {
set beresp.uncacheable = true;
set beresp.ttl = 120s;
return (deliver);
}
# A TTL of 24h
set beresp.ttl = 24h;
# Define the default grace period to serve cached content
set beresp.grace = 30s;
return (deliver);
}
# The routine when we deliver the HTTP request to the user
# Last chance to modify headers that are sent to the client
sub vcl_deliver {
if (obj.hits > 0) {
set resp.http.X-Cache = "cached";
} else {
set resp.http.x-Cache = "uncached";
}
# Remove some headers: PHP version
unset resp.http.X-Powered-By;
# Remove some headers: Apache version & OS
unset resp.http.Server;
# Remove some heanders: Varnish
unset resp.http.Via;
unset resp.http.X-Varnish;
return (deliver);
}
sub vcl_init {
return (ok);
}
sub vcl_fini {
return (ok);
}

View File

@ -7,6 +7,14 @@ do
sed -i "s|\${${name}}|${value}|g" /etc/varnish/default.vcl sed -i "s|\${${name}}|${value}|g" /etc/varnish/default.vcl
done done
echo "exec varnishd \
-a :$VARNISH_PORT \
-T localhost:6082 \
-F \
-f $VARNISH_CONFIG \
-s malloc,$CACHE_SIZE \
$VARNISHD_PARAMS"
exec bash -c \ exec bash -c \
"exec varnishd \ "exec varnishd \
-a :$VARNISH_PORT \ -a :$VARNISH_PORT \