diff --git a/DOCUMENTATION/content/documentation/index.md b/DOCUMENTATION/content/documentation/index.md index 14d5be03..91127676 100644 --- a/DOCUMENTATION/content/documentation/index.md +++ b/DOCUMENTATION/content/documentation/index.md @@ -693,6 +693,7 @@ You may wanna change the default security configuration, so go to `http://localh
+ ## Use Redis 1 - First make sure you run the Redis Container (`redis`) with the `docker-compose up` command. @@ -780,12 +781,81 @@ Read the [Laravel official documentation](https://laravel.com/docs/5.7/redis#con ``` +
+ +## Use Varnish +The goal was to proxy request to varnish server using nginx. So only nginx has been configured for Varnish proxy. +Nginx is on port 80 or 443. Nginx sends request through varnish server and varnish server sends request back to nginx on port 81 (external port is defined in `VARNISH_BACKEND_PORT`). +The idea was taken from this [post](https://www.linode.com/docs/websites/varnish/use-varnish-and-nginx-to-serve-wordpress-over-ssl-and-http-on-debian-8/) + +The Varnish configuration was developed and tested for Wordpress only. Probably it works with other systems. + +#### Steps to configure varnish proxy server: +1. You have to set domain name for VARNISH_PROXY1_BACKEND_HOST variable. +2. If you want to use varnish for different domains, you have to add new configuration section in your env file. + ``` + VARNISH_PROXY1_CACHE_SIZE=128m + VARNISH_PROXY1_BACKEND_HOST=replace_with_your_domain.name + VARNISH_PROXY1_SERVER=SERVER1 + ``` +3. Then you have to add new config section into docker-compose.yml with related variables: + ``` + custom_proxy_name: + container_name: custom_proxy_name + build: ./varnish + expose: + - ${VARNISH_PORT} + environment: + - VARNISH_CONFIG=${VARNISH_CONFIG} + - CACHE_SIZE=${VARNISH_PROXY2_CACHE_SIZE} + - VARNISHD_PARAMS=${VARNISHD_PARAMS} + - VARNISH_PORT=${VARNISH_PORT} + - BACKEND_HOST=${VARNISH_PROXY2_BACKEND_HOST} + - BACKEND_PORT=${VARNISH_BACKEND_PORT} + - VARNISH_SERVER=${VARNISH_PROXY2_SERVER} + ports: + - "${VARNISH_PORT}:${VARNISH_PORT}" + links: + - workspace + networks: + - frontend + ``` +4. change your varnish config and add nginx configuration. Example Nginx configuration is here: `nginx/sites/laravel_varnish.conf.example`. +5. `varnish/default.vcl` is old varnish configuration, which was used in the previous version. Use `default_wordpress.vcl` instead. + +#### How to run: +1. Rename `default_wordpress.vcl` to `default.vcl` +2. `docker-compose up -d nginx` +3. `docker-compose up -d proxy` + +Keep in mind that varnish server must be built after Nginx cause varnish checks domain affordability. + +#### FAQ: + +1. How to purge cache?
+run from any cli:
`curl -X PURGE https://yourwebsite.com/`. +2. How to reload varnish?
+`docker container exec proxy varnishreload` +3. Which varnish commands are allowed? + - varnishadm + - varnishd + - varnishhist + - varnishlog + - varnishncsa + - varnishreload + - varnishstat + - varnishtest + - varnishtop +4. How to reload Nginx?
+`docker exec Nginx nginx -t`
+`docker exec Nginx nginx -s reload`
+ ## Use Mongo 1 - First install `mongo` in the Workspace and the PHP-FPM Containers: diff --git a/docker-compose.yml b/docker-compose.yml index 7f1543fe..dfc7a2e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -322,6 +322,7 @@ services: ports: - "${NGINX_HOST_HTTP_PORT}:80" - "${NGINX_HOST_HTTPS_PORT}:443" + - "${VARNISH_BACKEND_PORT}:81" depends_on: - php-fpm networks: @@ -869,6 +870,7 @@ services: ### Varnish ########################################## proxy: + container_name: proxy build: ./varnish expose: - ${VARNISH_PORT} @@ -880,12 +882,15 @@ services: - BACKEND_HOST=${VARNISH_PROXY1_BACKEND_HOST} - BACKEND_PORT=${VARNISH_BACKEND_PORT} - VARNISH_SERVER=${VARNISH_PROXY1_SERVER} + ports: + - "${VARNISH_PORT}:${VARNISH_PORT}" links: - workspace networks: - frontend proxy2: + container_name: proxy2 build: ./varnish expose: - ${VARNISH_PORT} @@ -897,6 +902,8 @@ services: - BACKEND_HOST=${VARNISH_PROXY2_BACKEND_HOST} - BACKEND_PORT=${VARNISH_BACKEND_PORT} - VARNISH_SERVER=${VARNISH_PROXY2_SERVER} + ports: + - "${VARNISH_PORT}:${VARNISH_PORT}" links: - workspace networks: diff --git a/env-example b/env-example index feecbd65..720f94b9 100644 --- a/env-example +++ b/env-example @@ -386,8 +386,8 @@ MAILDEV_SMTP_PORT=25 ### VARNISH ############################################### VARNISH_CONFIG=/etc/varnish/default.vcl -VARNISH_PORT=8080 -VARNISH_BACKEND_PORT=8888 +VARNISH_PORT=6081 +VARNISH_BACKEND_PORT=81 VARNISHD_PARAMS="-p default_ttl=3600 -p default_grace=3600" ### Varnish ############################################### diff --git a/nginx/Dockerfile b/nginx/Dockerfile index eb1015bd..e51879c8 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -18,6 +18,8 @@ RUN apk update \ && apk add --no-cache openssl \ && apk add --no-cache bash +RUN apk add --no-cache curl + RUN set -x ; \ addgroup -g 82 -S www-data ; \ adduser -u 82 -D -S -G www-data www-data && exit 0 ; exit 1 @@ -39,4 +41,4 @@ ADD ./startup.sh /opt/startup.sh RUN sed -i 's/\r//g' /opt/startup.sh CMD ["/bin/bash", "/opt/startup.sh"] -EXPOSE 80 443 +EXPOSE 80 81 443 diff --git a/nginx/sites/laravel_varnish.conf.example b/nginx/sites/laravel_varnish.conf.example new file mode 100644 index 00000000..7d545872 --- /dev/null +++ b/nginx/sites/laravel_varnish.conf.example @@ -0,0 +1,110 @@ +server { + listen 80; + listen [::]:80; + server_name www.laravel.test; + rewrite ^(.*) https://laravel.test$1/ permanent; +} + +server { + listen 80; + listen [::]:80; + server_name laravel.test; + rewrite ^(.*) https://laravel.test$1/ permanent; +} + +server { + listen 443 ssl ; + listen [::]:443 ssl; + ssl_certificate /etc/nginx/ssl/laravel.test.crt; + ssl_certificate_key /etc/nginx/ssl/laravel.test.key; + server_name www.laravel.test; + rewrite ^(.*) https://laravel.test$1/ permanent; +} + +server { + server_name laravel.test; + + # For https + listen 443 ssl ; + listen [::]:443 ssl; + ssl_certificate /etc/nginx/ssl/laravel.test.crt; + ssl_certificate_key /etc/nginx/ssl/laravel.test.key; + + port_in_redirect off; + + add_header Strict-Transport-Security "max-age=31536000"; + add_header X-Content-Type-Options nosniff; + + location / { + proxy_pass http://proxy:6081; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header HTTPS "on"; + proxy_redirect off; + } +} + +server { + server_name laravel.test; + + listen 81; + listen [::]:81; + + root /var/www/laravel.test/www; + + index index.php index.html index.htm; + + location / { + try_files $uri $uri/ /index.php$is_args$args; + } + + location ~ \.php$ { + fastcgi_max_temp_file_size 4m; + fastcgi_pass php-upstream; + + # Additional configs + fastcgi_pass_header Set-Cookie; + fastcgi_pass_header Cookie; + fastcgi_ignore_headers Cache-Control Expires Set-Cookie; + try_files $uri /index.php =404; + fastcgi_index index.php; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_split_path_info ^(.+\.php)(/.+)$; + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_param PATH_TRANSLATED $document_root$fastcgi_path_info; + fastcgi_param HTTPS on; + + fastcgi_buffers 16 16k; + fastcgi_buffer_size 32k; + + fastcgi_intercept_errors on; + + #fixes timeouts + fastcgi_read_timeout 600; + include fastcgi_params; + } + + # Caching + location ~* \.(ico|jpg|webp|jpeg|gif|css|png|js|ico|bmp|zip|woff)$ { + access_log off; + log_not_found off; + add_header Pragma public; + add_header Cache-Control "public"; + expires 14d; + } + + location ~* \.(php|html)$ { + access_log on; + log_not_found on; + add_header Pragma public; + add_header Cache-Control "public"; + expires 14d; + } + + location ~ /\.ht { + deny all; + } +} diff --git a/varnish/Dockerfile b/varnish/Dockerfile index 8cc4fbfb..3139da78 100644 --- a/varnish/Dockerfile +++ b/varnish/Dockerfile @@ -1,16 +1,8 @@ -FROM debian:latest - -LABEL maintainer="ZeroC0D3 Team" +FROM varnish:6.3 # Set Environment Variables ENV DEBIAN_FRONTEND noninteractive -# Install Dependencies -RUN apt-get update && apt-get install -y apt-utils && apt-get upgrade -y -RUN mkdir /home/site && mkdir /home/site/cache -RUN apt-get install -y varnish -RUN rm -rf /var/lib/apt/lists/* - # Setting Configurations ENV VARNISH_CONFIG /etc/varnish/default.vcl ENV CACHE_SIZE 128m diff --git a/varnish/default.vcl b/varnish/default.vcl index 73099c39..9da3360c 100644 --- a/varnish/default.vcl +++ b/varnish/default.vcl @@ -52,6 +52,7 @@ sub vcl_init { # vdir.add_backend(servern); } +# This function is used when a request is send by a HTTP client (Browser) sub vcl_recv { # Called at the beginning of a request, after the complete request has been received and parsed. # Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable, @@ -75,8 +76,12 @@ sub vcl_recv { # Not from an allowed IP? Then die with an error. return (synth(405, "This IP is not allowed to send PURGE requests.")); } - # If you got this stage (and didn't error out above), purge the cached result - return (purge); + + ban("req.http.host == " + req.http.host); + # Throw a synthetic page so the request won't go to the backend. + return(synth(200, "Ban added")); + # If allowed, do a cache_lookup -> vlc_hit() or vlc_miss() + #return (purge); } # Only deal with "normal" types diff --git a/varnish/default_wordpress.vcl b/varnish/default_wordpress.vcl new file mode 100644 index 00000000..a304f97d --- /dev/null +++ b/varnish/default_wordpress.vcl @@ -0,0 +1,243 @@ +vcl 4.1; +# Based on: https://github.com/mattiasgeniar/varnish-6.0-configuration-templates/blob/master/default.vcl + +import std; +import directors; + +backend everpracticalsolutionsServer { # Define one backend + .host = "${BACKEND_HOST}"; # IP or Hostname of backend + .port = "${BACKEND_PORT}"; # Port Apache or whatever is listening + .max_connections = 300; # That's it + + .probe = { + #.url = "/"; # short easy way (GET /) + # We prefer to only do a HEAD / + .request = + "HEAD /health_check.php HTTP/1.1" + "Host: ${BACKEND_HOST}" + "Connection: close" + "User-Agent: Varnish Health Probe"; + + .interval = 5s; # check the health of each backend every 5 seconds + .timeout = 1s; # timing out after 1 second. + .window = 5; # If 3 out of the last 5 polls succeeded the backend is considered healthy, otherwise it will be marked as sick + .threshold = 3; + } + + .first_byte_timeout = 300s; # How long to wait before we receive a first byte from our backend? + .connect_timeout = 5s; # How long to wait for a backend connection? + .between_bytes_timeout = 2s; # How long to wait between bytes received from our backend? +} + +# Only allow purging from specific IPs +acl purge { + "localhost"; + "127.0.0.1"; + "192.168.16.5"; + "192.168.16.6"; + "185.228.234.203"; +} + +# This function is used when a request is send by a HTTP client (Browser) +sub vcl_recv { + # Normalize the header, remove the port (in case you're testing this on various TCP ports) + set req.http.Host = regsub(req.http.Host, ":[0-9]+", ""); + + # Allow purging from ACL + if (req.method == "PURGE") { + # If not allowed then a error 405 is returned + if (!client.ip ~ purge) { + return(synth(405, "This IP is not allowed to send PURGE requests.")); + } + + ban("req.http.host == " + req.http.host); + # Throw a synthetic page so the request won't go to the backend. + return(synth(200, "Ban added")); + # If allowed, do a cache_lookup -> vlc_hit() or vlc_miss() + #return (purge); + } + + # Post requests will not be cached + if (req.http.Authorization || req.method == "POST") { + return (pass); + } + + # --- WordPress specific configuration + + # Did not cache the RSS feed + if (req.url ~ "/feed") { + return (pass); + } + + # Blitz hack + if (req.url ~ "/mu-.*") { + return (pass); + } + + # Did not cache the admin and login pages + if (req.url ~ "/wp-(login|admin)") { + return (pass); + } + + # Remove the "has_js" cookie + set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", ""); + + # Remove any Google Analytics based cookies + set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", ""); + + # Remove the Quant Capital cookies (added by some plugin, all __qca) + set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", ""); + + # Remove the wp-settings-1 cookie + set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-1=[^;]+(; )?", ""); + + # Remove the wp-settings-time-1 cookie + set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-time-1=[^;]+(; )?", ""); + + # Remove the wp test cookie + set req.http.Cookie = regsuball(req.http.Cookie, "wordpress_test_cookie=[^;]+(; )?", ""); + + # Are there cookies left with only spaces or that are empty? + if (req.http.cookie ~ "^ *$") { + unset req.http.cookie; + } + + # Cache the following files extensions + if (req.url ~ "\.(css|js|png|gif|jp(e)?g|swf|ico)") { + unset req.http.cookie; + } + + # Normalize Accept-Encoding header and compression + # https://www.varnish-cache.org/docs/3.0/tutorial/vary.html + if (req.http.Accept-Encoding) { + # Do no compress compressed files... + if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") { + unset req.http.Accept-Encoding; + } elsif (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } elsif (req.http.Accept-Encoding ~ "deflate") { + set req.http.Accept-Encoding = "deflate"; + } else { + unset req.http.Accept-Encoding; + } + } + + # Check the cookies for wordpress-specific items + if (req.http.Cookie ~ "wordpress_" || req.http.Cookie ~ "comment_") { + return (pass); + } + if (!req.http.cookie) { + unset req.http.cookie; + } + + # --- End of WordPress specific configuration + + # Do not cache HTTP authentication and HTTP Cookie + if (req.http.Authorization || req.http.Cookie) { + # Not cacheable by default + return (pass); + } + + # Cache all others requests + return (hash); +} + +sub vcl_pipe { + return (pipe); +} + +sub vcl_pass { + return (fetch); +} + +# The data on which the hashing will take place +sub vcl_hash { + hash_data(req.url); + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + + # If the client supports compression, keep that in a different cache + if (req.http.Accept-Encoding) { + hash_data(req.http.Accept-Encoding); + } + + return (lookup); +} + +# This function is used when a request is sent by our backend (Nginx server) +sub vcl_backend_response { + # Remove some headers we never want to see + unset beresp.http.Server; + unset beresp.http.X-Powered-By; + + # For static content strip all backend cookies + if (bereq.url ~ "\.(css|js|png|gif|jp(e?)g)|swf|ico") { + unset beresp.http.cookie; + } + + # Only allow cookies to be set if we're in admin area + if (beresp.http.Set-Cookie && bereq.url !~ "^/wp-(login|admin)") { + unset beresp.http.Set-Cookie; + } + + # don't cache response to posted requests or those with basic auth + if ( bereq.method == "POST" || bereq.http.Authorization ) { + set beresp.uncacheable = true; + set beresp.ttl = 120s; + return (deliver); + } + + # don't cache search results + if ( bereq.url ~ "\?s=" ){ + set beresp.uncacheable = true; + set beresp.ttl = 120s; + return (deliver); + } + + # only cache status ok + if ( beresp.status != 200 ) { + set beresp.uncacheable = true; + set beresp.ttl = 120s; + return (deliver); + } + + # A TTL of 24h + set beresp.ttl = 24h; + # Define the default grace period to serve cached content + set beresp.grace = 30s; + + return (deliver); +} + +# The routine when we deliver the HTTP request to the user +# Last chance to modify headers that are sent to the client +sub vcl_deliver { + if (obj.hits > 0) { + set resp.http.X-Cache = "cached"; + } else { + set resp.http.x-Cache = "uncached"; + } + + # Remove some headers: PHP version + unset resp.http.X-Powered-By; + + # Remove some headers: Apache version & OS + unset resp.http.Server; + + # Remove some heanders: Varnish + unset resp.http.Via; + unset resp.http.X-Varnish; + + return (deliver); +} + +sub vcl_init { + return (ok); +} + +sub vcl_fini { + return (ok); +} \ No newline at end of file diff --git a/varnish/start.sh b/varnish/start.sh index 7e7c28a5..8bdad94a 100644 --- a/varnish/start.sh +++ b/varnish/start.sh @@ -7,6 +7,14 @@ do sed -i "s|\${${name}}|${value}|g" /etc/varnish/default.vcl done +echo "exec varnishd \ + -a :$VARNISH_PORT \ + -T localhost:6082 \ + -F \ + -f $VARNISH_CONFIG \ + -s malloc,$CACHE_SIZE \ + $VARNISHD_PARAMS" + exec bash -c \ "exec varnishd \ -a :$VARNISH_PORT \