update config for crawler-whitelist, cleanup a few things in the example

INSTALLATION.md

@@ -12,15 +12,15 @@ NOTE: Use either HCAPTCHA_ or RECAPTHCA_, not both.
 - HMAC_COOKIE_SECRET - different random string, a salt for pow cookies
 - TOR_CONTROL_PORT_PASSWORD - the control port password for tor daemon
 - RAY_ID - string to identify the HAProxy node by
-- CHALLENGE_EXPIRY - how long solution cookies last for, in seconds
-- CHALLENGE_INCLUDES_IP - any value, whether to lock solved challenges to IP or tor circuit
 - BACKEND_NAME - Optional, name of backend to build from hosts.map
 - SERVER_PREFIX - Optional, prefix of server names used in server-template
-- ARGON_TIME - argon2 iterations
-- ARGON_KB - argon2 memory usage in KB
-- POW_DIFFICULTY - pow difficulty
+- VERIFY_BACKEND_SSL - whether to verify backend ssl, requires you have a private CA, install the cert on the proxies, and CA signed certs on your origins.
+- CHALLENGE_EXPIRY - how long solution cookies last for, in seconds
+- CHALLENGE_INCLUDES_IP - any value, whether to lock solved challenges to IP or tor circuit
+- ARGON_TIME - default argon2 iterations
+- ARGON_KB - default argon2 memory usage in KB
+- POW_DIFFICULTY - default pow difficulty
 - POW_TYPE - type of ahsh algorithm for pow "argon2" or "sha256"
-- VERIFY_BACKEND_SSL - whether to verify backend ssl, requires you have a private CA on the proxy and using it to sign your backend certs
 
 #### Run in docker (for testing/development)
 
@@ -37,7 +37,6 @@ Requires HAProxy compiled with lua support, and version >=2.5 for the native lua
 
 - Clone the repo somewhere. `/var/www/haproxy-protection` works.
 - Copy [haproxy.cfg](haproxy/haproxy.cfg) to `/etc/haproxy/haproxy.cfg`.
-  - Please note this configuration is very minimal, and is simply an example configuration for haproxy-protection. You are expected to customise it significantly or otherwise copy the relevant parts into your own haproxy config.
 - Copy/link [scripts](src/lua/scripts) to `/etc/haproxy/scripts`.
 - Copy/link [libs](src/lua/libs) to `/etc/haproxy/libs`.
 - Copy/link [template](haproxy/template) to `/etc/haproxy/template`.
@@ -51,6 +50,8 @@ sudo luarocks install argon2
 ```
 - Test your haproxy config, `sudo haproxy -c -V -f /etc/haproxy/haproxy.cfg`. You should see "Configuration file is valid".
 
+NOTE: the provided configuration is only an example. You are expected to customise it significantly or otherwise copy the relevant parts into your own haproxy config.
+
 If you have problems, read the error messages before opening an issue that is simply a bad configuration.
 
 ### Tor

docker-compose.yml

@@ -12,6 +12,7 @@ services:
     volumes:
       - ./haproxy/haproxy.cfg:/etc/haproxy/haproxy.cfg
       - ./haproxy/dataplaneapi.hcl:/etc/haproxy/dataplaneapi.hcl
+      - ./haproxy/errors/:/etc/haproxy/errors/
       - ./haproxy/map/:/etc/haproxy/map/
       - ./haproxy/template/:/etc/haproxy/template/
       - ./src/lua/scripts/:/etc/haproxy/scripts/
@@ -38,12 +39,12 @@ services:
       - POW_TYPE=argon2
       - TOR_CONTROL_PORT_PASSWORD=changeme
 
-  nginx:
-    ports:
-      - 81:80
-    image: "nginx:latest"
-    volumes:
-      - ./nginx:/usr/share/nginx/html
+  # nginx:
+    # ports:
+      # - 81:80
+    # image: "nginx:latest"
+    # volumes:
+      # - ./nginx:/usr/share/nginx/html
 
 #  tor:
 #    build:

haproxy/dataplaneapi.hcl

@@ -7,11 +7,9 @@ mode = "single"
 dataplaneapi {
   host       = "127.0.0.1"
   port       = 2001
+  advertised = {}
 
-  user "admin" {
-    insecure = true
-    password = "admin"
-  }
+  scheme = ["http"]
 
   transaction {
     transaction_dir = "/tmp/haproxy"
@@ -22,7 +20,10 @@ dataplaneapi {
     ssl_certs_dir = "/etc/haproxy/ssl"
   }
 
-  advertised {}
+  user "admin" {
+    insecure = true
+    password = "admin"
+  }
 }
 
 haproxy {

haproxy/haproxy.cfg

@@ -18,6 +18,14 @@ defaults
 	timeout client 50000ms
 	timeout server 50000ms
 	timeout tarpit 5000ms
+	http-error status 400 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/400.http
+	http-error status 403 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/403.http
+	http-error status 408 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/408.http
+	http-error status 429 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/429.http
+	http-error status 500 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/500.http
+	http-error status 502 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/502.http
+	http-error status 503 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/503.http
+	http-error status 504 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/504.http
 
 program api
 	command dataplaneapi -f /etc/haproxy/dataplaneapi.hcl --update-map-files
@@ -68,6 +76,7 @@ frontend http-in
 
 	# acl for lua check whitelisted IPs/subnets and some excluded paths
 	acl is_excluded src,map_ip(/etc/haproxy/map/whitelist.map) -m found
+	acl is_excluded src -m found -f /etc/haproxy/map/crawler-whitelist.map
 	acl is_excluded path /favicon.ico /.basedflare/pow-icon #add more
 
 	# acl ORs for when ddos_mode_enabled
@@ -83,7 +92,7 @@ frontend http-in
 
 	# acl for domains in maintenance mode to return maintenance page (after challenge page htp-request return rules, for the footerlogo)
 	acl maintenance_mode hdr(host),lower,map_str(/etc/haproxy/map/maintenance.map) -m found
-	http-request lua.set-lang-json if maintenance_mode
+	http-request lua.set-lang-json
 	http-request return lf-file /etc/haproxy/template/maintenance.html status 200 content-type "text/html; charset=utf-8" hdr "Cache-Control" "private, max-age=30" if maintenance_mode
 
 	# rewrite specific domain+path to domain or domain+path
@@ -119,23 +128,17 @@ frontend http-in
 	# optional alt-svc header (done after cache so not set in cached responses
 	http-response set-header Alt-Svc %[var(txn.xcn),map(/etc/haproxy/map/alt-svc.map)]
 
-	acl c0 res.hdr(Cache-Control,0) -m sub max-age=0
-	acl c0 res.hdr(Cache-Control,1) -m sub max-age=0
-	acl c0 res.hdr(Cache-Control,2) -m sub max-age=0
-	acl cf0 res.fhdr(Cache-Control,0) -m sub max-age=0
-	acl cf0 res.fhdr(Cache-Control,1) -m sub max-age=0
-	acl cf0 res.fhdr(Cache-Control,2) -m sub max-age=0
-	http-response set-header X-c0 true if c0
-	http-response set-header X-cf0 true if cf0
-	http-response set-header X-res-hdr0-Cache-Control %[res.hdr(Cache-Control,0)]
-	http-response set-header X-res-hdr1-Cache-Control %[res.hdr(Cache-Control,1)]
-	http-response set-header X-res-hdr2-Cache-Control %[res.hdr(Cache-Control,2)]
-	http-response set-header X-res-fhdr0-Cache-Control %[res.fhdr(Cache-Control,0)]
-	http-response set-header X-res-fhdr1-Cache-Control %[res.fhdr(Cache-Control,1)]
-	http-response set-header X-res-fhdr2-Cache-Control %[res.fhdr(Cache-Control,2)]
+	# header checks for no caching
+	# acl auth_cookie_set res.hdr(Set-Cookie),lower -m found
+	# acl cache_control_max_age_0 res.fhdr(Cache-Control,0) -m sub "max-age=0"
+	# acl cache_control_max_age_0 res.fhdr(Cache-Control,1) -m sub "max-age=0"
+	# acl cache_control_max_age_0 res.fhdr(Cache-Control,2) -m sub "max-age=0"
+
+	# basic caching
+	# http-response set-header Cache-Control no-cache if auth_cookie_set
+	# http-response cache-store basic_cache if !auth_cookie_set !cache_control_max_age_0
+	# http-request cache-use basic_cache
 	
-	http-request cache-use basic_cache
-	http-response cache-store basic_cache
 	default_backend servers
 
 cache basic_cache

haproxy/map/crawler-whitelist.map

@@ -0,0 +1,10 @@
+2001:4860:4801:10::/64
+2001:4860:4801:11::/64
+2001:4860:4801:12::/64
+2001:4860:4801:13::/64
+2001:4860:4801:14::/64
+2001:4860:4801:15::/64
+2001:4860:4801:16::/64
+2001:4860:4801:17::/64
+2001:4860:4801:18::/64
+2001:4860:4801:19::/64

haproxy/map/ddos.map

@@ -1,3 +1,2 @@
-localhost 1
 127.0.0.1 1
 127.0.0.1/captcha 2

haproxy/map/ddos_config.map

@@ -1 +1 @@
-localhost {"pd":23,"pt":"sha256","cip":true,"cex":600}
+127.0.0.1 {"pd":"argon2","pt":23,"cip":false,"cex":43200}

haproxy/map/hosts.map

@@ -1,2 +1 @@
 127.0.0.1 127.0.0.1:81
-localhost 127.0.0.1:8200