From 10c875e689e7eabffab479d8e64a8ede422e3e2d Mon Sep 17 00:00:00 2001 From: Thomas Lynch Date: Sun, 4 Jun 2023 13:04:13 +1000 Subject: [PATCH] update config for crawler-whitelist, cleanup a few things in the example --- INSTALLATION.md | 15 +++++++------ docker-compose.yml | 13 ++++++----- haproxy/dataplaneapi.hcl | 15 +++++++------ haproxy/haproxy.cfg | 37 +++++++++++++++++-------------- haproxy/map/crawler-whitelist.map | 10 +++++++++ haproxy/map/ddos.map | 1 - haproxy/map/ddos_config.map | 2 +- haproxy/map/hosts.map | 1 - 8 files changed, 54 insertions(+), 40 deletions(-) create mode 100644 haproxy/map/crawler-whitelist.map diff --git a/INSTALLATION.md b/INSTALLATION.md index 75f4a3c..2a5d40c 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -12,15 +12,15 @@ NOTE: Use either HCAPTCHA_ or RECAPTHCA_, not both. - HMAC_COOKIE_SECRET - different random string, a salt for pow cookies - TOR_CONTROL_PORT_PASSWORD - the control port password for tor daemon - RAY_ID - string to identify the HAProxy node by -- CHALLENGE_EXPIRY - how long solution cookies last for, in seconds -- CHALLENGE_INCLUDES_IP - any value, whether to lock solved challenges to IP or tor circuit - BACKEND_NAME - Optional, name of backend to build from hosts.map - SERVER_PREFIX - Optional, prefix of server names used in server-template -- ARGON_TIME - argon2 iterations -- ARGON_KB - argon2 memory usage in KB -- POW_DIFFICULTY - pow difficulty +- VERIFY_BACKEND_SSL - whether to verify backend ssl, requires you have a private CA, install the cert on the proxies, and CA signed certs on your origins. +- CHALLENGE_EXPIRY - how long solution cookies last for, in seconds +- CHALLENGE_INCLUDES_IP - any value, whether to lock solved challenges to IP or tor circuit +- ARGON_TIME - default argon2 iterations +- ARGON_KB - default argon2 memory usage in KB +- POW_DIFFICULTY - default pow difficulty - POW_TYPE - type of ahsh algorithm for pow "argon2" or "sha256" -- VERIFY_BACKEND_SSL - whether to verify backend ssl, requires you have a private CA on the proxy and using it to sign your backend certs #### Run in docker (for testing/development) @@ -37,7 +37,6 @@ Requires HAProxy compiled with lua support, and version >=2.5 for the native lua - Clone the repo somewhere. `/var/www/haproxy-protection` works. - Copy [haproxy.cfg](haproxy/haproxy.cfg) to `/etc/haproxy/haproxy.cfg`. - - Please note this configuration is very minimal, and is simply an example configuration for haproxy-protection. You are expected to customise it significantly or otherwise copy the relevant parts into your own haproxy config. - Copy/link [scripts](src/lua/scripts) to `/etc/haproxy/scripts`. - Copy/link [libs](src/lua/libs) to `/etc/haproxy/libs`. - Copy/link [template](haproxy/template) to `/etc/haproxy/template`. @@ -51,6 +50,8 @@ sudo luarocks install argon2 ``` - Test your haproxy config, `sudo haproxy -c -V -f /etc/haproxy/haproxy.cfg`. You should see "Configuration file is valid". +NOTE: the provided configuration is only an example. You are expected to customise it significantly or otherwise copy the relevant parts into your own haproxy config. + If you have problems, read the error messages before opening an issue that is simply a bad configuration. ### Tor diff --git a/docker-compose.yml b/docker-compose.yml index 37329bd..c893c44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: volumes: - ./haproxy/haproxy.cfg:/etc/haproxy/haproxy.cfg - ./haproxy/dataplaneapi.hcl:/etc/haproxy/dataplaneapi.hcl + - ./haproxy/errors/:/etc/haproxy/errors/ - ./haproxy/map/:/etc/haproxy/map/ - ./haproxy/template/:/etc/haproxy/template/ - ./src/lua/scripts/:/etc/haproxy/scripts/ @@ -38,12 +39,12 @@ services: - POW_TYPE=argon2 - TOR_CONTROL_PORT_PASSWORD=changeme - nginx: - ports: - - 81:80 - image: "nginx:latest" - volumes: - - ./nginx:/usr/share/nginx/html + # nginx: + # ports: + # - 81:80 + # image: "nginx:latest" + # volumes: + # - ./nginx:/usr/share/nginx/html # tor: # build: diff --git a/haproxy/dataplaneapi.hcl b/haproxy/dataplaneapi.hcl index 7d61788..9407ebf 100644 --- a/haproxy/dataplaneapi.hcl +++ b/haproxy/dataplaneapi.hcl @@ -5,13 +5,11 @@ name = "basedflare" mode = "single" dataplaneapi { - host = "127.0.0.1" - port = 2001 + host = "127.0.0.1" + port = 2001 + advertised = {} - user "admin" { - insecure = true - password = "admin" - } + scheme = ["http"] transaction { transaction_dir = "/tmp/haproxy" @@ -22,7 +20,10 @@ dataplaneapi { ssl_certs_dir = "/etc/haproxy/ssl" } - advertised {} + user "admin" { + insecure = true + password = "admin" + } } haproxy { diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg index 70af7bd..db560b5 100644 --- a/haproxy/haproxy.cfg +++ b/haproxy/haproxy.cfg @@ -18,6 +18,14 @@ defaults timeout client 50000ms timeout server 50000ms timeout tarpit 5000ms + http-error status 400 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/400.http + http-error status 403 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/403.http + http-error status 408 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/408.http + http-error status 429 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/429.http + http-error status 500 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/500.http + http-error status 502 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/502.http + http-error status 503 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/503.http + http-error status 504 content-type "text/html; charset=utf-8" lf-file /etc/haproxy/errors/504.http program api command dataplaneapi -f /etc/haproxy/dataplaneapi.hcl --update-map-files @@ -68,6 +76,7 @@ frontend http-in # acl for lua check whitelisted IPs/subnets and some excluded paths acl is_excluded src,map_ip(/etc/haproxy/map/whitelist.map) -m found + acl is_excluded src -m found -f /etc/haproxy/map/crawler-whitelist.map acl is_excluded path /favicon.ico /.basedflare/pow-icon #add more # acl ORs for when ddos_mode_enabled @@ -83,7 +92,7 @@ frontend http-in # acl for domains in maintenance mode to return maintenance page (after challenge page htp-request return rules, for the footerlogo) acl maintenance_mode hdr(host),lower,map_str(/etc/haproxy/map/maintenance.map) -m found - http-request lua.set-lang-json if maintenance_mode + http-request lua.set-lang-json http-request return lf-file /etc/haproxy/template/maintenance.html status 200 content-type "text/html; charset=utf-8" hdr "Cache-Control" "private, max-age=30" if maintenance_mode # rewrite specific domain+path to domain or domain+path @@ -119,23 +128,17 @@ frontend http-in # optional alt-svc header (done after cache so not set in cached responses http-response set-header Alt-Svc %[var(txn.xcn),map(/etc/haproxy/map/alt-svc.map)] - acl c0 res.hdr(Cache-Control,0) -m sub max-age=0 - acl c0 res.hdr(Cache-Control,1) -m sub max-age=0 - acl c0 res.hdr(Cache-Control,2) -m sub max-age=0 - acl cf0 res.fhdr(Cache-Control,0) -m sub max-age=0 - acl cf0 res.fhdr(Cache-Control,1) -m sub max-age=0 - acl cf0 res.fhdr(Cache-Control,2) -m sub max-age=0 - http-response set-header X-c0 true if c0 - http-response set-header X-cf0 true if cf0 - http-response set-header X-res-hdr0-Cache-Control %[res.hdr(Cache-Control,0)] - http-response set-header X-res-hdr1-Cache-Control %[res.hdr(Cache-Control,1)] - http-response set-header X-res-hdr2-Cache-Control %[res.hdr(Cache-Control,2)] - http-response set-header X-res-fhdr0-Cache-Control %[res.fhdr(Cache-Control,0)] - http-response set-header X-res-fhdr1-Cache-Control %[res.fhdr(Cache-Control,1)] - http-response set-header X-res-fhdr2-Cache-Control %[res.fhdr(Cache-Control,2)] + # header checks for no caching + # acl auth_cookie_set res.hdr(Set-Cookie),lower -m found + # acl cache_control_max_age_0 res.fhdr(Cache-Control,0) -m sub "max-age=0" + # acl cache_control_max_age_0 res.fhdr(Cache-Control,1) -m sub "max-age=0" + # acl cache_control_max_age_0 res.fhdr(Cache-Control,2) -m sub "max-age=0" + + # basic caching + # http-response set-header Cache-Control no-cache if auth_cookie_set + # http-response cache-store basic_cache if !auth_cookie_set !cache_control_max_age_0 + # http-request cache-use basic_cache - http-request cache-use basic_cache - http-response cache-store basic_cache default_backend servers cache basic_cache diff --git a/haproxy/map/crawler-whitelist.map b/haproxy/map/crawler-whitelist.map new file mode 100644 index 0000000..8db745d --- /dev/null +++ b/haproxy/map/crawler-whitelist.map @@ -0,0 +1,10 @@ +2001:4860:4801:10::/64 +2001:4860:4801:11::/64 +2001:4860:4801:12::/64 +2001:4860:4801:13::/64 +2001:4860:4801:14::/64 +2001:4860:4801:15::/64 +2001:4860:4801:16::/64 +2001:4860:4801:17::/64 +2001:4860:4801:18::/64 +2001:4860:4801:19::/64 diff --git a/haproxy/map/ddos.map b/haproxy/map/ddos.map index a1ea520..7033bab 100644 --- a/haproxy/map/ddos.map +++ b/haproxy/map/ddos.map @@ -1,3 +1,2 @@ -localhost 1 127.0.0.1 1 127.0.0.1/captcha 2 diff --git a/haproxy/map/ddos_config.map b/haproxy/map/ddos_config.map index 54eaac6..86331df 100644 --- a/haproxy/map/ddos_config.map +++ b/haproxy/map/ddos_config.map @@ -1 +1 @@ -localhost {"pd":23,"pt":"sha256","cip":true,"cex":600} +127.0.0.1 {"pd":"argon2","pt":23,"cip":false,"cex":43200} diff --git a/haproxy/map/hosts.map b/haproxy/map/hosts.map index 3648f14..746aa2e 100644 --- a/haproxy/map/hosts.map +++ b/haproxy/map/hosts.map @@ -1,2 +1 @@ 127.0.0.1 127.0.0.1:81 -localhost 127.0.0.1:8200