Retry PROXY e2e reads while the tunnel handshake completes

The first connection through the tunnel can exceed the read deadline on a slow CI runner.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.dockerignore

@@ -0,0 +1,3 @@
+# The image only packages the binaries that deploy/build.sh already built.
+*
+!dist/bin

.gitea/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      GOPRIVATE: git.veillax.com
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      # Go fetches the other sharks modules straight from this server. This is harmless
+      # for public repositories and is what makes private ones work.
+      - name: Let Go fetch modules from this server
+        run: git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@git.veillax.com/".insteadOf "https://git.veillax.com/"
+
+      - name: Formatting
+        run: |
+          unformatted=$(gofmt -l .)
+          if [ -n "$unformatted" ]; then
+            echo "These files need gofmt:"
+            echo "$unformatted"
+            exit 1
+          fi
+
+      - name: Shell scripts parse
+        run: |
+          for f in deploy/*.sh; do sh -n "$f"; done
+
+      - name: Vet
+        run: go vet ./...
+
+      - name: Test
+        run: go test ./...
+
+      # Catches platform-specific breakage before a release is tagged.
+      - name: Every release platform compiles
+        run: |
+          for t in linux/amd64 linux/arm64 linux/arm windows/amd64 darwin/amd64 darwin/arm64; do
+            echo "building for $t"
+            GOOS=${t%/*} GOARCH=${t#*/} GOARM=7 CGO_ENABLED=0 go build -o /dev/null ./cmd/hammerhead
+          done

.gitea/workflows/release.yml

@@ -0,0 +1,62 @@
+name: Release
+
+# Push a tag such as v0.2.0 to build and publish a release:
+#   git tag -a v0.2.0 -m "hammerhead v0.2.0" && git push origin v0.2.0
+on:
+  push:
+    tags: ['v*']
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    env:
+      GOPRIVATE: git.veillax.com
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # release notes come from the history since the previous tag
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Let Go fetch modules from this server
+        run: git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@git.veillax.com/".insteadOf "https://git.veillax.com/"
+
+      - name: Test
+        run: |
+          go vet ./...
+          go test ./...
+
+      - name: Build release archives
+        run: sh deploy/build.sh "${GITHUB_REF_NAME}"
+
+      # Optional secret RELEASE_TOKEN: an access token that can write releases, for
+      # servers where the automatic job token can't.
+      - name: Publish the release
+        env:
+          TOKEN: ${{ secrets.RELEASE_TOKEN || secrets.GITHUB_TOKEN }}
+        run: sh deploy/publish-release.sh "${GITHUB_REF_NAME}"
+
+      # Skipped when the repository variable PUBLISH_DOCKER is "false", or fails alone
+      # if this runner has no Docker: the release above is already published by then.
+      # Optional secret REGISTRY_TOKEN (and variable REGISTRY_USER): an access token
+      # with package write permission, for servers where the job token can't push.
+      - name: Build and push the container image
+        if: ${{ vars.PUBLISH_DOCKER != 'false' }}
+        env:
+          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN || secrets.GITHUB_TOKEN }}
+          REGISTRY_USER: ${{ vars.REGISTRY_USER || github.actor }}
+        run: |
+          host=${GITHUB_SERVER_URL#*://}
+          owner=$(echo "$GITHUB_REPOSITORY_OWNER" | tr 'A-Z' 'a-z')
+          image="$host/$owner/hammerhead"
+          echo "$REGISTRY_TOKEN" | docker login "$host" -u "$REGISTRY_USER" --password-stdin
+          # a container-driver builder can build several architectures in one go
+          docker buildx create --name release --driver docker-container --use >/dev/null 2>&1 || docker buildx use release
+          tags="-t $image:$GITHUB_REF_NAME"
+          case "$GITHUB_REF_NAME" in
+            *-*) ;;                       # pre-releases (v1.0.0-rc1) don't move :latest
+            *) tags="$tags -t $image:latest" ;;
+          esac
+          docker buildx build --platform linux/amd64,linux/arm64 $tags --push .

.gitignore

@@ -5,3 +5,4 @@ bin/
 *.db-wal
 *.db-shm
 master.key
+dist/

Dockerfile

@@ -0,0 +1,22 @@
+# syntax=docker/dockerfile:1
+# Packages the binaries deploy/build.sh already built (dist/bin/linux_<arch>/hammerhead),
+# so the arm64 image needs no emulation. Build it with:  sh deploy/build.sh && docker buildx build .
+FROM --platform=$BUILDPLATFORM alpine:3.22 AS prep
+RUN apk add --no-cache ca-certificates \
+ && mkdir /data && chown 65532:65532 /data
+
+FROM scratch
+ARG TARGETARCH
+LABEL org.opencontainers.image.title="Hammerhead" \
+      org.opencontainers.image.description="Control plane for tunnelled ports and websites" \
+      org.opencontainers.image.source="https://git.veillax.com/Veillax/hammerhead" \
+      org.opencontainers.image.licenses="AGPL-3.0-only"
+COPY --from=prep /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=prep --chown=65532:65532 /data /data
+COPY dist/bin/linux_${TARGETARCH}/hammerhead /hammerhead
+USER 65532:65532
+ENV HAMMERHEAD_DATA=/data
+VOLUME ["/data"]
+EXPOSE 8080
+ENTRYPOINT ["/hammerhead"]
+CMD ["-listen", ":8080"]

README.md

@@ -39,6 +39,52 @@ bamboo -hammerhead https://hammerhead.example.com -token <site token> -docker
 Open the edge's WireGuard UDP port (and any ports you open) in its firewall. Bamboo needs no inbound
 ports. Then open ports on the **Resources** page.
 
+## Deploying
+
+Releases are built by Gitea Actions. Pushing a tag such as `v0.2.0` runs `.gitea/workflows/release.yml`, which tests the code, builds Hammerhead for Linux (amd64, arm64, armv7), Windows (amd64) and macOS (amd64, arm64), publishes the archives with a `SHA256SUMS` file on the Releases page, and pushes a multi-architecture container image to this server's registry (`git.veillax.com/veillax/hammerhead`). Every push and pull request runs `.gitea/workflows/ci.yml`: formatting, vet, tests, and a check that every release platform still compiles.
+
+**Fastest: one script on a fresh VPS**
+
+`deploy/setup.sh` (also inside the Linux archives) installs everything for you on a systemd Linux: the service, a generated master key, the firewall rules, and optionally HTTPS.
+
+```
+sudo sh setup.sh --domain hammerhead.example.com            # Hammerhead + Caddy for HTTPS
+sudo sh setup.sh edge  --hammerhead https://hammerhead.example.com --token <edge token> --ports 25565,30000-30100
+sudo sh setup.sh agent --hammerhead https://hammerhead.example.com --token <site token>
+```
+
+`edge` installs Blacktip (with its bundled Traefik) and lists every IP address on the machine that ports can be opened on; `agent` installs Bamboo on the machine that hosts your services. All three can run on one machine or on three.
+
+**On a Linux server (systemd)**
+
+```
+curl -LO https://git.veillax.com/Veillax/hammerhead/releases/download/v0.1.0/hammerhead_v0.1.0_linux_amd64.tar.gz
+curl -LO https://git.veillax.com/Veillax/hammerhead/releases/download/v0.1.0/SHA256SUMS
+sha256sum -c --ignore-missing SHA256SUMS
+tar xzf hammerhead_v0.1.0_linux_amd64.tar.gz && cd hammerhead_v0.1.0_linux_amd64
+sudo ./install.sh                 # installs the binary and a hardened systemd service
+sudoedit /etc/hammerhead/hammerhead.env   # your settings and secrets
+sudo systemctl start hammerhead
+```
+
+Use `linux_arm64` or `linux_arm` for Raspberry Pi and other ARM machines. To upgrade, repeat the same steps: your settings and data are kept. `sudo ./install.sh --uninstall` removes the service and binary but leaves `/etc/hammerhead` and `/var/lib/hammerhead` alone. `hammerhead -version` tells you which build is running.
+
+**With Docker**
+
+```
+cp .env.example .env              # fill it in
+docker compose -f deploy/docker-compose.yml up -d
+```
+
+The image is `git.veillax.com/veillax/hammerhead:<version>` (and `:latest` for full releases). The compose file is in the repository and inside each Linux archive.
+
+**Setting up the build (once)**
+
+- A Gitea Actions runner with the label `ubuntu-latest`, and Docker on it if you want the container image.
+- Nothing else is required: the workflows use the automatic job token. If your server doesn't let that token publish releases or packages, add the secrets `RELEASE_TOKEN` and `REGISTRY_TOKEN` (access tokens with release and package write permission) and optionally the variable `REGISTRY_USER`.
+- To skip the container image, set the repository variable `PUBLISH_DOCKER` to `false`.
+- Build locally with `sh deploy/build.sh` (archives in `dist/`).
+
 ## Configuration and secrets
 
 All three programs read a `.env` file from their working directory before they start (or the file

cmd/hammerhead/main.go

@@ -15,6 +15,9 @@ import (
 	"git.veillax.com/Veillax/reef/dotenv"
 )
 
+// version is set at build time (deploy/build.sh).
+var version = "dev"
+
 func env(k, def string) string {
 	if v := os.Getenv(k); v != "" {
 		return v
@@ -36,7 +39,12 @@ func main() {
 	url := flag.String("url", env("HAMMERHEAD_URL", ""), "public URL people reach hammerhead at (used for login redirects and SSO); can also be set in Settings")
 	flag.String("env", ".env", "file to read secrets and settings from")
 	genkey := flag.Bool("genkey", false, "print a new random HAMMERHEAD_MASTER_KEY and exit")
+	showVersion := flag.Bool("version", false, "print the version and exit")
 	flag.Parse()
+	if *showVersion {
+		os.Stdout.WriteString("hammerhead " + version + "\n")
+		return
+	}
 	if *genkey {
 		fmt.Println(secretbox.GenerateKey())
 		return
@@ -55,7 +63,7 @@ func main() {
 		log.Fatal(err)
 	}
 	defer srv.Close()
-	log.Printf("hammerhead listening on %s (open it in a browser to finish setup)", *listen)
+	log.Printf("hammerhead %s listening on %s (open it in a browser to finish setup)", version, *listen)
 	hs := &http.Server{Addr: *listen, Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second}
 	log.Fatal(hs.ListenAndServe())
 }

deploy/build.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+# Builds hammerhead for every release platform and packs the archives into dist/.
+#
+#   deploy/build.sh [version]      version defaults to `git describe`
+#
+# Output:
+#   dist/bin/<os>_<arch>/hammerhead             the raw binaries (the Docker image uses the linux ones)
+#   dist/hammerhead_<version>_<os>_<arch>.tar.gz or .zip
+#   dist/SHA256SUMS
+#
+# Set TARGETS="linux/amd64 linux/arm64" to build fewer platforms.
+set -eu
+NAME=hammerhead
+cd "$(dirname "$0")/.."
+VERSION=${1:-$(git describe --tags --always 2>/dev/null || echo dev)}
+TARGETS=${TARGETS:-"linux/amd64 linux/arm64 linux/arm windows/amd64 darwin/amd64 darwin/arm64"}
+
+rm -rf dist
+mkdir -p dist/bin dist/stage
+for t in $TARGETS; do
+	os=${t%/*}
+	arch=${t#*/}
+	ext=""
+	[ "$os" = windows ] && ext=".exe"
+	out="dist/bin/${os}_${arch}"
+	mkdir -p "$out"
+	echo "building $NAME $VERSION for $os/$arch"
+	CGO_ENABLED=0 GOOS=$os GOARCH=$arch GOARM=7 \
+		go build -trimpath -ldflags "-s -w -X main.version=$VERSION" -o "$out/$NAME$ext" "./cmd/$NAME"
+
+	pkg="${NAME}_${VERSION}_${os}_${arch}"
+	stage="dist/stage/$pkg"
+	mkdir -p "$stage"
+	cp "$out/$NAME$ext" "$stage/"
+	for f in LICENSE README.md .env.example; do
+		[ -f "$f" ] && cp "$f" "$stage/"
+	done
+	if [ "$os" = linux ]; then
+		cp deploy/install.sh deploy/setup.sh "deploy/$NAME.service" deploy/docker-compose.yml "$stage/"
+		chmod +x "$stage/install.sh"
+	fi
+	if [ "$os" = windows ] && command -v zip >/dev/null 2>&1; then
+		(cd dist/stage && zip -qr "../$pkg.zip" "$pkg")
+	else
+		tar -C dist/stage -czf "dist/$pkg.tar.gz" "$pkg"
+	fi
+done
+rm -rf dist/stage
+(cd dist && sha256sum $(ls | grep -E '\.(tar\.gz|zip)$') > SHA256SUMS)
+echo "done: $(ls dist | grep -cE '\.(tar\.gz|zip)$') archives in dist/"

deploy/docker-compose.yml

@@ -0,0 +1,18 @@
+# Run Hammerhead with Docker:
+#   cp .env.example .env     (then fill it in: at least HAMMERHEAD_MASTER_KEY, see `hammerhead -genkey`)
+#   docker compose up -d
+#
+# Put it behind HTTPS (a reverse proxy, or Blacktip once it's running) and set
+# HAMMERHEAD_URL to the public address in .env.
+services:
+  hammerhead:
+    image: git.veillax.com/veillax/hammerhead:latest
+    restart: unless-stopped
+    env_file: .env
+    ports:
+      - "8080:8080"
+    volumes:
+      - hammerhead-data:/data
+
+volumes:
+  hammerhead-data:

deploy/hammerhead.service

@@ -0,0 +1,26 @@
+[Unit]
+Description=Hammerhead: control plane for tunnelled ports and websites
+Documentation=https://git.veillax.com/Veillax/hammerhead
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+User=hammerhead
+Group=hammerhead
+EnvironmentFile=-/etc/hammerhead/hammerhead.env
+ExecStart=/usr/local/bin/hammerhead -listen :8080 -data /var/lib/hammerhead
+WorkingDirectory=/var/lib/hammerhead
+Restart=on-failure
+RestartSec=3
+
+# It only needs its own data folder and the network.
+NoNewPrivileges=true
+ProtectSystem=strict
+ReadWritePaths=/var/lib/hammerhead
+ProtectHome=true
+PrivateTmp=true
+ProtectKernelTunables=true
+ProtectControlGroups=true
+
+[Install]
+WantedBy=multi-user.target

deploy/install.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+# Installs hammerhead as a systemd service from the files in this folder.
+#
+#   sudo ./install.sh              install or upgrade (your settings and data are kept)
+#   sudo ./install.sh --start      the same, then start (or restart) the service
+#   sudo ./install.sh --uninstall  remove the service and binary (keeps the settings and data)
+#
+# Files: /usr/local/bin/hammerhead, /etc/systemd/system/hammerhead.service,
+#        /etc/hammerhead/hammerhead.env (your settings and secrets), /var/lib/hammerhead (data).
+# Set DESTDIR=/some/dir to stage everything there without touching the system.
+set -eu
+NAME=hammerhead
+HERE=$(cd "$(dirname "$0")" && pwd)
+ROOT=${DESTDIR:-}
+BIN=$ROOT/usr/local/bin/$NAME
+UNIT=$ROOT/etc/systemd/system/$NAME.service
+CONF=$ROOT/etc/$NAME
+ENVFILE=$CONF/$NAME.env
+DATA=$ROOT/var/lib/$NAME
+
+say() { printf '%s\n' "$*"; }
+real() { [ -z "${DESTDIR:-}" ]; } # are we touching the real system?
+
+if real; then
+	[ "$(id -u)" = 0 ] || { say "Run this as root:  sudo $0 $*"; exit 1; }
+	command -v systemctl > /dev/null 2>&1 || { say "systemd wasn't found. Run $NAME directly (see the README) or use Docker."; exit 1; }
+fi
+
+if [ "${1:-}" = "--uninstall" ]; then
+	if real; then
+		systemctl disable --now "$NAME" > /dev/null 2>&1 || true
+	fi
+	rm -f "$UNIT" "$BIN"
+	if real; then
+		systemctl daemon-reload || true
+	fi
+	say "Removed $NAME. Left in place: $CONF and $DATA (delete them yourself if you want them gone)."
+	exit 0
+fi
+
+[ -f "$HERE/$NAME" ] || { say "Can't find the $NAME binary next to this script."; exit 1; }
+
+if real && ! id "$NAME" > /dev/null 2>&1; then
+	useradd --system --home-dir "/var/lib/$NAME" --shell /usr/sbin/nologin "$NAME"
+	say "Created the system user '$NAME'."
+fi
+
+mkdir -p "$ROOT/usr/local/bin" "$ROOT/etc/systemd/system" "$CONF" "$DATA"
+install -m 0755 "$HERE/$NAME" "$BIN.new"
+mv -f "$BIN.new" "$BIN" # atomic, so upgrading a running service is safe
+install -m 0644 "$HERE/$NAME.service" "$UNIT"
+if [ ! -f "$ENVFILE" ]; then
+	install -m 0600 "$HERE/.env.example" "$ENVFILE"
+	say "Created $ENVFILE. Edit it before starting."
+fi
+if real; then
+	chown -R "$NAME:$NAME" "$DATA"
+fi
+
+if real; then
+	systemctl daemon-reload
+	systemctl enable "$NAME" > /dev/null 2>&1
+	if [ "${1:-}" = "--start" ] || systemctl is-active --quiet "$NAME"; then
+		systemctl restart "$NAME"
+		say "$NAME is running."
+	fi
+fi
+
+say "Installed $("$BIN" -version 2> /dev/null || echo "$NAME")."
+say "Next:"
+say "  1. edit $ENVFILE"
+say "  2. systemctl start $NAME      (or run this again with --start)"
+say "  3. journalctl -u $NAME -f     to watch it"

deploy/publish-release.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+# Creates (or updates) the Gitea release for a tag and uploads what deploy/build.sh made.
+#
+#   TOKEN=<access token that can write releases> deploy/publish-release.sh v0.2.0
+#
+# In Gitea Actions GITHUB_SERVER_URL and GITHUB_REPOSITORY are already set. Safe to
+# re-run: an existing release is reused and assets with the same name are replaced.
+set -eu
+TAG=${1:?usage: publish-release.sh <tag>}
+: "${TOKEN:?set TOKEN to a Gitea access token that can write releases}"
+SERVER=${GITHUB_SERVER_URL:-${GITEA_SERVER_URL:-https://git.veillax.com}}
+REPO=${GITHUB_REPOSITORY:-Veillax/hammerhead}
+API="$SERVER/api/v1/repos/$REPO"
+AUTH="Authorization: token $TOKEN"
+
+# a JSON helper without needing jq: any python will do
+PY=python3
+python3 -c 'pass' > /dev/null 2>&1 || PY=python
+json() { "$PY" -c "$1"; }
+
+# release notes: the commit subjects since the previous tag
+prev=$(git describe --tags --abbrev=0 "$TAG^" 2>/dev/null || true)
+if [ -n "$prev" ]; then
+	notes=$(git log --pretty='format:- %s' "$prev..$TAG")
+else
+	notes=$(git log --pretty='format:- %s' "$TAG")
+fi
+prerelease=false
+case "$TAG" in *-*) prerelease=true ;; esac
+
+body=$(NOTES="$notes" TAG="$TAG" PRE="$prerelease" json '
+import json, os
+print(json.dumps({"tag_name": os.environ["TAG"], "name": os.environ["TAG"], "body": os.environ["NOTES"],
+                  "draft": False, "prerelease": os.environ["PRE"] == "true"}))')
+
+# reuse the release when a failed job is re-run
+id=$(curl -fsS -H "$AUTH" "$API/releases/tags/$TAG" 2>/dev/null | json 'import sys, json; print(json.load(sys.stdin)["id"])' 2>/dev/null || true)
+if [ -z "$id" ]; then
+	echo "creating release $TAG"
+	id=$(curl -fsS -X POST -H "$AUTH" -H 'Content-Type: application/json' -d "$body" "$API/releases" | json 'import sys, json; print(json.load(sys.stdin)["id"])')
+else
+	echo "release $TAG already exists (id $id): replacing its files"
+fi
+
+existing=$(curl -fsS -H "$AUTH" "$API/releases/$id/assets" | json 'import sys, json
+for a in json.load(sys.stdin):
+    print(a["id"], a["name"])')
+
+for f in dist/*.tar.gz dist/*.zip dist/SHA256SUMS; do
+	[ -f "$f" ] || continue
+	name=$(basename "$f")
+	echo "$existing" | while read -r aid aname; do
+		if [ "$aname" = "$name" ]; then
+			curl -fsS -X DELETE -H "$AUTH" "$API/releases/$id/assets/$aid" > /dev/null
+		fi
+	done
+	echo "uploading $name"
+	curl -fsS -X POST -H "$AUTH" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" > /dev/null
+done
+echo "published $SERVER/$REPO/releases/tag/$TAG"

deploy/setup.sh

@@ -0,0 +1,215 @@
+#!/bin/sh
+# One-shot setup for a VPS. Run as root.
+#
+#   sh setup.sh [--domain hh.example.com] [--version v0.1.0]     Hammerhead (dashboard and API)
+#   sh setup.sh edge  --hammerhead URL --token TOKEN [--ports 25565,30000-30100] [--version V]
+#   sh setup.sh agent --hammerhead URL --token TOKEN [--version V]
+#
+# Hammerhead: installs it as a systemd service (with a generated master key, kept in
+#   /etc/hammerhead/hammerhead.env), and with --domain also Caddy, which gets and renews
+#   the HTTPS certificate for that name. DNS for the name must already point here.
+# edge:  installs Blacktip, with its bundled Traefik, on this machine (or any other VPS),
+#   opens the firewall and lists every IP address it can open ports on.
+# agent: installs Bamboo, the site agent, on the machine that hosts your services.
+#
+# Edge and agent need a token, which only exists once Hammerhead is up: create the edge under
+# Server -> Edges (or the site under Sites) in the dashboard and copy the values from "Show command".
+# Safe to run again: it upgrades and keeps your settings and data.
+set -eu
+
+BASE=${SHARKS_BASE:-https://git.veillax.com/Veillax}
+MODE=hammerhead
+case "${1:-}" in edge | agent) MODE=$1; shift ;; esac
+DOMAIN= VERSION= HH= TOKEN= PORTS=
+while [ $# -gt 0 ]; do
+	case "$1" in
+		--domain) DOMAIN=$2; shift 2 ;;
+		--version) VERSION=$2; shift 2 ;;
+		--hammerhead) HH=$2; shift 2 ;;
+		--token) TOKEN=$2; shift 2 ;;
+		--ports) PORTS=$2; shift 2 ;;
+		-h | --help) sed -n '2,16p' "$0"; exit 0 ;;
+		*) echo "unknown option $1 (see --help)" >&2; exit 2 ;;
+	esac
+done
+
+say() { printf '\n== %s\n' "$*"; }
+die() { echo "error: $*" >&2; exit 1; }
+[ "$(id -u)" = 0 ] || die "run as root: sudo sh $0 $*"
+command -v systemctl > /dev/null 2>&1 || die "this needs a systemd-based Linux (Debian, Ubuntu, Fedora, RHEL, Arch...)"
+
+case "$(uname -m)" in
+	x86_64 | amd64) ARCH=amd64 ;;
+	aarch64 | arm64) ARCH=arm64 ;;
+	armv7* | armv6* | arm) ARCH=arm ;;
+	*) die "unsupported CPU: $(uname -m)" ;;
+esac
+
+pkg_install() { # installs packages with whichever package manager there is
+	if command -v apt-get > /dev/null 2>&1; then
+		apt-get update -qq || true
+		DEBIAN_FRONTEND=noninteractive apt-get install -y "$@"
+	elif command -v dnf > /dev/null 2>&1; then
+		dnf install -y "$@"
+	elif command -v yum > /dev/null 2>&1; then
+		yum install -y "$@"
+	elif command -v pacman > /dev/null 2>&1; then
+		pacman -S --noconfirm --needed "$@"
+	else
+		die "install these yourself and run again: $*"
+	fi
+}
+need() { command -v "$1" > /dev/null 2>&1 || { say "installing $1"; pkg_install "$1"; }; }
+need curl
+need tar
+need git
+
+latest() { # newest release tag of repo $1 (read from git, so no API token is needed)
+	git ls-remote --tags --refs "$BASE/$1" 2> /dev/null | sed 's|.*refs/tags/||' | grep -E '^v[0-9]' | sort -V | tail -n 1
+}
+
+# fetch_install <app>: downloads that app's release for this CPU, verifies it and runs its install.sh
+fetch_install() {
+	app=$1
+	ver=${VERSION:-$(latest "$app")}
+	[ -n "$ver" ] || die "couldn't find a release of $app at $BASE/$app (use --version, or SHARKS_BASE for a mirror)"
+	dir=$(mktemp -d)
+	say "downloading $app $ver ($ARCH)"
+	name="${app}_${ver}_linux_${ARCH}"
+	curl -fsSL -o "$dir/$name.tar.gz" "$BASE/$app/releases/download/$ver/$name.tar.gz" || die "no $app $ver release for linux/$ARCH"
+	curl -fsSL -o "$dir/SHA256SUMS" "$BASE/$app/releases/download/$ver/SHA256SUMS" || die "couldn't download the checksums"
+	(cd "$dir" && grep " $name.tar.gz\$" SHA256SUMS | sha256sum -c -) || die "checksum mismatch: not installing"
+	tar -xzf "$dir/$name.tar.gz" -C "$dir"
+	(cd "$dir/$name" && sh ./install.sh)
+	rm -rf "$dir"
+}
+
+# set_env <file> <KEY> <value>: sets KEY=value, replacing the example line or an older value
+set_env() {
+	f=$1 k=$2 v=$3
+	if grep -q "^#\?$k=" "$f"; then
+		tmp=$(mktemp)
+		awk -v k="$k" -v v="$v" 'BEGIN{done=0} { if (!done && ($0 ~ "^#?"k"=")) { print k"="v; done=1 } else print }' "$f" > "$tmp"
+		cat "$tmp" > "$f"
+		rm -f "$tmp"
+	else
+		echo "$k=$v" >> "$f"
+	fi
+}
+
+firewall_open() { # firewall_open 80/tcp 443/tcp ...
+	if command -v ufw > /dev/null 2>&1 && ufw status 2> /dev/null | grep -q "Status: active"; then
+		for p in "$@"; do ufw allow "$p" > /dev/null; done
+		echo "opened in ufw: $*"
+	elif command -v firewall-cmd > /dev/null 2>&1 && firewall-cmd --state > /dev/null 2>&1; then
+		for p in "$@"; do firewall-cmd --permanent --add-port="$p" > /dev/null; done
+		firewall-cmd --reload > /dev/null
+		echo "opened in firewalld: $*"
+	else
+		echo "No active ufw or firewalld found. If this machine has a firewall, or your host has a cloud firewall / security group, open: $*"
+	fi
+}
+
+public_ips() { # global addresses actually configured on this machine
+	ip -o addr show scope global 2> /dev/null | awk '{print $4}' | cut -d/ -f1 || true
+}
+
+wait_active() { # wait_active <service>
+	sleep 2
+	systemctl is-active --quiet "$1" || { journalctl -u "$1" -n 20 --no-pager; die "$1 didn't start"; }
+}
+
+case $MODE in
+hammerhead)
+	fetch_install hammerhead
+	ENVF=/etc/hammerhead/hammerhead.env
+	if ! grep -q '^HAMMERHEAD_MASTER_KEY=.' "$ENVF"; then
+		say "generating the master key"
+		set_env "$ENVF" HAMMERHEAD_MASTER_KEY "$(/usr/local/bin/hammerhead -genkey)"
+		echo "Saved in $ENVF. Back that file up: without the key the encrypted secrets in the database can't be read."
+	fi
+	if [ -n "$DOMAIN" ]; then
+		set_env "$ENVF" HAMMERHEAD_URL "https://$DOMAIN"
+		# only Caddy needs to reach it
+		sed -i 's|-listen :8080|-listen 127.0.0.1:8080|' /etc/systemd/system/hammerhead.service
+		systemctl daemon-reload
+		if ! command -v caddy > /dev/null 2>&1; then
+			say "installing Caddy (HTTPS certificates for $DOMAIN)"
+			pkg_install caddy || die "couldn't install Caddy from this distribution; see https://caddyserver.com/docs/install and proxy $DOMAIN to 127.0.0.1:8080"
+		fi
+		mkdir -p /etc/caddy
+		printf '%s {\n\treverse_proxy 127.0.0.1:8080\n}\n' "$DOMAIN" > /etc/caddy/Caddyfile
+		systemctl enable caddy > /dev/null 2>&1 || true
+		systemctl restart caddy
+		firewall_open 80/tcp 443/tcp
+		URL="https://$DOMAIN"
+	else
+		firewall_open 8080/tcp
+		ip=$(public_ips | head -n 1)
+		URL="http://${ip:-this-server}:8080"
+		echo "No --domain given, so the dashboard is plain HTTP. Put it behind HTTPS before you create real logins (run this again with --domain)."
+	fi
+	systemctl restart hammerhead
+	wait_active hammerhead
+	say "Hammerhead is running"
+	echo "Open $URL to create the first administrator."
+	echo
+	echo "Next, in the dashboard: Server -> Edges -> add an edge, then on the machine that has your public IPs run"
+	echo "  sh setup.sh edge --hammerhead $URL --token <edge token>"
+	echo "and on each machine that hosts your services (home server, game server...) run"
+	echo "  sh setup.sh agent --hammerhead $URL --token <site token>"
+	echo "Either can be this same machine."
+	;;
+edge)
+	[ -n "$HH" ] && [ -n "$TOKEN" ] || die "edge needs --hammerhead URL and --token TOKEN (Server -> Edges -> Show command)"
+	fetch_install blacktip
+	ENVF=/etc/blacktip/blacktip.env
+	set_env "$ENVF" BLACKTIP_HAMMERHEAD "$HH"
+	set_env "$ENVF" BLACKTIP_TOKEN "$TOKEN"
+	chmod 600 "$ENVF"
+	say "network"
+	# WireGuard moves a lot of UDP; the stock socket buffers are small
+	printf 'net.core.rmem_max=7500000\nnet.core.wmem_max=7500000\n' > /etc/sysctl.d/90-blacktip.conf
+	sysctl -q -p /etc/sysctl.d/90-blacktip.conf > /dev/null 2>&1 || true
+	list="80/tcp 443/tcp 51820/udp"
+	if [ -n "$PORTS" ]; then
+		# 25565,30000-30100: each is opened for TCP and UDP (the dashboard decides which one is used)
+		for p in $(echo "$PORTS" | tr ',' ' '); do
+			list="$list $p/tcp $p/udp"
+		done
+	fi
+	# ufw wants a range as 30000:30100, firewalld as 30000-30100
+	if command -v ufw > /dev/null 2>&1; then list=$(echo "$list" | tr '-' ':'); fi
+	# shellcheck disable=SC2086
+	firewall_open $list
+	ips=$(public_ips)
+	echo
+	echo "Blacktip can open ports on these addresses of this machine:"
+	echo "$ips" | sed 's/^/  /'
+	if [ "$(echo "$ips" | grep -c .)" -lt 2 ]; then
+		echo "Only one address found. Extra IPs must be added to this machine's network configuration first."
+		echo "A provider's floating or 1:1 NAT address that isn't listed above can't be bound by Blacktip."
+	fi
+	echo "In the dashboard each port picks which of these addresses it listens on."
+	echo "Ports you open later in the dashboard must also be allowed in the firewall: run this again with --ports."
+	systemctl restart blacktip
+	wait_active blacktip
+	say "Blacktip is running"
+	;;
+agent)
+	[ -n "$HH" ] && [ -n "$TOKEN" ] || die "agent needs --hammerhead URL and --token TOKEN (Sites -> Show command)"
+	fetch_install bamboo
+	ENVF=/etc/bamboo/bamboo.env
+	set_env "$ENVF" BAMBOO_HAMMERHEAD "$HH"
+	set_env "$ENVF" BAMBOO_TOKEN "$TOKEN"
+	chmod 600 "$ENVF"
+	if command -v docker > /dev/null 2>&1 && getent group docker > /dev/null 2>&1; then
+		usermod -aG docker bamboo
+		set_env "$ENVF" BAMBOO_DOCKER 1
+		echo "Docker found: Bamboo will list your containers as targets."
+	fi
+	systemctl restart bamboo
+	wait_active bamboo
+	say "Bamboo is running (it only connects outward, so there is nothing to open)"
+	;;
+esac

e2e/proxyproto_test.go

@@ -91,7 +91,10 @@ func proxyBackend(t *testing.T, seen chan<- string) int {
 func askFrom(t *testing.T, from, addr, send string) string {
 	t.Helper()
 	var last error
-	for i := 0; i < 50; i++ {
+	// The first connection through the tunnel can wait for the WireGuard handshake, which is
+	// slow on a busy CI runner, so a read timeout is retried until the overall deadline.
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
 		d := net.Dialer{LocalAddr: &net.TCPAddr{IP: net.ParseIP(from)}, Timeout: time.Second}
 		c, err := d.Dial("tcp", addr)
 		if err == nil {
@@ -102,9 +105,6 @@ func askFrom(t *testing.T, from, addr, send string) string {
 			if rerr == nil {
 				return strings.TrimSpace(line)
 			}
-			if ne, ok := rerr.(net.Error); ok && ne.Timeout() {
-				t.Fatalf("%s accepted the connection but never answered: %v", addr, rerr) // a real hang, not a startup race
-			}
 			last = rerr
 		} else {
 			last = err