Add website analytics
Release / release (push) Successful in 3m49s
CI / test (push) Successful in 4m45s

A Tianji-style Websites section: a small tracking script, a public collection endpoint, and a
page per website with visitors, page views, visits, bounce rate, a chart and top lists for pages,
referrers, browsers, systems, devices, countries, languages and events. No cookies and no stored
IP address or user agent. SANDBAR_TRUST_PROXY makes the client address come from the proxy.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
VeillaxandClaude Sonnet 5.5 committed 2026-10-27 04:22:55 -06:00
1 parent 4ea4fe5289
commit b4fd31a182
13 files changed
+843 -22

No files matched your search

+431
View File
@@ -0,0 +1,431 @@
package sandbar
import (
"crypto/sha256"
_ "embed"
"encoding/hex"
"encoding/json"
"errors"
"io"
"net"
"net/http"
"net/url"
"regexp"
"sort"
"strings"
"time"
)
// Website analytics, in the manner of Tianji, Plausible and Umami: a small script on a site sends
// page views and custom events here, and the Websites page shows visitors, pages, referrers,
// browsers, systems, devices, countries and events.
//
// Privacy by construction: no cookies, no stored IP address or user agent. A visitor is a hash of the
// IP address, the user agent and the website, salted with a secret and the UTC date, so the same
// person is the same visitor within a day and cannot be followed beyond it.
//go:embed web/script.js
var trackerJS []byte
// Website is a site whose visits are counted.
type Website struct {
ID string `json:"id"`
Name string `json:"name"`
Domain string `json:"domain"` // optional: when set, only hits from this host and its subdomains count
}
func prepareWebsite(v any, id string) error {
w := v.(*Website)
w.ID = id
w.Name = strings.TrimSpace(w.Name)
if w.Name == "" {
return errors.New("give the website a name")
}
d := strings.ToLower(strings.TrimSpace(w.Domain))
if d != "" {
if !strings.Contains(d, "//") {
d = "//" + d
}
u, err := url.Parse(d)
if err != nil || u.Hostname() == "" {
return errors.New("the domain should look like example.com")
}
d = u.Hostname()
}
w.Domain = d
return nil
}
func (s *Server) deleteWebsiteData(id string) {
s.st.db.Exec(`DELETE FROM hits WHERE website=?`, id)
}
// realIP is the address of the client. Behind a reverse proxy, set TrustProxy so the address the
// proxy added (the last entry of X-Forwarded-For) is used; otherwise every visitor looks like the proxy.
func (s *Server) realIP(r *http.Request) string {
if s.opts.TrustProxy {
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
parts := strings.Split(xff, ",")
if ip := strings.TrimSpace(parts[len(parts)-1]); net.ParseIP(ip) != nil {
return ip
}
}
if ip := strings.TrimSpace(r.Header.Get("X-Real-IP")); net.ParseIP(ip) != nil {
return ip
}
}
h, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
return r.RemoteAddr
}
return h
}
func (s *Server) serveTracker(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
w.Header().Set("Cache-Control", "public, max-age=3600")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Write(trackerJS)
}
var botRe = regexp.MustCompile(`(?i)bot|crawl|spider|slurp|headless|lighthouse|preview|monitor|curl/|wget|python-requests|go-http-client|okhttp`)
// parseUA reduces a user agent to a browser, a system and a kind of device. Only these labels are stored.
func parseUA(ua string) (browser, os, device string) {
switch {
case strings.Contains(ua, "Edg/") || strings.Contains(ua, "EdgA/") || strings.Contains(ua, "EdgiOS/"):
browser = "Edge"
case strings.Contains(ua, "OPR/") || strings.Contains(ua, "Opera"):
browser = "Opera"
case strings.Contains(ua, "SamsungBrowser"):
browser = "Samsung Internet"
case strings.Contains(ua, "Firefox/") || strings.Contains(ua, "FxiOS/"):
browser = "Firefox"
case strings.Contains(ua, "Chrome/") || strings.Contains(ua, "CriOS/"):
browser = "Chrome"
case strings.Contains(ua, "Safari/"):
browser = "Safari"
default:
browser = "Other"
}
switch {
case strings.Contains(ua, "Windows"):
os = "Windows"
case strings.Contains(ua, "Android"):
os = "Android"
case strings.Contains(ua, "iPhone") || strings.Contains(ua, "iPad") || strings.Contains(ua, "iPod"):
os = "iOS"
case strings.Contains(ua, "Mac OS X") || strings.Contains(ua, "Macintosh"):
os = "macOS"
case strings.Contains(ua, "CrOS"):
os = "ChromeOS"
case strings.Contains(ua, "Linux") || strings.Contains(ua, "X11"):
os = "Linux"
default:
os = "Other"
}
switch {
case strings.Contains(ua, "iPad") || strings.Contains(ua, "Tablet") || (strings.Contains(ua, "Android") && !strings.Contains(ua, "Mobile")):
device = "Tablet"
case strings.Contains(ua, "Mobi") || strings.Contains(ua, "iPhone"):
device = "Mobile"
default:
device = "Desktop"
}
return
}
// secret is the random value that salts visitor hashes. It is made once and kept with the other settings.
func (s *Server) analyticsSecret() string {
s.anaMu.Lock()
defer s.anaMu.Unlock()
if s.anaSecret != "" {
return s.anaSecret
}
var v struct{ Secret string }
if s.st.get("setting", "analytics", &v) != nil || v.Secret == "" {
v.Secret = newToken()
s.st.put("setting", "analytics", &v)
}
s.anaSecret = v.Secret
return v.Secret
}
func (s *Server) visitorID(website, ip, ua string, now time.Time) string {
h := sha256.Sum256([]byte(s.analyticsSecret() + "|" + now.UTC().Format("2006-01-02") + "|" + website + "|" + ip + "|" + ua))
return hex.EncodeToString(h[:8])
}
// hitAllowed limits how fast one address can send hits: a script that loops must not fill the database.
func (s *Server) hitAllowed(ip string) bool {
now := s.now()
s.anaMu.Lock()
defer s.anaMu.Unlock()
if len(s.anaTry) > 20000 {
s.anaTry = map[string][]time.Time{}
}
recent := s.anaTry[ip][:0]
for _, t := range s.anaTry[ip] {
if now.Sub(t) < time.Minute {
recent = append(recent, t)
}
}
if len(recent) >= 120 {
s.anaTry[ip] = recent
return false
}
s.anaTry[ip] = append(recent, now)
return true
}
func clip(s string, n int) string {
s = strings.TrimSpace(s)
if len(s) > n {
s = strings.ToValidUTF8(s[:n], "")
}
return s
}
// collect receives a page view or event from the tracker script. It is public by design, so it
// answers 204 to everything it ignores and gives nothing away.
func (s *Server) collect(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
if r.Method == http.MethodOptions {
w.WriteHeader(204)
return
}
defer w.WriteHeader(204)
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 4<<10))
if err != nil {
return
}
var in struct {
Website, Path, Referrer, Hostname, Language, Event string
Data map[string]any
}
if json.Unmarshal(body, &in) != nil {
return
}
var site Website
if s.st.get("website", in.Website, &site) != nil {
return
}
host := strings.ToLower(in.Hostname)
if site.Domain != "" && host != site.Domain && !strings.HasSuffix(host, "."+site.Domain) {
return
}
ua := r.UserAgent()
if ua == "" || botRe.MatchString(ua) {
return
}
ip := s.realIP(r)
if !s.hitAllowed(ip) {
return
}
path := in.Path
if i := strings.IndexAny(path, "?"); i >= 0 {
path = path[:i]
}
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
ref := ""
if u, err := url.Parse(in.Referrer); err == nil && u.Host != "" {
if rh := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www."); rh != strings.TrimPrefix(host, "www.") {
ref = rh
}
}
country := strings.ToUpper(clip(r.Header.Get("CF-IPCountry"), 2))
if country == "" {
country = strings.ToUpper(clip(r.Header.Get("CloudFront-Viewer-Country"), 2))
}
if country == "XX" || country == "T1" {
country = ""
}
lang := strings.ToLower(clip(in.Language, 12))
if i := strings.IndexAny(lang, "-_"); i > 0 {
lang = lang[:i]
}
data := ""
if in.Event != "" && len(in.Data) > 0 {
if b, err := json.Marshal(in.Data); err == nil && len(b) <= 1024 {
data = string(b)
}
}
browser, osName, device := parseUA(ua)
now := s.now()
s.st.db.Exec(`INSERT INTO hits(website,time,visitor,event,path,referrer,browser,os,device,country,lang,data) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`,
site.ID, now.Unix(), s.visitorID(site.ID, ip, ua, now), clip(in.Event, 80), clip(path, 200), ref, browser, osName, device, country, lang, data)
}
// ---------- reading ----------
type topRow struct {
Name string `json:"name"`
Visitors int `json:"visitors"`
Views int `json:"views"` // page views, or the number of times for an event
}
type seriesPoint struct {
Time int64 `json:"t"`
Views int `json:"views"`
Visitors int `json:"visitors"`
}
type siteStats struct {
Range string `json:"range"`
Bucket string `json:"bucket"` // "hour" or "day"
Visitors int `json:"visitors"`
Views int `json:"views"`
Visits int `json:"visits"`
BounceRate float64 `json:"bounceRate"` // percent of visits with one page view; -1 if no visits
Online int `json:"online"` // distinct visitors in the last 5 minutes
Series []seriesPoint `json:"series"`
Pages []topRow `json:"pages"`
Referrers []topRow `json:"referrers"`
Browsers []topRow `json:"browsers"`
Systems []topRow `json:"systems"`
Devices []topRow `json:"devices"`
Countries []topRow `json:"countries"`
Languages []topRow `json:"languages"`
Events []topRow `json:"events"`
}
var ranges = map[string]time.Duration{"24h": 24 * time.Hour, "7d": 7 * 24 * time.Hour, "30d": 30 * 24 * time.Hour, "90d": 90 * 24 * time.Hour}
// visitGap is how long a visitor can be quiet before their next page view starts a new visit.
const visitGap = 30 * 60
func (s *Server) websiteStats(id, rng string) (*siteStats, error) {
span, ok := ranges[rng]
if !ok {
rng, span = "7d", ranges["7d"]
}
now := s.now()
from := now.Add(-span).Unix()
out := &siteStats{Range: rng, Bucket: "day", BounceRate: -1, Series: []seriesPoint{}}
step := int64(86400)
if rng == "24h" {
out.Bucket, step = "hour", 3600
}
db := s.st.db
db.QueryRow(`SELECT count(DISTINCT visitor), count(*) FROM hits WHERE website=? AND time>=? AND event=''`, id, from).Scan(&out.Visitors, &out.Views)
db.QueryRow(`SELECT count(DISTINCT visitor) FROM hits WHERE website=? AND time>=?`, id, now.Add(-5*time.Minute).Unix()).Scan(&out.Online)
// Visits and bounces: a visitor's page views, split wherever they were quiet for half an hour.
if rows, err := db.Query(`SELECT visitor,time FROM hits WHERE website=? AND time>=? AND event='' ORDER BY visitor,time`, id, from); err == nil {
var prev string
var prevT int64
views, bounces := 0, 0
closeVisit := func() {
if views > 0 {
out.Visits++
}
if views == 1 {
bounces++
}
}
for rows.Next() {
var v string
var t int64
if rows.Scan(&v, &t) != nil {
continue
}
if v != prev || t-prevT > visitGap {
closeVisit()
views = 0
}
views++
prev, prevT = v, t
}
closeVisit()
rows.Close()
if out.Visits > 0 {
out.BounceRate = float64(bounces) * 100 / float64(out.Visits)
}
}
// The chart: one point per hour or UTC day, empty ones included so the line has no gaps.
buckets := map[int64]seriesPoint{}
if rows, err := db.Query(`SELECT (time/?)*?, count(*), count(DISTINCT visitor) FROM hits WHERE website=? AND time>=? AND event='' GROUP BY 1`, step, step, id, from); err == nil {
for rows.Next() {
var p seriesPoint
if rows.Scan(&p.Time, &p.Views, &p.Visitors) == nil {
buckets[p.Time] = p
}
}
rows.Close()
}
for t := (from / step) * step; t <= now.Unix(); t += step {
p := buckets[t]
p.Time = t
out.Series = append(out.Series, p)
}
top := func(col string, events bool, empty string) []topRow {
cond := "event=''"
sel := col
if events {
cond, sel = "event!=''", "event"
}
rows, err := db.Query(`SELECT `+sel+`, count(DISTINCT visitor), count(*) FROM hits WHERE website=? AND time>=? AND `+cond+
` GROUP BY 1 ORDER BY 3 DESC, 1 LIMIT 10`, id, from)
list := []topRow{}
if err != nil {
return list
}
defer rows.Close()
for rows.Next() {
var r topRow
if rows.Scan(&r.Name, &r.Visitors, &r.Views) == nil {
if r.Name == "" {
r.Name = empty
}
list = append(list, r)
}
}
return list
}
out.Pages = top("path", false, "/")
out.Referrers = top("referrer", false, "Direct")
out.Browsers = top("browser", false, "Other")
out.Systems = top("os", false, "Other")
out.Devices = top("device", false, "Other")
out.Countries = top("country", false, "Unknown")
out.Languages = top("lang", false, "Unknown")
out.Events = top("", true, "")
sort.SliceStable(out.Series, func(i, j int) bool { return out.Series[i].Time < out.Series[j].Time })
return out, nil
}
func (s *Server) websiteStatsHandler(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
var site Website
if s.st.get("website", id, &site) != nil {
writeJSON(w, 404, map[string]string{"error": "No such website."})
return
}
st, _ := s.websiteStats(id, r.URL.Query().Get("range"))
writeJSON(w, 200, st)
}
// websiteSummary gives the list page a number for each site: visitors and page views in the last day.
func (s *Server) websiteSummary(w http.ResponseWriter, r *http.Request) {
out := map[string]map[string]int{}
since := s.now().Add(-24 * time.Hour).Unix()
rows, err := s.st.db.Query(`SELECT website, count(DISTINCT visitor), count(*) FROM hits WHERE time>=? AND event='' GROUP BY website`, since)
if err == nil {
defer rows.Close()
for rows.Next() {
var id string
var v, p int
if rows.Scan(&id, &v, &p) == nil {
out[id] = map[string]int{"visitors": v, "views": p}
}
}
}
writeJSON(w, 200, out)
}
+175
View File
@@ -0,0 +1,175 @@
package sandbar
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
const (
uaChrome = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
uaPhone = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1"
)
func TestParseUA(t *testing.T) {
for _, c := range []struct{ ua, browser, os, device string }{
{uaChrome, "Chrome", "Windows", "Desktop"},
{uaPhone, "Safari", "iOS", "Mobile"},
{"Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0", "Firefox", "Linux", "Desktop"},
{"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 Chrome/126.0 Safari/537.36 Edg/126.0", "Edge", "Windows", "Desktop"},
{"Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 Chrome/126.0 Mobile Safari/537.36", "Chrome", "Android", "Mobile"},
{"Mozilla/5.0 (Linux; Android 13; SM-X700) AppleWebKit/537.36 Chrome/126.0 Safari/537.36", "Chrome", "Android", "Tablet"},
} {
b, o, d := parseUA(c.ua)
if b != c.browser || o != c.os || d != c.device {
t.Errorf("%q: got %s/%s/%s, want %s/%s/%s", c.ua, b, o, d, c.browser, c.os, c.device)
}
}
}
func TestWebsiteAnalytics(t *testing.T) {
s, _, _ := newTestServer(t)
s.st.put("website", "w1", &Website{ID: "w1", Name: "Blog", Domain: "example.com"})
ts := httptest.NewServer(s.Handler())
defer ts.Close()
send := func(ua string, body string) int {
req, _ := http.NewRequest("POST", ts.URL+"/api/send", strings.NewReader(body))
req.Header.Set("Content-Type", "text/plain")
req.Header.Set("User-Agent", ua)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
return resp.StatusCode
}
view := func(ua, path, ref string) {
b, _ := json.Marshal(map[string]string{"website": "w1", "hostname": "www.example.com", "path": path, "referrer": ref, "language": "en-GB"})
if code := send(ua, string(b)); code != 204 {
t.Fatalf("send: %d", code)
}
}
view(uaChrome, "/", "https://news.example.org/story?id=1")
view(uaChrome, "/about?utm=x", "")
view(uaPhone, "/", "")
send(uaChrome, `{"website":"w1","hostname":"example.com","path":"/pricing","event":"signup","data":{"plan":"pro"}}`)
// Things that must not be counted.
send("Googlebot/2.1", `{"website":"w1","hostname":"example.com","path":"/bot"}`)
send(uaChrome, `{"website":"w1","hostname":"evil.test","path":"/wrong-host"}`)
send(uaChrome, `{"website":"nope","hostname":"example.com","path":"/unknown-site"}`)
send(uaChrome, `not json`)
c := newAPIClient(t, ts)
if code := c.do("GET", "/api/v1/websites/w1/stats", nil, nil); code != 401 {
t.Fatalf("stats without sign-in: %d", code)
}
c.setup()
var st siteStats
if code := c.do("GET", "/api/v1/websites/w1/stats?range=7d", nil, &st); code != 200 {
t.Fatalf("stats: %d", code)
}
if st.Views != 3 || st.Visitors != 2 || st.Visits != 2 || st.BounceRate != 50 {
t.Fatalf("totals: views %d visitors %d visits %d bounce %.0f", st.Views, st.Visitors, st.Visits, st.BounceRate)
}
has := func(rows []topRow, name string, views int) bool {
for _, r := range rows {
if r.Name == name && r.Views == views {
return true
}
}
return false
}
for _, c := range []struct {
what string
rows []topRow
name string
n int
}{
{"page /", st.Pages, "/", 2}, {"page /about without the query", st.Pages, "/about", 1},
{"referrer host only", st.Referrers, "news.example.org", 1}, {"direct", st.Referrers, "Direct", 2},
{"browser", st.Browsers, "Chrome", 2}, {"os", st.Systems, "iOS", 1},
{"device", st.Devices, "Mobile", 1}, {"language", st.Languages, "en", 3}, {"event", st.Events, "signup", 1},
} {
if !has(c.rows, c.name, c.n) {
t.Errorf("%s: %q x%d not in %+v", c.what, c.name, c.n, c.rows)
}
}
for _, p := range st.Pages {
if p.Name == "/bot" || p.Name == "/wrong-host" || p.Name == "/unknown-site" || p.Name == "/pricing" {
t.Errorf("counted something it should not: %+v", p)
}
}
if st.Online != 2 {
t.Fatalf("online %d", st.Online)
}
nonEmpty := 0
for _, p := range st.Series {
if p.Views > 0 {
nonEmpty++
}
}
if nonEmpty != 1 || len(st.Series) < 7 {
t.Fatalf("series: %d points, %d with views", len(st.Series), nonEmpty)
}
// No IP address or user agent is stored.
var n int
s.st.db.QueryRow(`SELECT count(*) FROM hits WHERE visitor LIKE '%127.0.0.1%' OR browser LIKE '%Mozilla%'`).Scan(&n)
if n != 0 {
t.Fatal("raw identifiers stored")
}
// The script and the browser preflight.
resp, _ := http.Get(ts.URL + "/script.js")
body := make([]byte, 64)
resp.Body.Read(body)
resp.Body.Close()
if resp.StatusCode != 200 || !strings.Contains(resp.Header.Get("Content-Type"), "javascript") {
t.Fatalf("script.js: %d %s", resp.StatusCode, resp.Header.Get("Content-Type"))
}
req, _ := http.NewRequest("OPTIONS", ts.URL+"/api/send", nil)
resp, _ = http.DefaultClient.Do(req)
resp.Body.Close()
if resp.StatusCode != 204 || resp.Header.Get("Access-Control-Allow-Origin") != "*" {
t.Fatalf("preflight: %d", resp.StatusCode)
}
// Deleting the website deletes its hits.
if code := c.do("DELETE", "/api/v1/websites/w1", nil, nil); code != 200 {
t.Fatalf("delete: %d", code)
}
s.st.db.QueryRow(`SELECT count(*) FROM hits`).Scan(&n)
if n != 0 {
t.Fatalf("%d hits left after deleting the website", n)
}
}
func TestHitRateLimitAndProxy(t *testing.T) {
s, _, _ := newTestServer(t)
for i := 0; i < 120; i++ {
if !s.hitAllowed("1.2.3.4") {
t.Fatalf("hit %d refused too early", i)
}
}
if s.hitAllowed("1.2.3.4") {
t.Fatal("the 121st hit in a minute should be refused")
}
if !s.hitAllowed("5.6.7.8") {
t.Fatal("another address has its own allowance")
}
r := httptest.NewRequest("GET", "/", nil)
r.RemoteAddr = "10.0.0.1:5555"
r.Header.Set("X-Forwarded-For", "6.6.6.6, 9.9.9.9")
if ip := s.realIP(r); ip != "10.0.0.1" {
t.Fatalf("without TrustProxy the connection address is used, got %s", ip)
}
s.opts.TrustProxy = true
if ip := s.realIP(r); ip != "9.9.9.9" {
t.Fatalf("the proxy's entry is the last one, got %s", ip)
}
}
+9
View File
@@ -25,6 +25,9 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("GET /status.json", s.statusJSON)
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "ok") })
mux.HandleFunc("GET /metrics", s.metrics)
mux.HandleFunc("GET /script.js", s.serveTracker)
mux.HandleFunc("POST /api/send", s.collect)
mux.HandleFunc("OPTIONS /api/send", s.collect)
mux.HandleFunc("GET /api/v1/hb/{token}", s.heartbeat)
mux.HandleFunc("POST /api/v1/hb/{token}", s.heartbeat)
@@ -105,6 +108,9 @@ func (s *Server) Handler() http.Handler {
}
return nil
}, nil)
s.crud(mux, "websites", "website", true, func() any { return &Website{} }, prepareWebsite, nil)
mux.HandleFunc("GET /api/v1/websites/{id}/stats", s.auth(false, s.websiteStatsHandler))
mux.HandleFunc("GET /api/v1/website-summary", s.auth(false, s.websiteSummary))
mux.HandleFunc("GET /api/v1/system/update", s.auth(true, s.updateStatus))
mux.HandleFunc("POST /api/v1/system/update", s.auth(true, s.updateApply))
mux.HandleFunc("POST /api/v1/channels/{id}/test", s.auth(true, s.testChannel))
@@ -349,6 +355,9 @@ func (s *Server) crud(mux *http.ServeMux, path, kind string, adminWrite bool, ne
if kind == "monitor" {
s.deleteMonitorData(id)
}
if kind == "website" {
s.deleteWebsiteData(id)
}
fail2(w, s.st.del(kind, id))
}))
}
+1 -10
View File
@@ -4,7 +4,6 @@ import (
"context"
"database/sql"
"errors"
"net"
"net/http"
"strings"
"time"
@@ -144,19 +143,11 @@ func (s *Server) noteFailure(ip string) {
s.loginMu.Unlock()
}
func clientIP(r *http.Request) string {
h, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
return r.RemoteAddr
}
return h
}
var errThrottled = errors.New("too many attempts, try again in a few minutes")
// login checks credentials and starts a session.
func (s *Server) login(w http.ResponseWriter, r *http.Request, email, password string) (*User, error) {
ip := clientIP(r)
ip := s.realIP(r)
if s.throttled(ip) {
return nil, errThrottled
}
+1 -1
View File
@@ -38,7 +38,7 @@ func TestUptimeBars(t *testing.T) {
}
// The rollup survives pruning the raw checks.
s.st.prune(clk.Now().AddDate(0, 0, 60), 30, 1)
s.st.prune(clk.Now().AddDate(0, 0, 60), 30, 1, 365)
if got := s.dailyUptime(m.ID, 5, clk.Now()); got[2] != [2]int{4, 1} {
t.Fatalf("rollup lost after pruning: %v", got)
}
+13 -4
View File
@@ -23,8 +23,10 @@ type Options struct {
MasterKey string
MetricsToken string
SMTP SMTPConfig
MessageDays int // how long topic messages are kept (default 30)
CheckDays int // how long monitor results are kept (default 30)
MessageDays int // how long topic messages are kept (default 30)
CheckDays int // how long monitor results are kept (default 30)
HitDays int // how long website analytics are kept (default 365)
TrustProxy bool // take the client address from X-Forwarded-For, for use behind a reverse proxy
// Self-update (see update.go). Version is the running release, such as v0.2.0; anything else
// ("dev") never updates. Restart is called after a new binary is installed and should close
@@ -55,6 +57,10 @@ type Server struct {
upd updateState
anaMu sync.Mutex // website analytics: the visitor salt and a per-address rate limit
anaSecret string
anaTry map[string][]time.Time
stop chan struct{}
wg sync.WaitGroup
}
@@ -66,7 +72,10 @@ func New(opts Options, st *Store) *Server {
if opts.CheckDays <= 0 {
opts.CheckDays = 30
}
return &Server{opts: opts, st: st, hub: newHub(), running: map[string]bool{}, loginTry: map[string][]time.Time{}, stop: make(chan struct{})}
if opts.HitDays <= 0 {
opts.HitDays = 365
}
return &Server{opts: opts, st: st, hub: newHub(), running: map[string]bool{}, loginTry: map[string][]time.Time{}, anaTry: map[string][]time.Time{}, stop: make(chan struct{})}
}
func (s *Server) now() time.Time {
@@ -80,7 +89,7 @@ func (s *Server) now() time.Time {
func (s *Server) Start() {
s.every(time.Second, func() { s.runDueMonitors(s.now()) })
s.every(5*time.Second, func() { s.escalate(s.now()) })
s.every(time.Hour, func() { s.st.prune(s.now(), s.opts.MessageDays, s.opts.CheckDays) })
s.every(time.Hour, func() { s.st.prune(s.now(), s.opts.MessageDays, s.opts.CheckDays, s.opts.HitDays) })
}
func (s *Server) Close() {
+8 -1
View File
@@ -48,6 +48,12 @@ CREATE INDEX IF NOT EXISTS checks_monitor ON checks(monitor_id, time);
-- One row per monitor per UTC day, kept far longer than raw checks: it feeds the status page's uptime bars.
CREATE TABLE IF NOT EXISTS daily (
monitor_id TEXT NOT NULL, day TEXT NOT NULL, checks INTEGER NOT NULL, fails INTEGER NOT NULL, PRIMARY KEY (monitor_id, day));
-- Website analytics: one row per page view or event. No IP address or user agent is kept, only labels.
CREATE TABLE IF NOT EXISTS hits (
website TEXT NOT NULL, time INTEGER NOT NULL, visitor TEXT NOT NULL, event TEXT NOT NULL, path TEXT NOT NULL,
referrer TEXT NOT NULL, browser TEXT NOT NULL, os TEXT NOT NULL, device TEXT NOT NULL, country TEXT NOT NULL,
lang TEXT NOT NULL, data TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS hits_site ON hits(website, time);
CREATE TABLE IF NOT EXISTS mstate (
monitor_id TEXT PRIMARY KEY, status TEXT NOT NULL, since INTEGER NOT NULL, last_check INTEGER NOT NULL,
last_ms INTEGER NOT NULL, fails INTEGER NOT NULL, last_ping INTEGER NOT NULL, detail TEXT NOT NULL);
@@ -246,7 +252,8 @@ func (s *Store) events(scope, ref string) []Event {
}
// prune deletes data past its retention.
func (s *Store) prune(now time.Time, messageDays, checkDays int) {
func (s *Store) prune(now time.Time, messageDays, checkDays, hitDays int) {
s.db.Exec(`DELETE FROM hits WHERE time < ?`, now.AddDate(0, 0, -hitDays).Unix())
s.db.Exec(`DELETE FROM messages WHERE time < ?`, now.AddDate(0, 0, -messageDays).Unix())
s.db.Exec(`DELETE FROM checks WHERE time < ?`, now.AddDate(0, 0, -checkDays).Unix())
s.db.Exec(`DELETE FROM daily WHERE day < ?`, now.UTC().AddDate(0, 0, -400).Format("2006-01-02"))
+94 -1
View File
@@ -227,6 +227,28 @@
.feed .empty { margin:0; }
.msg { border:var(--border-w) solid var(--border); border-radius:var(--radius); padding:var(--s2) var(--s3); display:flex; flex-direction:column; gap:var(--s1); }
.msg .tag { margin-right:var(--s1); }
/* website analytics */
.stats { display:grid; grid-template-columns:repeat(auto-fit,minmax(104px,1fr)); gap:var(--s3); margin-bottom:var(--s4); }
.stat { background:var(--surface); border:var(--border-w) solid var(--border); border-radius:var(--radius-lg); padding:var(--s3) var(--s4); }
.stat .n { font-size:var(--text-xl); font-weight:600; font-variant-numeric:tabular-nums; }
.stat .l { font-size:var(--text-sm); color:var(--text-muted); }
.chart { display:flex; align-items:flex-end; gap:2px; height:180px; margin-top:var(--s3); }
.chart .col { position:relative; flex:1; min-width:0; height:100%; display:flex; align-items:flex-end; }
.chart .col:hover { background:var(--surface-2); }
.chart .v, .chart .u { position:absolute; left:0; right:0; bottom:0; border-radius:2px 2px 0 0; }
.chart .v { background:color-mix(in srgb, var(--accent) 35%, var(--surface)); }
.chart .u { background:var(--accent); }
.axis2 { display:flex; justify-content:space-between; margin-top:var(--s1); font-size:var(--text-xs); color:var(--text-faint); }
.legend { display:flex; gap:var(--s4); font-size:var(--text-sm); color:var(--text-muted); flex-wrap:wrap; }
.legend i { display:inline-block; width:10px; height:10px; border-radius:2px; margin-right:var(--s1); vertical-align:baseline; }
.cols { display:grid; grid-template-columns:repeat(auto-fit,minmax(300px,1fr)); gap:var(--s4); margin-top:var(--s4); }
.cols .card { margin:0; }
.top { width:100%; border-collapse:collapse; font-size:var(--text-sm); }
.top td { padding:var(--s1) var(--s2); }
.top td:first-child { background:linear-gradient(to right, var(--accent-soft) var(--w), transparent var(--w)); overflow-wrap:anywhere; }
.top td.num { text-align:right; white-space:nowrap; width:1%; }
.top th { text-align:right; font-weight:500; color:var(--text-muted); padding:var(--s1) var(--s2); }
.top th:first-child { text-align:left; }
</style>
</head>
<body>
@@ -360,6 +382,7 @@ const ICONS = {
incidents:'<path d="m21.73 18-8-14a2 2 0 0 0-3.48 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3"/><path d="M12 9v4M12 17h.01"/>',
oncall:'<rect x="3" y="4" width="18" height="18" rx="2"/><path d="M16 2v4M8 2v4M3 10h18"/>',
monitors:'<path d="M22 12h-4l-3 9L9 3l-3 9H2"/>',
websites:'<path d="M3 3v18h18"/><path d="M18 17V9"/><path d="M13 17V5"/><path d="M8 17v-3"/>',
topics:'<path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/>',
channels:'<path d="m22 2-7 20-4-9-9-4z"/><path d="M22 2 11 13"/>',
status:'<circle cx="12" cy="12" r="10"/><path d="M2 12h20"/><path d="M12 2a15.3 15.3 0 0 1 4 10 15.3 15.3 0 0 1-4 10 15.3 15.3 0 0 1-4-10 15.3 15.3 0 0 1 4-10z"/>',
@@ -420,6 +443,7 @@ const PAGES = {
incidents: ['Incidents', 'Coordinate the response when something is wrong, and keep a record of what happened.'],
oncall: ['On-call', 'Who is responsible right now, and who is told next when nobody answers.'],
monitors: ['Monitors', 'Checks that tell you when a website, port, certificate or scheduled job stops working.'],
websites: ['Websites', 'Count visitors and page views on your sites, without cookies or personal data.'],
topics: ['Topics', 'Publish and subscribe to messages with plain HTTP requests. Phones can use the ntfy app.'],
channels: ['Channels', 'Email addresses and webhooks that receive every alert and incident update.'],
users: ['People', 'Everyone who can sign in, and how to reach them.'],
@@ -427,7 +451,7 @@ const PAGES = {
};
const NAV = [
['Respond', [['alerts', 'Alerts', 'alerts'], ['incidents', 'Incidents', 'incidents'], ['oncall', 'On-call', 'oncall']]],
['Observe', [['monitors', 'Monitors', 'monitors'], ['topics', 'Topics', 'topics']]],
['Observe', [['monitors', 'Monitors', 'monitors'], ['websites', 'Websites', 'websites'], ['topics', 'Topics', 'topics']]],
['Set up', [['channels', 'Channels', 'channels'], ['users', 'People', 'users']]],
];
function route() {
@@ -754,6 +778,75 @@ function passwordDialog(u) {
dialogForm('Set a password for ' + u.name, 'They are signed out everywhere.', [field('New password', pw, 'At least 10 characters')], 'Set password', async () => { await api('POST', `${V1}/users/${u.id}/password`, {password:pw.value}); toast('Password changed.'); });
}
// ---------- websites (analytics) ----------
const RANGES = [['24h', 'Last 24 hours'], ['7d', 'Last 7 days'], ['30d', 'Last 30 days'], ['90d', 'Last 90 days']];
const num = n => Number(n || 0).toLocaleString();
function regionName(code) { try { return new Intl.DisplayNames(undefined, {type:'region'}).of(code) || code; } catch (e) { return code; } }
function languageName(code) { try { return new Intl.DisplayNames(undefined, {type:'language'}).of(code) || code; } catch (e) { return code; } }
views.websites = async () => {
if (S.arg) return websiteDetail(S.arg);
const admin = S.me.role === 'admin';
S.headerAction = admin ? ['Add website', () => websiteDialog()] : null;
const [sites, sum] = await Promise.all([api('GET', V1 + '/websites'), api('GET', V1 + '/website-summary')]);
if (!sites.length) return emptyState('websites', 'No websites yet', 'Add a website, then paste one line of code into its pages. Visitors and page views show up here. No cookies are set and no personal data is kept.',
admin ? h('button', {class:'btn', onclick:() => websiteDialog()}, 'Add website') : null);
return table(['Website', 'Visitors, last 24 hours', 'Page views, last 24 hours', ''], sites.map(w => { const s = sum[w.id] || {};
return h('tr', {}, h('td', {}, h('a', {href:'#websites/' + w.id}, w.name), w.domain ? h('div', {class:'small dim'}, w.domain) : null),
h('td', {class:'num'}, num(s.visitors)), h('td', {class:'num'}, num(s.views)),
h('td', {}, h('div', {class:'actions'}, h('button', {class:'ghost', onclick:() => trackingDialog(w)}, 'Tracking code'),
admin ? h('button', {class:'ghost', onclick:() => websiteDialog(w)}, 'Edit') : null,
admin ? h('button', {class:'ghost danger', onclick:ask({title:`Delete ${w.name}?`, body:'The website and all of its visits and events are removed. This cannot be undone.', confirm:'Delete website'}, async () => { await api('DELETE', `${V1}/websites/${w.id}`); toast('Website deleted.'); refresh(); })}, 'Delete') : null)));
}));
};
function websiteDialog(w) {
const e = !!w, f = { name: h('input', {value:w ? w.name : ''}), domain: h('input', {value:w ? w.domain : '', placeholder:'example.com'}) };
dialogForm(e ? 'Edit ' + w.name : 'Add a website', null, [field('Name', f.name), field('Domain', f.domain, 'Optional. When set, visits from other domains are ignored, which stops others counting against your site.')], e ? 'Save changes' : 'Add website',
async () => { const saved = await api(e ? 'PUT' : 'POST', e ? `${V1}/websites/${w.id}` : V1 + '/websites', {...(w || {}), name:f.name.value, domain:f.domain.value}); toast(e ? 'Website saved.' : 'Website added.'); await refresh(); if (!e) trackingDialog(saved); });
}
function trackingDialog(w) {
const tag = `<script defer src="${origin()}/script.js" data-website="${w.id}"><\/script>`;
const done = modal('Tracking code for ' + w.name, h('div', {class:'stack'},
h('p', {class:'dim', style:'margin:0'}, 'Paste this inside the head of every page you want to count.'), cmdBox(tag),
h('p', {class:'dim', style:'margin:0'}, 'Single-page sites are counted as people move between pages. To count something else, such as a sign-up, call this from your code:'),
cmdBox('sandbar.track("signup", { plan: "pro" })'),
h('p', {class:'small dim', style:'margin:0'}, 'The script sets no cookies, keeps no IP address, and does nothing when the visitor has Do Not Track on. Behind a reverse proxy, set SANDBAR_TRUST_PROXY=1 so visitors are told apart.'),
h('div', {class:'actions form-actions'}, h('button', {class:'btn', onclick:() => done()}, 'Done'))));
}
async function websiteDetail(id) {
S.noHeader = true;
const range = S.siteRange || '7d';
const [sites, st] = await Promise.all([api('GET', V1 + '/websites'), api('GET', `${V1}/websites/${id}/stats?range=${range}`)]);
setTimeout(() => { S.noHeader = false; });
const w = sites.find(x => x.id === id);
if (!w) return emptyState('websites', 'That website no longer exists', 'It may have been deleted.', h('a', {class:'btn', href:'#websites'}, 'Back to websites'));
const tabs = h('div', {class:'tabs2', role:'group', 'aria-label':'Time range'}, RANGES.map(([k, l]) => h('button', {class:range === k ? 'on' : '', 'aria-pressed':range === k, onclick:() => { S.siteRange = k; refresh(); }}, l)));
const tile = (n, l) => h('div', {class:'stat'}, h('div', {class:'n'}, n), h('div', {class:'l'}, l));
const stats = h('div', {class:'stats'}, tile(num(st.visitors), 'Visitors'), tile(num(st.views), 'Page views'), tile(num(st.visits), 'Visits'),
tile(st.bounceRate >= 0 ? Math.round(st.bounceRate) + '%' : 'No data', 'Bounce rate'), tile(num(st.online), 'Online now'));
const max = Math.max(1, ...st.series.map(p => p.views));
const fmt = p => new Date(p.t * 1000).toLocaleString(undefined, st.bucket === 'hour' ? {month:'short', day:'numeric', hour:'numeric'} : {month:'short', day:'numeric', year:'numeric'});
const chart = h('div', {class:'chart', role:'img', 'aria-label':`${num(st.views)} page views from ${num(st.visitors)} visitors in the ${RANGES.find(r => r[0] === range)[1].toLowerCase()}`},
st.series.map(p => h('div', {class:'col', title:`${fmt(p)}: ${num(p.visitors)} visitors, ${num(p.views)} page views`},
h('div', {class:'v', style:`height:${p.views ? Math.max(2, p.views * 100 / max) : 0}%`}), h('div', {class:'u', style:`height:${p.visitors ? Math.max(2, p.visitors * 100 / max) : 0}%`}))));
const list = (title, rows, label, nameOf) => {
const top = Math.max(1, ...rows.map(r => r.views));
return h('div', {class:'card'}, h('h2', {}, title), rows.length ? h('table', {class:'top'}, h('thead', {}, h('tr', {}, h('th', {scope:'col'}, label), h('th', {scope:'col'}, 'Visitors'), h('th', {scope:'col'}, 'Views'))),
h('tbody', {}, rows.map(r => h('tr', {}, h('td', {style:`--w:${Math.round(r.views * 100 / top)}%`}, nameOf ? nameOf(r.name) : r.name), h('td', {class:'num'}, num(r.visitors)), h('td', {class:'num'}, num(r.views))))))
: h('p', {class:'dim', style:'margin:0'}, 'Nothing yet.'));
};
const nothing = !st.views && !st.events.length;
return h('div', {}, h('div', {class:'page-h'}, h('div', {}, h('a', {href:'#websites', class:'small'}, 'Back to websites'), h('h1', {}, w.name), h('p', {}, w.domain || 'Any domain')),
h('button', {class:'ghost', onclick:() => trackingDialog(w)}, 'Tracking code')),
tabs, h('div', {style:'height:16px'}), stats,
nothing ? emptyState('websites', 'No visits in this period', 'Paste the tracking code into your pages, then open one. The first visit appears within a few seconds.', h('button', {class:'btn', onclick:() => trackingDialog(w)}, 'Show tracking code')) : h('div', {},
card('Visitors and page views', h('div', {class:'legend'}, h('span', {}, h('i', {style:'background:var(--accent)'}), 'Visitors'), h('span', {}, h('i', {style:'background:color-mix(in srgb, var(--accent) 35%, var(--surface))'}), 'Page views')), chart,
h('div', {class:'axis2', 'aria-hidden':'true'}, h('span', {}, fmt(st.series[0])), h('span', {}, 'Now'))),
h('div', {class:'cols'}, list('Pages', st.pages, 'Path'), list('Referrers', st.referrers, 'Source'), list('Browsers', st.browsers, 'Browser'), list('Operating systems', st.systems, 'System'),
list('Devices', st.devices, 'Device'), list('Countries', st.countries, 'Country', n => n.length === 2 ? regionName(n) : n), list('Languages', st.languages, 'Language', n => n.length <= 3 ? languageName(n) : n),
list('Events', st.events, 'Event'))));
}
// ---------- account ----------
// Administrators see which release is running and can install a newer one. The server restarts
// into the new binary, so the page waits for it to come back.
+66
View File
@@ -0,0 +1,66 @@
/* Sandbar website analytics. Add to a page:
<script defer src="https://sandbar.example.com/script.js" data-website="WEBSITE_ID"></script>
It sets no cookies and stores nothing in the browser. It sends the page path, the referrer, the
language and the browser's own user agent, and it honors Do Not Track.
Options on the tag: data-hash (count #hash changes as pages), data-auto="false" (send nothing
until you call sandbar.view() yourself). Custom events: sandbar.track("signup", {plan: "pro"}). */
(function () {
var tag = document.currentScript;
if (!tag) return;
var site = tag.getAttribute('data-website');
if (!site) return;
var base = tag.src.replace(/\/script\.js(\?.*)?$/, '');
var dnt = navigator.doNotTrack === '1' || window.doNotTrack === '1' || navigator.msDoNotTrack === '1';
var withHash = tag.hasAttribute('data-hash');
var last = '';
var first = true;
function send(body) {
if (dnt) return;
body.website = site;
body.hostname = location.hostname;
body.language = navigator.language || '';
var json = JSON.stringify(body);
// text/plain keeps this a "simple" cross-origin request, so no preflight is needed
try {
if (navigator.sendBeacon && navigator.sendBeacon(base + '/api/send', new Blob([json], { type: 'text/plain' }))) return;
} catch (e) { /* fall through to fetch */ }
try {
fetch(base + '/api/send', { method: 'POST', body: json, headers: { 'Content-Type': 'text/plain' }, keepalive: true, mode: 'cors', credentials: 'omit' });
} catch (e) { /* analytics must never break the page */ }
}
function view() {
var path = location.pathname + (withHash ? location.hash : '');
if (path === last) return;
last = path;
send({ path: path, referrer: first ? document.referrer : '' });
first = false;
}
function hook(name) {
var orig = history[name];
if (!orig) return;
history[name] = function () {
var r = orig.apply(this, arguments);
setTimeout(view, 0);
return r;
};
}
hook('pushState');
hook('replaceState');
window.addEventListener('popstate', function () { setTimeout(view, 0); });
if (withHash) window.addEventListener('hashchange', function () { setTimeout(view, 0); });
window.sandbar = {
view: view,
track: function (name, data) {
if (!name) return;
send({ path: location.pathname, event: String(name).slice(0, 80), data: data });
}
};
if (tag.getAttribute('data-auto') === 'false') return;
if (document.readyState === 'complete') view();
else window.addEventListener('load', view);
})();