Sandbar/internal/sandbar/analytics.go

package sandbar

import (
	"crypto/sha256"
	_ "embed"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"
)

// Website analytics, in the manner of Tianji, Plausible and Umami: a small script on a site sends
// page views and custom events here, and the Websites page shows visitors, pages, referrers,
// browsers, systems, devices, countries and events.
//
// Privacy by construction: no cookies, no stored IP address or user agent. A visitor is a hash of the
// IP address, the user agent and the website, salted with a secret and the UTC date, so the same
// person is the same visitor within a day and cannot be followed beyond it.

//go:embed web/script.js
var trackerJS []byte

// Website is a site whose visits are counted.
type Website struct {
	ID     string `json:"id"`
	Name   string `json:"name"`
	Domain string `json:"domain"` // optional: when set, only hits from this host and its subdomains count
}

func prepareWebsite(v any, id string) error {
	w := v.(*Website)
	w.ID = id
	w.Name = strings.TrimSpace(w.Name)
	if w.Name == "" {
		return errors.New("give the website a name")
	}
	d := strings.ToLower(strings.TrimSpace(w.Domain))
	if d != "" {
		if !strings.Contains(d, "//") {
			d = "//" + d
		}
		u, err := url.Parse(d)
		if err != nil || u.Hostname() == "" {
			return errors.New("the domain should look like example.com")
		}
		d = u.Hostname()
	}
	w.Domain = d
	return nil
}

func (s *Server) deleteWebsiteData(id string) {
	s.st.db.Exec(`DELETE FROM hits WHERE website=?`, id)
}

// realIP is the address of the client. Behind a reverse proxy, set TrustProxy so the address the
// proxy added (the last entry of X-Forwarded-For) is used; otherwise every visitor looks like the proxy.
func (s *Server) realIP(r *http.Request) string {
	if s.opts.TrustProxy {
		if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
			parts := strings.Split(xff, ",")
			if ip := strings.TrimSpace(parts[len(parts)-1]); net.ParseIP(ip) != nil {
				return ip
			}
		}
		if ip := strings.TrimSpace(r.Header.Get("X-Real-IP")); net.ParseIP(ip) != nil {
			return ip
		}
	}
	h, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		return r.RemoteAddr
	}
	return h
}

func (s *Server) serveTracker(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
	w.Header().Set("Cache-Control", "public, max-age=3600")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(trackerJS)
}

var botRe = regexp.MustCompile(`(?i)bot|crawl|spider|slurp|headless|lighthouse|preview|monitor|curl/|wget|python-requests|go-http-client|okhttp`)

// parseUA reduces a user agent to a browser, a system and a kind of device. Only these labels are stored.
func parseUA(ua string) (browser, os, device string) {
	switch {
	case strings.Contains(ua, "Edg/") || strings.Contains(ua, "EdgA/") || strings.Contains(ua, "EdgiOS/"):
		browser = "Edge"
	case strings.Contains(ua, "OPR/") || strings.Contains(ua, "Opera"):
		browser = "Opera"
	case strings.Contains(ua, "SamsungBrowser"):
		browser = "Samsung Internet"
	case strings.Contains(ua, "Firefox/") || strings.Contains(ua, "FxiOS/"):
		browser = "Firefox"
	case strings.Contains(ua, "Chrome/") || strings.Contains(ua, "CriOS/"):
		browser = "Chrome"
	case strings.Contains(ua, "Safari/"):
		browser = "Safari"
	default:
		browser = "Other"
	}
	switch {
	case strings.Contains(ua, "Windows"):
		os = "Windows"
	case strings.Contains(ua, "Android"):
		os = "Android"
	case strings.Contains(ua, "iPhone") || strings.Contains(ua, "iPad") || strings.Contains(ua, "iPod"):
		os = "iOS"
	case strings.Contains(ua, "Mac OS X") || strings.Contains(ua, "Macintosh"):
		os = "macOS"
	case strings.Contains(ua, "CrOS"):
		os = "ChromeOS"
	case strings.Contains(ua, "Linux") || strings.Contains(ua, "X11"):
		os = "Linux"
	default:
		os = "Other"
	}
	switch {
	case strings.Contains(ua, "iPad") || strings.Contains(ua, "Tablet") || (strings.Contains(ua, "Android") && !strings.Contains(ua, "Mobile")):
		device = "Tablet"
	case strings.Contains(ua, "Mobi") || strings.Contains(ua, "iPhone"):
		device = "Mobile"
	default:
		device = "Desktop"
	}
	return
}

// secret is the random value that salts visitor hashes. It is made once and kept with the other settings.
func (s *Server) analyticsSecret() string {
	s.anaMu.Lock()
	defer s.anaMu.Unlock()
	if s.anaSecret != "" {
		return s.anaSecret
	}
	var v struct{ Secret string }
	if s.st.get("setting", "analytics", &v) != nil || v.Secret == "" {
		v.Secret = newToken()
		s.st.put("setting", "analytics", &v)
	}
	s.anaSecret = v.Secret
	return v.Secret
}

func (s *Server) visitorID(website, ip, ua string, now time.Time) string {
	h := sha256.Sum256([]byte(s.analyticsSecret() + "|" + now.UTC().Format("2006-01-02") + "|" + website + "|" + ip + "|" + ua))
	return hex.EncodeToString(h[:8])
}

// hitAllowed limits how fast one address can send hits: a script that loops must not fill the database.
func (s *Server) hitAllowed(ip string) bool {
	now := s.now()
	s.anaMu.Lock()
	defer s.anaMu.Unlock()
	if len(s.anaTry) > 20000 {
		s.anaTry = map[string][]time.Time{}
	}
	recent := s.anaTry[ip][:0]
	for _, t := range s.anaTry[ip] {
		if now.Sub(t) < time.Minute {
			recent = append(recent, t)
		}
	}
	if len(recent) >= 120 {
		s.anaTry[ip] = recent
		return false
	}
	s.anaTry[ip] = append(recent, now)
	return true
}

func clip(s string, n int) string {
	s = strings.TrimSpace(s)
	if len(s) > n {
		s = strings.ToValidUTF8(s[:n], "")
	}
	return s
}

// collect receives a page view or event from the tracker script. It is public by design, so it
// answers 204 to everything it ignores and gives nothing away.
func (s *Server) collect(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
	w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
	if r.Method == http.MethodOptions {
		w.WriteHeader(204)
		return
	}
	defer w.WriteHeader(204)
	body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 4<<10))
	if err != nil {
		return
	}
	var in struct {
		Website, Path, Referrer, Hostname, Language, Event string
		Data                                               map[string]any
	}
	if json.Unmarshal(body, &in) != nil {
		return
	}
	var site Website
	if s.st.get("website", in.Website, &site) != nil {
		return
	}
	host := strings.ToLower(in.Hostname)
	if site.Domain != "" && host != site.Domain && !strings.HasSuffix(host, "."+site.Domain) {
		return
	}
	ua := r.UserAgent()
	if ua == "" || botRe.MatchString(ua) {
		return
	}
	ip := s.realIP(r)
	if !s.hitAllowed(ip) {
		return
	}
	path := in.Path
	if i := strings.IndexAny(path, "?"); i >= 0 {
		path = path[:i]
	}
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	ref := ""
	if u, err := url.Parse(in.Referrer); err == nil && u.Host != "" {
		if rh := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www."); rh != strings.TrimPrefix(host, "www.") {
			ref = rh
		}
	}
	country := strings.ToUpper(clip(r.Header.Get("CF-IPCountry"), 2))
	if country == "" {
		country = strings.ToUpper(clip(r.Header.Get("CloudFront-Viewer-Country"), 2))
	}
	if country == "XX" || country == "T1" {
		country = ""
	}
	lang := strings.ToLower(clip(in.Language, 12))
	if i := strings.IndexAny(lang, "-_"); i > 0 {
		lang = lang[:i]
	}
	data := ""
	if in.Event != "" && len(in.Data) > 0 {
		if b, err := json.Marshal(in.Data); err == nil && len(b) <= 1024 {
			data = string(b)
		}
	}
	browser, osName, device := parseUA(ua)
	now := s.now()
	s.st.db.Exec(`INSERT INTO hits(website,time,visitor,event,path,referrer,browser,os,device,country,lang,data) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`,
		site.ID, now.Unix(), s.visitorID(site.ID, ip, ua, now), clip(in.Event, 80), clip(path, 200), ref, browser, osName, device, country, lang, data)
}

// ---------- reading ----------

type topRow struct {
	Name     string `json:"name"`
	Visitors int    `json:"visitors"`
	Views    int    `json:"views"` // page views, or the number of times for an event
}

type seriesPoint struct {
	Time     int64 `json:"t"`
	Views    int   `json:"views"`
	Visitors int   `json:"visitors"`
}

type siteStats struct {
	Range      string        `json:"range"`
	Bucket     string        `json:"bucket"` // "hour" or "day"
	Visitors   int           `json:"visitors"`
	Views      int           `json:"views"`
	Visits     int           `json:"visits"`
	BounceRate float64       `json:"bounceRate"` // percent of visits with one page view; -1 if no visits
	Online     int           `json:"online"`     // distinct visitors in the last 5 minutes
	Series     []seriesPoint `json:"series"`
	Pages      []topRow      `json:"pages"`
	Referrers  []topRow      `json:"referrers"`
	Browsers   []topRow      `json:"browsers"`
	Systems    []topRow      `json:"systems"`
	Devices    []topRow      `json:"devices"`
	Countries  []topRow      `json:"countries"`
	Languages  []topRow      `json:"languages"`
	Events     []topRow      `json:"events"`
}

var ranges = map[string]time.Duration{"24h": 24 * time.Hour, "7d": 7 * 24 * time.Hour, "30d": 30 * 24 * time.Hour, "90d": 90 * 24 * time.Hour}

// visitGap is how long a visitor can be quiet before their next page view starts a new visit.
const visitGap = 30 * 60

func (s *Server) websiteStats(id, rng string) (*siteStats, error) {
	span, ok := ranges[rng]
	if !ok {
		rng, span = "7d", ranges["7d"]
	}
	now := s.now()
	from := now.Add(-span).Unix()
	out := &siteStats{Range: rng, Bucket: "day", BounceRate: -1, Series: []seriesPoint{}}
	step := int64(86400)
	if rng == "24h" {
		out.Bucket, step = "hour", 3600
	}
	db := s.st.db

	db.QueryRow(`SELECT count(DISTINCT visitor), count(*) FROM hits WHERE website=? AND time>=? AND event=''`, id, from).Scan(&out.Visitors, &out.Views)
	db.QueryRow(`SELECT count(DISTINCT visitor) FROM hits WHERE website=? AND time>=?`, id, now.Add(-5*time.Minute).Unix()).Scan(&out.Online)

	// Visits and bounces: a visitor's page views, split wherever they were quiet for half an hour.
	if rows, err := db.Query(`SELECT visitor,time FROM hits WHERE website=? AND time>=? AND event='' ORDER BY visitor,time`, id, from); err == nil {
		var prev string
		var prevT int64
		views, bounces := 0, 0
		closeVisit := func() {
			if views > 0 {
				out.Visits++
			}
			if views == 1 {
				bounces++
			}
		}
		for rows.Next() {
			var v string
			var t int64
			if rows.Scan(&v, &t) != nil {
				continue
			}
			if v != prev || t-prevT > visitGap {
				closeVisit()
				views = 0
			}
			views++
			prev, prevT = v, t
		}
		closeVisit()
		rows.Close()
		if out.Visits > 0 {
			out.BounceRate = float64(bounces) * 100 / float64(out.Visits)
		}
	}

	// The chart: one point per hour or UTC day, empty ones included so the line has no gaps.
	buckets := map[int64]seriesPoint{}
	if rows, err := db.Query(`SELECT (time/?)*?, count(*), count(DISTINCT visitor) FROM hits WHERE website=? AND time>=? AND event='' GROUP BY 1`, step, step, id, from); err == nil {
		for rows.Next() {
			var p seriesPoint
			if rows.Scan(&p.Time, &p.Views, &p.Visitors) == nil {
				buckets[p.Time] = p
			}
		}
		rows.Close()
	}
	for t := (from / step) * step; t <= now.Unix(); t += step {
		p := buckets[t]
		p.Time = t
		out.Series = append(out.Series, p)
	}

	top := func(col string, events bool, empty string) []topRow {
		cond := "event=''"
		sel := col
		if events {
			cond, sel = "event!=''", "event"
		}
		rows, err := db.Query(`SELECT `+sel+`, count(DISTINCT visitor), count(*) FROM hits WHERE website=? AND time>=? AND `+cond+
			` GROUP BY 1 ORDER BY 3 DESC, 1 LIMIT 10`, id, from)
		list := []topRow{}
		if err != nil {
			return list
		}
		defer rows.Close()
		for rows.Next() {
			var r topRow
			if rows.Scan(&r.Name, &r.Visitors, &r.Views) == nil {
				if r.Name == "" {
					r.Name = empty
				}
				list = append(list, r)
			}
		}
		return list
	}
	out.Pages = top("path", false, "/")
	out.Referrers = top("referrer", false, "Direct")
	out.Browsers = top("browser", false, "Other")
	out.Systems = top("os", false, "Other")
	out.Devices = top("device", false, "Other")
	out.Countries = top("country", false, "Unknown")
	out.Languages = top("lang", false, "Unknown")
	out.Events = top("", true, "")
	sort.SliceStable(out.Series, func(i, j int) bool { return out.Series[i].Time < out.Series[j].Time })
	return out, nil
}

func (s *Server) websiteStatsHandler(w http.ResponseWriter, r *http.Request) {
	id := r.PathValue("id")
	var site Website
	if s.st.get("website", id, &site) != nil {
		writeJSON(w, 404, map[string]string{"error": "No such website."})
		return
	}
	st, _ := s.websiteStats(id, r.URL.Query().Get("range"))
	writeJSON(w, 200, st)
}

// websiteSummary gives the list page a number for each site: visitors and page views in the last day.
func (s *Server) websiteSummary(w http.ResponseWriter, r *http.Request) {
	out := map[string]map[string]int{}
	since := s.now().Add(-24 * time.Hour).Unix()
	rows, err := s.st.db.Query(`SELECT website, count(DISTINCT visitor), count(*) FROM hits WHERE time>=? AND event='' GROUP BY website`, since)
	if err == nil {
		defer rows.Close()
		for rows.Next() {
			var id string
			var v, p int
			if rows.Scan(&id, &v, &p) == nil {
				out[id] = map[string]int{"visitors": v, "views": p}
			}
		}
	}
	writeJSON(w, 200, out)
}