Add website analytics
A Tianji-style Websites section: a small tracking script, a public collection endpoint, and a page per website with visitors, page views, visits, bounce rate, a chart and top lists for pages, referrers, browsers, systems, devices, countries, languages and events. No cookies and no stored IP address or user agent. SANDBAR_TRUST_PROXY makes the client address come from the proxy. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
VeillaxandClaude Sonnet 5.5 committed
1 parent
4ea4fe5289
commit
b4fd31a182
13 files changed
+843
-22
No files matched your search
@@ -0,0 +1,431 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
_ "embed"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Website analytics, in the manner of Tianji, Plausible and Umami: a small script on a site sends
|
||||
// page views and custom events here, and the Websites page shows visitors, pages, referrers,
|
||||
// browsers, systems, devices, countries and events.
|
||||
//
|
||||
// Privacy by construction: no cookies, no stored IP address or user agent. A visitor is a hash of the
|
||||
// IP address, the user agent and the website, salted with a secret and the UTC date, so the same
|
||||
// person is the same visitor within a day and cannot be followed beyond it.
|
||||
|
||||
//go:embed web/script.js
|
||||
var trackerJS []byte
|
||||
|
||||
// Website is a site whose visits are counted.
|
||||
type Website struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Domain string `json:"domain"` // optional: when set, only hits from this host and its subdomains count
|
||||
}
|
||||
|
||||
func prepareWebsite(v any, id string) error {
|
||||
w := v.(*Website)
|
||||
w.ID = id
|
||||
w.Name = strings.TrimSpace(w.Name)
|
||||
if w.Name == "" {
|
||||
return errors.New("give the website a name")
|
||||
}
|
||||
d := strings.ToLower(strings.TrimSpace(w.Domain))
|
||||
if d != "" {
|
||||
if !strings.Contains(d, "//") {
|
||||
d = "//" + d
|
||||
}
|
||||
u, err := url.Parse(d)
|
||||
if err != nil || u.Hostname() == "" {
|
||||
return errors.New("the domain should look like example.com")
|
||||
}
|
||||
d = u.Hostname()
|
||||
}
|
||||
w.Domain = d
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) deleteWebsiteData(id string) {
|
||||
s.st.db.Exec(`DELETE FROM hits WHERE website=?`, id)
|
||||
}
|
||||
|
||||
// realIP is the address of the client. Behind a reverse proxy, set TrustProxy so the address the
|
||||
// proxy added (the last entry of X-Forwarded-For) is used; otherwise every visitor looks like the proxy.
|
||||
func (s *Server) realIP(r *http.Request) string {
|
||||
if s.opts.TrustProxy {
|
||||
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
||||
parts := strings.Split(xff, ",")
|
||||
if ip := strings.TrimSpace(parts[len(parts)-1]); net.ParseIP(ip) != nil {
|
||||
return ip
|
||||
}
|
||||
}
|
||||
if ip := strings.TrimSpace(r.Header.Get("X-Real-IP")); net.ParseIP(ip) != nil {
|
||||
return ip
|
||||
}
|
||||
}
|
||||
h, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
return r.RemoteAddr
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func (s *Server) serveTracker(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Write(trackerJS)
|
||||
}
|
||||
|
||||
var botRe = regexp.MustCompile(`(?i)bot|crawl|spider|slurp|headless|lighthouse|preview|monitor|curl/|wget|python-requests|go-http-client|okhttp`)
|
||||
|
||||
// parseUA reduces a user agent to a browser, a system and a kind of device. Only these labels are stored.
|
||||
func parseUA(ua string) (browser, os, device string) {
|
||||
switch {
|
||||
case strings.Contains(ua, "Edg/") || strings.Contains(ua, "EdgA/") || strings.Contains(ua, "EdgiOS/"):
|
||||
browser = "Edge"
|
||||
case strings.Contains(ua, "OPR/") || strings.Contains(ua, "Opera"):
|
||||
browser = "Opera"
|
||||
case strings.Contains(ua, "SamsungBrowser"):
|
||||
browser = "Samsung Internet"
|
||||
case strings.Contains(ua, "Firefox/") || strings.Contains(ua, "FxiOS/"):
|
||||
browser = "Firefox"
|
||||
case strings.Contains(ua, "Chrome/") || strings.Contains(ua, "CriOS/"):
|
||||
browser = "Chrome"
|
||||
case strings.Contains(ua, "Safari/"):
|
||||
browser = "Safari"
|
||||
default:
|
||||
browser = "Other"
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(ua, "Windows"):
|
||||
os = "Windows"
|
||||
case strings.Contains(ua, "Android"):
|
||||
os = "Android"
|
||||
case strings.Contains(ua, "iPhone") || strings.Contains(ua, "iPad") || strings.Contains(ua, "iPod"):
|
||||
os = "iOS"
|
||||
case strings.Contains(ua, "Mac OS X") || strings.Contains(ua, "Macintosh"):
|
||||
os = "macOS"
|
||||
case strings.Contains(ua, "CrOS"):
|
||||
os = "ChromeOS"
|
||||
case strings.Contains(ua, "Linux") || strings.Contains(ua, "X11"):
|
||||
os = "Linux"
|
||||
default:
|
||||
os = "Other"
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(ua, "iPad") || strings.Contains(ua, "Tablet") || (strings.Contains(ua, "Android") && !strings.Contains(ua, "Mobile")):
|
||||
device = "Tablet"
|
||||
case strings.Contains(ua, "Mobi") || strings.Contains(ua, "iPhone"):
|
||||
device = "Mobile"
|
||||
default:
|
||||
device = "Desktop"
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// secret is the random value that salts visitor hashes. It is made once and kept with the other settings.
|
||||
func (s *Server) analyticsSecret() string {
|
||||
s.anaMu.Lock()
|
||||
defer s.anaMu.Unlock()
|
||||
if s.anaSecret != "" {
|
||||
return s.anaSecret
|
||||
}
|
||||
var v struct{ Secret string }
|
||||
if s.st.get("setting", "analytics", &v) != nil || v.Secret == "" {
|
||||
v.Secret = newToken()
|
||||
s.st.put("setting", "analytics", &v)
|
||||
}
|
||||
s.anaSecret = v.Secret
|
||||
return v.Secret
|
||||
}
|
||||
|
||||
func (s *Server) visitorID(website, ip, ua string, now time.Time) string {
|
||||
h := sha256.Sum256([]byte(s.analyticsSecret() + "|" + now.UTC().Format("2006-01-02") + "|" + website + "|" + ip + "|" + ua))
|
||||
return hex.EncodeToString(h[:8])
|
||||
}
|
||||
|
||||
// hitAllowed limits how fast one address can send hits: a script that loops must not fill the database.
|
||||
func (s *Server) hitAllowed(ip string) bool {
|
||||
now := s.now()
|
||||
s.anaMu.Lock()
|
||||
defer s.anaMu.Unlock()
|
||||
if len(s.anaTry) > 20000 {
|
||||
s.anaTry = map[string][]time.Time{}
|
||||
}
|
||||
recent := s.anaTry[ip][:0]
|
||||
for _, t := range s.anaTry[ip] {
|
||||
if now.Sub(t) < time.Minute {
|
||||
recent = append(recent, t)
|
||||
}
|
||||
}
|
||||
if len(recent) >= 120 {
|
||||
s.anaTry[ip] = recent
|
||||
return false
|
||||
}
|
||||
s.anaTry[ip] = append(recent, now)
|
||||
return true
|
||||
}
|
||||
|
||||
func clip(s string, n int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) > n {
|
||||
s = strings.ToValidUTF8(s[:n], "")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// collect receives a page view or event from the tracker script. It is public by design, so it
|
||||
// answers 204 to everything it ignores and gives nothing away.
|
||||
func (s *Server) collect(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(204)
|
||||
return
|
||||
}
|
||||
defer w.WriteHeader(204)
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 4<<10))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var in struct {
|
||||
Website, Path, Referrer, Hostname, Language, Event string
|
||||
Data map[string]any
|
||||
}
|
||||
if json.Unmarshal(body, &in) != nil {
|
||||
return
|
||||
}
|
||||
var site Website
|
||||
if s.st.get("website", in.Website, &site) != nil {
|
||||
return
|
||||
}
|
||||
host := strings.ToLower(in.Hostname)
|
||||
if site.Domain != "" && host != site.Domain && !strings.HasSuffix(host, "."+site.Domain) {
|
||||
return
|
||||
}
|
||||
ua := r.UserAgent()
|
||||
if ua == "" || botRe.MatchString(ua) {
|
||||
return
|
||||
}
|
||||
ip := s.realIP(r)
|
||||
if !s.hitAllowed(ip) {
|
||||
return
|
||||
}
|
||||
path := in.Path
|
||||
if i := strings.IndexAny(path, "?"); i >= 0 {
|
||||
path = path[:i]
|
||||
}
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
path = "/" + path
|
||||
}
|
||||
ref := ""
|
||||
if u, err := url.Parse(in.Referrer); err == nil && u.Host != "" {
|
||||
if rh := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www."); rh != strings.TrimPrefix(host, "www.") {
|
||||
ref = rh
|
||||
}
|
||||
}
|
||||
country := strings.ToUpper(clip(r.Header.Get("CF-IPCountry"), 2))
|
||||
if country == "" {
|
||||
country = strings.ToUpper(clip(r.Header.Get("CloudFront-Viewer-Country"), 2))
|
||||
}
|
||||
if country == "XX" || country == "T1" {
|
||||
country = ""
|
||||
}
|
||||
lang := strings.ToLower(clip(in.Language, 12))
|
||||
if i := strings.IndexAny(lang, "-_"); i > 0 {
|
||||
lang = lang[:i]
|
||||
}
|
||||
data := ""
|
||||
if in.Event != "" && len(in.Data) > 0 {
|
||||
if b, err := json.Marshal(in.Data); err == nil && len(b) <= 1024 {
|
||||
data = string(b)
|
||||
}
|
||||
}
|
||||
browser, osName, device := parseUA(ua)
|
||||
now := s.now()
|
||||
s.st.db.Exec(`INSERT INTO hits(website,time,visitor,event,path,referrer,browser,os,device,country,lang,data) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
site.ID, now.Unix(), s.visitorID(site.ID, ip, ua, now), clip(in.Event, 80), clip(path, 200), ref, browser, osName, device, country, lang, data)
|
||||
}
|
||||
|
||||
// ---------- reading ----------
|
||||
|
||||
type topRow struct {
|
||||
Name string `json:"name"`
|
||||
Visitors int `json:"visitors"`
|
||||
Views int `json:"views"` // page views, or the number of times for an event
|
||||
}
|
||||
|
||||
type seriesPoint struct {
|
||||
Time int64 `json:"t"`
|
||||
Views int `json:"views"`
|
||||
Visitors int `json:"visitors"`
|
||||
}
|
||||
|
||||
type siteStats struct {
|
||||
Range string `json:"range"`
|
||||
Bucket string `json:"bucket"` // "hour" or "day"
|
||||
Visitors int `json:"visitors"`
|
||||
Views int `json:"views"`
|
||||
Visits int `json:"visits"`
|
||||
BounceRate float64 `json:"bounceRate"` // percent of visits with one page view; -1 if no visits
|
||||
Online int `json:"online"` // distinct visitors in the last 5 minutes
|
||||
Series []seriesPoint `json:"series"`
|
||||
Pages []topRow `json:"pages"`
|
||||
Referrers []topRow `json:"referrers"`
|
||||
Browsers []topRow `json:"browsers"`
|
||||
Systems []topRow `json:"systems"`
|
||||
Devices []topRow `json:"devices"`
|
||||
Countries []topRow `json:"countries"`
|
||||
Languages []topRow `json:"languages"`
|
||||
Events []topRow `json:"events"`
|
||||
}
|
||||
|
||||
var ranges = map[string]time.Duration{"24h": 24 * time.Hour, "7d": 7 * 24 * time.Hour, "30d": 30 * 24 * time.Hour, "90d": 90 * 24 * time.Hour}
|
||||
|
||||
// visitGap is how long a visitor can be quiet before their next page view starts a new visit.
|
||||
const visitGap = 30 * 60
|
||||
|
||||
func (s *Server) websiteStats(id, rng string) (*siteStats, error) {
|
||||
span, ok := ranges[rng]
|
||||
if !ok {
|
||||
rng, span = "7d", ranges["7d"]
|
||||
}
|
||||
now := s.now()
|
||||
from := now.Add(-span).Unix()
|
||||
out := &siteStats{Range: rng, Bucket: "day", BounceRate: -1, Series: []seriesPoint{}}
|
||||
step := int64(86400)
|
||||
if rng == "24h" {
|
||||
out.Bucket, step = "hour", 3600
|
||||
}
|
||||
db := s.st.db
|
||||
|
||||
db.QueryRow(`SELECT count(DISTINCT visitor), count(*) FROM hits WHERE website=? AND time>=? AND event=''`, id, from).Scan(&out.Visitors, &out.Views)
|
||||
db.QueryRow(`SELECT count(DISTINCT visitor) FROM hits WHERE website=? AND time>=?`, id, now.Add(-5*time.Minute).Unix()).Scan(&out.Online)
|
||||
|
||||
// Visits and bounces: a visitor's page views, split wherever they were quiet for half an hour.
|
||||
if rows, err := db.Query(`SELECT visitor,time FROM hits WHERE website=? AND time>=? AND event='' ORDER BY visitor,time`, id, from); err == nil {
|
||||
var prev string
|
||||
var prevT int64
|
||||
views, bounces := 0, 0
|
||||
closeVisit := func() {
|
||||
if views > 0 {
|
||||
out.Visits++
|
||||
}
|
||||
if views == 1 {
|
||||
bounces++
|
||||
}
|
||||
}
|
||||
for rows.Next() {
|
||||
var v string
|
||||
var t int64
|
||||
if rows.Scan(&v, &t) != nil {
|
||||
continue
|
||||
}
|
||||
if v != prev || t-prevT > visitGap {
|
||||
closeVisit()
|
||||
views = 0
|
||||
}
|
||||
views++
|
||||
prev, prevT = v, t
|
||||
}
|
||||
closeVisit()
|
||||
rows.Close()
|
||||
if out.Visits > 0 {
|
||||
out.BounceRate = float64(bounces) * 100 / float64(out.Visits)
|
||||
}
|
||||
}
|
||||
|
||||
// The chart: one point per hour or UTC day, empty ones included so the line has no gaps.
|
||||
buckets := map[int64]seriesPoint{}
|
||||
if rows, err := db.Query(`SELECT (time/?)*?, count(*), count(DISTINCT visitor) FROM hits WHERE website=? AND time>=? AND event='' GROUP BY 1`, step, step, id, from); err == nil {
|
||||
for rows.Next() {
|
||||
var p seriesPoint
|
||||
if rows.Scan(&p.Time, &p.Views, &p.Visitors) == nil {
|
||||
buckets[p.Time] = p
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
for t := (from / step) * step; t <= now.Unix(); t += step {
|
||||
p := buckets[t]
|
||||
p.Time = t
|
||||
out.Series = append(out.Series, p)
|
||||
}
|
||||
|
||||
top := func(col string, events bool, empty string) []topRow {
|
||||
cond := "event=''"
|
||||
sel := col
|
||||
if events {
|
||||
cond, sel = "event!=''", "event"
|
||||
}
|
||||
rows, err := db.Query(`SELECT `+sel+`, count(DISTINCT visitor), count(*) FROM hits WHERE website=? AND time>=? AND `+cond+
|
||||
` GROUP BY 1 ORDER BY 3 DESC, 1 LIMIT 10`, id, from)
|
||||
list := []topRow{}
|
||||
if err != nil {
|
||||
return list
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var r topRow
|
||||
if rows.Scan(&r.Name, &r.Visitors, &r.Views) == nil {
|
||||
if r.Name == "" {
|
||||
r.Name = empty
|
||||
}
|
||||
list = append(list, r)
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
out.Pages = top("path", false, "/")
|
||||
out.Referrers = top("referrer", false, "Direct")
|
||||
out.Browsers = top("browser", false, "Other")
|
||||
out.Systems = top("os", false, "Other")
|
||||
out.Devices = top("device", false, "Other")
|
||||
out.Countries = top("country", false, "Unknown")
|
||||
out.Languages = top("lang", false, "Unknown")
|
||||
out.Events = top("", true, "")
|
||||
sort.SliceStable(out.Series, func(i, j int) bool { return out.Series[i].Time < out.Series[j].Time })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Server) websiteStatsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
var site Website
|
||||
if s.st.get("website", id, &site) != nil {
|
||||
writeJSON(w, 404, map[string]string{"error": "No such website."})
|
||||
return
|
||||
}
|
||||
st, _ := s.websiteStats(id, r.URL.Query().Get("range"))
|
||||
writeJSON(w, 200, st)
|
||||
}
|
||||
|
||||
// websiteSummary gives the list page a number for each site: visitors and page views in the last day.
|
||||
func (s *Server) websiteSummary(w http.ResponseWriter, r *http.Request) {
|
||||
out := map[string]map[string]int{}
|
||||
since := s.now().Add(-24 * time.Hour).Unix()
|
||||
rows, err := s.st.db.Query(`SELECT website, count(DISTINCT visitor), count(*) FROM hits WHERE time>=? AND event='' GROUP BY website`, since)
|
||||
if err == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var v, p int
|
||||
if rows.Scan(&id, &v, &p) == nil {
|
||||
out[id] = map[string]int{"visitors": v, "views": p}
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
Reference in new issue
Block a user