Reef/model/model.go

// Package model holds the types shared by hammerhead, blacktip and bamboo.
package model

import (
	"sort"
	"strings"
	"time"
)

const (
	ProtoTCP   = "tcp"   // raw TCP on ip:port
	ProtoUDP   = "udp"   // raw UDP on ip:port
	ProtoHTTP  = "http"  // reverse proxy, routed by Host header (many hostnames per ip:port)
	ProtoHTTPS = "https" // reverse proxy with automatic Let's Encrypt certificates
	ProtoTLS   = "tls"   // TLS passthrough, routed by SNI (many hostnames per ip:port)
)

// LastPort is the last public port a mapping occupies (Port itself unless it is a range).
func (m Mapping) LastPort() int {
	if m.PortEnd > m.Port {
		return m.PortEnd
	}
	return m.Port
}

// IsRedirect reports whether the mapping answers with a redirect instead of forwarding to a target.
func (m Mapping) IsRedirect() bool { return m.RedirectTo != "" }

// RedirectLocation is where a request for requestURI (path and query) is sent. With KeepPath the
// request's path and query follow the destination, so example.com/a?b=1 goes to to/a?b=1.
func (m Mapping) RedirectLocation(requestURI string) string {
	if !m.RedirectKeepPath {
		return m.RedirectTo
	}
	if requestURI == "" || requestURI[0] != '/' {
		requestURI = "/" + requestURI
	}
	return strings.TrimRight(m.RedirectTo, "/") + requestURI
}

// HostBased reports whether mappings of this protocol are routed by hostname.
func HostBased(proto string) bool {
	return proto == ProtoHTTP || proto == ProtoHTTPS || proto == ProtoTLS
}

// Web reports whether the protocol is served by the HTTP reverse proxy.
func Web(proto string) bool { return proto == ProtoHTTP || proto == ProtoHTTPS }

// Edge is a blacktip node: a public server that owns one or more IP addresses.
type Edge struct {
	ID         string    `json:"id"`
	Name       string    `json:"name"`
	Token      string    `json:"token,omitempty"`
	AuthKey    string    `json:"authKey,omitempty"` // signs resource-auth tokens; never sent to browsers
	TunnelAddr string    `json:"tunnelAddr"`        // public host:port (UDP) of the edge's WireGuard endpoint
	TunnelIP   string    `json:"tunnelIp"`          // the edge's address inside the tunnel network
	PublicKey  string    `json:"publicKey"`         // WireGuard public key, reported by the edge
	OrgIDs     []string  `json:"orgIds"`            // orgs allowed to use this edge; empty = all
	IPs        []string  `json:"ips"`               // addresses the edge reported it can bind
	LastSeen   time.Time `json:"lastSeen"`

	// Version is the release the program reports (e.g. v0.1.5). AutoUpdate says whether it may update
	// itself; only then will it act on an update request (UpdateID). UpdateMsg is the outcome of the last requested update.
	Version    string `json:"version,omitempty"`
	AutoUpdate bool   `json:"autoUpdate,omitempty"`
	UpdateID   string `json:"updateId,omitempty"` // set while an update requested by an administrator is pending
	UpdateMsg  string `json:"updateMsg,omitempty"`
}

// Site is a bamboo agent: a private network that holds the real services.
type Site struct {
	ID        string    `json:"id"`
	OrgID     string    `json:"orgId"`
	Name      string    `json:"name"`
	Token     string    `json:"token,omitempty"`
	Connected []string  `json:"connected"` // edge IDs the agent currently has tunnels to
	LastSeen  time.Time `json:"lastSeen"`
	// Local sites are created automatically with each edge. Mappings that use
	// one are dialed directly from the edge machine, so no bamboo is needed.
	Local  bool   `json:"local"`
	EdgeID string `json:"edgeId,omitempty"` // the edge a local site belongs to

	Tags []string `json:"tags,omitempty"` // labels for grouping and filtering in the dashboard

	// What the agent last said about its SSH server (see SiteReport.SSH and SSHRelay).
	SSH      string `json:"ssh,omitempty"`
	SSHRelay bool   `json:"sshRelay,omitempty"`

	TunnelIP  string `json:"tunnelIp,omitempty"`  // the site's address inside the tunnel network
	PublicKey string `json:"publicKey,omitempty"` // WireGuard public key, reported by the agent

	// Version is the release the program reports (e.g. v0.1.5). AutoUpdate says whether it may update
	// itself; only then will it act on an update request (UpdateID). UpdateMsg is the outcome of the last requested update.
	Version    string `json:"version,omitempty"`
	AutoUpdate bool   `json:"autoUpdate,omitempty"`
	UpdateID   string `json:"updateId,omitempty"` // set while an update requested by an administrator is pending
	UpdateMsg  string `json:"updateMsg,omitempty"`
}

// Domain is a base domain an org owns. Hostnames of web/tls mappings must sit under one.
type Domain struct {
	ID     string `json:"id"`
	OrgID  string `json:"orgId"`
	Name   string `json:"name"`
	EdgeID string `json:"edgeId"`
	IP     string `json:"ip"`
	Note   string `json:"note"`
}

// Target is one backend a mapping forwards to.
type Target struct {
	SiteID string `json:"siteId"`
	Host   string `json:"host"`
	Port   int    `json:"port"`
}

// Auth is the access policy in front of a web mapping. Secrets (password, PIN)
// live only in hammerhead; these flags say which methods are switched on.
type Auth struct {
	SSO          bool     `json:"sso"`      // sign in with a hammerhead account
	Password     bool     `json:"password"` // shared password
	Pin          bool     `json:"pin"`      // 4-12 digit PIN
	Email        bool     `json:"email"`    // one-time code emailed to an allowed address
	AllowedRoles []string `json:"allowedRoles"`
	AllowedUsers []string `json:"allowedUsers"`
	AllowedMails []string `json:"allowedMails"` // addresses allowed for email codes ("*@example.com" ok)
	SessionHours int      `json:"sessionHours"` // 0 = 24
}

// Protected reports whether visitors must authenticate.
func (a Auth) Protected() bool { return a.SSO || a.Password || a.Pin || a.Email }

// Rule is evaluated before authentication, top to bottom.
type Rule struct {
	Action string `json:"action"` // "allow" (skip auth) or "deny"
	Match  string `json:"match"`  // "path" (prefix) or "ip" (address or CIDR)
	Value  string `json:"value"`
}

// Mapping opens one public port (or a hostname on a shared port) and sends it to one or more targets.
type Mapping struct {
	ID       string   `json:"id"`
	OrgID    string   `json:"orgId"`
	Name     string   `json:"name"`
	Proto    string   `json:"proto"`
	EdgeID   string   `json:"edgeId"`
	BindIP   string   `json:"bindIp"` // 0.0.0.0 = every address on the edge
	Port     int      `json:"port"`
	PortEnd  int      `json:"portEnd,omitempty"`  // tcp/udp only: Port..PortEnd are all opened, each sent to the target port shifted by the same offset
	Hostname string   `json:"hostname,omitempty"` // web/tls only; "*.example.com" allowed
	Targets  []Target `json:"targets"`            // udp uses only the first
	Enabled  bool     `json:"enabled"`
	Tags     []string `json:"tags,omitempty"` // labels for grouping and filtering in the dashboard
	// ProxyProtocol ("v1" or "v2") makes the edge start each tcp/tls connection to the
	// target with a PROXY protocol header carrying the real client address.
	ProxyProtocol string `json:"proxyProtocol,omitempty"`

	// A web mapping with RedirectTo set sends visitors there instead of to a target (so it has no
	// targets, login or health checks). It answers 301 when RedirectPermanent, otherwise 302.
	RedirectTo        string `json:"redirectTo,omitempty"`
	RedirectPermanent bool   `json:"redirectPermanent,omitempty"`
	RedirectKeepPath  bool   `json:"redirectKeepPath,omitempty"`

	// web (http/https) options
	Auth          Auth   `json:"auth"`
	AuthEpoch     int64  `json:"authEpoch"` // bumped to sign every visitor out; set by hammerhead only
	Rules         []Rule `json:"rules"`
	RedirectHTTP  bool   `json:"redirectHttp"`  // https: also answer on port 80 with a redirect
	BackendHTTPS  bool   `json:"backendHttps"`  // talk TLS (unverified) to the targets
	RewriteHost   string `json:"rewriteHost"`   // Host header sent to the target; "" keeps the original
	HealthPath    string `json:"healthPath"`    // http(s): GET this path; "" = TCP connect check
	HealthSeconds int    `json:"healthSeconds"` // 0 = 30; -1 = off
}

// MailEgress lets a mail server at a site send its outgoing mail from an edge's
// public IP. Bamboo runs a small SMTP relay on the site's machine; each message
// travels through the WireGuard tunnel and Blacktip delivers it straight to the
// recipient's mail server (found by MX lookup) from SourceIP, introducing itself
// as HeloName. Reverse DNS, SPF and the HELO name can then all agree, which is
// what receiving servers check.
type MailEgress struct {
	ID            string `json:"id"`
	OrgID         string `json:"orgId"`
	Name          string `json:"name"`
	SiteID        string `json:"siteId"`
	EdgeID        string `json:"edgeId"`
	Listen        string `json:"listen"`        // host:port of the SMTP relay on the site's machine
	SourceIP      string `json:"sourceIp"`      // edge address mail leaves from (its PTR should match HeloName)
	HeloName      string `json:"heloName"`      // name announced to receiving servers, e.g. mail.example.com
	Smarthost     string `json:"smarthost"`     // optional host:port: relay through this server instead of looking up MX
	RequireTLS    bool   `json:"requireTls"`    // refuse to send to servers that don't offer verifiable STARTTLS
	RatePerMinute int    `json:"ratePerMinute"` // messages per minute the edge will send; 0 = 60
	Username      string `json:"username,omitempty"`
	Password      string `json:"password,omitempty"`
	Enabled       bool   `json:"enabled"`
}

// MailSpec is what a site agent needs to run the SMTP relay.
type MailSpec struct {
	ID       string `json:"id"`
	EdgeID   string `json:"edgeId"`
	Listen   string `json:"listen"`
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
}

// DKIMKey is a signing key for one sending domain. The edge signs outgoing mail
// with it; the public half goes in DNS at <selector>._domainkey.<domain>.
type DKIMKey struct {
	ID         string    `json:"id"`
	OrgID      string    `json:"orgId"`
	Domain     string    `json:"domain"`
	Selector   string    `json:"selector"`
	PrivateKey string    `json:"privateKey,omitempty"` // PEM, never sent to browsers
	PublicKey  string    `json:"publicKey"`            // base64 DER, the p= value of the TXT record
	Created    time.Time `json:"created"`
}

// DKIMSigner is what an edge needs to sign mail for one domain.
type DKIMSigner struct {
	KeyID      string `json:"keyId"`
	Domain     string `json:"domain"`
	Selector   string `json:"selector"`
	PrivateKey string `json:"privateKey"`
	PublicKey  string `json:"publicKey"`
}

// MailPolicy is what an edge needs to deliver mail for one MailEgress.
type MailPolicy struct {
	ID            string `json:"id"`
	SiteID        string `json:"siteId"`
	SourceIP      string `json:"sourceIp"`
	HeloName      string `json:"heloName"`
	Smarthost     string `json:"smarthost"`
	RequireTLS    bool   `json:"requireTls"`
	RatePerMinute int    `json:"ratePerMinute"`

	DKIM []DKIMSigner `json:"dkim"` // keys for the org's sending domains
}

// MailCheck is the edge's report on how deliverable mail from a source IP is.
type MailCheck struct {
	CheckedAt      time.Time         `json:"checkedAt"`
	PTR            []string          `json:"ptr"`            // reverse DNS names of the source IP
	PTRMatchesHelo bool              `json:"ptrMatchesHelo"` // one of them equals the HELO name
	HeloResolves   bool              `json:"heloResolves"`   // the HELO name resolves back to the source IP
	Port25         string            `json:"port25"`         // "open" or why not
	DKIM           map[string]string `json:"dkim"`           // key ID -> "published", "missing" or "mismatch"

	Domains    map[string]DomainCheck `json:"domains,omitempty"`    // sending domain -> its DNS checks
	Blocklists []string               `json:"blocklists,omitempty"` // DNS blocklists that list the source IP
	Listable   int                    `json:"listable,omitempty"`   // how many blocklists answered, so "none listed" means something
	Outcomes   *MailOutcomes          `json:"outcomes,omitempty"`   // what happened to the mail this egress sent
}

// Finding statuses.
const (
	FindingPass = "pass"
	FindingWarn = "warn"
	FindingFail = "fail"
	FindingNone = "none" // nothing published
)

// MailFinding is the result of one deliverability check, with what to do about it.
type MailFinding struct {
	Status string `json:"status"`
	Detail string `json:"detail"`
	Hint   string `json:"hint,omitempty"`
}

// DomainCheck is how the DNS of one sending domain looks to a receiving server.
type DomainCheck struct {
	SPF    MailFinding `json:"spf"`
	DMARC  MailFinding `json:"dmarc"`
	MTASTS MailFinding `json:"mtaSts"`
}

// MailOutcomes counts what became of the mail an egress handled over a window.
type MailOutcomes struct {
	Since     time.Time               `json:"since"`
	Delivered int                     `json:"delivered"`
	Deferred  int                     `json:"deferred"`           // 4xx: the receiver asked us to try later
	Failed    int                     `json:"failed"`             // 5xx: refused for good
	ByDomain  map[string]OutcomeCount `json:"byDomain,omitempty"` // recipient domain
	Recent    []OutcomeFailure        `json:"recent,omitempty"`   // latest problems, newest first
}

// OutcomeCount is delivered/deferred/failed for one recipient domain.
type OutcomeCount struct {
	Delivered int `json:"delivered"`
	Deferred  int `json:"deferred"`
	Failed    int `json:"failed"`
}

// OutcomeFailure is one deferral or refusal, for the dashboard to explain.
type OutcomeFailure struct {
	Time   time.Time `json:"time"`
	Domain string    `json:"domain"`
	Code   int       `json:"code"`
	Text   string    `json:"text"`
}

// ---------- edge <-> hammerhead ----------

// EdgeConfig is what blacktip long-polls from hammerhead.
type EdgeConfig struct {
	Version    int64     `json:"version"`
	EdgeID     string    `json:"edgeId"`
	AuthKey    string    `json:"authKey"`
	Dashboard  string    `json:"dashboard"` // public URL of hammerhead, for login redirects
	Mappings   []Mapping `json:"mappings"`
	TunnelIP   string    `json:"tunnelIp"` // the edge's WireGuard address
	Peers      []WGPeer  `json:"peers"`    // sites the edge may reach through the tunnel
	Local      []string  `json:"local"`    // site IDs the edge dials directly on its own machine
	AcmeEmail  string    `json:"acmeEmail"`
	AcmeStages bool      `json:"acmeStaging"`
	UpdateID   string    `json:"updateId,omitempty"` // an administrator asked this edge to update itself now; echoed back in its report

	MailEgress []MailPolicy `json:"mailEgress"`
}

// WGPeer is a site as seen from an edge.
type WGPeer struct {
	SiteID    string `json:"siteId"`
	PublicKey string `json:"publicKey"`
	IP        string `json:"ip"`
}

// TargetHealth is the latest check result for one target.
type TargetHealth struct {
	Healthy bool   `json:"healthy"`
	Detail  string `json:"detail,omitempty"`
}

// MappingStatus is what an edge reports per mapping.
type MappingStatus struct {
	Error   string         `json:"error"`          // "" = listening
	Cert    string         `json:"cert,omitempty"` // https: certificate state
	Targets []TargetHealth `json:"targets,omitempty"`
}

// Stat is traffic since the last report.
type Stat struct {
	Conns    int64 `json:"conns"`
	BytesIn  int64 `json:"bytesIn"`  // client -> target
	BytesOut int64 `json:"bytesOut"` // target -> client
	// Sources are the busiest client addresses behind these numbers (at most MaxSources),
	// busiest first. Addresses beyond that are only counted in the totals above.
	Sources []SourceStat `json:"sources,omitempty"`
}

// MaxSources is how many client addresses an edge names per port in one report.
const MaxSources = 10

// SourceStat is one client address's traffic on a port since the last report.
type SourceStat struct {
	IP       string `json:"ip"`
	Conns    int64  `json:"conns"`
	BytesIn  int64  `json:"bytesIn"`
	BytesOut int64  `json:"bytesOut"`
}

// TopSources keeps the n busiest of a port's client addresses (by bytes, then connections), busiest first.
func TopSources(all map[string]*SourceStat, n int) []SourceStat {
	out := make([]SourceStat, 0, len(all))
	for _, s := range all {
		out = append(out, *s)
	}
	sort.Slice(out, func(i, j int) bool {
		a, b := out[i], out[j]
		if ta, tb := a.BytesIn+a.BytesOut, b.BytesIn+b.BytesOut; ta != tb {
			return ta > tb
		}
		if a.Conns != b.Conns {
			return a.Conns > b.Conns
		}
		return a.IP < b.IP
	})
	if len(out) > n {
		out = out[:n]
	}
	return out
}

// LogEntry is one access-log line.
type LogEntry struct {
	ID       int64     `json:"id,omitempty"` // assigned by hammerhead
	Time     time.Time `json:"time"`
	Mapping  string    `json:"mapping"`
	OrgID    string    `json:"orgId,omitempty"` // filled in by hammerhead
	Remote   string    `json:"remote"`
	Method   string    `json:"method,omitempty"`
	Host     string    `json:"host,omitempty"`
	Path     string    `json:"path,omitempty"`
	Status   int       `json:"status,omitempty"`
	User     string    `json:"user,omitempty"`
	Result   string    `json:"result,omitempty"` // allowed, denied, auth-required, no-target
	BytesIn  int64     `json:"bytesIn"`
	BytesOut int64     `json:"bytesOut"`
	Millis   int64     `json:"ms"`
}

// Container is a Docker container seen by an agent.
type Container struct {
	ID       string            `json:"id"`
	Name     string            `json:"name"`
	Image    string            `json:"image"`
	State    string            `json:"state"`
	Ports    []ContainerPort   `json:"ports"`
	Networks map[string]string `json:"networks"` // network name -> container IP
}

type ContainerPort struct {
	Private int    `json:"private"`
	Public  int    `json:"public,omitempty"` // 0 = not published
	Type    string `json:"type"`
}

// DiscoveredService is a listening service an agent found on its own network (local sockets or an
// opt-in scan of configured subnets). Hammerhead offers these as suggested targets; nothing is
// proxied until an administrator picks one.
type DiscoveredService struct {
	Host    string `json:"host"` // address to dial, as seen from the agent
	Port    int    `json:"port"`
	Proto   string `json:"proto"`             // "tcp"
	Service string `json:"service,omitempty"` // best guess: http, https, ssh, smtp, ftp, mysql, ... (empty = unknown)
	Detail  string `json:"detail,omitempty"`  // banner or Server header, trimmed
	Source  string `json:"source"`            // "local" (listening socket on the agent's host) or "scan"
}

// EdgeReport is what blacktip posts back.
type EdgeReport struct {
	IPs       []string                 `json:"ips"`
	PublicKey string                   `json:"publicKey"` // the edge's WireGuard public key
	Status    map[string]MappingStatus `json:"status"`
	Stats     map[string]Stat          `json:"stats"` // deltas since last report
	// StatsSeconds is how long Stats covers (the time since the previous report that carried them),
	// so a rate can be worked out however irregular the reports are.
	StatsSeconds float64              `json:"statsSeconds,omitempty"`
	Containers   []Container          `json:"containers"`
	MailChecks   map[string]MailCheck `json:"mailChecks"` // by mail-egress ID
	Version      string               `json:"version,omitempty"`
	AutoUpdate   bool                 `json:"autoUpdate,omitempty"`
	UpdateMsg    string               `json:"updateMsg,omitempty"` // outcome of the update asked for by UpdateID
	UpdateID     string               `json:"updateId,omitempty"`
	// progress of that update, for the dashboard's progress bar and log
	UpdateStage string   `json:"updateStage,omitempty"`
	UpdatePct   int      `json:"updatePct,omitempty"`
	UpdateLog   []string `json:"updateLog,omitempty"`
}

// EdgeRef tells a site where to dial.
type EdgeRef struct {
	ID        string `json:"id"`
	Name      string `json:"name"`
	Endpoint  string `json:"endpoint"` // host:port (UDP) to send WireGuard packets to
	PublicKey string `json:"publicKey"`
	IP        string `json:"ip"` // the edge's address inside the tunnel
}

// SiteConfig is what bamboo long-polls from hammerhead.
type SiteConfig struct {
	Version  int64     `json:"version"`
	SiteID   string    `json:"siteId"`
	TunnelIP string    `json:"tunnelIp"` // this site's WireGuard address
	Edges    []EdgeRef `json:"edges"`
	UpdateID string    `json:"updateId,omitempty"` // an administrator asked this agent to update itself now; echoed back in its report

	// Terminals lists terminal sessions waiting for this agent. For each, the agent opens a connection
	// to hammerhead at /site/terminal/<id> and joins it to its own SSH server, so a shell can be opened
	// from the dashboard without any port being open on an edge.
	Terminals []string `json:"terminals,omitempty"`

	MailEgress []MailSpec `json:"mailEgress"`
}

// SiteReport is what bamboo posts back.
type SiteReport struct {
	PublicKey  string              `json:"publicKey"`  // the agent's WireGuard public key
	MailEgress map[string]string   `json:"mailEgress"` // egress ID -> "" (listening) or an error
	Connected  []string            `json:"connected"`
	Containers []Container         `json:"containers"`
	Discovered []DiscoveredService `json:"discovered,omitempty"`
	Version    string              `json:"version,omitempty"`
	AutoUpdate bool                `json:"autoUpdate,omitempty"`
	UpdateMsg  string              `json:"updateMsg,omitempty"`
	UpdateID   string              `json:"updateId,omitempty"`
	// SSH is where this machine's SSH server listens ("" when it is not running). SSHRelay says this
	// agent can answer SiteConfig.Terminals, which is how an agent that is too old to relay shows itself.
	SSH      string `json:"ssh,omitempty"`
	SSHRelay bool   `json:"sshRelay,omitempty"`
	// progress of that update, for the dashboard's progress bar and log
	UpdateStage string   `json:"updateStage,omitempty"`
	UpdatePct   int      `json:"updatePct,omitempty"`
	UpdateLog   []string `json:"updateLog,omitempty"`
}

// HostMatches reports whether host matches an exact name or a "*.example.com" pattern.
func HostMatches(pattern, host string) bool {
	if pattern == host {
		return true
	}
	return len(pattern) > 2 && pattern[:2] == "*." && len(host) > len(pattern)-1 && host[len(host)-len(pattern)+1:] == pattern[1:]
}