Merge branch 'feature/discovered-services'

agent.go

@@ -259,6 +259,7 @@ func (s *Server) siteReport(w http.ResponseWriter, r *http.Request) {
 	answered := noteUpdateReport(&st.Version, &st.AutoUpdate, &st.UpdateID, &st.UpdateMsg, in.Version, in.AutoUpdate, in.UpdateID, in.UpdateMsg)
 	s.recordUpdateProgress(st.ID, in.UpdateID, in.UpdateStage, in.UpdatePct, in.UpdateLog, oldVersion, st.Version)
 	s.containers[st.ID] = in.Containers
+	s.discovered[st.ID] = cleanDiscovered(in.Discovered)
 	for _, x := range s.st.MailEgress {
 		if x.SiteID != st.ID {
 			continue

discovered_test.go

@@ -0,0 +1,36 @@
+package hammerhead
+
+import (
+	"strings"
+	"testing"
+
+	"git.veillax.com/Veillax/reef/model"
+)
+
+func TestCleanDiscoveredDropsBadEntriesAndBoundsText(t *testing.T) {
+	in := []model.DiscoveredService{
+		{Host: "192.168.1.5", Port: 22, Proto: "udp", Service: "ssh", Detail: "SSH-2.0", Source: "scan"},
+		{Host: "127.0.0.1", Port: 8080, Service: strings.Repeat("x", 100), Detail: strings.Repeat("d", 1000), Source: "local"},
+		{Host: "<script>", Port: 80},
+		{Host: "10.0.0.1", Port: 0},
+		{Host: "10.0.0.1", Port: 70000},
+		{Host: "10.0.0.2", Port: 80, Source: "whatever"},
+	}
+	got := cleanDiscovered(in)
+	if len(got) != 3 {
+		t.Fatalf("want 3 valid entries, got %+v", got)
+	}
+	if got[0].Proto != "tcp" || got[1].Source != "local" || len(got[1].Service) != 20 || len(got[1].Detail) != 160 || got[2].Source != "scan" {
+		t.Errorf("not normalised: %+v", got)
+	}
+	many := make([]model.DiscoveredService, 900)
+	for i := range many {
+		many[i] = model.DiscoveredService{Host: "10.0.0.1", Port: i%60000 + 1}
+	}
+	if n := len(cleanDiscovered(many)); n != 500 {
+		t.Errorf("not capped: %d", n)
+	}
+	if cleanDiscovered(nil) == nil {
+		t.Error("should be an empty list, not nil, so the API returns []")
+	}
+}

infra.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"net"
 	"net/http"
+	"net/netip"
 	"sort"
 	"strings"
 	"time"
@@ -302,6 +303,47 @@ func (s *Server) siteContainers(w http.ResponseWriter, r *http.Request, c *orgCt
 	writeJSON(w, 200, out)
 }
 
+// cleanDiscovered keeps what an agent reports as suggestions only if it is well formed: it is shown in
+// the dashboard, so lengths are bounded and anything that is not a plain address is dropped.
+func cleanDiscovered(in []model.DiscoveredService) []model.DiscoveredService {
+	out := []model.DiscoveredService{}
+	clip := func(s string, n int) string {
+		if len(s) > n {
+			s = s[:n]
+		}
+		return strings.ToValidUTF8(s, "")
+	}
+	for _, d := range in {
+		if len(out) >= 500 {
+			break
+		}
+		if _, err := netip.ParseAddr(d.Host); err != nil || d.Port < 1 || d.Port > 65535 {
+			continue
+		}
+		src := "scan"
+		if d.Source == "local" {
+			src = "local"
+		}
+		out = append(out, model.DiscoveredService{Host: d.Host, Port: d.Port, Proto: "tcp", Service: clip(d.Service, 20), Detail: clip(d.Detail, 160), Source: src})
+	}
+	return out
+}
+
+func (s *Server) siteDiscovered(w http.ResponseWriter, r *http.Request, c *orgCtx) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	st := s.findSite(r.PathValue("id"))
+	if st == nil || !s.siteVisible(c, st) {
+		fail(w, 404, errors.New("no such site"))
+		return
+	}
+	out := s.discovered[st.ID]
+	if out == nil {
+		out = []model.DiscoveredService{}
+	}
+	writeJSON(w, 200, out)
+}
+
 // ---------- domains ----------
 
 func domainsOverlap(a, b string) bool {

server.go

@@ -134,6 +134,7 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("PUT /api/orgs/{org}/sites/{id}", s.withOrg(true, s.putSite))
 	mux.HandleFunc("DELETE /api/orgs/{org}/sites/{id}", s.withOrg(true, s.deleteSite))
 	mux.HandleFunc("GET /api/orgs/{org}/sites/{id}/containers", s.withOrg(true, s.siteContainers))
+	mux.HandleFunc("GET /api/orgs/{org}/sites/{id}/discovered", s.withOrg(true, s.siteDiscovered))
 	mux.HandleFunc("POST /api/orgs/{org}/domains", s.withOrg(true, s.addDomain))
 	mux.HandleFunc("DELETE /api/orgs/{org}/domains/{id}", s.withOrg(true, s.deleteDomain))
 	mux.HandleFunc("POST /api/orgs/{org}/domains/{id}/check", s.withOrg(true, s.checkDomain))

store.go

@@ -199,9 +199,10 @@ type Server struct {
 	opts Options
 
 	status     map[string]model.MappingStatus
-	mailStatus map[string]string            // mail egress ID -> "" (listening) or an error
-	mailChecks map[string]model.MailCheck   // mail egress ID -> the edge's deliverability report
-	containers map[string][]model.Container // by site ID
+	mailStatus map[string]string                    // mail egress ID -> "" (listening) or an error
+	mailChecks map[string]model.MailCheck           // mail egress ID -> the edge's deliverability report
+	containers map[string][]model.Container         // by site ID
+	discovered map[string][]model.DiscoveredService // by site ID: suggested targets the agent found
 	changed    chan struct{}
 
 	tickets map[string]ticket
@@ -249,6 +250,7 @@ func New(dir string, opts Options) (*Server, error) {
 		mailStatus: map[string]string{},
 		mailChecks: map[string]model.MailCheck{},
 		containers: map[string][]model.Container{},
+		discovered: map[string][]model.DiscoveredService{},
 		changed:    make(chan struct{}),
 		tickets:    map[string]ticket{},
 		oauth:      map[string]oauthState{},

web/index.html

@@ -788,21 +788,22 @@ function portForm(existing) {
       const th = h('input', {value:t.host, oninput:e => t.host = e.target.value, placeholder:'127.0.0.1 or 192.168.1.20'});
       const tp = h('input', {type:'number', value:t.port, placeholder:'port', oninput:e => t.port = +e.target.value});
       return h('div', {class:'tgt'}, field(i ? 'Target ' + (i + 1) : 'Target site', site), field('Host', th), field('Port', tp),
-        h('button', {class:'ghost', type:'button', title:'Pick a Docker container on that site', onclick:() => discover(t, th, tp)}, 'Containers'),
+        h('button', {class:'ghost', type:'button', title:'Pick a Docker container or discovered service on that site', onclick:() => discover(t, th, tp)}, 'Containers'),
         targets.length > 1 ? h('button', {class:'ghost danger', type:'button', 'aria-label':'Remove target', onclick:() => { targets.splice(i, 1); drawTargets(); }}, icon('close', 16)) : h('span'));
     }), many ? h('button', {class:'ghost', type:'button', style:'align-self:flex-start', title:'Spread visitors across targets, or fail over when one is down', onclick:() => { targets.push({siteId:targets[0] ? targets[0].siteId : '', host:'127.0.0.1', port:''}); drawTargets(); }}, icon('add', 16), 'Add another target') : null);
   };
   const discover = act(async (t, th, tp) => {
     if (!t.siteId) throw new Error('Choose a site first');
-    const list = await api('GET', O(`/sites/${t.siteId}/containers`));
-    if (!list.length) throw new Error('No containers reported. Run the agent with --docker (and make sure Docker is running).');
+    const [list, found] = await Promise.all([api('GET', O(`/sites/${t.siteId}/containers`)), api('GET', O(`/sites/${t.siteId}/discovered`))]);
+    if (!list.length && !found.length) throw new Error('Nothing reported. Run the agent with --docker for containers, or --discover (optionally --discover-subnets) for services on the network.');
     const opts = [];
+    for (const d of found) opts.push({label:`${d.service || 'tcp'}  ${d.host}:${d.port}${d.detail ? '  ' + d.detail : ''} (${d.source === 'local' ? 'listening here' : 'found by scan'})`, host:d.host, port:d.port});
     for (const c of list) for (const p of c.ports) {
       if (p.public) opts.push({label:`${c.name}  127.0.0.1:${p.public} (published)`, host:'127.0.0.1', port:p.public});
       for (const [n, ip] of Object.entries(c.networks)) opts.push({label:`${c.name}  ${ip}:${p.private} (${n} network)`, host:ip, port:p.private});
     }
     const sel = h('select', {size:Math.min(12, opts.length + 1)}, opts.map((o, i) => h('option', {value:i}, o.label)));
-    const close = modal('Pick a container', h('div', {class:'stack'}, sel, h('button', {class:'btn', onclick:() => { const o = opts[sel.value]; if (!o) return; t.host = th.value = o.host; t.port = o.port; tp.value = o.port; close(); }}, 'Use this')));
+    const close = modal('Pick a container or discovered service', h('div', {class:'stack'}, sel, h('button', {class:'btn', onclick:() => { const o = opts[sel.value]; if (!o) return; t.host = th.value = o.host; t.port = o.port; tp.value = o.port; close(); }}, 'Use this')));
   });
   const drawRules = () => ruleBox.replaceChildren(...rules.map((r, i) => h('div', {class:'grid', style:'grid-template-columns:110px 110px 1fr auto'},
     h('select', {onchange:e => r.action = e.target.value}, ['allow', 'deny'].map(v => h('option', {value:v, selected:r.action === v}, v === 'allow' ? 'Skip login' : 'Block'))),