Terminal without an open port: the agent calls back, and the SSH dialog shows its state

Bamboo reports where its SSH server listens. When a terminal is opened,
Hammerhead lists the session in the site's config and the agent connects out and
joins that connection to its SSH server; Hammerhead signs in over it. No port
resource or edge is involved. A site whose Bamboo does not report an SSH server
still works through an SSH port. The long poll now returns at once for a session
the agent has not been told about, so a request between two polls is not missed.

The SSH dialog shows whether the SSH server is running, says when no port is
needed, and offers the port only for an ordinary ssh client. Builds against Reef
v0.1.12 and Bamboo v0.1.9.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

terminal.go

@@ -280,16 +280,11 @@ func (s *Server) runTerminal(ws *websocket.Conn, c *orgCtx, siteID string, cols,
 	}
 	defer ws.Close()
 
-	addr, err := s.terminalAddr(c, siteID)
-	if err != nil {
-		problem(err.Error())
-		return
-	}
 	s.mu.Lock()
 	signer, err := s.terminalSignerLocked()
-	siteName := ""
+	siteName, viaAgent, agentCanRelay := "", false, false
 	if st := s.findSite(siteID); st != nil {
-		siteName = st.Name
+		siteName, viaAgent, agentCanRelay = st.Name, st.SSH != "" && st.SSHRelay, st.SSHRelay
 	}
 	s.mu.Unlock()
 	if err != nil {
@@ -304,14 +299,39 @@ func (s *Server) runTerminal(ws *websocket.Conn, c *orgCtx, siteID string, cols,
 		return
 	}
 
-	client, err := ssh.Dial("tcp", addr, &ssh.ClientConfig{User: "hammerhead", Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
-		HostKeyCallback: s.trustHostKey(siteID, siteName), Timeout: termDialTimeout})
+	cfg := &ssh.ClientConfig{User: "hammerhead", Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		HostKeyCallback: s.trustHostKey(siteID, siteName), Timeout: termDialTimeout}
+	var client *ssh.Client
+	if viaAgent {
+		// Bamboo reports its SSH server, so the agent calls back and no port has to be open anywhere.
+		var release func()
+		client, release, err = s.dialViaAgent(siteID, cfg)
+		defer func() {
+			if release != nil {
+				release()
+			}
+		}()
+	} else {
+		addr, aerr := s.terminalAddr(c, siteID)
+		if aerr != nil {
+			switch {
+			case agentCanRelay: // a current Bamboo whose SSH server is not running
+				problem("That machine's SSH server isn't running. Start Bamboo there with -ssh, and add Hammerhead's public key to authorized_keys.")
+			default:
+				problem(aerr.Error() + " Updating Bamboo on that machine lets the terminal work without any port.")
+			}
+			return
+		}
+		client, err = ssh.Dial("tcp", addr, cfg)
+	}
 	if err != nil {
 		var hk *hostKeyChanged
 		switch {
 		case errors.As(err, &hk):
 			control(termControl{Type: "error", Message: hk.Error(), HostKeyChanged: true})
 			time.Sleep(200 * time.Millisecond)
+		case errors.Is(err, errAgentSilent):
+			problem("The site didn't answer. Is Bamboo running there and connected to Hammerhead?")
 		case strings.Contains(err.Error(), "unable to authenticate"):
 			problem("The site refused Hammerhead's key. Add Hammerhead's public key to authorized_keys on that machine.")
 		default:
@@ -416,3 +436,133 @@ func (s *Server) runTerminal(ws *websocket.Conn, c *orgCtx, siteID string, cols,
 		}
 	}
 }
+
+// ---------- the call-back: a terminal without any open port ----------
+
+// termCallBack is how long Hammerhead waits for an agent to answer a terminal request. The agent
+// notices the request within a moment, because it is long-polling for its configuration.
+const termCallBack = 15 * time.Second
+
+var errAgentSilent = errors.New("the agent did not call back")
+
+// pendingTerm is a terminal session waiting for the site's agent to call back.
+type pendingTerm struct {
+	site    string
+	created time.Time
+	claimed bool
+	offered bool          // an agent has been sent this session in its configuration
+	conn    chan net.Conn // the agent's call-back connection, once it arrives
+	done    chan struct{} // closed when Hammerhead has finished with that connection
+}
+
+// wakeAgentsLocked ends every long poll now, so an agent hears about a change at once.
+// The configuration version does not move, so nothing is saved or re-applied.
+func (s *Server) wakeAgentsLocked() {
+	close(s.changed)
+	s.changed = make(chan struct{})
+}
+
+// waitingTerminalsLocked lists the sessions an agent should answer, and forgets stale ones.
+func (s *Server) waitingTerminalsLocked(siteID string) []string {
+	var ids []string
+	for id, p := range s.terms {
+		if time.Since(p.created) > 2*termCallBack {
+			delete(s.terms, id)
+			continue
+		}
+		if p.site == siteID && !p.claimed {
+			ids = append(ids, id)
+			p.offered = true
+		}
+	}
+	return ids
+}
+
+// newTerminalsLocked reports whether the site has a session its agent has not been told about yet,
+// which makes that agent's long poll return straight away instead of waiting for a change.
+func (s *Server) newTerminalsLocked(siteID string) bool {
+	for _, p := range s.terms {
+		if p.site == siteID && !p.claimed && !p.offered {
+			return true
+		}
+	}
+	return false
+}
+
+// dialViaAgent asks the site's agent to call back and signs in to its SSH server over that
+// connection. release must be called when the session is over.
+func (s *Server) dialViaAgent(siteID string, cfg *ssh.ClientConfig) (*ssh.Client, func(), error) {
+	id := randHex(12)
+	p := &pendingTerm{site: siteID, created: time.Now(), conn: make(chan net.Conn, 1), done: make(chan struct{})}
+	s.mu.Lock()
+	s.terms[id] = p
+	s.wakeAgentsLocked()
+	s.mu.Unlock()
+	var once sync.Once
+	release := func() {
+		once.Do(func() {
+			s.mu.Lock()
+			delete(s.terms, id)
+			s.mu.Unlock()
+			close(p.done)
+		})
+	}
+	var conn net.Conn
+	select {
+	case conn = <-p.conn:
+	case <-time.After(termCallBack):
+		release()
+		return nil, nil, errAgentSilent
+	}
+	conn.SetDeadline(time.Now().Add(termDialTimeout))
+	cc, chans, reqs, err := ssh.NewClientConn(conn, "site", cfg)
+	if err != nil {
+		conn.Close()
+		release()
+		return nil, nil, err
+	}
+	conn.SetDeadline(time.Time{})
+	return ssh.NewClient(cc, chans, reqs), release, nil
+}
+
+// terminalCallBack is where an agent connects to answer a terminal request. The connection is
+// handed to the session that asked for it and carries that session's SSH traffic.
+func (s *Server) terminalCallBack(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	st := s.siteByToken(r)
+	var p *pendingTerm
+	if st != nil {
+		if q := s.terms[r.PathValue("id")]; q != nil && q.site == st.ID && !q.claimed {
+			q.claimed, p = true, q
+		}
+	}
+	s.mu.Unlock()
+	switch {
+	case st == nil:
+		fail(w, 401, errors.New("bad site token"))
+		return
+	case p == nil:
+		fail(w, 404, errors.New("no such terminal"))
+		return
+	}
+	websocket.Server{
+		Handshake: func(*websocket.Config, *http.Request) error { return nil }, // authenticated by the site token
+		Handler: func(ws *websocket.Conn) {
+			ws.PayloadType = websocket.BinaryFrame
+			ws.MaxPayloadBytes = termMaxFrame
+			select {
+			case p.conn <- ws:
+				<-p.done // the websocket closes when this handler returns
+			case <-time.After(termCallBack):
+			}
+		},
+	}.ServeHTTP(w, r)
+}
+
+// clipText cuts s to at most n bytes.
+func clipText(s string, n int) string {
+	if len(s) > n {
+		return s[:n]
+	}
+	return s
+}