blob: 9f2d339dee29709d24e224804dec2ce061cb175c [file]
package ssh
import (
"context"
"errors"
"log/slog"
"sync"
"time"
gossh "golang.org/x/crypto/ssh"
)
// poolIdleTimeout is how long a connection nobody is using stays open for the next
// caller. Long enough to carry a burst of staging activities for one process, short
// enough that an abandoned host does not hold a login session for good.
const poolIdleTimeout = 5 * time.Minute
// poolKeepaliveInterval paces the probe that notices a connection the network dropped
// without a reset, which would otherwise sit in the pool looking healthy.
const poolKeepaliveInterval = 30 * time.Second
// poolKey names what a connection was authenticated as. It is derived from the dial
// config rather than being the config itself: the credential carries pointers, so two
// configs for the same key would compare unequal by identity alone.
type poolKey struct {
host string
port int
sshUsername string
credentialID string
userId string
}
func poolKeyFor(dialConfig SSHDialConfig) poolKey {
return poolKey{
host: dialConfig.host,
port: dialConfig.port,
sshUsername: dialConfig.sshUsername,
credentialID: dialConfig.cred.ID,
userId: dialConfig.userId,
}
}
// poolEntry is one connection, or the dial that will produce it. ready is closed once
// the dial settles, after which exactly one of client and err is set.
type poolEntry struct {
// ready is closed once the dial settles, after which exactly one of client and err is set.
ready chan struct{}
client *gossh.Client
err error
refs int
idle *time.Timer
}
// SSHConnectionPool shares one authenticated connection between every caller dialing
// with the same config. Besides saving handshakes, this keeps an interactive login
// from asking the user for a password or OTP once per command.
type SSHConnectionPool struct {
entries map[poolKey]*poolEntry
lock sync.Mutex
}
func newSSHConnectionPool() *SSHConnectionPool {
return &SSHConnectionPool{
entries: make(map[poolKey]*poolEntry),
}
}
// pooledClient is a borrowed connection. Close hands it back to the pool rather than
// closing it, so callers keep the defer client.Close() they would write anyway.
type pooledClient struct {
*gossh.Client
once sync.Once
release func()
}
func (c *pooledClient) Close() error {
c.once.Do(c.release)
return nil
}
// acquire returns a live connection for dialConfig, dialing one if there is none.
// Callers that arrive while a dial is in flight wait for it rather than starting their
// own, so concurrent work against one host shares a single login.
func (p *SSHConnectionPool) acquire(ctx context.Context, dialConfig SSHDialConfig, dial func(context.Context) (*gossh.Client, error)) (*pooledClient, error) {
key := poolKeyFor(dialConfig)
for {
p.lock.Lock()
entry, ok := p.entries[key]
if !ok {
entry = &poolEntry{ready: make(chan struct{})}
p.entries[key] = entry
}
entry.refs++
if entry.idle != nil {
entry.idle.Stop()
entry.idle = nil
}
p.lock.Unlock()
if !ok {
client, err := dial(ctx)
p.settle(key, entry, client, err)
}
select {
case <-entry.ready:
case <-ctx.Done():
p.release(key, entry)
return nil, ctx.Err()
}
if entry.err == nil {
return &pooledClient{Client: entry.client, release: func() { p.release(key, entry) }}, nil
}
p.release(key, entry)
// The dial ran on whichever caller got there first, so it may have failed only
// because that caller gave up. That says nothing about this caller's chances.
if ok && isContextError(entry.err) && ctx.Err() == nil {
continue
}
return nil, entry.err
}
}
// settle records the outcome of a dial and wakes everyone waiting on it.
func (p *SSHConnectionPool) settle(key poolKey, entry *poolEntry, client *gossh.Client, err error) {
p.lock.Lock()
entry.client, entry.err = client, err
if entry.err != nil {
// A failed dial is not cached: the next caller should try again.
p.evictLocked(key, entry)
}
p.lock.Unlock()
close(entry.ready)
if entry.client != nil {
go p.watch(key, entry)
}
}
// watch drops the entry once its connection dies, whether the server closed it or the
// keepalive gave up on it, so the next caller dials afresh instead of failing.
func (p *SSHConnectionPool) watch(key poolKey, entry *poolEntry) {
closed := make(chan struct{})
go func() {
entry.client.Wait()
slog.Info("SSH connection exited after wait", "host", key.host, "port", key.port, "user", key.sshUsername)
close(closed)
}()
ticker := time.NewTicker(poolKeepaliveInterval)
defer ticker.Stop()
for {
select {
case <-closed:
p.lock.Lock()
p.evictLocked(key, entry)
p.lock.Unlock()
return
case <-ticker.C:
if !keepalive(entry.client, poolKeepaliveInterval) {
slog.Info("Closing unresponsive pooled ssh connection", "host", key.host, "port", key.port, "user", key.sshUsername)
entry.client.Close()
p.lock.Lock()
p.evictLocked(key, entry)
p.lock.Unlock()
}
}
}
}
// keepalive reports whether the server answered a no-op global request within timeout.
// SendRequest has no deadline of its own, so a stalled one is abandoned to the caller,
// which closes the connection and so unblocks it.
func keepalive(client *gossh.Client, timeout time.Duration) bool {
answered := make(chan error, 1)
go func() {
_, _, err := client.SendRequest("keepalive@openssh.com", true, nil)
answered <- err
}()
select {
case err := <-answered:
return err == nil
case <-time.After(timeout):
return false
}
}
// release returns one reference. The last one out leaves the connection open for
// poolIdleTimeout in case another caller wants it.
func (p *SSHConnectionPool) release(key poolKey, entry *poolEntry) {
p.lock.Lock()
defer p.lock.Unlock()
entry.refs--
if entry.refs > 0 || entry.client == nil {
return
}
var timer *time.Timer
timer = time.AfterFunc(poolIdleTimeout, func() {
p.lock.Lock()
defer p.lock.Unlock()
// A caller may have taken the entry, and perhaps released it again under a
// fresh timer, between this one firing and the lock.
if entry.idle != timer {
return
}
p.evictLocked(key, entry)
entry.client.Close()
})
entry.idle = timer
}
// evictLocked removes entry from the pool if it is still the one held for key. A
// connection that is evicted while borrowed stays open until its borrowers finish
// with it; only new callers are sent elsewhere.
func (p *SSHConnectionPool) evictLocked(key poolKey, entry *poolEntry) {
slog.Info("Evicting pooled ssh connection", "host", key.host, "port", key.port, "user", key.sshUsername)
if p.entries[key] == entry {
delete(p.entries, key)
}
slog.Info("SSH connection pool size after eviction", "size", len(p.entries))
}
// Close closes every pooled connection, borrowed or not.
func (p *SSHConnectionPool) Close() {
p.lock.Lock()
entries := p.entries
p.entries = make(map[poolKey]*poolEntry)
p.lock.Unlock()
for _, entry := range entries {
select {
case <-entry.ready:
if entry.client != nil {
entry.client.Close()
}
default:
// Still dialing: the connection lands in an entry nobody can reach and is
// closed by the idle timer once its callers are done with it.
}
}
}
func isContextError(err error) bool {
return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
}