blob: 4f261fbdeeff8f437da2cfa3b68f1fa14f592c7d [file]
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package ssh
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"log/slog"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"time"
credmodel "github.com/apache/airavata/api/credentials/model"
gossh "golang.org/x/crypto/ssh"
)
// ========================== Utility section =====================================
// scpOK is the byte each side of the SCP wire protocol writes to acknowledge the
// other's last message. A leading 0x01 or 0x02 instead introduces a one-line error,
// which is the only way a remote scp reports a missing or unreadable path.
const scpOK = 0x00
// scpDialTimeout bounds the TCP connect and the SSH handshake behind it. A staging
// activity is retried by the workflow, so a host that is merely down should fail the
// attempt quickly rather than hold a worker slot until the OS gives up on the socket.
const scpDialTimeout = 30 * time.Second
type FileMetadata struct {
Path string
Size int64
Mode os.FileMode
}
type scpTransfer struct {
client *gossh.Client
session *gossh.Session
stdin io.WriteCloser
stdout *bufio.Reader
stderr *bytes.Buffer
ctx context.Context
done chan struct{}
target string
}
type SSHDialConfig struct {
host string
port int
sshUsername string
cred credmodel.SSHCredential
userId string
}
func NewSSHDialConfig(host string, port int, sshUsername string, cred credmodel.SSHCredential, userId string) SSHDialConfig {
return SSHDialConfig{
host: host,
port: port,
sshUsername: sshUsername,
cred: cred,
userId: userId,
}
}
func dialSSH(ctx context.Context, dialConfig SSHDialConfig, sshContext *GlobalSSHContext) (*gossh.Client, error) {
hostKeys := gossh.InsecureIgnoreHostKey()
if dialConfig.port == 0 {
dialConfig.port = 22
}
addr := net.JoinHostPort(dialConfig.host, strconv.Itoa(dialConfig.port))
// DialContext rather than ssh.Dial: the activity's context has to be able to
// abandon a connect that is going nowhere.
dialer := net.Dialer{Timeout: scpDialTimeout}
conn, err := dialer.DialContext(ctx, "tcp", addr)
if err != nil {
return nil, fmt.Errorf("connecting to %s: %w", addr, err)
}
if deadline, ok := ctx.Deadline(); ok {
_ = conn.SetDeadline(deadline)
}
// banner collects the server's pre-login text for the prompts that follow. The
// banner and auth callbacks both run on the handshake goroutine, so no lock.
var banner string
authMethods := []gossh.AuthMethod{}
if dialConfig.cred.PrivateKey != "" {
signer, err := signerFor(dialConfig.cred)
if err != nil {
return nil, err
}
authMethods = append(authMethods, gossh.PublicKeys(signer))
}
if dialConfig.cred.FullInteractive {
ki := gossh.KeyboardInteractive(func(name, instruction string, questions []string, echos []bool) ([]string, error) {
// RFC 4256 allows an INFO_REQUEST with zero prompts (servers use it to
// display a banner-like instruction). It still requires a response.
slog.Info("Keyboard interactive challenge received", "name", name, "instruction", instruction, "questions", questions, "echos", echos)
prompts := make([]Prompt, len(questions))
for i, q := range questions {
prompts[i] = Prompt{Text: q, Echo: echos[i]}
}
if len(prompts) == 0 && instruction == "" && name == "" {
return []string{}, nil
}
answers, err := askUser(ctx, dialConfig, sshContext, banner, name, instruction, prompts)
if err != nil {
return nil, err
}
if len(answers) != len(questions) {
return nil, fmt.Errorf("expected %d answers, got %d", len(questions), len(answers))
}
return answers, nil
})
// keyboard-interactive, so the password is asked for as a one-prompt challenge.
// The server only tries it if it advertises the method.
password := gossh.PasswordCallback(func() (string, error) {
prompt := fmt.Sprintf("%s@%s's password: ", dialConfig.sshUsername, dialConfig.host)
answers, err := askUser(ctx, dialConfig, sshContext, banner, "", "", []Prompt{{Text: prompt, Echo: false}})
if err != nil {
return "", err
}
if len(answers) != 1 {
return "", fmt.Errorf("expected 1 answer, got %d", len(answers))
}
return answers[0], nil
})
authMethods = append(authMethods, ki, password)
}
sshConn, chans, reqs, err := gossh.NewClientConn(conn, addr, &gossh.ClientConfig{
User: dialConfig.sshUsername,
Auth: authMethods,
HostKeyCallback: hostKeys,
Timeout: scpDialTimeout,
BannerCallback: func(message string) error {
banner += message
return nil
},
})
if err != nil {
conn.Close()
return nil, fmt.Errorf("ssh handshake with %s: %w", addr, err)
}
// The handshake deadline must not outlive the handshake: a transfer reads for as
// long as the file takes.
_ = conn.SetDeadline(time.Time{})
return gossh.NewClient(sshConn, chans, reqs), nil
}
func intializeScpSession(ctx context.Context, dialConfig SSHDialConfig, sshContext *GlobalSSHContext, direction, remotePath, target string) (*scpTransfer, error) {
client, err := dialSSH(ctx, dialConfig, sshContext)
if err != nil {
return nil, fmt.Errorf("%s: %w", target, err)
}
session, err := client.NewSession() // client is *gossh.Client, so this is fine
if err != nil {
client.Close()
return nil, fmt.Errorf("%s: opening session: %w", target, err)
}
t := &scpTransfer{client: client, session: session, ctx: ctx, target: target, done: make(chan struct{})}
if t.stdin, err = session.StdinPipe(); err != nil {
session.Close()
client.Close()
return nil, fmt.Errorf("%s: %w", target, err)
}
stdout, err := session.StdoutPipe()
if err != nil {
session.Close()
client.Close()
return nil, fmt.Errorf("%s: %w", target, err)
}
t.stdout = bufio.NewReader(stdout)
t.stderr = &bytes.Buffer{}
session.Stderr = t.stderr
// A session read blocks on the network, so cancellation has to arrive by closing
// the session under it. done stops the watcher when the transfer finishes first.
go func() {
select {
case <-ctx.Done():
session.Close()
case <-t.done:
}
}()
if err := session.Start("scp " + direction + " " + ShellQuote(remotePath)); err != nil {
t.close()
return nil, fmt.Errorf("%s: starting remote scp: %w", target, err)
}
return t, nil
}
// runSSHCommand runs one command on the host and waits for it to finish, returning
// what it wrote to stdout and to stderr.
//
// Both streams are returned whether the command succeeded or not: what a failing
// command printed is usually what explains it, and a caller that reads the output —
// the job id sbatch announces, say — still wants to see what came back. Whatever went
// to stderr is folded into the error as well, so a caller that only checks err is no
// worse off than before. The streams are returned as the command wrote them, trailing
// newline included; a caller parsing them should trim.
func RunSSHCommand(ctx context.Context, dialConfig SSHDialConfig, sshContext *GlobalSSHContext, command, target string) (string, string, error) {
client, err := dialSSH(ctx, dialConfig, sshContext)
if err != nil {
return "", "", fmt.Errorf("%s: %w", target, err)
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return "", "", fmt.Errorf("%s: opening session: %w", target, err)
}
defer session.Close()
var stdout, stderr bytes.Buffer
session.Stdout = &stdout
session.Stderr = &stderr
done := make(chan struct{})
defer close(done)
go func() {
select {
case <-ctx.Done():
session.Close()
case <-done:
}
}()
if err := session.Run(command); err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return stdout.String(), stderr.String(), fmt.Errorf("%s: %w", target, ctxErr)
}
if remote := strings.TrimSpace(stderr.String()); remote != "" {
return stdout.String(), stderr.String(), fmt.Errorf("%s: %w (remote: %s)", target, err, remote)
}
return stdout.String(), stderr.String(), fmt.Errorf("%s: %w", target, err)
}
return stdout.String(), stderr.String(), nil
}
func (t *scpTransfer) close() {
close(t.done)
t.session.Close()
t.client.Close()
}
// fail names the transfer in err. A cancelled context takes precedence, because every
// read and write fails once the session is closed under it and the resulting I/O error
// says nothing useful. Whatever the remote scp wrote to stderr is folded in.
func (t *scpTransfer) fail(err error) error {
if ctxErr := t.ctx.Err(); ctxErr != nil {
return fmt.Errorf("%s: %w", t.target, ctxErr)
}
if remote := strings.TrimSpace(t.stderr.String()); remote != "" {
return fmt.Errorf("%s: %w (remote: %s)", t.target, err, remote)
}
return fmt.Errorf("%s: %w", t.target, err)
}
// Graceful completion of the scp transfer. It closes the stdin to signal the end of the transfer
// and waits for the remote scp process to exit, returning any errors encountered.
func (t *scpTransfer) finish() error {
if err := t.stdin.Close(); err != nil {
return t.fail(err)
}
if err := t.session.Wait(); err != nil {
return t.fail(fmt.Errorf("remote scp exited: %w", err))
}
return nil
}
// sendSCPFile drives the source half of the protocol: wait to be invited, announce the
// file, write exactly the announced bytes, close it with a zero byte of our own, and
// read the remote's verdict.
//
// Every step waits for an acknowledgement before the next, so a refusal — no such
// directory, permission denied, disk full — surfaces as an error here rather than as a
// transfer that appears to succeed and leaves nothing on the host.
func sendSCPFile(r *bufio.Reader, w io.Writer, src io.Reader, name string, mode os.FileMode, size int64) error {
// A name is interpolated into a header line the remote parses, so a newline would
// inject a second header and a slash would move the file out of the directory the
// caller named.
if name == "" || strings.ContainsAny(name, "\n/") {
return fmt.Errorf("unusable remote file name %q", name)
}
if err := readSCPAck(r); err != nil {
return err
}
if _, err := fmt.Fprintf(w, "C%04o %d %s\n", mode.Perm(), size, name); err != nil {
return fmt.Errorf("sending scp file header: %w", err)
}
if err := readSCPAck(r); err != nil {
return err
}
// Exactly size bytes: the header already committed to that count, so a file that
// grew under us must not spill into the next protocol message, and one that shrank
// has to fail rather than leave the remote waiting.
if _, err := io.CopyN(w, src, size); err != nil {
return fmt.Errorf("sending %d bytes: %w", size, err)
}
if err := scpAck(w); err != nil {
return err
}
return readSCPAck(r)
}
// readSCPAck reads the remote's verdict on what was just sent. A zero byte accepts it;
// 0x01 or 0x02 introduce a one-line description of the refusal.
func readSCPAck(r *bufio.Reader) error {
b, err := r.ReadByte()
if err != nil {
if errors.Is(err, io.EOF) {
return errors.New("remote closed the connection without acknowledging (is scp installed on the host?)")
}
return fmt.Errorf("reading scp acknowledgement: %w", err)
}
if b == scpOK {
return nil
}
line, err := r.ReadString('\n')
if err != nil && !errors.Is(err, io.EOF) {
return fmt.Errorf("reading scp error message: %w", err)
}
if line = strings.TrimRight(line, "\n"); line == "" {
line = "unspecified error"
}
return fmt.Errorf("remote scp: %s", line)
}
func receiveSCPFile(r *bufio.Reader, w io.Writer, localPath string) error {
if err := scpAck(w); err != nil {
return err
}
for {
msg, err := readSCPMessage(r)
if err != nil {
return err
}
switch msg[0] {
case 'T':
// Modification times, sent only under -p, which is not requested here.
// Accept and read on rather than fail if a remote sends them anyway.
if err := scpAck(w); err != nil {
return err
}
case 'C':
mode, size, _, err := parseSCPFileHeader(msg)
if err != nil {
return err
}
if err := scpAck(w); err != nil {
return err
}
if err := writeLocalFile(r, localPath, mode, size); err != nil {
return err
}
// The remote closes the file with its own acknowledgement byte.
b, err := r.ReadByte()
if err != nil {
return fmt.Errorf("reading end of file marker: %w", err)
}
if b != scpOK {
return fmt.Errorf("remote reported a transfer error after sending %d bytes", size)
}
return scpAck(w)
case 'D':
return fmt.Errorf("remote path is a directory: recursive scp download is not supported")
default:
return fmt.Errorf("unexpected scp message %q", msg)
}
}
}
// readSCPMessage reads one control line, turning the remote's error messages into
// errors. The returned string keeps its leading type byte.
func readSCPMessage(r *bufio.Reader) (string, error) {
kind, err := r.ReadByte()
if err != nil {
if errors.Is(err, io.EOF) {
return "", errors.New("remote closed the connection without sending a file")
}
return "", fmt.Errorf("reading scp message: %w", err)
}
line, err := r.ReadString('\n')
if err != nil && !errors.Is(err, io.EOF) {
return "", fmt.Errorf("reading scp message: %w", err)
}
line = strings.TrimRight(line, "\n")
switch kind {
case 0x01, 0x02:
if line == "" {
line = "unspecified error"
}
return "", fmt.Errorf("remote scp: %s", line)
case scpOK:
return "", errors.New("unexpected scp acknowledgement while awaiting a file header")
}
return string(kind) + line, nil
}
// parseSCPFileHeader reads a "C<mode> <size> <name>" header. The name is split off as
// the last field rather than on every space, because a remote file name may contain
// them. A download ignores the name — the caller chose where the bytes land, and
// honouring a name the remote picked would let it write outside that path — but a
// metadata probe reports it.
func parseSCPFileHeader(msg string) (os.FileMode, int64, string, error) {
fields := strings.SplitN(msg[1:], " ", 3)
if len(fields) != 3 {
return 0, 0, "", fmt.Errorf("malformed scp file header %q", msg)
}
mode, err := strconv.ParseUint(fields[0], 8, 32)
if err != nil {
return 0, 0, "", fmt.Errorf("malformed mode in scp file header %q", msg)
}
size, err := strconv.ParseInt(fields[1], 10, 64)
if err != nil || size < 0 {
return 0, 0, "", fmt.Errorf("malformed size in scp file header %q", msg)
}
return os.FileMode(mode).Perm(), size, fields[2], nil
}
// writeLocalFile stages size bytes into a sibling temporary file and renames it into
// place, so a transfer that fails part way through cannot leave a truncated file where
// a later task would read it as the staged input.
func writeLocalFile(r io.Reader, localPath string, mode os.FileMode, size int64) error {
dir := filepath.Dir(localPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("creating %s: %w", dir, err)
}
tmp, err := os.CreateTemp(dir, filepath.Base(localPath)+".part-*")
if err != nil {
return fmt.Errorf("creating staging file in %s: %w", dir, err)
}
tmpName := tmp.Name()
// Both are no-ops once the transfer has closed and renamed the file.
defer tmp.Close()
defer os.Remove(tmpName)
if _, err := io.CopyN(tmp, r, size); err != nil {
return fmt.Errorf("copying %d bytes into %s: %w", size, tmpName, err)
}
if err := tmp.Chmod(mode); err != nil {
return fmt.Errorf("setting mode on %s: %w", tmpName, err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("closing %s: %w", tmpName, err)
}
if err := os.Rename(tmpName, localPath); err != nil {
return fmt.Errorf("moving %s into place at %s: %w", tmpName, localPath, err)
}
return nil
}
func scpAck(w io.Writer) error {
if _, err := w.Write([]byte{scpOK}); err != nil {
return fmt.Errorf("acknowledging scp message: %w", err)
}
return nil
}
func signerFor(key credmodel.SSHCredential) (gossh.Signer, error) {
pem := []byte(key.PrivateKey)
if key.Passphrase != nil && *key.Passphrase != "" {
signer, err := gossh.ParsePrivateKeyWithPassphrase(pem, []byte(*key.Passphrase))
if err != nil {
return nil, fmt.Errorf("ssh credential %s could not be decrypted with its registered passphrase", key.ID)
}
return signer, nil
}
signer, err := gossh.ParsePrivateKey(pem)
if err != nil {
return nil, fmt.Errorf("ssh credential %s could not be parsed (is it passphrase protected without a registered passphrase?)", key.ID)
}
return signer, nil
}
// ShellQuote wraps s for the remote login shell, which is what runs the scp command
// line. A path is user supplied, so it cannot be interpolated bare.
func ShellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}