| /** | |
| * | |
| * Licensed to the Apache Software Foundation (ASF) under one | |
| * or more contributor license agreements. See the NOTICE file | |
| * distributed with this work for additional information | |
| * regarding copyright ownership. The ASF licenses this file | |
| * to you under the Apache License, Version 2.0 (the | |
| * "License"); you may not use this file except in compliance | |
| * with the License. You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, | |
| * software distributed under the License is distributed on an | |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| * KIND, either express or implied. See the License for the | |
| * specific language governing permissions and limitations | |
| * under the License. | |
| */ | |
| package ssh | |
| import ( | |
| "bufio" | |
| "bytes" | |
| "context" | |
| "errors" | |
| "fmt" | |
| "io" | |
| "log/slog" | |
| "net" | |
| "os" | |
| "path/filepath" | |
| "strconv" | |
| "strings" | |
| "time" | |
| credmodel "github.com/apache/airavata/api/credentials/model" | |
| gossh "golang.org/x/crypto/ssh" | |
| ) | |
| // ========================== Utility section ===================================== | |
| // scpOK is the byte each side of the SCP wire protocol writes to acknowledge the | |
| // other's last message. A leading 0x01 or 0x02 instead introduces a one-line error, | |
| // which is the only way a remote scp reports a missing or unreadable path. | |
| const scpOK = 0x00 | |
| // scpDialTimeout bounds the TCP connect and the SSH handshake behind it. A staging | |
| // activity is retried by the workflow, so a host that is merely down should fail the | |
| // attempt quickly rather than hold a worker slot until the OS gives up on the socket. | |
| const scpDialTimeout = 30 * time.Second | |
| type FileMetadata struct { | |
| Path string | |
| Size int64 | |
| Mode os.FileMode | |
| } | |
| type scpTransfer struct { | |
| client *gossh.Client | |
| session *gossh.Session | |
| stdin io.WriteCloser | |
| stdout *bufio.Reader | |
| stderr *bytes.Buffer | |
| ctx context.Context | |
| done chan struct{} | |
| target string | |
| } | |
| type SSHDialConfig struct { | |
| host string | |
| port int | |
| sshUsername string | |
| cred credmodel.SSHCredential | |
| userId string | |
| } | |
| func NewSSHDialConfig(host string, port int, sshUsername string, cred credmodel.SSHCredential, userId string) SSHDialConfig { | |
| return SSHDialConfig{ | |
| host: host, | |
| port: port, | |
| sshUsername: sshUsername, | |
| cred: cred, | |
| userId: userId, | |
| } | |
| } | |
| func dialSSH(ctx context.Context, dialConfig SSHDialConfig, sshContext *GlobalSSHContext) (*gossh.Client, error) { | |
| hostKeys := gossh.InsecureIgnoreHostKey() | |
| if dialConfig.port == 0 { | |
| dialConfig.port = 22 | |
| } | |
| addr := net.JoinHostPort(dialConfig.host, strconv.Itoa(dialConfig.port)) | |
| // DialContext rather than ssh.Dial: the activity's context has to be able to | |
| // abandon a connect that is going nowhere. | |
| dialer := net.Dialer{Timeout: scpDialTimeout} | |
| conn, err := dialer.DialContext(ctx, "tcp", addr) | |
| if err != nil { | |
| return nil, fmt.Errorf("connecting to %s: %w", addr, err) | |
| } | |
| if deadline, ok := ctx.Deadline(); ok { | |
| _ = conn.SetDeadline(deadline) | |
| } | |
| // banner collects the server's pre-login text for the prompts that follow. The | |
| // banner and auth callbacks both run on the handshake goroutine, so no lock. | |
| var banner string | |
| authMethods := []gossh.AuthMethod{} | |
| if dialConfig.cred.PrivateKey != "" { | |
| signer, err := signerFor(dialConfig.cred) | |
| if err != nil { | |
| return nil, err | |
| } | |
| authMethods = append(authMethods, gossh.PublicKeys(signer)) | |
| } | |
| if dialConfig.cred.FullInteractive { | |
| ki := gossh.KeyboardInteractive(func(name, instruction string, questions []string, echos []bool) ([]string, error) { | |
| // RFC 4256 allows an INFO_REQUEST with zero prompts (servers use it to | |
| // display a banner-like instruction). It still requires a response. | |
| slog.Info("Keyboard interactive challenge received", "name", name, "instruction", instruction, "questions", questions, "echos", echos) | |
| prompts := make([]Prompt, len(questions)) | |
| for i, q := range questions { | |
| prompts[i] = Prompt{Text: q, Echo: echos[i]} | |
| } | |
| if len(prompts) == 0 && instruction == "" && name == "" { | |
| return []string{}, nil | |
| } | |
| answers, err := askUser(ctx, dialConfig, sshContext, banner, name, instruction, prompts) | |
| if err != nil { | |
| return nil, err | |
| } | |
| if len(answers) != len(questions) { | |
| return nil, fmt.Errorf("expected %d answers, got %d", len(questions), len(answers)) | |
| } | |
| return answers, nil | |
| }) | |
| // keyboard-interactive, so the password is asked for as a one-prompt challenge. | |
| // The server only tries it if it advertises the method. | |
| password := gossh.PasswordCallback(func() (string, error) { | |
| prompt := fmt.Sprintf("%s@%s's password: ", dialConfig.sshUsername, dialConfig.host) | |
| answers, err := askUser(ctx, dialConfig, sshContext, banner, "", "", []Prompt{{Text: prompt, Echo: false}}) | |
| if err != nil { | |
| return "", err | |
| } | |
| if len(answers) != 1 { | |
| return "", fmt.Errorf("expected 1 answer, got %d", len(answers)) | |
| } | |
| return answers[0], nil | |
| }) | |
| authMethods = append(authMethods, ki, password) | |
| } | |
| sshConn, chans, reqs, err := gossh.NewClientConn(conn, addr, &gossh.ClientConfig{ | |
| User: dialConfig.sshUsername, | |
| Auth: authMethods, | |
| HostKeyCallback: hostKeys, | |
| Timeout: scpDialTimeout, | |
| BannerCallback: func(message string) error { | |
| banner += message | |
| return nil | |
| }, | |
| }) | |
| if err != nil { | |
| conn.Close() | |
| return nil, fmt.Errorf("ssh handshake with %s: %w", addr, err) | |
| } | |
| // The handshake deadline must not outlive the handshake: a transfer reads for as | |
| // long as the file takes. | |
| _ = conn.SetDeadline(time.Time{}) | |
| return gossh.NewClient(sshConn, chans, reqs), nil | |
| } | |
| func intializeScpSession(ctx context.Context, dialConfig SSHDialConfig, sshContext *GlobalSSHContext, direction, remotePath, target string) (*scpTransfer, error) { | |
| client, err := dialSSH(ctx, dialConfig, sshContext) | |
| if err != nil { | |
| return nil, fmt.Errorf("%s: %w", target, err) | |
| } | |
| session, err := client.NewSession() // client is *gossh.Client, so this is fine | |
| if err != nil { | |
| client.Close() | |
| return nil, fmt.Errorf("%s: opening session: %w", target, err) | |
| } | |
| t := &scpTransfer{client: client, session: session, ctx: ctx, target: target, done: make(chan struct{})} | |
| if t.stdin, err = session.StdinPipe(); err != nil { | |
| session.Close() | |
| client.Close() | |
| return nil, fmt.Errorf("%s: %w", target, err) | |
| } | |
| stdout, err := session.StdoutPipe() | |
| if err != nil { | |
| session.Close() | |
| client.Close() | |
| return nil, fmt.Errorf("%s: %w", target, err) | |
| } | |
| t.stdout = bufio.NewReader(stdout) | |
| t.stderr = &bytes.Buffer{} | |
| session.Stderr = t.stderr | |
| // A session read blocks on the network, so cancellation has to arrive by closing | |
| // the session under it. done stops the watcher when the transfer finishes first. | |
| go func() { | |
| select { | |
| case <-ctx.Done(): | |
| session.Close() | |
| case <-t.done: | |
| } | |
| }() | |
| if err := session.Start("scp " + direction + " " + ShellQuote(remotePath)); err != nil { | |
| t.close() | |
| return nil, fmt.Errorf("%s: starting remote scp: %w", target, err) | |
| } | |
| return t, nil | |
| } | |
| // runSSHCommand runs one command on the host and waits for it to finish, returning | |
| // what it wrote to stdout and to stderr. | |
| // | |
| // Both streams are returned whether the command succeeded or not: what a failing | |
| // command printed is usually what explains it, and a caller that reads the output — | |
| // the job id sbatch announces, say — still wants to see what came back. Whatever went | |
| // to stderr is folded into the error as well, so a caller that only checks err is no | |
| // worse off than before. The streams are returned as the command wrote them, trailing | |
| // newline included; a caller parsing them should trim. | |
| func RunSSHCommand(ctx context.Context, dialConfig SSHDialConfig, sshContext *GlobalSSHContext, command, target string) (string, string, error) { | |
| client, err := dialSSH(ctx, dialConfig, sshContext) | |
| if err != nil { | |
| return "", "", fmt.Errorf("%s: %w", target, err) | |
| } | |
| defer client.Close() | |
| session, err := client.NewSession() | |
| if err != nil { | |
| return "", "", fmt.Errorf("%s: opening session: %w", target, err) | |
| } | |
| defer session.Close() | |
| var stdout, stderr bytes.Buffer | |
| session.Stdout = &stdout | |
| session.Stderr = &stderr | |
| done := make(chan struct{}) | |
| defer close(done) | |
| go func() { | |
| select { | |
| case <-ctx.Done(): | |
| session.Close() | |
| case <-done: | |
| } | |
| }() | |
| if err := session.Run(command); err != nil { | |
| if ctxErr := ctx.Err(); ctxErr != nil { | |
| return stdout.String(), stderr.String(), fmt.Errorf("%s: %w", target, ctxErr) | |
| } | |
| if remote := strings.TrimSpace(stderr.String()); remote != "" { | |
| return stdout.String(), stderr.String(), fmt.Errorf("%s: %w (remote: %s)", target, err, remote) | |
| } | |
| return stdout.String(), stderr.String(), fmt.Errorf("%s: %w", target, err) | |
| } | |
| return stdout.String(), stderr.String(), nil | |
| } | |
| func (t *scpTransfer) close() { | |
| close(t.done) | |
| t.session.Close() | |
| t.client.Close() | |
| } | |
| // fail names the transfer in err. A cancelled context takes precedence, because every | |
| // read and write fails once the session is closed under it and the resulting I/O error | |
| // says nothing useful. Whatever the remote scp wrote to stderr is folded in. | |
| func (t *scpTransfer) fail(err error) error { | |
| if ctxErr := t.ctx.Err(); ctxErr != nil { | |
| return fmt.Errorf("%s: %w", t.target, ctxErr) | |
| } | |
| if remote := strings.TrimSpace(t.stderr.String()); remote != "" { | |
| return fmt.Errorf("%s: %w (remote: %s)", t.target, err, remote) | |
| } | |
| return fmt.Errorf("%s: %w", t.target, err) | |
| } | |
| // Graceful completion of the scp transfer. It closes the stdin to signal the end of the transfer | |
| // and waits for the remote scp process to exit, returning any errors encountered. | |
| func (t *scpTransfer) finish() error { | |
| if err := t.stdin.Close(); err != nil { | |
| return t.fail(err) | |
| } | |
| if err := t.session.Wait(); err != nil { | |
| return t.fail(fmt.Errorf("remote scp exited: %w", err)) | |
| } | |
| return nil | |
| } | |
| // sendSCPFile drives the source half of the protocol: wait to be invited, announce the | |
| // file, write exactly the announced bytes, close it with a zero byte of our own, and | |
| // read the remote's verdict. | |
| // | |
| // Every step waits for an acknowledgement before the next, so a refusal — no such | |
| // directory, permission denied, disk full — surfaces as an error here rather than as a | |
| // transfer that appears to succeed and leaves nothing on the host. | |
| func sendSCPFile(r *bufio.Reader, w io.Writer, src io.Reader, name string, mode os.FileMode, size int64) error { | |
| // A name is interpolated into a header line the remote parses, so a newline would | |
| // inject a second header and a slash would move the file out of the directory the | |
| // caller named. | |
| if name == "" || strings.ContainsAny(name, "\n/") { | |
| return fmt.Errorf("unusable remote file name %q", name) | |
| } | |
| if err := readSCPAck(r); err != nil { | |
| return err | |
| } | |
| if _, err := fmt.Fprintf(w, "C%04o %d %s\n", mode.Perm(), size, name); err != nil { | |
| return fmt.Errorf("sending scp file header: %w", err) | |
| } | |
| if err := readSCPAck(r); err != nil { | |
| return err | |
| } | |
| // Exactly size bytes: the header already committed to that count, so a file that | |
| // grew under us must not spill into the next protocol message, and one that shrank | |
| // has to fail rather than leave the remote waiting. | |
| if _, err := io.CopyN(w, src, size); err != nil { | |
| return fmt.Errorf("sending %d bytes: %w", size, err) | |
| } | |
| if err := scpAck(w); err != nil { | |
| return err | |
| } | |
| return readSCPAck(r) | |
| } | |
| // readSCPAck reads the remote's verdict on what was just sent. A zero byte accepts it; | |
| // 0x01 or 0x02 introduce a one-line description of the refusal. | |
| func readSCPAck(r *bufio.Reader) error { | |
| b, err := r.ReadByte() | |
| if err != nil { | |
| if errors.Is(err, io.EOF) { | |
| return errors.New("remote closed the connection without acknowledging (is scp installed on the host?)") | |
| } | |
| return fmt.Errorf("reading scp acknowledgement: %w", err) | |
| } | |
| if b == scpOK { | |
| return nil | |
| } | |
| line, err := r.ReadString('\n') | |
| if err != nil && !errors.Is(err, io.EOF) { | |
| return fmt.Errorf("reading scp error message: %w", err) | |
| } | |
| if line = strings.TrimRight(line, "\n"); line == "" { | |
| line = "unspecified error" | |
| } | |
| return fmt.Errorf("remote scp: %s", line) | |
| } | |
| func receiveSCPFile(r *bufio.Reader, w io.Writer, localPath string) error { | |
| if err := scpAck(w); err != nil { | |
| return err | |
| } | |
| for { | |
| msg, err := readSCPMessage(r) | |
| if err != nil { | |
| return err | |
| } | |
| switch msg[0] { | |
| case 'T': | |
| // Modification times, sent only under -p, which is not requested here. | |
| // Accept and read on rather than fail if a remote sends them anyway. | |
| if err := scpAck(w); err != nil { | |
| return err | |
| } | |
| case 'C': | |
| mode, size, _, err := parseSCPFileHeader(msg) | |
| if err != nil { | |
| return err | |
| } | |
| if err := scpAck(w); err != nil { | |
| return err | |
| } | |
| if err := writeLocalFile(r, localPath, mode, size); err != nil { | |
| return err | |
| } | |
| // The remote closes the file with its own acknowledgement byte. | |
| b, err := r.ReadByte() | |
| if err != nil { | |
| return fmt.Errorf("reading end of file marker: %w", err) | |
| } | |
| if b != scpOK { | |
| return fmt.Errorf("remote reported a transfer error after sending %d bytes", size) | |
| } | |
| return scpAck(w) | |
| case 'D': | |
| return fmt.Errorf("remote path is a directory: recursive scp download is not supported") | |
| default: | |
| return fmt.Errorf("unexpected scp message %q", msg) | |
| } | |
| } | |
| } | |
| // readSCPMessage reads one control line, turning the remote's error messages into | |
| // errors. The returned string keeps its leading type byte. | |
| func readSCPMessage(r *bufio.Reader) (string, error) { | |
| kind, err := r.ReadByte() | |
| if err != nil { | |
| if errors.Is(err, io.EOF) { | |
| return "", errors.New("remote closed the connection without sending a file") | |
| } | |
| return "", fmt.Errorf("reading scp message: %w", err) | |
| } | |
| line, err := r.ReadString('\n') | |
| if err != nil && !errors.Is(err, io.EOF) { | |
| return "", fmt.Errorf("reading scp message: %w", err) | |
| } | |
| line = strings.TrimRight(line, "\n") | |
| switch kind { | |
| case 0x01, 0x02: | |
| if line == "" { | |
| line = "unspecified error" | |
| } | |
| return "", fmt.Errorf("remote scp: %s", line) | |
| case scpOK: | |
| return "", errors.New("unexpected scp acknowledgement while awaiting a file header") | |
| } | |
| return string(kind) + line, nil | |
| } | |
| // parseSCPFileHeader reads a "C<mode> <size> <name>" header. The name is split off as | |
| // the last field rather than on every space, because a remote file name may contain | |
| // them. A download ignores the name — the caller chose where the bytes land, and | |
| // honouring a name the remote picked would let it write outside that path — but a | |
| // metadata probe reports it. | |
| func parseSCPFileHeader(msg string) (os.FileMode, int64, string, error) { | |
| fields := strings.SplitN(msg[1:], " ", 3) | |
| if len(fields) != 3 { | |
| return 0, 0, "", fmt.Errorf("malformed scp file header %q", msg) | |
| } | |
| mode, err := strconv.ParseUint(fields[0], 8, 32) | |
| if err != nil { | |
| return 0, 0, "", fmt.Errorf("malformed mode in scp file header %q", msg) | |
| } | |
| size, err := strconv.ParseInt(fields[1], 10, 64) | |
| if err != nil || size < 0 { | |
| return 0, 0, "", fmt.Errorf("malformed size in scp file header %q", msg) | |
| } | |
| return os.FileMode(mode).Perm(), size, fields[2], nil | |
| } | |
| // writeLocalFile stages size bytes into a sibling temporary file and renames it into | |
| // place, so a transfer that fails part way through cannot leave a truncated file where | |
| // a later task would read it as the staged input. | |
| func writeLocalFile(r io.Reader, localPath string, mode os.FileMode, size int64) error { | |
| dir := filepath.Dir(localPath) | |
| if err := os.MkdirAll(dir, 0o755); err != nil { | |
| return fmt.Errorf("creating %s: %w", dir, err) | |
| } | |
| tmp, err := os.CreateTemp(dir, filepath.Base(localPath)+".part-*") | |
| if err != nil { | |
| return fmt.Errorf("creating staging file in %s: %w", dir, err) | |
| } | |
| tmpName := tmp.Name() | |
| // Both are no-ops once the transfer has closed and renamed the file. | |
| defer tmp.Close() | |
| defer os.Remove(tmpName) | |
| if _, err := io.CopyN(tmp, r, size); err != nil { | |
| return fmt.Errorf("copying %d bytes into %s: %w", size, tmpName, err) | |
| } | |
| if err := tmp.Chmod(mode); err != nil { | |
| return fmt.Errorf("setting mode on %s: %w", tmpName, err) | |
| } | |
| if err := tmp.Close(); err != nil { | |
| return fmt.Errorf("closing %s: %w", tmpName, err) | |
| } | |
| if err := os.Rename(tmpName, localPath); err != nil { | |
| return fmt.Errorf("moving %s into place at %s: %w", tmpName, localPath, err) | |
| } | |
| return nil | |
| } | |
| func scpAck(w io.Writer) error { | |
| if _, err := w.Write([]byte{scpOK}); err != nil { | |
| return fmt.Errorf("acknowledging scp message: %w", err) | |
| } | |
| return nil | |
| } | |
| func signerFor(key credmodel.SSHCredential) (gossh.Signer, error) { | |
| pem := []byte(key.PrivateKey) | |
| if key.Passphrase != nil && *key.Passphrase != "" { | |
| signer, err := gossh.ParsePrivateKeyWithPassphrase(pem, []byte(*key.Passphrase)) | |
| if err != nil { | |
| return nil, fmt.Errorf("ssh credential %s could not be decrypted with its registered passphrase", key.ID) | |
| } | |
| return signer, nil | |
| } | |
| signer, err := gossh.ParsePrivateKey(pem) | |
| if err != nil { | |
| return nil, fmt.Errorf("ssh credential %s could not be parsed (is it passphrase protected without a registered passphrase?)", key.ID) | |
| } | |
| return signer, nil | |
| } | |
| // ShellQuote wraps s for the remote login shell, which is what runs the scp command | |
| // line. A path is user supplied, so it cannot be interpolated bare. | |
| func ShellQuote(s string) string { | |
| return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" | |
| } |