From: djm Date: Mon, 2 Aug 2021 23:38:27 +0000 (+0000) Subject: support for using the SFTP protocol for file transfers in scp, via a X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=dea9b0bf579739817193e4539c14b76920d03f00;p=openbsd support for using the SFTP protocol for file transfers in scp, via a new "-M sftp" option. Marked as experimental for now. Some corner-cases exist, in particular there is no attempt to provide bug-compatibility with scp's weird "double shell" quoting rules. Mostly by Jakub Jelen in GHPR#194 with some tweaks by me. ok markus@ Thanks jmc@ for improving the scp.1 bits. --- diff --git a/usr.bin/ssh/scp.1 b/usr.bin/ssh/scp.1 index d9a9bb92bce..54285b700fe 100644 --- a/usr.bin/ssh/scp.1 +++ b/usr.bin/ssh/scp.1 @@ -8,9 +8,9 @@ .\" .\" Created: Sun May 7 00:14:37 1995 ylo .\" -.\" $OpenBSD: scp.1,v 1.96 2021/07/02 05:11:21 dtucker Exp $ +.\" $OpenBSD: scp.1,v 1.97 2021/08/02 23:38:27 djm Exp $ .\" -.Dd $Mdocdate: July 2 2021 $ +.Dd $Mdocdate: August 2 2021 $ .Dt SCP 1 .Os .Sh NAME @@ -20,10 +20,12 @@ .Nm scp .Op Fl 346ABCpqrTv .Op Fl c Ar cipher +.Op Fl D Ar sftp_server_path .Op Fl F Ar ssh_config .Op Fl i Ar identity_file .Op Fl J Ar destination .Op Fl l Ar limit +.Op Fl M Ar scp | sftp .Op Fl o Ar ssh_option .Op Fl P Ar port .Op Fl S Ar program @@ -108,6 +110,13 @@ to enable compression. Selects the cipher to use for encrypting the data transfer. This option is directly passed to .Xr ssh 1 . +.It Fl D Ar sftp_server_path +When using the experimental SFTP protocol support via +.Fl M , +connect directly to a local SFTP server program rather than a +remote one via +.Xr ssh 1 . +This option may be useful in debugging the client and server. .It Fl F Ar ssh_config Specifies an alternative per-user configuration file for @@ -134,6 +143,14 @@ This option is directly passed to .Xr ssh 1 . .It Fl l Ar limit Limits the used bandwidth, specified in Kbit/s. +.It Fl M Ar scp | sftp +Specifies a mode which will be used to transfer files. +The default is to use the original +.Cm scp +protocol. +Alternately, experimental support for using the +.Cm sftp +protocol is available. .It Fl o Ar ssh_option Can be used to pass options to .Nm ssh diff --git a/usr.bin/ssh/scp.c b/usr.bin/ssh/scp.c index fc84791d65a..21ca77e50ae 100644 --- a/usr.bin/ssh/scp.c +++ b/usr.bin/ssh/scp.c @@ -1,4 +1,4 @@ -/* $OpenBSD: scp.c,v 1.215 2021/07/05 00:25:42 djm Exp $ */ +/* $OpenBSD: scp.c,v 1.216 2021/08/02 23:38:27 djm Exp $ */ /* * scp - secure remote copy. This is basically patched BSD rcp which * uses ssh to do the data transfer (instead of using rcmd). @@ -83,6 +83,8 @@ #include #include #include +#include +#include #include #include #include @@ -105,10 +107,15 @@ #include "progressmeter.h" #include "utf8.h" +#include "sftp-common.h" +#include "sftp-client.h" + #define COPY_BUFLEN 16384 -int do_cmd(char *host, char *remuser, int port, char *cmd, int *fdin, int *fdout); -int do_cmd2(char *host, char *remuser, int port, char *cmd, int fdin, int fdout); +int do_cmd(char *program, char *host, char *remuser, int port, char *cmd, + int *fdin, int *fdout); +int do_cmd2(char *host, char *remuser, int port, char *cmd, + int fdin, int fdout); /* Struct for addargs */ arglist args; @@ -123,6 +130,7 @@ char *curfile; /* This is set to non-zero to enable verbose mode. */ int verbose_mode = 0; +LogLevel log_level = SYSLOG_LEVEL_INFO; /* This is set to zero if the progressmeter is not desired. */ int showprogress = 1; @@ -142,6 +150,12 @@ char *ssh_program = _PATH_SSH_PROGRAM; /* This is used to store the pid of ssh_program */ pid_t do_cmd_pid = -1; +/* Needed for sftp */ +volatile sig_atomic_t interrupted = 0; + +int remote_glob(struct sftp_conn *, const char *, int, + int (*)(const char *, int), glob_t *); /* proto for sftp-glob.c */ + static void killchild(int signo) { @@ -218,14 +232,15 @@ do_local_cmd(arglist *a) */ int -do_cmd(char *host, char *remuser, int port, char *cmd, int *fdin, int *fdout) +do_cmd(char *program, char *host, char *remuser, int port, char *cmd, + int *fdin, int *fdout) { int pin[2], pout[2], reserved[2]; if (verbose_mode) fmprintf(stderr, "Executing: program %s host %s, user %s, command %s\n", - ssh_program, host, + program, host, remuser ? remuser : "(unspecified)", cmd); if (port == -1) @@ -263,7 +278,7 @@ do_cmd(char *host, char *remuser, int port, char *cmd, int *fdin, int *fdout) close(pin[0]); close(pout[1]); - replacearg(&args, 0, "%s", ssh_program); + replacearg(&args, 0, "%s", program); if (port != -1) { addargs(&args, "-p"); addargs(&args, "%d", port); @@ -276,8 +291,8 @@ do_cmd(char *host, char *remuser, int port, char *cmd, int *fdin, int *fdout) addargs(&args, "%s", host); addargs(&args, "%s", cmd); - execvp(ssh_program, args.list); - perror(ssh_program); + execvp(program, args.list); + perror(program); exit(1); } else if (do_cmd_pid == -1) { fatal("fork: %s", strerror(errno)); @@ -368,22 +383,33 @@ int Tflag, pflag, iamremote, iamrecursive, targetshouldbedirectory; #define CMDNEEDS 64 char cmd[CMDNEEDS]; /* must hold "rcp -r -p -d\0" */ +enum scp_mode_e { + MODE_SCP, + MODE_SFTP +}; + int response(void); void rsource(char *, struct stat *); void sink(int, char *[], const char *); void source(int, char *[]); -void tolocal(int, char *[]); -void toremote(int, char *[]); +void tolocal(int, char *[], enum scp_mode_e, char *sftp_direct); +void toremote(int, char *[], enum scp_mode_e, char *sftp_direct); void usage(void); +void source_sftp(int, char *, char *, struct sftp_conn *, char **); +void sink_sftp(int, char *, const char *, struct sftp_conn *); + int main(int argc, char **argv) { int ch, fflag, tflag, status, n; - char **newargv; + char **newargv, *argv0; const char *errstr; extern char *optarg; extern int optind; + /* For now, keep SCP as default */ + enum scp_mode_e mode = MODE_SCP; + char *sftp_direct = NULL; /* Ensure that fds 0, 1 and 2 are open or directed to /dev/null */ sanitise_stdfd(); @@ -391,6 +417,7 @@ main(int argc, char **argv) setlocale(LC_CTYPE, ""); /* Copy argv, because we modify it */ + argv0 = argv[0]; newargv = xcalloc(MAXIMUM(argc + 1, 1), sizeof(*newargv)); for (n = 0; n < argc; n++) newargv[n] = xstrdup(argv[n]); @@ -408,7 +435,7 @@ main(int argc, char **argv) fflag = Tflag = tflag = 0; while ((ch = getopt(argc, argv, - "12346ABCTdfpqrtvF:J:P:S:c:i:l:o:")) != -1) { + "12346ABCTdfpqrtvD:F:J:M:P:S:c:i:l:o:")) != -1) { switch (ch) { /* User-visible flags. */ case '1': @@ -424,6 +451,9 @@ main(int argc, char **argv) addargs(&args, "-%c", ch); addargs(&remote_remote_args, "-%c", ch); break; + case 'D': + sftp_direct = optarg; + break; case '3': throughlocal = 1; break; @@ -446,6 +476,14 @@ main(int argc, char **argv) addargs(&remote_remote_args, "-oBatchmode=yes"); addargs(&args, "-oBatchmode=yes"); break; + case 'M': + if (strcmp(optarg, "sftp") == 0) + mode = MODE_SFTP; + else if (strcmp(optarg, "scp") == 0) + mode = MODE_SCP; + else + usage(); + break; case 'l': limit_kbps = strtonum(optarg, 1, 100 * 1024 * 1024, &errstr); @@ -466,6 +504,10 @@ main(int argc, char **argv) case 'v': addargs(&args, "-v"); addargs(&remote_remote_args, "-v"); + if (verbose_mode == 0) + log_level = SYSLOG_LEVEL_DEBUG1; + else if (log_level < SYSLOG_LEVEL_DEBUG3) + log_level++; verbose_mode = 1; break; case 'q': @@ -496,9 +538,17 @@ main(int argc, char **argv) argc -= optind; argv += optind; + log_init(argv0, log_level, SYSLOG_FACILITY_USER, 1); + /* Do this last because we want the user to be able to override it */ addargs(&args, "-oForwardAgent=no"); + if (mode != MODE_SFTP && sftp_direct != NULL) + fatal("SFTP direct can be used only in SFTP mode"); + + if (mode == MODE_SFTP && iamremote) + fatal("The server can not be ran in SFTP mode"); + if ((pwd = getpwuid(userid = getuid())) == NULL) fatal("unknown user %u", (u_int) userid); @@ -545,11 +595,11 @@ main(int argc, char **argv) (void) ssh_signal(SIGPIPE, lostconn); if (colon(argv[argc - 1])) /* Dest is remote host. */ - toremote(argc, argv); + toremote(argc, argv, mode, sftp_direct); else { if (targetshouldbedirectory) verifydir(argv[argc - 1]); - tolocal(argc, argv); /* Dest is local host. */ + tolocal(argc, argv, mode, sftp_direct); /* Dest is local host. */ } /* * Finally check the exit status of the ssh process, if one was forked @@ -860,12 +910,33 @@ brace_expand(const char *pattern, char ***patternsp, size_t *npatternsp) return ret; } +static struct sftp_conn * +do_sftp_connect(char *host, char *user, int port, char *sftp_direct) +{ + if (sftp_direct == NULL) { + addargs(&args, "-s"); + if (do_cmd(ssh_program, host, user, port, "sftp", + &remin, &remout) < 0) + return NULL; + + } else { + args.list = NULL; + addargs(&args, "sftp-server"); + if (do_cmd(sftp_direct, host, NULL, -1, "sftp", + &remin, &remout) < 0) + return NULL; + } + return do_init(remin, remout, 32768, 64, limit_kbps); +} + void -toremote(int argc, char **argv) +toremote(int argc, char **argv, enum scp_mode_e mode, char *sftp_direct) { char *suser = NULL, *host = NULL, *src = NULL; char *bp, *tuser, *thost, *targ; + char *remote_path = NULL; int sport = -1, tport = -1; + struct sftp_conn *conn = NULL; arglist alist; int i, r; u_int j; @@ -908,9 +979,15 @@ toremote(int argc, char **argv) continue; } if (host && throughlocal) { /* extended remote to remote */ + if (mode == MODE_SFTP) { + /* TODO */ + fatal("Extended remote to remote through local " + "is not yet supported with SFTP"); + } xasprintf(&bp, "%s -f %s%s", cmd, *src == '-' ? "-- " : "", src); - if (do_cmd(host, suser, sport, bp, &remin, &remout) < 0) + if (do_cmd(ssh_program, host, suser, sport, bp, + &remin, &remout) < 0) exit(1); free(bp); xasprintf(&bp, "%s -t %s%s", cmd, @@ -958,6 +1035,14 @@ toremote(int argc, char **argv) addargs(&alist, "--"); addargs(&alist, "%s", host); addargs(&alist, "%s", cmd); + /* + * This will work only if the first remote scp + * supports sftp mode + */ + if (mode == MODE_SFTP) { + addargs(&alist, "-M"); + addargs(&alist, "sftp"); + } addargs(&alist, "%s", src); addargs(&alist, "%s%s%s:%s", tuser ? tuser : "", tuser ? "@" : "", @@ -965,11 +1050,28 @@ toremote(int argc, char **argv) if (do_local_cmd(&alist) != 0) errs = 1; } else { /* local to remote */ + if (mode == MODE_SFTP) { + if (remin == -1) { + /* Connect to remote now */ + conn = do_sftp_connect(thost, tuser, + tport, sftp_direct); + if (conn == NULL) { + fatal("Unable to open sftp " + "connection"); + } + } + + /* The protocol */ + source_sftp(1, argv[i], targ, conn, + &remote_path); + continue; + } + /* SCP */ if (remin == -1) { xasprintf(&bp, "%s -t %s%s", cmd, *targ == '-' ? "-- " : "", targ); - if (do_cmd(thost, tuser, tport, bp, &remin, - &remout) < 0) + if (do_cmd(ssh_program, thost, tuser, tport, bp, + &remin, &remout) < 0) exit(1); if (response() < 0) exit(1); @@ -979,6 +1081,10 @@ toremote(int argc, char **argv) } } out: + if (mode == MODE_SFTP) { + free(remote_path); + free(conn); + } free(tuser); free(thost); free(targ); @@ -988,10 +1094,11 @@ out: } void -tolocal(int argc, char **argv) +tolocal(int argc, char **argv, enum scp_mode_e mode, char *sftp_direct) { char *bp, *host = NULL, *src = NULL, *suser = NULL; arglist alist; + struct sftp_conn *conn = NULL; int i, r, sport = -1; memset(&alist, '\0', sizeof(alist)); @@ -1028,9 +1135,29 @@ tolocal(int argc, char **argv) continue; } /* Remote to local. */ + if (mode == MODE_SFTP) { + conn = do_sftp_connect(host, suser, sport, sftp_direct); + if (conn == NULL) { + error("Couldn't make sftp connection " + "to server"); + ++errs; + continue; + } + + /* The protocol */ + sink_sftp(1, argv[argc - 1], src, conn); + + free(conn); + (void) close(remin); + (void) close(remout); + remin = remout = -1; + continue; + } + /* SCP */ xasprintf(&bp, "%s -f %s%s", cmd, *src == '-' ? "-- " : "", src); - if (do_cmd(host, suser, sport, bp, &remin, &remout) < 0) { + if (do_cmd(ssh_program, host, suser, sport, bp, &remin, + &remout) < 0) { free(bp); ++errs; continue; @@ -1045,6 +1172,53 @@ tolocal(int argc, char **argv) free(src); } +void +source_sftp(int argc, char *src, char *targ, + struct sftp_conn *conn, char **remote_path) +{ + char *target = NULL, *filename = NULL, *abs_dst = NULL; + int target_is_dir; + + if (*remote_path == NULL) { + *remote_path = do_realpath(conn, "."); + if (*remote_path == NULL) + fatal("Unable to determine remote working directory"); + } + + if ((filename = basename(src)) == NULL) + fatal("basename %s: %s", src, strerror(errno)); + + /* + * No need to glob here - the local shell already took care of + * the expansions + */ + target = xstrdup(targ); + target = make_absolute(target, *remote_path); + target_is_dir = remote_is_dir(conn, target); + if (targetshouldbedirectory && !target_is_dir) { + fatal("Target is not a directory, but more files selected " + "for upload"); + } + if (target_is_dir) + abs_dst = path_append(target, filename); + else { + abs_dst = target; + target = NULL; + } + debug3_f("copying local %s to remote %s", src, abs_dst); + + if (local_is_dir(src) && iamrecursive) { + if (upload_dir(conn, src, abs_dst, pflag, 1, 0, 0) != 0) { + fatal("failed to upload directory %s to %s", + src, abs_dst); + } + } else if (do_upload(conn, src, abs_dst, pflag, 0, 0) != 0) + fatal("failed to upload file %s to %s", src, abs_dst); + + free(abs_dst); + free(target); +} + void source(int argc, char **argv) { @@ -1206,6 +1380,88 @@ rsource(char *name, struct stat *statp) (void) response(); } +void +sink_sftp(int argc, char *dst, const char *src, struct sftp_conn *conn) +{ + char *abs_src = NULL; + char *abs_dst = NULL; + glob_t g; + char *filename, *tmp = NULL, *remote_path = NULL; + int i, r, err = 0; + + /* + * Here, we need remote glob as SFTP can not depend on remote shell + * expansions + */ + + remote_path = do_realpath(conn, "."); + if (remote_path == NULL) { + error("Could not obtain remote working directory"); + /* TODO - gracefully degrade by using relative paths ? */ + err = -1; + goto out; + } + + abs_src = xstrdup(src); + abs_src = make_absolute(abs_src, remote_path); + free(remote_path); + memset(&g, 0, sizeof(g)); + + debug3_f("copying remote %s to local %s", abs_src, dst); + if ((r = remote_glob(conn, abs_src, GLOB_MARK, NULL, &g)) != 0) { + if (r == GLOB_NOSPACE) + error("Too many glob matches for \"%s\".", abs_src); + else + error("File \"%s\" not found.", abs_src); + err = -1; + goto out; + } + + if (g.gl_matchc > 1 && !local_is_dir(dst)) { + error("Multiple files match pattern, but destination " + "\"%s\" is not a directory", dst); + err = -1; + goto out; + } + + for (i = 0; g.gl_pathv[i] && !interrupted; i++) { + tmp = xstrdup(g.gl_pathv[i]); + if ((filename = basename(tmp)) == NULL) { + error("basename %s: %s", tmp, strerror(errno)); + free(tmp); + err = -1; + goto out; + } + free(tmp); + + if (local_is_dir(dst)) + abs_dst = path_append(dst, filename); + else + abs_dst = xstrdup(dst); + + debug("Fetching %s to %s\n", g.gl_pathv[i], abs_dst); + if (globpath_is_dir(g.gl_pathv[i]) && iamrecursive) { + if (download_dir(conn, g.gl_pathv[i], abs_dst, NULL, + pflag, 1, 0, 0) == -1) + err = -1; + } else { + if (do_download(conn, g.gl_pathv[i], abs_dst, NULL, + pflag, 0, 0) == -1) + err = -1; + } + free(abs_dst); + abs_dst = NULL; + } + +out: + free(abs_src); + globfree(&g); + if (err == -1) { + fatal("Failed to download file '%s'", src); + } +} + + #define TYPE_OVERFLOW(type, val) \ ((sizeof(type) == 4 && (val) > INT32_MAX) || \ (sizeof(type) == 8 && (val) > INT64_MAX) || \ @@ -1565,9 +1821,9 @@ void usage(void) { (void) fprintf(stderr, - "usage: scp [-346ABCpqrTv] [-c cipher] [-F ssh_config] [-i identity_file]\n" - " [-J destination] [-l limit] [-o ssh_option] [-P port]\n" - " [-S program] source ... target\n"); + "usage: scp [-346ABCpqrTv] [-c cipher] [-D sftp_server_path] [-F ssh_config]\n" + " [-i identity_file] [-J destination] [-l limit] [-M scp|sftp]\n" + " [-o ssh_option] [-P port] [-S program] source ... target\n"); exit(1); } diff --git a/usr.bin/ssh/scp/Makefile b/usr.bin/ssh/scp/Makefile index 4f0c3f4f980..705f59af796 100644 --- a/usr.bin/ssh/scp/Makefile +++ b/usr.bin/ssh/scp/Makefile @@ -1,9 +1,10 @@ -# $OpenBSD: Makefile,v 1.21 2018/07/25 17:12:35 deraadt Exp $ +# $OpenBSD: Makefile,v 1.22 2021/08/02 23:38:27 djm Exp $ .PATH: ${.CURDIR}/.. SRCS= scp.c SRCS+= atomicio.c cleanup.c fatal.c progressmeter.c utf8.c +SRCS+= sftp-common.c sftp-client.c sftp-glob.c SRCS+= ${SRCS_BASE} PROG= scp @@ -11,3 +12,6 @@ PROG= scp BINDIR= /usr/bin .include + +LDADD+= -lutil +DPADD+= ${LIBUTIL}