introduce support for sending the If-Modified-Since header while

fetching over http(s) and use the timestamps from the remote server's
Last-Modified header if available when saving local files
this makes it possible to mirror files better with ftp(1)

the new timestamp behaviour can be disabled with the new '-u' flag

ok sthen@, input from sthen@ and gnezdo@

diff --git a/usr.bin/ftp/extern.h b/usr.bin/ftp/extern.h
index 5390898..85136ef 100644 (file)
--- a/usr.bin/ftp/extern.h
+++ b/usr.bin/ftp/extern.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: extern.h,v 1.51 2019/05/16 12:44:17 florian Exp $     */
+/*     $OpenBSD: extern.h,v 1.52 2021/02/02 12:58:42 robert Exp $      */
 /*     $NetBSD: extern.h,v 1.17 1997/08/18 10:20:19 lukem Exp $        */
 
 /*
@@ -125,6 +125,7 @@ void        parse_list(char **, char *);
 char   *remglob2(char **, int, char **, FILE **ftemp, char *type);
 int    ruserpass(const char *, char **, char **, char **);
 void   sendrequest(const char *, const char *, const char *, int);
+ssize_t http_time(time_t, char *, size_t);
 #endif /* !SMALL */
 
 extern jmp_buf abortprox;
@@ -144,6 +145,7 @@ extern char *action;
 
 #ifndef SMALL
 extern int     NCMDS;
+extern int     server_timestamps;
 #endif /* !SMALL */
 
 extern char *__progname;               /* from crt0.o */

diff --git a/usr.bin/ftp/fetch.c b/usr.bin/ftp/fetch.c
index 16f14b0..c965512 100644 (file)
--- a/usr.bin/ftp/fetch.c
+++ b/usr.bin/ftp/fetch.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: fetch.c,v 1.199 2021/01/01 17:39:54 chrisz Exp $      */
+/*     $OpenBSD: fetch.c,v 1.200 2021/02/02 12:58:42 robert Exp $      */
 /*     $NetBSD: fetch.c,v 1.14 1997/08/18 10:20:20 lukem Exp $ */
 
 /*-
@@ -58,6 +58,7 @@
 #include <unistd.h>
 #include <util.h>
 #include <resolv.h>
+#include <utime.h>
 
 #ifndef NOSSL
 #include <tls.h>
@@ -339,6 +340,11 @@ url_get(const char *origline, const char *proxyenv, const char *outfile, int las
        const char *scheme;
        char *locbase;
        struct addrinfo *ares = NULL;
+       char tmbuf[32];
+       time_t mtime = 0;
+       struct stat stbuf;
+       struct tm lmt = { 0 };
+       struct timespec ts[2];
 #endif /* !SMALL */
        struct tls *tls = NULL;
        int status;
@@ -731,13 +737,16 @@ noslash:
                if (verbose)
                        fprintf(ttyout, "Requesting %s\n", origline);
 #ifndef SMALL
-               if (resume) {
-                       struct stat stbuf;
-
-                       if (stat(savefile, &stbuf) == 0)
-                               restart_point = stbuf.st_size;
-                       else
+               if (resume || timestamp) {
+                       if (stat(savefile, &stbuf) == 0) {
+                               if (resume)
+                                       restart_point = stbuf.st_size;
+                               if (timestamp)
+                                       mtime = stbuf.st_mtime;
+                       } else {
                                restart_point = 0;
+                               mtime = 0;
+                       }
                }
 #endif /* SMALL */
                ftp_printf(fin,
@@ -777,6 +786,12 @@ noslash:
                if (port && strcmp(port, "80") != 0)
                        ftp_printf(fin, ":%s", port);
 #endif /* !NOSSL */
+
+#ifndef SMALL
+               if (mtime && (http_time(mtime, tmbuf, sizeof(tmbuf)) != 0))
+                       ftp_printf(fin, "\r\nIf-Modified-Since: %s", tmbuf);
+#endif /* SMALL */
+
                ftp_printf(fin, "\r\n%s%s\r\n",
                    buf ? buf : "", httpuseragent);
                if (credentials)
@@ -851,6 +866,9 @@ noslash:
                }
                break;
 #ifndef SMALL
+       case 304:       /* Not Modified */
+               warnx("File is not modified on the server");
+               goto cleanup_url_get;
        case 416:       /* Requested Range Not Satisfiable */
                warnx("File is already fully retrieved.");
                goto cleanup_url_get;
@@ -973,6 +991,15 @@ noslash:
                        cp[strcspn(cp, " \t")] = '\0';
                        if (strcasecmp(cp, "chunked") == 0)
                                chunked = 1;
+#ifndef SMALL
+#define LAST_MODIFIED "Last-Modified: "
+               } else if (strncasecmp(cp, LAST_MODIFIED,
+                           sizeof(LAST_MODIFIED) - 1) == 0) {
+                       cp += sizeof(LAST_MODIFIED) - 1;
+                       cp[strcspn(cp, "\t")] = '\0';
+                       if (strptime(cp, "%a, %d %h %Y %T %Z", &lmt) == NULL)
+                               server_timestamps = 0;
+#endif /* !SMALL */
                }
                free(buf);
        }
@@ -1114,8 +1141,19 @@ cleanup_url_get:
        free(sslhost);
 #endif /* !NOSSL */
        ftp_close(&fin, &tls, &fd);
-       if (out >= 0 && out != fileno(stdout))
+       if (out >= 0 && out != fileno(stdout)) {
+#ifndef SMALL
+               if (server_timestamps && lmt.tm_zone != 0) {
+                       ts[0].tv_nsec = UTIME_NOW;
+                       ts[1].tv_nsec = 0;
+                       setenv("TZ", lmt.tm_zone, 1);
+                       if (((ts[1].tv_sec = mktime(&lmt)) != -1) &&
+                           (futimens(out, ts) == -1))
+                               warnx("Unable to set file modification time");
+               }
+#endif /* !SMALL */
                close(out);
+       }
        free(buf);
        free(pathbuf);
        free(proxyhost);

diff --git a/usr.bin/ftp/ftp.1 b/usr.bin/ftp/ftp.1
index 4f4bfd8..6fad63a 100644 (file)
--- a/usr.bin/ftp/ftp.1
+++ b/usr.bin/ftp/ftp.1
@@ -1,4 +1,4 @@
-.\"    $OpenBSD: ftp.1,v 1.121 2020/09/06 09:15:04 tb Exp $
+.\"    $OpenBSD: ftp.1,v 1.122 2021/02/02 12:58:42 robert Exp $
 .\"    $NetBSD: ftp.1,v 1.22 1997/08/18 10:20:22 lukem Exp $
 .\"
 .\" Copyright (c) 1985, 1989, 1990, 1993
@@ -30,7 +30,7 @@
 .\"
 .\"    @(#)ftp.1       8.3 (Berkeley) 10/9/94
 .\"
-.Dd $Mdocdate: September 6 2020 $
+.Dd $Mdocdate: February 2 2021 $
 .Dt FTP 1
 .Os
 .Sh NAME
@@ -57,7 +57,7 @@
 .Sm on
 .Ar ...
 .Nm ftp
-.Op Fl C
+.Op Fl CTu
 .Op Fl c Ar cookie
 .Op Fl N Ar name
 .Op Fl o Ar output
@@ -289,6 +289,11 @@ will be used.
 .It Fl s Ar sourceaddr
 Set the source address for connections, which is useful on machines
 with multiple interfaces.
+.It Fl T
+Send an
+.Dq If-Modified-Since
+header to the remote to determine if the remote file's timestamp
+has changed.
 .It Fl t
 Enables packet tracing.
 .It Fl U Ar useragent
@@ -297,6 +302,13 @@ Set
 as the User-Agent for HTTP(S) URL requests.
 If not specified, the default User-Agent is
 .Dq OpenBSD ftp .
+.It Fl u
+Disable setting the local file's timestamps based
+on the
+.Dq Last-Modified
+header.
+By default the local file's timestamps are set to match those
+from the remote.
 .It Fl V
 Disable verbose mode, overriding the default of enabled when input
 is from a terminal.

diff --git a/usr.bin/ftp/ftp_var.h b/usr.bin/ftp/ftp_var.h
index 2f5e952..405a247 100644 (file)
--- a/usr.bin/ftp/ftp_var.h
+++ b/usr.bin/ftp/ftp_var.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ftp_var.h,v 1.45 2020/09/01 12:33:48 jca Exp $        */
+/*     $OpenBSD: ftp_var.h,v 1.46 2021/02/02 12:58:42 robert Exp $     */
 /*     $NetBSD: ftp_var.h,v 1.18 1997/08/18 10:20:25 lukem Exp $       */
 
 /*
@@ -157,6 +157,7 @@ extern char  *cursor_pos;   /* cursor position we're looking for */
 extern size_t    cursor_argc;  /* location of cursor in margv */
 extern size_t    cursor_argo;  /* offset of cursor in margv[cursor_argc] */
 extern int       resume;       /* continue transfer */
+extern int       timestamp;    /* send an If-Modified-Since header */
 extern char     *srcaddr;      /* source address to bind to */
 #endif /* !SMALL */
 

diff --git a/usr.bin/ftp/main.c b/usr.bin/ftp/main.c
index 2d2f66b..dcae11a 100644 (file)
--- a/usr.bin/ftp/main.c
+++ b/usr.bin/ftp/main.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: main.c,v 1.135 2020/09/06 09:49:11 tb Exp $   */
+/*     $OpenBSD: main.c,v 1.136 2021/02/02 12:58:42 robert Exp $       */
 /*     $NetBSD: main.c,v 1.24 1997/08/18 10:20:26 lukem Exp $  */
 
 /*
@@ -144,6 +144,7 @@ size_t        cursor_argc;
 size_t   cursor_argo;
 int      resume;
 char    *srcaddr;
+int      timestamp;
 #endif /* !SMALL */
 
 char    *cookiefile;
@@ -189,6 +190,11 @@ FILE       *ttyout;
 
 int connect_timeout;
 
+#ifndef SMALL
+/* enable using server timestamps by default */
+int server_timestamps = 1;
+#endif
+
 #ifndef NOSSL
 char * const ssl_verify_opts[] = {
 #define SSL_CAFILE             0
@@ -341,6 +347,7 @@ main(volatile int argc, char *argv[])
        el = NULL;
        hist = NULL;
        resume = 0;
+       timestamp = 0;
        srcaddr = NULL;
        marg_sl = sl_init();
 #endif /* !SMALL */
@@ -415,7 +422,7 @@ main(volatile int argc, char *argv[])
        httpuseragent = NULL;
 
        while ((ch = getopt(argc, argv,
-                   "46AaCc:dD:EeN:gik:Mmno:pP:r:S:s:tU:vVw:")) != -1) {
+                   "46AaCc:dD:EeN:gik:Mmno:pP:r:S:s:TtU:uvVw:")) != -1) {
                switch (ch) {
                case '4':
                        family = PF_INET;
@@ -537,6 +544,11 @@ main(volatile int argc, char *argv[])
 #endif /* !SMALL */
                        break;
 
+#ifndef SMALL
+               case 'T':
+                       timestamp = 1;
+                       break;
+#endif /* !SMALL */
                case 't':
                        trace = 1;
                        break;
@@ -551,6 +563,9 @@ main(volatile int argc, char *argv[])
                                errx(1, "Can't allocate memory for HTTP(S) "
                                    "User-Agent");
                        break;
+               case 'u':
+                       server_timestamps = 0;
+                       break;
 #endif /* !SMALL */
 
                case 'v':

diff --git a/usr.bin/ftp/util.c b/usr.bin/ftp/util.c
index 930402c..ef187dc 100644 (file)
--- a/usr.bin/ftp/util.c
+++ b/usr.bin/ftp/util.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: util.c,v 1.94 2020/10/18 20:35:18 naddy Exp $ */
+/*     $OpenBSD: util.c,v 1.95 2021/02/02 12:58:42 robert Exp $        */
 /*     $NetBSD: util.c,v 1.12 1997/08/18 10:20:27 lukem Exp $  */
 
 /*-
@@ -1099,3 +1099,17 @@ connect_wait(int s)
        }
        return 0;
 }
+
+#ifndef SMALL
+ssize_t
+http_time(time_t t, char *tmbuf, size_t len)
+{
+       struct tm tm;
+ 
+       /* New HTTP/1.1 RFC 7231 prefers IMF-fixdate from RFC 5322 */
+       if (gmtime_r(&t, &tm) == NULL)
+               return 0;
+       else
+               return (strftime(tmbuf, len, "%a, %d %h %Y %T %Z", &tm));
+}
+#endif /* !SMALL */