From 49dd5d56f6defd80a07dbcd0295b36eb2c07ab52 Mon Sep 17 00:00:00 2001 From: robert Date: Tue, 2 Feb 2021 12:58:42 +0000 Subject: [PATCH] introduce support for sending the If-Modified-Since header while fetching over http(s) and use the timestamps from the remote server's Last-Modified header if available when saving local files this makes it possible to mirror files better with ftp(1) the new timestamp behaviour can be disabled with the new '-u' flag ok sthen@, input from sthen@ and gnezdo@ --- usr.bin/ftp/extern.h | 4 +++- usr.bin/ftp/fetch.c | 54 ++++++++++++++++++++++++++++++++++++------- usr.bin/ftp/ftp.1 | 18 ++++++++++++--- usr.bin/ftp/ftp_var.h | 3 ++- usr.bin/ftp/main.c | 19 +++++++++++++-- usr.bin/ftp/util.c | 16 ++++++++++++- 6 files changed, 98 insertions(+), 16 deletions(-) diff --git a/usr.bin/ftp/extern.h b/usr.bin/ftp/extern.h index 53908981b4e..85136eff7fa 100644 --- a/usr.bin/ftp/extern.h +++ b/usr.bin/ftp/extern.h @@ -1,4 +1,4 @@ -/* $OpenBSD: extern.h,v 1.51 2019/05/16 12:44:17 florian Exp $ */ +/* $OpenBSD: extern.h,v 1.52 2021/02/02 12:58:42 robert Exp $ */ /* $NetBSD: extern.h,v 1.17 1997/08/18 10:20:19 lukem Exp $ */ /* @@ -125,6 +125,7 @@ void parse_list(char **, char *); char *remglob2(char **, int, char **, FILE **ftemp, char *type); int ruserpass(const char *, char **, char **, char **); void sendrequest(const char *, const char *, const char *, int); +ssize_t http_time(time_t, char *, size_t); #endif /* !SMALL */ extern jmp_buf abortprox; @@ -144,6 +145,7 @@ extern char *action; #ifndef SMALL extern int NCMDS; +extern int server_timestamps; #endif /* !SMALL */ extern char *__progname; /* from crt0.o */ diff --git a/usr.bin/ftp/fetch.c b/usr.bin/ftp/fetch.c index 16f14b0b159..c9655122716 100644 --- a/usr.bin/ftp/fetch.c +++ b/usr.bin/ftp/fetch.c @@ -1,4 +1,4 @@ -/* $OpenBSD: fetch.c,v 1.199 2021/01/01 17:39:54 chrisz Exp $ */ +/* $OpenBSD: fetch.c,v 1.200 2021/02/02 12:58:42 robert Exp $ */ /* $NetBSD: fetch.c,v 1.14 1997/08/18 10:20:20 lukem Exp $ */ /*- @@ -58,6 +58,7 @@ #include #include #include +#include #ifndef NOSSL #include @@ -339,6 +340,11 @@ url_get(const char *origline, const char *proxyenv, const char *outfile, int las const char *scheme; char *locbase; struct addrinfo *ares = NULL; + char tmbuf[32]; + time_t mtime = 0; + struct stat stbuf; + struct tm lmt = { 0 }; + struct timespec ts[2]; #endif /* !SMALL */ struct tls *tls = NULL; int status; @@ -731,13 +737,16 @@ noslash: if (verbose) fprintf(ttyout, "Requesting %s\n", origline); #ifndef SMALL - if (resume) { - struct stat stbuf; - - if (stat(savefile, &stbuf) == 0) - restart_point = stbuf.st_size; - else + if (resume || timestamp) { + if (stat(savefile, &stbuf) == 0) { + if (resume) + restart_point = stbuf.st_size; + if (timestamp) + mtime = stbuf.st_mtime; + } else { restart_point = 0; + mtime = 0; + } } #endif /* SMALL */ ftp_printf(fin, @@ -777,6 +786,12 @@ noslash: if (port && strcmp(port, "80") != 0) ftp_printf(fin, ":%s", port); #endif /* !NOSSL */ + +#ifndef SMALL + if (mtime && (http_time(mtime, tmbuf, sizeof(tmbuf)) != 0)) + ftp_printf(fin, "\r\nIf-Modified-Since: %s", tmbuf); +#endif /* SMALL */ + ftp_printf(fin, "\r\n%s%s\r\n", buf ? buf : "", httpuseragent); if (credentials) @@ -851,6 +866,9 @@ noslash: } break; #ifndef SMALL + case 304: /* Not Modified */ + warnx("File is not modified on the server"); + goto cleanup_url_get; case 416: /* Requested Range Not Satisfiable */ warnx("File is already fully retrieved."); goto cleanup_url_get; @@ -973,6 +991,15 @@ noslash: cp[strcspn(cp, " \t")] = '\0'; if (strcasecmp(cp, "chunked") == 0) chunked = 1; +#ifndef SMALL +#define LAST_MODIFIED "Last-Modified: " + } else if (strncasecmp(cp, LAST_MODIFIED, + sizeof(LAST_MODIFIED) - 1) == 0) { + cp += sizeof(LAST_MODIFIED) - 1; + cp[strcspn(cp, "\t")] = '\0'; + if (strptime(cp, "%a, %d %h %Y %T %Z", &lmt) == NULL) + server_timestamps = 0; +#endif /* !SMALL */ } free(buf); } @@ -1114,8 +1141,19 @@ cleanup_url_get: free(sslhost); #endif /* !NOSSL */ ftp_close(&fin, &tls, &fd); - if (out >= 0 && out != fileno(stdout)) + if (out >= 0 && out != fileno(stdout)) { +#ifndef SMALL + if (server_timestamps && lmt.tm_zone != 0) { + ts[0].tv_nsec = UTIME_NOW; + ts[1].tv_nsec = 0; + setenv("TZ", lmt.tm_zone, 1); + if (((ts[1].tv_sec = mktime(&lmt)) != -1) && + (futimens(out, ts) == -1)) + warnx("Unable to set file modification time"); + } +#endif /* !SMALL */ close(out); + } free(buf); free(pathbuf); free(proxyhost); diff --git a/usr.bin/ftp/ftp.1 b/usr.bin/ftp/ftp.1 index 4f4bfd8d5d5..6fad63a2b06 100644 --- a/usr.bin/ftp/ftp.1 +++ b/usr.bin/ftp/ftp.1 @@ -1,4 +1,4 @@ -.\" $OpenBSD: ftp.1,v 1.121 2020/09/06 09:15:04 tb Exp $ +.\" $OpenBSD: ftp.1,v 1.122 2021/02/02 12:58:42 robert Exp $ .\" $NetBSD: ftp.1,v 1.22 1997/08/18 10:20:22 lukem Exp $ .\" .\" Copyright (c) 1985, 1989, 1990, 1993 @@ -30,7 +30,7 @@ .\" .\" @(#)ftp.1 8.3 (Berkeley) 10/9/94 .\" -.Dd $Mdocdate: September 6 2020 $ +.Dd $Mdocdate: February 2 2021 $ .Dt FTP 1 .Os .Sh NAME @@ -57,7 +57,7 @@ .Sm on .Ar ... .Nm ftp -.Op Fl C +.Op Fl CTu .Op Fl c Ar cookie .Op Fl N Ar name .Op Fl o Ar output @@ -289,6 +289,11 @@ will be used. .It Fl s Ar sourceaddr Set the source address for connections, which is useful on machines with multiple interfaces. +.It Fl T +Send an +.Dq If-Modified-Since +header to the remote to determine if the remote file's timestamp +has changed. .It Fl t Enables packet tracing. .It Fl U Ar useragent @@ -297,6 +302,13 @@ Set as the User-Agent for HTTP(S) URL requests. If not specified, the default User-Agent is .Dq OpenBSD ftp . +.It Fl u +Disable setting the local file's timestamps based +on the +.Dq Last-Modified +header. +By default the local file's timestamps are set to match those +from the remote. .It Fl V Disable verbose mode, overriding the default of enabled when input is from a terminal. diff --git a/usr.bin/ftp/ftp_var.h b/usr.bin/ftp/ftp_var.h index 2f5e952b991..405a247b45a 100644 --- a/usr.bin/ftp/ftp_var.h +++ b/usr.bin/ftp/ftp_var.h @@ -1,4 +1,4 @@ -/* $OpenBSD: ftp_var.h,v 1.45 2020/09/01 12:33:48 jca Exp $ */ +/* $OpenBSD: ftp_var.h,v 1.46 2021/02/02 12:58:42 robert Exp $ */ /* $NetBSD: ftp_var.h,v 1.18 1997/08/18 10:20:25 lukem Exp $ */ /* @@ -157,6 +157,7 @@ extern char *cursor_pos; /* cursor position we're looking for */ extern size_t cursor_argc; /* location of cursor in margv */ extern size_t cursor_argo; /* offset of cursor in margv[cursor_argc] */ extern int resume; /* continue transfer */ +extern int timestamp; /* send an If-Modified-Since header */ extern char *srcaddr; /* source address to bind to */ #endif /* !SMALL */ diff --git a/usr.bin/ftp/main.c b/usr.bin/ftp/main.c index 2d2f66b0812..dcae11a5b48 100644 --- a/usr.bin/ftp/main.c +++ b/usr.bin/ftp/main.c @@ -1,4 +1,4 @@ -/* $OpenBSD: main.c,v 1.135 2020/09/06 09:49:11 tb Exp $ */ +/* $OpenBSD: main.c,v 1.136 2021/02/02 12:58:42 robert Exp $ */ /* $NetBSD: main.c,v 1.24 1997/08/18 10:20:26 lukem Exp $ */ /* @@ -144,6 +144,7 @@ size_t cursor_argc; size_t cursor_argo; int resume; char *srcaddr; +int timestamp; #endif /* !SMALL */ char *cookiefile; @@ -189,6 +190,11 @@ FILE *ttyout; int connect_timeout; +#ifndef SMALL +/* enable using server timestamps by default */ +int server_timestamps = 1; +#endif + #ifndef NOSSL char * const ssl_verify_opts[] = { #define SSL_CAFILE 0 @@ -341,6 +347,7 @@ main(volatile int argc, char *argv[]) el = NULL; hist = NULL; resume = 0; + timestamp = 0; srcaddr = NULL; marg_sl = sl_init(); #endif /* !SMALL */ @@ -415,7 +422,7 @@ main(volatile int argc, char *argv[]) httpuseragent = NULL; while ((ch = getopt(argc, argv, - "46AaCc:dD:EeN:gik:Mmno:pP:r:S:s:tU:vVw:")) != -1) { + "46AaCc:dD:EeN:gik:Mmno:pP:r:S:s:TtU:uvVw:")) != -1) { switch (ch) { case '4': family = PF_INET; @@ -537,6 +544,11 @@ main(volatile int argc, char *argv[]) #endif /* !SMALL */ break; +#ifndef SMALL + case 'T': + timestamp = 1; + break; +#endif /* !SMALL */ case 't': trace = 1; break; @@ -551,6 +563,9 @@ main(volatile int argc, char *argv[]) errx(1, "Can't allocate memory for HTTP(S) " "User-Agent"); break; + case 'u': + server_timestamps = 0; + break; #endif /* !SMALL */ case 'v': diff --git a/usr.bin/ftp/util.c b/usr.bin/ftp/util.c index 930402c8c55..ef187dc800f 100644 --- a/usr.bin/ftp/util.c +++ b/usr.bin/ftp/util.c @@ -1,4 +1,4 @@ -/* $OpenBSD: util.c,v 1.94 2020/10/18 20:35:18 naddy Exp $ */ +/* $OpenBSD: util.c,v 1.95 2021/02/02 12:58:42 robert Exp $ */ /* $NetBSD: util.c,v 1.12 1997/08/18 10:20:27 lukem Exp $ */ /*- @@ -1099,3 +1099,17 @@ connect_wait(int s) } return 0; } + +#ifndef SMALL +ssize_t +http_time(time_t t, char *tmbuf, size_t len) +{ + struct tm tm; + + /* New HTTP/1.1 RFC 7231 prefers IMF-fixdate from RFC 5322 */ + if (gmtime_r(&t, &tm) == NULL) + return 0; + else + return (strftime(tmbuf, len, "%a, %d %h %Y %T %Z", &tm)); +} +#endif /* !SMALL */ -- 2.20.1