From: robert <robert@openbsd.org>
Date: Tue, 2 Feb 2021 12:58:42 +0000 (+0000)
Subject: introduce support for sending the If-Modified-Since header while
X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=49dd5d56f6defd80a07dbcd0295b36eb2c07ab52;p=openbsd

introduce support for sending the If-Modified-Since header while
fetching over http(s) and use the timestamps from the remote server's
Last-Modified header if available when saving local files
this makes it possible to mirror files better with ftp(1)

the new timestamp behaviour can be disabled with the new '-u' flag

ok sthen@, input from sthen@ and gnezdo@
---

diff --git a/usr.bin/ftp/extern.h b/usr.bin/ftp/extern.h
index 53908981b4e..85136eff7fa 100644
--- a/usr.bin/ftp/extern.h
+++ b/usr.bin/ftp/extern.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: extern.h,v 1.51 2019/05/16 12:44:17 florian Exp $	*/
+/*	$OpenBSD: extern.h,v 1.52 2021/02/02 12:58:42 robert Exp $	*/
 /*	$NetBSD: extern.h,v 1.17 1997/08/18 10:20:19 lukem Exp $	*/
 
 /*
@@ -125,6 +125,7 @@ void	parse_list(char **, char *);
 char   *remglob2(char **, int, char **, FILE **ftemp, char *type);
 int	ruserpass(const char *, char **, char **, char **);
 void	sendrequest(const char *, const char *, const char *, int);
+ssize_t http_time(time_t, char *, size_t);
 #endif /* !SMALL */
 
 extern jmp_buf	abortprox;
@@ -144,6 +145,7 @@ extern char	*action;
 
 #ifndef SMALL
 extern int	NCMDS;
+extern int	server_timestamps;
 #endif /* !SMALL */
 
 extern char *__progname;		/* from crt0.o */
diff --git a/usr.bin/ftp/fetch.c b/usr.bin/ftp/fetch.c
index 16f14b0b159..c9655122716 100644
--- a/usr.bin/ftp/fetch.c
+++ b/usr.bin/ftp/fetch.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: fetch.c,v 1.199 2021/01/01 17:39:54 chrisz Exp $	*/
+/*	$OpenBSD: fetch.c,v 1.200 2021/02/02 12:58:42 robert Exp $	*/
 /*	$NetBSD: fetch.c,v 1.14 1997/08/18 10:20:20 lukem Exp $	*/
 
 /*-
@@ -58,6 +58,7 @@
 #include <unistd.h>
 #include <util.h>
 #include <resolv.h>
+#include <utime.h>
 
 #ifndef NOSSL
 #include <tls.h>
@@ -339,6 +340,11 @@ url_get(const char *origline, const char *proxyenv, const char *outfile, int las
 	const char *scheme;
 	char *locbase;
 	struct addrinfo *ares = NULL;
+	char tmbuf[32];
+	time_t mtime = 0;
+	struct stat stbuf;
+	struct tm lmt = { 0 };
+	struct timespec ts[2];
 #endif /* !SMALL */
 	struct tls *tls = NULL;
 	int status;
@@ -731,13 +737,16 @@ noslash:
 		if (verbose)
 			fprintf(ttyout, "Requesting %s\n", origline);
 #ifndef SMALL
-		if (resume) {
-			struct stat stbuf;
-
-			if (stat(savefile, &stbuf) == 0)
-				restart_point = stbuf.st_size;
-			else
+		if (resume || timestamp) {
+			if (stat(savefile, &stbuf) == 0) {
+				if (resume)
+					restart_point = stbuf.st_size;
+				if (timestamp)
+					mtime = stbuf.st_mtime;
+			} else {
 				restart_point = 0;
+				mtime = 0;
+			}
 		}
 #endif	/* SMALL */
 		ftp_printf(fin,
@@ -777,6 +786,12 @@ noslash:
 		if (port && strcmp(port, "80") != 0)
 			ftp_printf(fin, ":%s", port);
 #endif /* !NOSSL */
+
+#ifndef SMALL
+		if (mtime && (http_time(mtime, tmbuf, sizeof(tmbuf)) != 0))
+			ftp_printf(fin, "\r\nIf-Modified-Since: %s", tmbuf);
+#endif /* SMALL */
+
 		ftp_printf(fin, "\r\n%s%s\r\n",
 		    buf ? buf : "", httpuseragent);
 		if (credentials)
@@ -851,6 +866,9 @@ noslash:
 		}
 		break;
 #ifndef SMALL
+	case 304:	/* Not Modified */
+		warnx("File is not modified on the server");
+		goto cleanup_url_get;
 	case 416:	/* Requested Range Not Satisfiable */
 		warnx("File is already fully retrieved.");
 		goto cleanup_url_get;
@@ -973,6 +991,15 @@ noslash:
 			cp[strcspn(cp, " \t")] = '\0';
 			if (strcasecmp(cp, "chunked") == 0)
 				chunked = 1;
+#ifndef SMALL
+#define LAST_MODIFIED "Last-Modified: "
+		} else if (strncasecmp(cp, LAST_MODIFIED,
+			    sizeof(LAST_MODIFIED) - 1) == 0) {
+			cp += sizeof(LAST_MODIFIED) - 1;
+			cp[strcspn(cp, "\t")] = '\0';
+			if (strptime(cp, "%a, %d %h %Y %T %Z", &lmt) == NULL)
+				server_timestamps = 0;
+#endif /* !SMALL */
 		}
 		free(buf);
 	}
@@ -1114,8 +1141,19 @@ cleanup_url_get:
 	free(sslhost);
 #endif /* !NOSSL */
 	ftp_close(&fin, &tls, &fd);
-	if (out >= 0 && out != fileno(stdout))
+	if (out >= 0 && out != fileno(stdout)) {
+#ifndef SMALL
+		if (server_timestamps && lmt.tm_zone != 0) {
+			ts[0].tv_nsec = UTIME_NOW;
+			ts[1].tv_nsec = 0;
+			setenv("TZ", lmt.tm_zone, 1);
+			if (((ts[1].tv_sec = mktime(&lmt)) != -1) &&
+			    (futimens(out, ts) == -1))
+				warnx("Unable to set file modification time");
+		}
+#endif /* !SMALL */
 		close(out);
+	}
 	free(buf);
 	free(pathbuf);
 	free(proxyhost);
diff --git a/usr.bin/ftp/ftp.1 b/usr.bin/ftp/ftp.1
index 4f4bfd8d5d5..6fad63a2b06 100644
--- a/usr.bin/ftp/ftp.1
+++ b/usr.bin/ftp/ftp.1
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: ftp.1,v 1.121 2020/09/06 09:15:04 tb Exp $
+.\"	$OpenBSD: ftp.1,v 1.122 2021/02/02 12:58:42 robert Exp $
 .\"	$NetBSD: ftp.1,v 1.22 1997/08/18 10:20:22 lukem Exp $
 .\"
 .\" Copyright (c) 1985, 1989, 1990, 1993
@@ -30,7 +30,7 @@
 .\"
 .\"	@(#)ftp.1	8.3 (Berkeley) 10/9/94
 .\"
-.Dd $Mdocdate: September 6 2020 $
+.Dd $Mdocdate: February 2 2021 $
 .Dt FTP 1
 .Os
 .Sh NAME
@@ -57,7 +57,7 @@
 .Sm on
 .Ar ...
 .Nm ftp
-.Op Fl C
+.Op Fl CTu
 .Op Fl c Ar cookie
 .Op Fl N Ar name
 .Op Fl o Ar output
@@ -289,6 +289,11 @@ will be used.
 .It Fl s Ar sourceaddr
 Set the source address for connections, which is useful on machines
 with multiple interfaces.
+.It Fl T
+Send an
+.Dq If-Modified-Since
+header to the remote to determine if the remote file's timestamp
+has changed.
 .It Fl t
 Enables packet tracing.
 .It Fl U Ar useragent
@@ -297,6 +302,13 @@ Set
 as the User-Agent for HTTP(S) URL requests.
 If not specified, the default User-Agent is
 .Dq OpenBSD ftp .
+.It Fl u
+Disable setting the local file's timestamps based
+on the
+.Dq Last-Modified
+header.
+By default the local file's timestamps are set to match those
+from the remote.
 .It Fl V
 Disable verbose mode, overriding the default of enabled when input
 is from a terminal.
diff --git a/usr.bin/ftp/ftp_var.h b/usr.bin/ftp/ftp_var.h
index 2f5e952b991..405a247b45a 100644
--- a/usr.bin/ftp/ftp_var.h
+++ b/usr.bin/ftp/ftp_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ftp_var.h,v 1.45 2020/09/01 12:33:48 jca Exp $	*/
+/*	$OpenBSD: ftp_var.h,v 1.46 2021/02/02 12:58:42 robert Exp $	*/
 /*	$NetBSD: ftp_var.h,v 1.18 1997/08/18 10:20:25 lukem Exp $	*/
 
 /*
@@ -157,6 +157,7 @@ extern char	 *cursor_pos;	/* cursor position we're looking for */
 extern size_t	  cursor_argc;	/* location of cursor in margv */
 extern size_t	  cursor_argo;	/* offset of cursor in margv[cursor_argc] */
 extern int	  resume;	/* continue transfer */
+extern int	  timestamp;    /* send an If-Modified-Since header */
 extern char	 *srcaddr;	/* source address to bind to */
 #endif /* !SMALL */
 
diff --git a/usr.bin/ftp/main.c b/usr.bin/ftp/main.c
index 2d2f66b0812..dcae11a5b48 100644
--- a/usr.bin/ftp/main.c
+++ b/usr.bin/ftp/main.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: main.c,v 1.135 2020/09/06 09:49:11 tb Exp $	*/
+/*	$OpenBSD: main.c,v 1.136 2021/02/02 12:58:42 robert Exp $	*/
 /*	$NetBSD: main.c,v 1.24 1997/08/18 10:20:26 lukem Exp $	*/
 
 /*
@@ -144,6 +144,7 @@ size_t	  cursor_argc;
 size_t	  cursor_argo;
 int	  resume;
 char	 *srcaddr;
+int  	  timestamp;
 #endif /* !SMALL */
 
 char	 *cookiefile;
@@ -189,6 +190,11 @@ FILE	*ttyout;
 
 int connect_timeout;
 
+#ifndef SMALL
+/* enable using server timestamps by default */
+int server_timestamps = 1;
+#endif
+
 #ifndef NOSSL
 char * const ssl_verify_opts[] = {
 #define SSL_CAFILE		0
@@ -341,6 +347,7 @@ main(volatile int argc, char *argv[])
 	el = NULL;
 	hist = NULL;
 	resume = 0;
+	timestamp = 0;
 	srcaddr = NULL;
 	marg_sl = sl_init();
 #endif /* !SMALL */
@@ -415,7 +422,7 @@ main(volatile int argc, char *argv[])
 	httpuseragent = NULL;
 
 	while ((ch = getopt(argc, argv,
-		    "46AaCc:dD:EeN:gik:Mmno:pP:r:S:s:tU:vVw:")) != -1) {
+		    "46AaCc:dD:EeN:gik:Mmno:pP:r:S:s:TtU:uvVw:")) != -1) {
 		switch (ch) {
 		case '4':
 			family = PF_INET;
@@ -537,6 +544,11 @@ main(volatile int argc, char *argv[])
 #endif /* !SMALL */
 			break;
 
+#ifndef SMALL
+		case 'T':
+			timestamp = 1;
+			break;
+#endif /* !SMALL */
 		case 't':
 			trace = 1;
 			break;
@@ -551,6 +563,9 @@ main(volatile int argc, char *argv[])
 				errx(1, "Can't allocate memory for HTTP(S) "
 				    "User-Agent");
 			break;
+		case 'u':
+			server_timestamps = 0;
+			break;
 #endif /* !SMALL */
 
 		case 'v':
diff --git a/usr.bin/ftp/util.c b/usr.bin/ftp/util.c
index 930402c8c55..ef187dc800f 100644
--- a/usr.bin/ftp/util.c
+++ b/usr.bin/ftp/util.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: util.c,v 1.94 2020/10/18 20:35:18 naddy Exp $	*/
+/*	$OpenBSD: util.c,v 1.95 2021/02/02 12:58:42 robert Exp $	*/
 /*	$NetBSD: util.c,v 1.12 1997/08/18 10:20:27 lukem Exp $	*/
 
 /*-
@@ -1099,3 +1099,17 @@ connect_wait(int s)
 	}
 	return 0;
 }
+
+#ifndef SMALL
+ssize_t
+http_time(time_t t, char *tmbuf, size_t len)
+{
+	struct tm tm;
+ 
+	/* New HTTP/1.1 RFC 7231 prefers IMF-fixdate from RFC 5322 */
+	if (gmtime_r(&t, &tm) == NULL)
+		return 0;
+	else
+		return (strftime(tmbuf, len, "%a, %d %h %Y %T %Z", &tm));
+}
+#endif /* !SMALL */