additional directories to check for files to be available.
OK benno@
-# $OpenBSD: Makefile,v 1.11 2021/08/29 13:43:46 claudio Exp $
+# $OpenBSD: Makefile,v 1.12 2021/10/22 11:10:34 claudio Exp $
PROG= openrsync
-SRCS= blocks.c client.c downloader.c fargs.c flist.c hash.c ids.c \
+SRCS= blocks.c client.c copy.c downloader.c fargs.c flist.c hash.c ids.c \
io.c log.c main.c misc.c mkpath.c mktemp.c receiver.c rmatch.c \
rules.c sender.c server.c session.c socket.c symlinks.c uploader.c
LDADD+= -lcrypto -lm
DPADD+= ${LIBCRYPTO} ${LIBM}
MAN= openrsync.1
-CFLAGS+=-g -W -Wall -Wextra
+CFLAGS+= -Wall -Wextra
+CFLAGS+= -Wstrict-prototypes -Wmissing-prototypes
+CFLAGS+= -Wmissing-declarations
+CFLAGS+= -Wshadow
+
openrsync.1: rsync.1
ln -sf ${.CURDIR}/rsync.1 openrsync.1
--- /dev/null
+/* $OpenBSD: copy.c,v 1.1 2021/10/22 11:10:34 claudio Exp $ */
+/*
+ * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h> /* MAXBSIZE */
+
+#include <err.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "extern.h"
+
+/*
+ * Return true if all bytes in buffer are zero.
+ * A buffer of zero lenght is also considered a zero buffer.
+ */
+static int
+iszero(const void *b, size_t len)
+{
+ const unsigned char *c = b;
+
+ for (; len > 0; len--) {
+ if (*c++ != '\0')
+ return 0;
+ }
+ return 1;
+}
+
+static int
+copy_internal(int fromfd, int tofd)
+{
+ char buf[MAXBSIZE];
+ ssize_t r, w;
+
+ while ((r = read(fromfd, buf, sizeof(buf))) > 0) {
+ if (iszero(buf, sizeof(buf))) {
+ if (lseek(tofd, r, SEEK_CUR) == -1)
+ return -1;
+ } else {
+ w = write(tofd, buf, r);
+ if (r != w || w == -1)
+ return -1;
+ }
+ }
+ if (r == -1)
+ return -1;
+ if (ftruncate(tofd, lseek(tofd, 0, SEEK_CUR)) == -1)
+ return -1;
+ return 0;
+}
+
+void
+copy_file(int rootfd, const char *basedir, const struct flist *f)
+{
+ int fromfd, tofd, dfd;
+
+ dfd = openat(rootfd, basedir, O_RDONLY | O_DIRECTORY, 0);
+ if (dfd == -1)
+ err(ERR_FILE_IO, "%s: openat", basedir);
+
+ fromfd = openat(dfd, f->path, O_RDONLY | O_NOFOLLOW, 0);
+ if (fromfd == -1)
+ err(ERR_FILE_IO, "%s/%s: openat", basedir, f->path);
+ close(dfd);
+
+ tofd = openat(rootfd, f->path,
+ O_WRONLY | O_NOFOLLOW | O_TRUNC | O_CREAT | O_EXCL,
+ 0600);
+ if (tofd == -1)
+ err(ERR_FILE_IO, "%s: openat", f->path);
+
+ if (copy_internal(fromfd, tofd) == -1)
+ err(ERR_FILE_IO, "%s: copy file", f->path);
+
+ close(fromfd);
+ close(tofd);
+}
-/* $OpenBSD: extern.h,v 1.41 2021/09/01 09:48:08 claudio Exp $ */
+/* $OpenBSD: extern.h,v 1.42 2021/10/22 11:10:34 claudio Exp $ */
/*
* Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
*
*/
#define BLOCK_SIZE_MIN (700)
+/*
+ * Maximum number of base directories that can be used.
+ */
+#define MAX_BASEDIR 20
+
+#define BASE_MODE_COMPARE 1
+#define BASE_MODE_COPY 2
+#define BASE_MODE_LINK 3
+
/*
* The sender and receiver use a two-phase synchronisation process.
* The first uses two-byte hashes; the second, 16-byte.
int no_motd; /* --no-motd */
int numeric_ids; /* --numeric-ids */
int one_file_system; /* -x */
+ int alt_base_mode;
char *rsync_path; /* --rsync-path */
char *ssh_prog; /* --rsh or -e */
char *port; /* --port */
char *address; /* --address */
+ char *basedir[MAX_BASEDIR];
};
enum rule_type {
int flist_gen_dels(struct sess *, const char *, struct flist **, size_t *,
const struct flist *, size_t);
-char **fargs_cmdline(struct sess *, const struct fargs *, size_t *);
+const char *alt_base_mode(int);
+char **fargs_cmdline(struct sess *, const struct fargs *, size_t *);
int io_read_buf(struct sess *, int, void *, size_t);
int io_read_byte(struct sess *, int, uint8_t *);
void hash_file(const void *, size_t, unsigned char *,
const struct sess *);
+void copy_file(int, const char *, const struct flist *);
+
int mkpath(char *);
int mkstempat(int, char *);
-/* $OpenBSD: fargs.c,v 1.19 2021/06/30 13:10:04 claudio Exp $ */
+/* $OpenBSD: fargs.c,v 1.20 2021/10/22 11:10:34 claudio Exp $ */
/*
* Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
*
#define RSYNC_PATH "rsync"
+const char *
+alt_base_mode(int mode)
+{
+ switch(mode) {
+ case BASE_MODE_COMPARE:
+ return "--compare-dest";
+ case BASE_MODE_COPY:
+ return "--copy-dest";
+ case BASE_MODE_LINK:
+ return "--link-dest";
+ default:
+ errx(1, "unknown base mode %d", mode);
+ }
+}
+
char **
fargs_cmdline(struct sess *sess, const struct fargs *f, size_t *skip)
{
/* --devices is sent as -D --no-specials */
addargs(&args, "--no-specials");
+ /* only add --compare-dest, etc if this is the sender */
+ if (sess->opts->alt_base_mode != 0 &&
+ f->mode == FARGS_SENDER) {
+ for (j = 0; j < MAX_BASEDIR; j++) {
+ if (sess->opts->basedir[j] == NULL)
+ break;
+ addargs(&args, "%s=%s",
+ alt_base_mode(sess->opts->alt_base_mode),
+ sess->opts->basedir[j]);
+ }
+ }
+
/* Terminate with a full-stop for reasons unknown. */
addargs(&args, ".");
-/* $OpenBSD: main.c,v 1.59 2021/09/01 09:48:08 claudio Exp $ */
+/* $OpenBSD: main.c,v 1.60 2021/10/22 11:10:34 claudio Exp $ */
/*
* Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
*
#define OP_INCLUDE 1006
#define OP_EXCLUDE_FROM 1007
#define OP_INCLUDE_FROM 1008
+#define OP_COMP_DEST 1009
+#define OP_COPY_DEST 1010
+#define OP_LINK_DEST 1011
const struct option lopts[] = {
{ "address", required_argument, NULL, OP_ADDRESS },
{ "archive", no_argument, NULL, 'a' },
+ { "compare-dest", required_argument, NULL, OP_COMP_DEST },
+#if 0
+ { "copy-dest", required_argument, NULL, OP_COPY_DEST },
+ { "link-dest", required_argument, NULL, OP_LINK_DEST },
+#endif
{ "compress", no_argument, NULL, 'z' },
{ "del", no_argument, &opts.del, 1 },
{ "delete", no_argument, &opts.del, 1 },
main(int argc, char *argv[])
{
pid_t child;
- int fds[2], sd = -1, rc, c, st, i;
+ int fds[2], sd = -1, rc, c, st, i, lidx;
+ size_t basedir_cnt = 0;
struct sess sess;
struct fargs *fargs;
char **args;
NULL) == -1)
err(ERR_IPC, "pledge");
- while ((c = getopt_long(argc, argv, "Dae:ghlnoprtvxz", lopts, NULL))
+ while ((c = getopt_long(argc, argv, "Dae:ghlnoprtvxz", lopts, &lidx))
!= -1) {
switch (c) {
case 'D':
case OP_INCLUDE_FROM:
parse_file(optarg, RULE_INCLUDE);
break;
+ case OP_COMP_DEST:
+ if (opts.alt_base_mode !=0 &&
+ opts.alt_base_mode != BASE_MODE_COMPARE) {
+ errx(1, "option --%s conflicts with %s",
+ lopts[lidx].name,
+ alt_base_mode(opts.alt_base_mode));
+ }
+ opts.alt_base_mode = BASE_MODE_COMPARE;
+#if 0
+ goto basedir;
+ case OP_COPY_DEST:
+ if (opts.alt_base_mode !=0 &&
+ opts.alt_base_mode != BASE_MODE_COPY) {
+ errx(1, "option --%s conflicts with %s",
+ lopts[lidx].name,
+ alt_base_mode(opts.alt_base_mode));
+ }
+ opts.alt_base_mode = BASE_MODE_COPY;
+ goto basedir;
+ case OP_LINK_DEST:
+ if (opts.alt_base_mode !=0 &&
+ opts.alt_base_mode != BASE_MODE_LINK) {
+ errx(1, "option --%s conflicts with %s",
+ lopts[lidx].name,
+ alt_base_mode(opts.alt_base_mode));
+ }
+ opts.alt_base_mode = BASE_MODE_LINK;
+
+basedir:
+#endif
+ if (basedir_cnt >= MAX_BASEDIR)
+ errx(1, "too many --%s directories specified",
+ lopts[lidx].name);
+ opts.basedir[basedir_cnt++] = optarg;
+ break;
case OP_VERSION:
fprintf(stderr, "openrsync: protocol version %u\n",
RSYNC_PROTOCOL);
exit(rc);
usage:
fprintf(stderr, "usage: %s"
- " [-aDglnoprtvx] [-e program] [--address=sourceaddr] [--del]\n"
- "\t[--exclude] [--exclude-from=file] [--include] "
- "[--include-from=file]\n"
- "\t[--no-motd] [--numeric-ids] [--port=portnumber] "
- "[--rsync-path=program]\n\t[--timeout=seconds] [--version] "
- "source ... directory\n",
+ " [-aDglnoprtvx] [-e program] [--address=sourceaddr]\n"
+ "\t[--compare-dest=dir] [--del] [--exclude] [--exclude-from=file]\n"
+ "\t[--include] [--include-from=file] [--no-motd] [--numeric-ids]\n"
+ "\t[--port=portnumber] [--rsync-path=program] [--timeout=seconds]\n"
+ "\t[--version] source ... directory\n",
getprogname());
exit(ERR_SYNTAX);
}
-/* $OpenBSD: receiver.c,v 1.29 2021/08/29 13:43:46 claudio Exp $ */
+/* $OpenBSD: receiver.c,v 1.30 2021/10/22 11:10:34 claudio Exp $ */
/*
* Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
if (pledge("stdio unix rpath wpath cpath dpath fattr chown getpw unveil", NULL) == -1)
err(ERR_IPC, "pledge");
+ /*
+ * Create the path for our destination directory, if we're not
+ * in dry-run mode (which would otherwise crash w/the pledge).
+ * This uses our current umask: we might set the permissions on
+ * this directory in post_dir().
+ */
+
+ if (!sess->opts->dry_run) {
+ if ((tofree = strdup(root)) == NULL)
+ err(ERR_NOMEM, NULL);
+ if (mkpath(tofree) < 0)
+ err(ERR_FILE_IO, "%s: mkpath", tofree);
+ free(tofree);
+ }
+
+ /*
+ * Make our entire view of the file-system be limited to what's
+ * in the root directory.
+ * This prevents us from accidentally (or "under the influence")
+ * writing into other parts of the file-system.
+ */
+ if (sess->opts->basedir[0]) {
+ /*
+ * XXX just unveil everything for read
+ * Could unveil each basedir or maybe a common path
+ * also the fact that relative path are relative to the
+ * root does not help.
+ */
+ if (unveil("/", "r") == -1)
+ err(ERR_IPC, "%s: unveil", root);
+ }
+
+ if (unveil(root, "rwc") == -1)
+ err(ERR_IPC, "%s: unveil", root);
+
+ if (unveil(NULL, NULL) == -1)
+ err(ERR_IPC, "unveil");
+
/* Client sends exclusions. */
if (!sess->opts->server)
send_rules(sess, fdout);
LOG2("%s: receiver destination", root);
- /*
- * Create the path for our destination directory, if we're not
- * in dry-run mode (which would otherwise crash w/the pledge).
- * This uses our current umask: we might set the permissions on
- * this directory in post_dir().
- */
-
- if (!sess->opts->dry_run) {
- if ((tofree = strdup(root)) == NULL)
- err(ERR_NOMEM, NULL);
- if (mkpath(tofree) < 0)
- err(ERR_FILE_IO, "%s: mkpath", tofree);
- free(tofree);
- }
-
/*
* Disable umask() so we can set permissions fully.
* Then open the directory iff we're not in dry_run.
goto out;
}
- /*
- * Make our entire view of the file-system be limited to what's
- * in the root directory.
- * This prevents us from accidentally (or "under the influence")
- * writing into other parts of the file-system.
- */
-
- if (unveil(root, "rwc") == -1)
- err(ERR_IPC, "%s: unveil", root);
- if (unveil(NULL, NULL) == -1)
- err(ERR_IPC, "unveil");
-
/* If we have a local set, go for the deletion. */
if (!flist_del(sess, dfd, dfl, dflsz)) {
-.\" $OpenBSD: rsync.1,v 1.25 2021/08/30 20:25:24 job Exp $
+.\" $OpenBSD: rsync.1,v 1.26 2021/10/22 11:10:34 claudio Exp $
.\"
.\" Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
.\"
.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
.\"
-.Dd $Mdocdate: August 30 2021 $
+.Dd $Mdocdate: October 22 2021 $
.Dt OPENRSYNC 1
.Os
.Sh NAME
.Op Fl aDglnoprtvx
.Op Fl e Ar program
.Op Fl -address Ns = Ns Ar sourceaddr
+.Op Fl -compare-dest Ns = Ns Ar directory
.Op Fl -del
.Op Fl -exclude Ar pattern
.Op Fl -exclude-from Ns = Ns Ar file
.Ar sourceaddr
as the source address for connections, which is useful on machines with
multiple interfaces.
+.It Fl -compare-dest Ns = Ns Ar directory
+Use directory as an alternate base directory to compare files against on the
+destination machine.
+If file in
+.Ar directory
+is found and identical to the senders file the file will not be transferred.
+Multiple
+.Fl -compare-dest
+directories may be provided.
+If
+.Ar directory
+is a relative path, it is relative to the destination directory.
.It Fl D
Also transfer device and special files.
Shorthand for
-/* $OpenBSD: uploader.c,v 1.29 2021/06/30 13:10:04 claudio Exp $ */
+/* $OpenBSD: uploader.c,v 1.30 2021/10/22 11:10:34 claudio Exp $ */
/*
* Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
* Copyright (c) 2019 Florian Obser <florian@openbsd.org>
#include <sys/stat.h>
#include <assert.h>
+#include <err.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
* Return <0 on failure 0 on success.
*/
static int
-pre_link(struct upload *p, struct sess *sess)
+pre_symlink(struct upload *p, struct sess *sess)
{
struct stat st;
const struct flist *f;
}
/*
- * See pre_link(), but for devices.
+ * See pre_symlink(), but for devices.
* FIXME: this is very similar to the other pre_xxx() functions.
* Return <0 on failure 0 on success.
*/
}
/*
- * See pre_link(), but for FIFOs.
+ * See pre_symlink(), but for FIFOs.
* FIXME: this is very similar to the other pre_xxx() functions.
* Return <0 on failure 0 on success.
*/
}
/*
- * See pre_link(), but for socket files.
+ * See pre_symlink(), but for socket files.
* FIXME: this is very similar to the other pre_xxx() functions.
* Return <0 on failure 0 on success.
*/
return 1;
}
+/*
+ * Check if file exists in the specified root directory.
+ * Returns:
+ * -1 on error
+ * 0 if file is considered the same
+ * 1 if file exists and is possible match
+ * 2 if file exists but quick check failed
+ * 3 if file does not exist
+ * The stat pointer st is only valid for 0, 1, and 2 returns.
+ */
+static int
+check_file(int rootfd, const struct flist *f, struct stat *st)
+{
+ if (fstatat(rootfd, f->path, st, AT_SYMLINK_NOFOLLOW) == -1) {
+ if (errno == ENOENT)
+ return 3;
+
+ ERR("%s: fstatat", f->path);
+ return -1;
+ }
+
+ /* non-regular file needs attention */
+ if (!S_ISREG(st->st_mode))
+ return 2;
+
+ /* quick check if file is the same */
+ /* TODO: add support for --checksum, --size-only and --ignore-times */
+ if (st->st_size == f->st.size) {
+ if (st->st_mtime == f->st.mtime)
+ return 0;
+ return 1;
+ }
+
+ /* file needs attention */
+ return 2;
+}
+
/*
* Try to open the file at the current index.
* If the file does not exist, returns with >0.
* success and the file needs attention.
*/
static int
-pre_file(const struct upload *p, int *filefd, struct stat *st,
+pre_file(const struct upload *p, int *filefd, off_t *size,
struct sess *sess)
{
const struct flist *f;
- int rc;
+ struct stat st;
+ int i, rc, match = -1;
f = &p->fl[p->idx];
assert(S_ISREG(f->st.mode));
* in the rsync_uploader() function.
*/
+ *size = 0;
*filefd = -1;
- rc = fstatat(p->rootfd, f->path, st, AT_SYMLINK_NOFOLLOW);
- if (rc == -1) {
- if (errno == ENOENT)
- return 1;
-
- ERR("%s: fstatat", f->path);
+ rc = check_file(p->rootfd, f, &st);
+ if (rc == -1)
return -1;
- }
- if (!S_ISREG(st->st_mode)) {
- if (S_ISDIR(st->st_mode) &&
+ if (rc == 2 && !S_ISREG(st.st_mode)) {
+ if (S_ISDIR(st.st_mode) &&
unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) {
ERR("%s: unlinkat", f->path);
return -1;
}
- return 1;
}
-
- /* quick check if file is the same */
- if (st->st_size == f->st.size &&
- st->st_mtime == f->st.mtime) {
- LOG3("%s: skipping: up to date", f->path);
+ if (rc == 0) {
if (!rsync_set_metadata_at(sess, 0, p->rootfd, f, f->path)) {
ERRX1("rsync_set_metadata");
return -1;
}
+ LOG3("%s: skipping: up to date", f->path);
return 0;
}
+ /* check alternative locations for better match */
+ for (i = 0; sess->opts->basedir[i] != NULL; i++) {
+ const char *root = sess->opts->basedir[i];
+ int dfd, x;
+
+ dfd = openat(p->rootfd, root, O_RDONLY | O_DIRECTORY, 0);
+ if (dfd == -1)
+ err(ERR_FILE_IO, "%s: openat", root);
+ x = check_file(dfd, f, &st);
+ /* found a match */
+ if (x == 0) {
+ if (rc >= 0) {
+ /* found better match, delete file in rootfd */
+ if (unlinkat(p->rootfd, f->path, 0) == -1 &&
+ errno != ENOENT) {
+ ERR("%s: unlinkat", f->path);
+ return -1;
+ }
+ }
+ LOG3("%s: skipping: up to date in %s", f->path, root);
+ /* TODO: depending on mode link or copy file */
+ close(dfd);
+ return 0;
+ } else if (x == 1 && match == -1) {
+ /* found a local file that is a close match */
+ match = i;
+ }
+ close(dfd);
+ }
+ if (match != -1) {
+ /* copy match from basedir into root as a start point */
+ copy_file(p->rootfd, sess->opts->basedir[match], f);
+ if (fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW) ==
+ -1) {
+ ERR("%s: fstatat", f->path);
+ return -1;
+ }
+ }
+
+ *size = st.st_size;
*filefd = openat(p->rootfd, f->path, O_RDONLY | O_NOFOLLOW, 0);
if (*filefd == -1 && errno != ENOENT) {
ERR("%s: openat", f->path);
struct sess *sess, int *fileoutfd)
{
struct blkset blk;
- struct stat st;
void *mbuf, *bufp;
ssize_t msz;
size_t i, pos, sz;
- off_t offs;
+ off_t offs, filesize;
int c;
/* Once finished this should never get called again. */
if (S_ISDIR(u->fl[u->idx].st.mode))
c = pre_dir(u, sess);
else if (S_ISLNK(u->fl[u->idx].st.mode))
- c = pre_link(u, sess);
+ c = pre_symlink(u, sess);
else if (S_ISREG(u->fl[u->idx].st.mode))
- c = pre_file(u, fileinfd, &st, sess);
+ c = pre_file(u, fileinfd, &filesize, sess);
else if (S_ISBLK(u->fl[u->idx].st.mode) ||
S_ISCHR(u->fl[u->idx].st.mode))
c = pre_dev(u, sess);
memset(&blk, 0, sizeof(struct blkset));
blk.csum = u->csumlen;
- if (*fileinfd != -1 && st.st_size > 0) {
- init_blkset(&blk, st.st_size);
+ if (*fileinfd != -1 && filesize > 0) {
+ init_blkset(&blk, filesize);
assert(blk.blksz);
blk.blks = calloc(blk.blksz, sizeof(struct blk));
projects / openbsd / commitdiff