receiver: add --delay-symlinks

NEWS.md

@@ -9,6 +9,8 @@
   `B/s`. Example output includes `46.7M`, `48.8Mi`, `782,448B` and
   `113,295B/s`. File-list, created-directory and skipped-deletion counts also
   use thousands separators.
+- Added `--delay-symlinks` to defer creating new and changed symlinks until
+  other transferred files are in place.
 
 ------------------------------------------------------------------------------
 

compat.c

@@ -36,6 +36,7 @@ extern int fuzzy_basis;
 extern int read_batch;
 extern int write_batch;
 extern int delay_updates;
+extern int delay_symlinks;
 extern int checksum_seed;
 extern int basis_dir_cnt;
 extern int prune_empty_dirs;
@@ -173,7 +174,7 @@ void set_allow_inc_recurse(void)
 		allow_inc_recurse = 0;
 	else if (!am_sender
 	 && (delete_before || delete_after
-	  || delay_updates || prune_empty_dirs))
+	  || delay_updates || delay_symlinks || prune_empty_dirs))
 		allow_inc_recurse = 0;
 	else if (am_server && strchr(client_info, 'i') == NULL)
 		allow_inc_recurse = 0;
@@ -719,6 +720,14 @@ void setup_protocol(int f_out,int f_in)
 				protocol_version);
 			exit_cleanup(RERR_PROTOCOL);
 		}
+
+		if (delay_symlinks) {
+			rprintf(FERROR,
+				"--delay-symlinks requires protocol 29 or higher"
+				" (negotiated %d).\n",
+				protocol_version);
+			exit_cleanup(RERR_PROTOCOL);
+		}
 	} else if (protocol_version >= 30) {
 		if (am_server) {
 			compat_flags = allow_inc_recurse ? CF_INC_RECURSE : 0;

generator.c

@@ -58,6 +58,7 @@ extern int msgdone_cnt;
 extern int ignore_errors;
 extern int remove_source_files;
 extern int delay_updates;
+extern int delay_symlinks;
 extern int update_only;
 extern int human_readable;
 extern int ignore_existing;
@@ -117,14 +118,48 @@ static int need_retouch_dir_times;
 static int need_retouch_dir_perms;
 static const char *solo_file = NULL;
 
+/* With --delay-symlinks, new and changed symlinks are deferred here and created
+ * only after the receiver has put every file into place (including any
+ * --delay-updates renames).  Each entry keeps what recv_generator() needs to
+ * itemize and log the link once it has been created. */
+struct deferred_symlink {
+	struct file_struct *file;
+	int ndx;
+	char *fname;
+	int itemizing;
+	enum logcode code;
+};
+static int defer_symlinks;
+static item_list deferred_symlinks = EMPTY_ITEM_LIST;
+
+#ifdef SUPPORT_LINKS
+static void defer_symlink(struct file_struct *file, int ndx, const char *fname,
+			  int itemizing, enum logcode code)
+{
+	struct deferred_symlink *ds
+	    = EXPAND_ITEM_LIST(&deferred_symlinks, struct deferred_symlink, 100);
+
+	ds->file = file;
+	ds->ndx = ndx;
+	if (!(ds->fname = strdup(fname)))
+		out_of_memory("defer_symlink");
+	ds->itemizing = itemizing;
+	ds->code = code;
+}
+#endif
+
 /* Forward declarations. */
 #ifdef SUPPORT_HARD_LINKS
 static void handle_skipped_hlink(struct file_struct *file, int itemizing,
 				 enum logcode code, int f_out);
 #endif
 
-#define EARLY_DELAY_DONE_MSG() (!delay_updates)
-#define EARLY_DELETE_DONE_MSG() (!(delete_during == 2 || delete_after))
+/* Deferred symlinks are created and itemized in the delay-updates phase, so
+ * that phase must not be ended early either, and the deletion stats must wait
+ * for any directory that a deferred symlink replaces. */
+#define EARLY_DELAY_DONE_MSG() (!delay_updates && !defer_symlinks)
+#define EARLY_DELETE_DONE_MSG() \
+	(!(delete_during == 2 || delete_after || defer_symlinks))
 
 static int start_delete_delay_temp(void)
 {
@@ -2001,6 +2036,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 				fnamecmp = fnamecmpbuf;
 			}
 		}
+		if (defer_symlinks) {
+			defer_symlink(file, ndx, fname, itemizing, code);
+			goto cleanup;
+		}
 		if (atomic_create(file, fname, sl, NULL, MAKEDEV(0, 0), &sx, statret == 0 ? DEL_FOR_SYMLINK : 0)) {
 			set_file_attrs(fname, file, NULL, NULL, 0);
 			if (itemizing) {
@@ -2715,6 +2754,46 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 	}
 }
 
+/* Create the symlinks deferred by recv_generator(), then itemize and log each
+ * one as recv_generator() would have.  We re-stat each destination because the
+ * transfer may have changed what's there.  A link that can't be created is
+ * reported as an error and not itemized. */
+static void create_deferred_symlinks(void)
+{
+#ifdef SUPPORT_LINKS
+	struct deferred_symlink *ds = deferred_symlinks.items;
+	size_t i;
+
+	for (i = 0; i < deferred_symlinks.count; i++) {
+		struct file_struct *file = ds[i].file;
+		/* atomic_create() can delete a directory in the way, which builds
+		 * its entries' paths in this buffer, so it must be MAXPATHLEN. */
+		char fname[MAXPATHLEN];
+		stat_x sx;
+		int statret;
+
+		strlcpy(fname, ds[i].fname, sizeof fname);
+		free(ds[i].fname);
+		init_stat_x(&sx);
+		statret = gen_entry_stat(fname, file, &sx.st, 0);
+		if (atomic_create(file, fname, F_SYMLINK(file), NULL, MAKEDEV(0, 0),
+				  &sx, statret == 0 ? DEL_FOR_SYMLINK : 0)) {
+			set_file_attrs(fname, file, NULL, NULL, 0);
+			if (ds[i].itemizing) {
+				if (statret == 0 && !S_ISLNK(sx.st.st_mode))
+					statret = -1;
+				itemize(fname, file, ds[i].ndx, statret, &sx,
+					ITEM_LOCAL_CHANGE|ITEM_REPORT_CHANGE, 0, NULL);
+			}
+			if (ds[i].code != FNONE && INFO_GTE(NAME, 1))
+				rprintf(ds[i].code, "%s -> %s\n", fname, F_SYMLINK(file));
+		}
+		free_stat_x(&sx);
+	}
+	deferred_symlinks.count = 0;
+#endif
+}
+
 void generate_files(int f_out, const char *local_name)
 {
 	int i, ndx, next_loopchk = 0;
@@ -2748,6 +2827,8 @@ void generate_files(int f_out, const char *local_name)
 	symlink_timeset_failed_flags = ITEM_REPORT_TIME
 	    | (protocol_version >= 30 || !am_server ? ITEM_REPORT_TIMEFAIL : 0);
 	implied_dirs_are_missing = relative_paths && !implied_dirs && protocol_version < 30;
+	/* A dry run creates nothing, so its symlinks take the normal path. */
+	defer_symlinks = delay_symlinks && !dry_run;
 
 	if (DEBUG_GTE(GENR, 1))
 		rprintf(FINFO, "generator starting pid=%d\n", (int)getpid());
@@ -2881,6 +2962,12 @@ void generate_files(int f_out, const char *local_name)
 		wait_for_receiver();
 	}
 
+	/* The receiver has now put every file into place, including any
+	 * --delay-updates renames, and the sender is still reading the itemized
+	 * output, so the deferred symlinks can be created and reported. */
+	if (defer_symlinks)
+		create_deferred_symlinks();
+
 	if (protocol_version >= 29) {
 		phase++;
 		if (DEBUG_GTE(GENR, 1))

options.c

@@ -151,6 +151,7 @@ int blocking_io = -1;
 int checksum_seed = 0;
 int inplace = 0;
 int delay_updates = 0;
+int delay_symlinks = 0;
 int32 block_size = 0;
 time_t stop_at_utime = 0;
 char *skip_compress = NULL;
@@ -790,6 +791,8 @@ static struct poptOption long_options[] = {
   {"partial-dir",      0,  POPT_ARG_STRING, &partial_dir, 0, 0, 0 },
   {"delay-updates",    0,  POPT_ARG_VAL,    &delay_updates, 1, 0, 0 },
   {"no-delay-updates", 0,  POPT_ARG_VAL,    &delay_updates, 0, 0, 0 },
+  {"delay-symlinks",   0,  POPT_ARG_VAL,    &delay_symlinks, 1, 0, 0 },
+  {"no-delay-symlinks",0,  POPT_ARG_VAL,    &delay_symlinks, 0, 0, 0 },
   {"prune-empty-dirs",'m', POPT_ARG_VAL,    &prune_empty_dirs, 1, 0, 0 },
   {"no-prune-empty-dirs",0,POPT_ARG_VAL,    &prune_empty_dirs, 0, 0, 0 },
   {"no-m",             0,  POPT_ARG_VAL,    &prune_empty_dirs, 0, 0, 0 },
@@ -2292,6 +2295,22 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			remove_source_files == 1 ? "source" : "sent");
 		goto cleanup;
 	}
+	if (delay_symlinks && remove_source_files) {
+		snprintf(err_buf, sizeof err_buf,
+			"--delay-symlinks cannot be used with --remove-%s-files\n",
+			remove_source_files == 1 ? "source" : "sent");
+		goto cleanup;
+	}
+	if (delay_symlinks && preserve_hard_links) {
+		snprintf(err_buf, sizeof err_buf,
+			"--delay-symlinks cannot be used with --hard-links\n");
+		goto cleanup;
+	}
+	if (delay_symlinks && basis_dir_cnt) {
+		snprintf(err_buf, sizeof err_buf,
+			"--delay-symlinks cannot be used with %s\n", alt_dest_opt(0));
+		goto cleanup;
+	}
 	if (batch_name && strlen(batch_name) > MAX_BATCH_NAME_LEN) {
 		snprintf(err_buf, sizeof err_buf,
 			"the batch-file name must be %d characters or less.\n",
@@ -3069,6 +3088,9 @@ void server_options(char **args, int *argc_p)
 	} else if (keep_partial && am_sender)
 		args[ac++] = "--partial";
 
+	if (delay_symlinks && am_sender)
+		args[ac++] = "--delay-symlinks";
+
 	if (ignore_errors)
 		args[ac++] = "--ignore-errors";
 

rsync.1.md

@@ -621,6 +621,7 @@ has its own detailed description later in this manpage.
 --partial                keep partially transferred files
 --partial-dir=DIR        put a partially transferred file into DIR
 --delay-updates          put all updated files into place at end
+--delay-symlinks         create symlinks after all files are in place
 --prune-empty-dirs, -m   prune empty directory chains from file-list
 --numeric-ids            don't map uid/gid values by user/group name
 --usermap=STRING         custom username mapping
@@ -1012,6 +1013,7 @@ sign) if you want the local shell to expand it.
     - [`--delete-after`](#opt)
     - [`--prune-empty-dirs`](#opt)
     - [`--delay-updates`](#opt)
+    - [`--delay-symlinks`](#opt)
 
     In order to be compatible with incremental recursion,
     [`--delete-during`](#opt) is the default delete mode for [`--delete`](#opt).
@@ -3745,6 +3747,34 @@ sign) if you want the local shell to expand it.
     update algorithm that is even closer to atomic (it uses [`--link-dest`](#opt)
     and a parallel hierarchy of files).
 
+0.  `--delay-symlinks`
+
+    This option tells the receiving rsync to hold back the creation of new and
+    changed symlinks until every other file in the transfer has been put into
+    place, including the renames done by [`--delay-updates`](#opt).  Without
+    it, a symlink is created as soon as rsync reaches it in the file list, so
+    for the rest of the transfer it can point at a file that hasn't arrived
+    yet.
+
+    This narrows the window in which the destination is inconsistent; it does
+    not make the update atomic.  A transfer that is interrupted before the end
+    leaves new symlinks missing and changed symlinks pointing at their old
+    targets.
+
+    If a symlink is moved to a new target and the old target is deleted, also
+    use [`--delete-delay`](#opt) or [`--delete-after`](#opt), since
+    [`--delete-before`](#opt) and the default [`--delete-during`](#opt) can
+    remove the old target while the old symlink still points at it.
+
+    This option implies [`--no-inc-recursive`](#opt) since it needs the full
+    file list in memory in order to be able to iterate over it at the end.
+
+    Conflicts with [`--remove-source-files`](#opt), [`--hard-links`](#opt),
+    [`--compare-dest`](#opt), [`--copy-dest`](#opt), and [`--link-dest`](#opt).
+    This option is incompatible with rsync versions prior to 2.6.4 (March
+    2005), and when the receiving side is remote, the remote rsync must also
+    support this option.
+
 0.  `--prune-empty-dirs`, `-m`
 
     This option tells the receiving rsync to get rid of empty directories from

support/rrsync

@@ -52,6 +52,7 @@ long_opts = {
   'copy-unsafe-links': 0,
   'daemon': -1,
   'debug': -1,
+  'delay-symlinks': 0,
   'delay-updates': 0,
   'delete': 0,
   'delete-after': 0,
@@ -519,7 +520,11 @@ def main():
         # backup onto a name that already exists deletes what is there
         # (backup.c make_backup()), and deletion backs files up as well
         # (delete.c), so an unrelated --delete can land on a protected name.
+        # --delay-symlinks creates its symlinks at the end of the transfer and
+        # replaces whatever has appeared at that name since the
+        # --ignore-existing check.
         long_opts['log-file'] = long_opts['partial-dir'] = long_opts['delay-updates'] = -1
+        long_opts['delay-symlinks'] = -1
         long_opts['backup-dir'] = -1
         short_disabled += 'b'   # must precede the short_no_arg_re build below
 

support/rrsync.1.md

@@ -98,11 +98,13 @@ The remainder of this manpage is dedicated to using the rrsync script.
 
     Because `--ignore-existing` protects only the file being transferred, this
     also refuses the options that can reach a *different* existing file in the
-    restricted dir: `--log-file`, `--partial-dir`, `--delay-updates`, and
-    backup mode (`-b`, `--backup-dir`, whose published backup replaces whatever
+    restricted dir: `--log-file`, `--partial-dir`, `--delay-updates`,
+    `--delay-symlinks` (which creates its symlinks at the end of the transfer,
+    replacing whatever has appeared at that name since the check), and backup
+    mode (`-b`, `--backup-dir`, whose published backup replaces whatever
     already occupies the backup name). Resumable uploads with an explicit
-    `--partial-dir`, `--delay-updates`, and server-side logging are therefore
-    unavailable under this option.
+    `--partial-dir`, `--delay-updates`, `--delay-symlinks`, and server-side
+    logging are therefore unavailable under this option.
 
 0.  `-help`, `-h`