Fix handling of files removed during incremental dumps.

Changes to src/create.c and src/incremen.c are partially
based on patch from Alexander Peslyak <solar at openwall.com>.

The new testcases require paxutils commit f653a2b or later.

* src/common.h (struct name): New member `cmdline'.
(dump_file): Change type of the 2nd argument to bool.
(file_removed_diag, dir_removed_diag): New prototypes.
(addname): New argument `cmdline'.
(name_from_list): Change return value.
* src/create.c (dump_dir0, dump_dir): top_level is bool.
(create_archive): Update calls to name_from_list.
Take advantage of the name->cmdline to set top_level argument
during incremental backups.
(dump_file0): top_level is bool.
Do not bail out if a no-top-level file disappears during incremental
backup, use file_removed_diag instead.
(dump_filed): top_level is bool.
* src/incremen.c (update_parent_directory): Silently ignore
ENOENT.  It should have already been reported elsewhere.
(scan_directory): Use dir_removed_diag to report missing directories.
* src/misc.c (file_removed_diag, dir_removed_diag): New functions.
* src/names.c (name_gather): Set ->cmdname.
(addname): Likewise. All uses updated.
(name_from_list): Return struct name const *. All uses updated.

* tests/filerem01.at: New testcase.
* tests/filerem02.at: New testcase.
* tests/Makefile.am, tests/testsuite.at: Add filerem01.at, filerem02.at
* tests/grow.at, test/truncate.at: Use new syntax for genfile --run.

* NEWS: Update.
* doc/tar.texi: Minor fix.

NEWS

@@ -31,6 +31,23 @@ options.  So far the only meaningful value for N is 0.  The
 `--level=0' option forces creating the level 0 dump, by truncating
 the snapshot file if it exists.
 
+* Files removed during incremental dumps
+
+If a file or directory is removed while incremental dump is
+in progress, tar exact actions depend on whether this file
+was explicitly listed in the command line, or was gathered
+during file system scan.
+
+If the file was explicitly listed in the command line, tar
+issues error messages and exits with the code 2, meaning
+fatal error.
+
+Otherwise, if the file was gathered during the file system
+scan, tar issues a warning, saying "File removed before we read it",
+and sets the exit code to 1, which means "some files differ".
+If the --warning=no-file-removed option is given, no warning
+is issued and the exit code remains 0.
+
 * Bugfixes
 ** Fix handling of hard link targets by -c --transform.
 ** Fix hard links recognition with -c --remove-files.

doc/tar.texi

@@ -2472,7 +2472,7 @@ messages as it reads through the archive.  It is intended for when you
 want a visual indication that @command{tar} is still running, but
 don't want to see @option{--verbose} output.  You can also instruct
 @command{tar} to execute a list of actions on each checkpoint, see
-@option{--checklist-action} below.  For a detailed description, see
+@option{--checkpoint-action} below.  For a detailed description, see
 @ref{checkpoints}.
 
 @opsummary{checkpoint-action}

src/common.h

@@ -331,8 +331,10 @@ struct name
 
     char *name;                 /* File name or globbing pattern */
     size_t length;		/* cached strlen (name) */
-    int matching_flags;		/* wildcard flags if name is a pattern */
-
+    int matching_flags;         /* wildcard flags if name is a pattern */
+    bool cmdline;               /* true if this name was given in the
+				   command line */
+    
     int change_dir;		/* Number of the directory to change to.
 				   Set with the -C option. */
     uintmax_t found_count;	/* number of times a matching file has
@@ -443,7 +445,7 @@ bool cachedir_file_p (const char *name);
 bool file_dumpable_p (struct tar_stat_info *st);
 void create_archive (void);
 void pad_archive (off_t size_left);
-void dump_file (const char *st, int top_level, dev_t parent_device);
+void dump_file (const char *st, bool top_level, dev_t parent_device);
 union block *start_header (struct tar_stat_info *st);
 void finish_header (struct tar_stat_info *st, union block *header,
 		    off_t block_ordinal);
@@ -629,6 +631,10 @@ void readlink_diag (char const *name);
 void savedir_diag (char const *name);
 void seek_diag_details (char const *name, off_t offset);
 void stat_diag (char const *name);
+void file_removed_diag (const char *name, bool top_level,
+			void (*diagfn) (char const *name));
+void dir_removed_diag (char const *name, bool top_level,
+		       void (*diagfn) (char const *name));
 void write_error_details (char const *name, size_t status, size_t size);
 void write_fatal (char const *name) __attribute__ ((noreturn));
 void write_fatal_details (char const *name, ssize_t status, size_t size)
@@ -656,12 +662,13 @@ void name_add_dir (const char *name);
 void name_term (void);
 const char *name_next (int change_dirs);
 void name_gather (void);
-struct name *addname (char const *string, int change_dir, struct name *parent);
+struct name *addname (char const *string, int change_dir,
+		      bool cmdline, struct name *parent);
 bool name_match (const char *name);
 void names_notfound (void);
 void collect_and_sort_names (void);
 struct name *name_scan (const char *name);
-char *name_from_list (void);
+struct name const *name_from_list (void);
 void blank_name_list (void);
 char *new_name (const char *dir_name, const char *name);
 size_t stripped_prefix_len (char const *file_name, size_t num);

src/create.c

@@ -1092,7 +1092,7 @@ dump_regular_file (int fd, struct tar_stat_info *st)
 
 static void
 dump_dir0 (char *directory,
-	   struct tar_stat_info *st, int top_level, dev_t parent_device)
+	   struct tar_stat_info *st, bool top_level, dev_t parent_device)
 {
   dev_t our_device = st->stat.st_dev;
   const char *tag_file_name;
@@ -1210,7 +1210,7 @@ dump_dir0 (char *directory,
 		  }
 		strcpy (name_buf + name_len, entry);
 		if (!excluded_name (name_buf))
-		  dump_file (name_buf, 0, our_device);
+		  dump_file (name_buf, false, our_device);
 	      }
 	    
 	    free (name_buf);
@@ -1224,7 +1224,7 @@ dump_dir0 (char *directory,
 	  name_buf = xmalloc (name_size);
 	  strcpy (name_buf, st->orig_file_name);
 	  strcat (name_buf, tag_file_name);
-	  dump_file (name_buf, 0, our_device);
+	  dump_file (name_buf, false, our_device);
 	  free (name_buf);
 	  break;
       
@@ -1250,7 +1250,8 @@ ensure_slash (char **pstr)
 }
 
 static bool
-dump_dir (int fd, struct tar_stat_info *st, int top_level, dev_t parent_device)
+dump_dir (int fd, struct tar_stat_info *st, bool top_level,
+	  dev_t parent_device)
 {
   char *directory = fdsavedir (fd);
   if (!directory)
@@ -1271,7 +1272,7 @@ dump_dir (int fd, struct tar_stat_info *st, int top_level, dev_t parent_device)
 void
 create_archive (void)
 {
-  const char *p;
+  struct name const *p;
 
   open_archive (ACCESS_WRITE);
   buffer_write_global_xheader ();
@@ -1285,21 +1286,21 @@ create_archive (void)
       collect_and_sort_names ();
 
       while ((p = name_from_list ()) != NULL)
-	if (!excluded_name (p))
-	  dump_file (p, -1, (dev_t) 0);
+	if (!excluded_name (p->name))
+	  dump_file (p->name, p->cmdline, (dev_t) 0);
 
       blank_name_list ();
       while ((p = name_from_list ()) != NULL)
-	if (!excluded_name (p))
+	if (!excluded_name (p->name))
 	  {
-	    size_t plen = strlen (p);
+	    size_t plen = strlen (p->name);
 	    if (buffer_size <= plen)
 	      {
 		while ((buffer_size *= 2) <= plen)
 		  continue;
 		buffer = xrealloc (buffer, buffer_size);
 	      }
-	    memcpy (buffer, p, plen);
+	    memcpy (buffer, p->name, plen);
 	    if (! ISSLASH (buffer[plen - 1]))
 	      buffer[plen++] = DIRECTORY_SEPARATOR;
 	    q = directory_contents (gnu_list_name->directory);
@@ -1316,7 +1317,7 @@ create_archive (void)
 			  buffer = xrealloc (buffer, buffer_size);
  			}
 		      strcpy (buffer + plen, q + 1);
-		      dump_file (buffer, -1, (dev_t) 0);
+		      dump_file (buffer, false, (dev_t) 0);
 		    }
 		  q += qlen + 1;
 		}
@@ -1325,9 +1326,10 @@ create_archive (void)
     }
   else
     {
-      while ((p = name_next (1)) != NULL)
-	if (!excluded_name (p))
-	  dump_file (p, 1, (dev_t) 0);
+      const char *name;
+      while ((name = name_next (1)) != NULL)
+	if (!excluded_name (name))
+	  dump_file (name, true, (dev_t) 0);
     }
 
   write_eot ();
@@ -1475,7 +1477,6 @@ check_links (void)
     }
 }
 
-
 /* Dump a single file, recursing on directories.  P is the file name
    to dump.  TOP_LEVEL tells whether this is a top-level call; zero
    means no, positive means yes, and negative means the top level
@@ -1487,7 +1488,7 @@ check_links (void)
 
 static void
 dump_file0 (struct tar_stat_info *st, const char *p,
-	    int top_level, dev_t parent_device)
+	    bool top_level, dev_t parent_device)
 {
   union block *header;
   char type;
@@ -1508,7 +1509,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
 
   if (deref_stat (dereference_option, p, &st->stat) != 0)
     {
-      stat_diag (p);
+      file_removed_diag (p, top_level, stat_diag);
       return;
     }
   st->archive_file_size = original_size = st->stat.st_size;
@@ -1580,12 +1581,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
 			 : 0)));
 	  if (fd < 0)
 	    {
-	      if (!top_level && errno == ENOENT)
-		WARNOPT (WARN_FILE_REMOVED,
-			 (0, 0, _("%s: File removed before we read it"),
-			  quotearg_colon (p)));
-	      else
-		open_diag (p);
+	      file_removed_diag (p, top_level, open_diag);
 	      return;
 	    }
 	}
@@ -1655,7 +1651,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
 	       : fstat (fd, &final_stat))
 	      != 0)
 	    {
-	      stat_diag (p);
+	      file_removed_diag (p, top_level, stat_diag);
 	      ok = false;
 	    }
 	}
@@ -1713,7 +1709,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
       size = readlink (p, buffer, linklen + 1);
       if (size < 0)
 	{
-	  readlink_diag (p);
+	  file_removed_diag (p, top_level, readlink_diag);
 	  return;
 	}
       buffer[size] = '\0';
@@ -1795,7 +1791,7 @@ dump_file0 (struct tar_stat_info *st, const char *p,
 }
 
 void
-dump_file (const char *p, int top_level, dev_t parent_device)
+dump_file (const char *p, bool top_level, dev_t parent_device)
 {
   struct tar_stat_info st;
   tar_stat_init (&st);

src/incremen.c

@@ -413,7 +413,11 @@ update_parent_directory (const char *name)
     {
       struct stat st;
       if (deref_stat (dereference_option, p, &st) != 0)
-	stat_diag (name);
+	{
+	  if (errno != ENOENT) 
+	    stat_diag (directory->name);
+	  /* else: should have been already reported */
+	}
       else
 	directory->mtime = get_stat_mtime (&st);
     }
@@ -549,6 +553,12 @@ procdir (const char *name_buffer, struct stat *stat_data,
   if (one_file_system_option && device != stat_data->st_dev
       /* ... except if it was explicitely given in the command line */
       && !is_individual_file (name_buffer))
+    /* FIXME: 
+	WARNOPT (WARN_XDEV,
+		 (0, 0,
+		  _("%s: directory is on a different filesystem; not dumped"),
+		  quotearg_colon (directory->name)));
+    */
     directory->children = NO_CHILDREN;
   else if (flag & PD_FORCE_CHILDREN)
     {
@@ -699,7 +709,7 @@ scan_directory (char *dir, dev_t device, bool cmdline)
   
   if (deref_stat (dereference_option, name_buffer, &stat_data))
     {
-      stat_diag (name_buffer);
+      dir_removed_diag (name_buffer, false, stat_diag);
       /* FIXME: used to be
            children = CHANGED_CHILDREN;
 	 but changed to: */

src/misc.c

@@ -745,6 +745,30 @@ stat_diag (char const *name)
     stat_error (name);
 }
 
+void
+file_removed_diag (const char *name, bool top_level,
+		   void (*diagfn) (char const *name))
+{
+  if (!top_level && errno == ENOENT)
+    WARNOPT (WARN_FILE_REMOVED,
+	     (0, 0, _("%s: File removed before we read it"),
+	      quotearg_colon (name)));
+  else
+    diagfn (name);
+}
+
+void
+dir_removed_diag (const char *name, bool top_level,
+		   void (*diagfn) (char const *name))
+{
+  if (!top_level && errno == ENOENT)
+    WARNOPT (WARN_FILE_REMOVED,
+	     (0, 0, _("%s: Directory removed before we read it"),
+	      quotearg_colon (name)));
+  else
+    diagfn (name);
+}
+
 void
 write_fatal_details (char const *name, ssize_t status, size_t size)
 {

src/names.c

@@ -420,12 +420,13 @@ name_gather (void)
 	  buffer->matching_flags = matching_flags;
 	  buffer->directory = NULL;
 	  buffer->parent = NULL;
+	  buffer->cmdline = true;
 	  
 	  namelist = buffer;
 	  nametail = &namelist->next;
 	}
       else if (change_dir)
-	addname (0, change_dir, NULL);
+	addname (0, change_dir, false, NULL);
     }
   else
     {
@@ -439,11 +440,11 @@ name_gather (void)
 	    change_dir = chdir_arg (xstrdup (ep->v.name));
 
 	  if (ep)
-	    addname (ep->v.name, change_dir, NULL);
+	    addname (ep->v.name, change_dir, true, NULL);
 	  else
 	    {
 	      if (change_dir != change_dir0)
-		addname (0, change_dir, NULL);
+		addname (NULL, change_dir, false, NULL);
 	      break;
 	    }
 	}
@@ -452,7 +453,7 @@ name_gather (void)
 
 /*  Add a name to the namelist.  */
 struct name *
-addname (char const *string, int change_dir, struct name *parent)
+addname (char const *string, int change_dir, bool cmdline, struct name *parent)
 {
   struct name *name = make_name (string);
 
@@ -463,6 +464,7 @@ addname (char const *string, int change_dir, struct name *parent)
   name->change_dir = change_dir;
   name->directory = NULL;
   name->parent = parent;
+  name->cmdline = cmdline;
   
   *nametail = name;
   nametail = &name->next;
@@ -811,7 +813,7 @@ add_hierarchy_to_namelist (struct name *name, dev_t device, bool cmdline)
 		  namebuf = xrealloc (namebuf, allocated_length + 1);
 		}
 	      strcpy (namebuf + name_length, string + 1);
-	      np = addname (namebuf, change_dir, name);
+	      np = addname (namebuf, change_dir, false, name);
 	      if (!child_head)
 		child_head = np;
 	      else
@@ -886,7 +888,7 @@ collect_and_sort_names (void)
   name_gather ();
 
   if (!namelist)
-    addname (".", 0, NULL);
+    addname (".", 0, false, NULL);
 
   if (listed_incremental_option)
     {
@@ -1030,8 +1032,8 @@ name_scan (const char *file_name)
    find and return all the non-found names in the namelist.  */
 struct name *gnu_list_name;
 
-char *
-name_from_list (void)
+struct name const *
+name_from_list ()
 {
   if (!gnu_list_name)
     gnu_list_name = namelist;
@@ -1042,9 +1044,9 @@ name_from_list (void)
     {
       gnu_list_name->found_count++;
       chdir_do (gnu_list_name->change_dir);
-      return gnu_list_name->name;
+      return gnu_list_name;
     }
-  return 0;
+  return NULL;
 }
 
 void

src/tar.c

@@ -1388,7 +1388,7 @@ parse_opt (int key, char *arg, struct argp_state *state)
 
     case 'K':
       starting_file_option = true;
-      addname (arg, 0, NULL);
+      addname (arg, 0, true, NULL);
       break;
 
     case ONE_FILE_SYSTEM_OPTION:

src/update.c

@@ -189,10 +189,10 @@ update_archive (void)
   output_start = current_block->buffer;
 
   {
-    char *file_name;
-
-    while ((file_name = name_from_list ()) != NULL)
+    struct name const *p;
+    while ((p = name_from_list ()) != NULL)
       {
+	char *file_name = p->name;
 	if (excluded_name (file_name))
 	  continue;
 	if (interactive_option && !confirm ("add", file_name))

tests/Makefile.am

@@ -69,6 +69,8 @@ TESTSUITE_AT = \
  extrac06.at\
  extrac07.at\
  extrac08.at\
+ filerem01.at\
+ filerem02.at\
  gzip.at\
  grow.at\
  incremental.at\

tests/filerem01.at

@@ -0,0 +1,88 @@
+# Process this file with autom4te to create testsuite. -*- Autotest -*-
+
+# Test suite for GNU tar.
+# Copyright (C) 2009 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+AT_SETUP([file removed as we read it (ca. 22 seconds)])
+AT_KEYWORDS([create incremental filechange filerem filerem01])
+
+AT_TAR_CHECK([
+mkdir dir
+mkdir dir/sub
+genfile --file dir/file1
+genfile --file dir/sub/file2
+
+genfile --run --checkpoint=3 --unlink dir/file1 -- \
+       tar --blocking-factor=1 --checkpoint=1 --checkpoint-action='sleep=1' \
+       --checkpoint-action='echo' -c -f archive.tar \
+       --listed-incremental db -v dir >/dev/null
+],
+[0],
+[ignore],
+[tar: dir: Directory is new
+tar: dir/sub: Directory is new
+tar: dir/file1: File removed before we read it
+],[],[],[gnu, posix])
+
+# Timing information:
+#
+# For -Hgnu the above command line takes about 8 seconds to execute and
+# produces:
+#
+# tar: dir: Directory is new
+# tar: dir/sub: Directory is new
+# dir/
+# tar: Write checkpoint 1
+# tar: Write checkpoint 2
+# dir/sub/
+# tar: Write checkpoint 3
+# tar: Write checkpoint 4
+# dir/file1
+# tar: Write checkpoint 5
+# dir/sub/file2
+# tar: Write checkpoint 6
+# tar: Write checkpoint 7
+# tar: Write checkpoint 8
+#
+# For -Hposix the above command line takes about 14 seconds to execute and
+# produces:
+#
+# ./tar: dir: Directory is new
+# ./tar: dir/sub: Directory is new
+# dir/
+# ./tar: Write checkpoint 1
+# ./tar: Write checkpoint 2
+# ./tar: Write checkpoint 3
+# dir/sub/
+# ./tar: Write checkpoint 4
+# ./tar: Write checkpoint 5
+# ./tar: Write checkpoint 6
+# dir/file1
+# ./tar: Write checkpoint 7
+# ./tar: Write checkpoint 8
+# ./tar: Write checkpoint 9
+# dir/sub/file2
+# ./tar: Write checkpoint 10
+# ./tar: Write checkpoint 11
+# ./tar: Write checkpoint 12
+# ./tar: Write checkpoint 13
+# ./tar: Write checkpoint 14
+
+
+AT_CLEANUP
+

tests/filerem02.at

@@ -0,0 +1,50 @@
+# Process this file with autom4te to create testsuite. -*- Autotest -*-
+
+# Test suite for GNU tar.
+# Copyright (C) 2009 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Description: see filerem01.at
+# This test case checks if the tar exit code is still 2 if a
+# file or directory disappears that is explicitly mentioned
+# in the command line.
+
+AT_SETUP([toplevel file removed (ca. 24 seconds)])
+AT_KEYWORDS([create incremental filechange filerem filerem02])
+
+AT_TAR_CHECK([
+mkdir dir
+mkdir dir/sub
+genfile --file dir/file1
+genfile --file dir/sub/file2
+mkdir dir2
+genfile --file dir2/file1
+
+genfile --run --checkpoint=3 --exec 'rm -rf dir2' -- \
+       tar --blocking-factor=1 --checkpoint=1 --checkpoint-action='sleep=1' \
+       --checkpoint-action='echo' -c -f archive.tar \
+       --listed-incremental db -v --warning=no-new-dir dir dir2 >/dev/null
+],
+[2],
+[ignore],
+[tar: dir2: Cannot stat: No such file or directory
+tar: dir2/file1: File removed before we read it
+tar: Exiting with failure status due to previous errors
+],[],[],[gnu, posix])
+
+# Timing information: see filerem01.at
+
+AT_CLEANUP
+

tests/grow.at

@@ -27,8 +27,7 @@ AT_KEYWORDS([grow filechange])
 AT_TAR_CHECK([
 genfile --file foo --length 50000k
 genfile --file baz
-genfile --run 'tar -vcf bar foo baz' --checkpoint 10 --length 1024 \
-        --append foo
+genfile --run --checkpoint 10 --length 1024 --append foo -- tar --checkpoint -vcf bar foo baz
 ],
 [1],
 [foo

tests/testsuite.at

@@ -154,6 +154,10 @@ m4_include([incr03.at])
 m4_include([incr04.at])
 m4_include([incr05.at])
 m4_include([incr06.at])
+
+m4_include([filerem01.at])
+m4_include([filerem02.at])
+
 m4_include([rename01.at])
 m4_include([rename02.at])
 m4_include([rename03.at])

tests/truncate.at

@@ -32,7 +32,7 @@ AT_KEYWORDS([truncate filechange])
 AT_TAR_CHECK([
 genfile --file foo --length 50000k
 genfile --file baz
-genfile --run 'tar -vcf bar foo baz' --checkpoint 10 --length 49995k --truncate foo
+genfile --run --checkpoint 10 --length 49995k --truncate foo -- tar --checkpoint -vcf bar foo baz
 echo Exit status: $?
 echo separator
 sleep 1