Merge pull request #2195 from duplicati/feature/handle_initial_backup_incomplete

Fixes for #2038 and #2101

7 files changed, 173 insertions, 79 deletions

diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
index e529ccdc2..746f4cacf 100644
--- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
@@ -652,6 +652,23 @@ namespace Duplicati.Library.Main.Database
                     return null;

         }

 

+        public RemoteVolumeEntry GetRemoteVolumeFromID(long id)

+        {

+            using (var cmd = m_connection.CreateCommand())

+            using (var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""RemoteVolume"" WHERE ""ID"" = ?", id))

+                if (rd.Read())

+                    return new RemoteVolumeEntry(

+                        rd.GetValue(0).ToString(),

+                        (rd.GetValue(3) == null || rd.GetValue(3) == DBNull.Value) ? null : rd.GetValue(3).ToString(),

+                        rd.ConvertValueToInt64(2, -1),

+                        (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(1).ToString()),

+                        (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(4).ToString()),

+                        new DateTime(rd.ConvertValueToInt64(5, 0), DateTimeKind.Utc)

+                    );

+                else

+                    return default(RemoteVolumeEntry);

+        }

+

         public IEnumerable<string> GetMissingIndexFiles()

         {

             using(var cmd = m_connection.CreateCommand())

diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs
index 9d57f9202..29515ebd8 100644
--- a/Duplicati/Library/Main/Database/LocalDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalDatabase.cs
@@ -365,9 +365,9 @@ namespace Duplicati.Library.Main.Database
             RemoveRemoteVolumes(new string[] { name }, transaction);

         }

 

-        public void RemoveRemoteVolumes(ICollection<string> names, System.Data.IDbTransaction transaction = null)

+        public void RemoveRemoteVolumes(IEnumerable<string> names, System.Data.IDbTransaction transaction = null)

         {

-            if (names.Count == 0) return;

+            if (names == null || !names.Any()) return;

 

             using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))

             using (var deletecmd = m_connection.CreateCommand())

diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs
index 7cfde1140..69bddca36 100644
--- a/Duplicati/Library/Main/Operation/BackupHandler.cs
+++ b/Duplicati/Library/Main/Operation/BackupHandler.cs
@@ -300,86 +300,87 @@ namespace Duplicati.Library.Main.Operation
 

         private void UploadSyntheticFilelist(BackendManager backend) 

         {

-            var incompleteFilesets = m_database.GetIncompleteFilesets(null).OrderBy(x => x.Value).ToArray();                        

-            if (incompleteFilesets.Length != 0)

+            var incompleteFilesets = m_database.GetIncompleteFilesets(null).OrderBy(x => x.Value).ToList();

+

+            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize);

+            m_result.AddMessage(string.Format("Uploading filelist from previous interrupted backup"));

+            using(var trn = m_database.BeginTransaction())

             {

-                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize);

-                m_result.AddMessage(string.Format("Uploading filelist from previous interrupted backup"));

-                using(var trn = m_database.BeginTransaction())

-                {

-                    var incompleteSet = incompleteFilesets.Last();

-                    var badIds = from n in incompleteFilesets select n.Key;

+                var incompleteSet = incompleteFilesets.Last();

+                var badIds = from n in incompleteFilesets select n.Key;

 

-                    var prevs = (from n in m_database.FilesetTimes 

-                        where 

-                        n.Key < incompleteSet.Key

-                        &&

-                        !badIds.Contains(n.Key)

-                        orderby n.Key                                                

-                        select n.Key).ToArray();

+                var prevs = (from n in m_database.FilesetTimes 

+                    where 

+                    n.Key < incompleteSet.Key

+                    &&

+                    !badIds.Contains(n.Key)

+                    orderby n.Key                                                

+                    select n.Key).ToArray();

 

-                    var prevId = prevs.Length == 0 ? -1 : prevs.Last();

+                var prevId = prevs.Length == 0 ? -1 : prevs.Last();

 

-                    FilesetVolumeWriter fsw = null;

-                    try

-                    {

-                        var s = 1;

-                        var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);

-                        var oldFilesetID = incompleteSet.Key;

+                FilesetVolumeWriter fsw = null;

+                try

+                {

+                    var s = 1;

+                    var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);

+                    var oldFilesetID = incompleteSet.Key;

 

-                        // Probe for an unused filename

-                        while (s < 60)

-                        {

-                            var id = m_database.GetRemoteVolumeID(VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, fileTime));

-                            if (id < 0)

-                                break;

+                    // Probe for an unused filename

+                    while (s < 60)

+                    {

+                        var id = m_database.GetRemoteVolumeID(VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, fileTime));

+                        if (id < 0)

+                            break;

 

-                            fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s);

-                        }

+                        fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s);

+                    }

 

-                        fsw = new FilesetVolumeWriter(m_options, fileTime);

-                        fsw.VolumeID = m_database.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);

+                    fsw = new FilesetVolumeWriter(m_options, fileTime);

+                    fsw.VolumeID = m_database.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);

 

-                        if (!string.IsNullOrEmpty(m_options.ControlFiles))

-                            foreach(var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))

-                                fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p));

+                    if (!string.IsNullOrEmpty(m_options.ControlFiles))

+                        foreach(var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))

+                            fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p));

 

-                        var newFilesetID = m_database.CreateFileset(fsw.VolumeID, fileTime, trn);

-                        m_database.LinkFilesetToVolume(newFilesetID, fsw.VolumeID, trn);

-                        m_database.AppendFilesFromPreviousSet(trn, null, newFilesetID, prevId, fileTime);

+                    var newFilesetID = m_database.CreateFileset(fsw.VolumeID, fileTime, trn);

+                    m_database.LinkFilesetToVolume(newFilesetID, fsw.VolumeID, trn);

+                    m_database.AppendFilesFromPreviousSet(trn, null, newFilesetID, prevId, fileTime);

 

-                        m_database.WriteFileset(fsw, trn, newFilesetID);

+                    m_database.WriteFileset(fsw, trn, newFilesetID);

 

-                        if (m_options.Dryrun)

-                        {

-                            m_result.AddDryrunMessage(string.Format("Would upload fileset: {0}, size: {1}", fsw.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(fsw.LocalFilename).Length)));

-                        }

-                        else

-                        {

-                            m_database.UpdateRemoteVolume(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null, trn);

+                    if (m_options.Dryrun)

+                    {

+                        m_result.AddDryrunMessage(string.Format("Would upload fileset: {0}, size: {1}", fsw.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(fsw.LocalFilename).Length)));

+                    }

+                    else

+                    {

+                        m_database.UpdateRemoteVolume(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null, trn);

 

-                            using(new Logging.Timer("CommitUpdateFilelistVolume"))

-                                trn.Commit();

+                        using(new Logging.Timer("CommitUpdateFilelistVolume"))

+                            trn.Commit();

 

-                            backend.Put(fsw);

-                            fsw = null;

-                        }

+                        backend.Put(fsw);

+                        fsw = null;

                     }

-                    finally

-                    {

-                        if (fsw != null)

-                            try { fsw.Dispose(); }

-                        catch { fsw = null; }

-                    }                          

                 }

+                finally

+                {

+                    if (fsw != null)

+                        try { fsw.Dispose(); }

+                    catch { fsw = null; }

+                }                          

             }

+        }

 

+        private void RecreateMissingIndexFiles(BackendManager backend)

+        {

             if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)

             {

                 var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);

                 var hashsize = blockhasher.HashSize / 8;

 

-                foreach(var blockfile in m_database.GetMissingIndexFiles())

+                foreach (var blockfile in m_database.GetMissingIndexFiles())

                 {

                     m_result.AddMessage(string.Format("Re-creating missing index file for {0}", blockfile));

                     var w = new IndexVolumeWriter(m_options);

@@ -389,13 +390,13 @@ namespace Duplicati.Library.Main.Operation
                     w.StartVolume(blockvolume.Name);

                     var volumeid = m_database.GetRemoteVolumeID(blockvolume.Name);

 

-                    foreach(var b in m_database.GetBlocks(volumeid))

+                    foreach (var b in m_database.GetBlocks(volumeid))

                         w.AddBlock(b.Hash, b.Size);

 

                     w.FinishVolume(blockvolume.Hash, blockvolume.Size);

 

                     if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)

-                        foreach(var b in m_database.GetBlocklists(volumeid, m_options.Blocksize, hashsize))

+                        foreach (var b in m_database.GetBlocklists(volumeid, m_options.Blocksize, hashsize))

                             w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);

 

                     w.Close();

@@ -413,7 +414,7 @@ namespace Duplicati.Library.Main.Operation
             }

         }

 

-        private void PreBackupVerify(BackendManager backend)

+        private void PreBackupVerify(BackendManager backend, string protectedfile)

         {

             m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupVerify);

             using(new Logging.Timer("PreBackupVerify"))

@@ -426,7 +427,7 @@ namespace Duplicati.Library.Main.Operation
 						UpdateStorageStatsFromDatabase();

 					}

 					else

-						FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter);

+                        FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter, protectedfile);

                 }

                 catch (Exception ex)

                 {

@@ -729,10 +730,48 @@ namespace Duplicati.Library.Main.Operation
                                 parallelScanner.Start(snapshot);

                             }

 

-                            PreBackupVerify(backend);

+                            string lasttempfilelist = null;

+                            long lasttempfileid = -1;

+                            if (!m_options.DisableSyntheticFilelist)

+                            {

+                                var candidates = m_database.GetIncompleteFilesets(null).OrderBy(x => x.Value).ToArray();

+                                if (candidates.Length > 0)

+                                {

+                                    lasttempfileid = candidates.Last().Key;

+                                    lasttempfilelist = m_database.GetRemoteVolumeFromID(lasttempfileid).Name;

+                                }

+                            }

 

                             // Verify before uploading a synthetic list

-                            UploadSyntheticFilelist(backend);

+                            PreBackupVerify(backend, lasttempfilelist);

+

+                            // If we have an incomplete entry, upload it now

+                            if (!m_options.DisableSyntheticFilelist && !string.IsNullOrWhiteSpace(lasttempfilelist) && lasttempfileid >= 0)

+                            {

+                                // Check that we still need to process this after the cleanup has performed its duties

+                                var syntbase = m_database.GetRemoteVolumeFromID(lasttempfileid);

+                                if (syntbase.Name != null && (syntbase.State == RemoteVolumeState.Uploading || syntbase.State == RemoteVolumeState.Temporary))

+                                {

+                                    UploadSyntheticFilelist(backend);

+

+                                    // Remove the protected file

+                                    if (syntbase.State == RemoteVolumeState.Uploading)

+                                    {

+                                        m_result.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", syntbase.State, syntbase.Name));

+                                        backend.Delete(syntbase.Name, syntbase.Size);

+                                    }

+                                    else if (syntbase.State == RemoteVolumeState.Temporary)

+                                    {

+                                        m_result.AddMessage(string.Format("removing file listed as {0}: {1}", syntbase.State, syntbase.Name));

+                                        m_database.RemoveRemoteVolume(syntbase.Name);

+                                    }

+                                }

+                                else if (syntbase.Name == null || syntbase.State != RemoteVolumeState.Uploaded)

+                                    m_result.AddWarning(string.Format("Expected there to be a temporary fileset for synthetic filelist ({0}, {1}), but none was found?", lasttempfileid, lasttempfilelist), null);

+                            }

+

+                            // Rebuild any index files that are missing

+                            RecreateMissingIndexFiles(backend);

 

                             m_database.BuildLookupTable(m_options);

                             m_transaction = m_database.BeginTransaction();

diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs
index 85d4e062f..a6a9b4dae 100644
--- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs
+++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs
@@ -75,9 +75,10 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="options">The options used</param>

         /// <param name="database">The database to compare with</param>

         /// <param name="log">The log instance to use</param>

-        public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)

+        /// <param name="protectedfile">A filename that should be excempted for deletion</param>

+        public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, string protectedfile = null)

         {

-            var tp = RemoteListAnalysis(backend, options, database, log);

+            var tp = RemoteListAnalysis(backend, options, database, log, protectedfile);

             long extraCount = 0;

             long missingCount = 0;

             

@@ -189,10 +190,12 @@ namespace Duplicati.Library.Main.Operation
         /// <param name="backend">The backend instance to use</param>

         /// <param name="options">The options used</param>

         /// <param name="database">The database to compare with</param>

-        public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log)

+        /// <param name="protectedfile">A filename that should be excempted for deletion</param>

+        public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, string protectedfile)

         {

             var rawlist = backend.List();

             var lookup = new Dictionary<string, Volumes.IParsedVolume>();

+            protectedfile = protectedfile ?? string.Empty;

 

             var remotelist = (from n in rawlist

                                        let p = Volumes.VolumeBase.ParseFilename(n)

@@ -277,8 +280,15 @@ namespace Duplicati.Library.Main.Operation
                             }

                             else

                             {

-                                log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));

-                                cleanupRemovedRemoteVolumes.Add(i.Name);

+                                if (string.Equals(i.Name, protectedfile) && i.State == RemoteVolumeState.Temporary)

+                                {

+                                    log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));

+                                }

+                                else

+                                {

+                                    log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name));

+                                    cleanupRemovedRemoteVolumes.Add(i.Name);

+                                }

                             }

                         }

                         break;

@@ -290,14 +300,30 @@ namespace Duplicati.Library.Main.Operation
                         }

                         else if (!remoteFound)

                         {

-                            log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name));

-                            cleanupRemovedRemoteVolumes.Add(i.Name);

-                            database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);

+

+                            if (string.Equals(i.Name, protectedfile))

+                            {

+                                log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));

+                                database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null);

+                            }

+                            else

+                            {

+                                log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name));

+                                cleanupRemovedRemoteVolumes.Add(i.Name);

+                                database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);

+                            }

                         }

                         else

                         {

-                            log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name));

-                            backend.Delete(i.Name, i.Size, true);

+                            if (string.Equals(i.Name, protectedfile))

+                            {

+                                log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name));

+                            }

+                            else

+                            {

+                                log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name));

+                                backend.Delete(i.Name, i.Size, true);

+                            }

                         }

                         break;

 

diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs
index 9cc993bdb..06511de70 100644
--- a/Duplicati/Library/Main/Operation/RepairHandler.cs
+++ b/Duplicati/Library/Main/Operation/RepairHandler.cs
@@ -110,7 +110,7 @@ namespace Duplicati.Library.Main.Operation
                 if (db.RepairInProgress)

                     throw new Exception("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.");

 

-                var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter);

+                var tp = FilelistProcessor.RemoteListAnalysis(backend, m_options, db, m_result.BackendWriter, null);

                 var buffer = new byte[m_options.Blocksize];

                 var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);

                 var hashsize = blockhasher.HashSize / 8;

diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs
index 889b51d95..9dc997967 100644
--- a/Duplicati/Library/Main/Options.cs
+++ b/Duplicati/Library/Main/Options.cs
@@ -248,7 +248,8 @@ namespace Duplicati.Library.Main
                     "hardlink-policy",

                     "exclude-files-attributes",

                     "compression-extension-file",

-                    "full-remote-verification"

+                    "full-remote-verification",

+                    "disable-synthetic-filelist"

                 };

             }

         }

@@ -477,6 +478,8 @@ namespace Duplicati.Library.Main
                     new CommandLineArgument("disable-filepath-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilepathcacheShort, Strings.Options.DisablefilepathcacheLong, "true"),

                     new CommandLineArgument("changed-files", CommandLineArgument.ArgumentType.Path, Strings.Options.ChangedfilesShort, Strings.Options.ChangedfilesLong),

                     new CommandLineArgument("deleted-files", CommandLineArgument.ArgumentType.Path, Strings.Options.DeletedfilesShort, Strings.Options.DeletedfilesLong("changed-files")),

+                    new CommandLineArgument("disable-synthetic-filelist", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablesyntheticfilelistShort, Strings.Options.DisablesyntehticfilelistLong, "false"),

+

 

                     new CommandLineArgument("threshold", CommandLineArgument.ArgumentType.Integer, Strings.Options.ThresholdShort, Strings.Options.ThresholdLong, DEFAULT_THRESHOLD.ToString()),

                     new CommandLineArgument("index-file-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.IndexfilepolicyShort, Strings.Options.IndexfilepolicyLong, IndexFileStrategy.Full.ToString(), null, Enum.GetNames(typeof(IndexFileStrategy))),

@@ -1409,7 +1412,14 @@ namespace Duplicati.Library.Main
             get { return !Library.Utility.Utility.ParseBoolOption(m_options, "skip-restore-verification"); }

         }

 

-        

+        /// <summary>

+        /// Gets a flag indicating if synthetic filelist generation is disabled

+        /// </summary>

+        public bool DisableSyntheticFilelist

+        {

+            get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-synthetic-filelist"); }

+        }

+

         /// <summary>

         /// Gets the file hash size

         /// </summary>

diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs
index bdc72914d..328262590 100644
--- a/Duplicati/Library/Main/Strings.cs
+++ b/Duplicati/Library/Main/Strings.cs
@@ -214,6 +214,8 @@ namespace Duplicati.Library.Main.Strings
         public static string DisablepipingLong { get { return LC.L(@"Use this option to disable multithreaded handling of up- and downloads, that can significantly speed up backend operations depending on the hardware you're running on and the transfer rate of your backend."); } }

         public static string HypervbackupvmShort { get { return LC.L(@"Perform backup of Hyper-V machines (Windows only)"); } }

         public static string HypervbackupvmLong { get { return LC.L(@"Use this option to specify the IDs of machines to include in the backup. Specify multiple machine IDs with a semicolon separator. (You can use this Powershell command to get ID 'Get-VM | ft VMName, ID')"); } }

+        public static string DisablesyntehticfilelistLong { get { return LC.L(@"If Duplicati detects that the previous backup did not complete, it will generate a filelist that is a merge of the last completed backup and the contents that were uploaded in the incomplete backup session."); } }

+        public static string DisablesyntheticfilelistShort { get { return LC.L(@"Disables synethic filelist"); } }

     }

 

     internal static class Common