1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
|
#region Disclaimer / License
// Copyright (C) 2019, The Duplicati Team
// http://www.duplicati.com, info@duplicati.com
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
#endregion
using System;
using System.Linq;
using System.Collections.Generic;
using Duplicati.Library.Interface;
using Duplicati.Library.Main.Database;
namespace Duplicati.Library.Main.Operation
{
internal class DeleteHandler
{
/// <summary>
/// The tag used for logging
/// </summary>
internal static readonly string LOGTAG = Logging.Log.LogTagFromType<DeleteHandler>();
private readonly DeleteResults m_result;
protected readonly string m_backendurl;
protected readonly Options m_options;
public DeleteHandler(string backend, Options options, DeleteResults result)
{
m_backendurl = backend;
m_options = options;
m_result = result;
}
public void Run()
{
if (!System.IO.File.Exists(m_options.Dbpath))
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing");
using(var db = new Database.LocalDeleteDatabase(m_options.Dbpath, "Delete"))
{
var tr = db.BeginTransaction();
try
{
m_result.SetDatabase(db);
Utility.UpdateOptionsFromDb(db, m_options);
Utility.VerifyParameters(db, m_options);
DoRun(db, ref tr, false, false, null);
if (!m_options.Dryrun)
{
using(new Logging.Timer(LOGTAG, "CommitDelete", "CommitDelete"))
tr.Commit();
db.WriteResults();
}
else
tr.Rollback();
tr = null;
}
finally
{
if (tr != null)
try { tr.Rollback(); }
catch { }
}
}
}
public void DoRun(Database.LocalDeleteDatabase db, ref System.Data.IDbTransaction transaction, bool hasVerifiedBacked, bool forceCompact, BackendManager sharedManager)
{
// Workaround where we allow a running backendmanager to be used
using(var bk = sharedManager == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null)
{
var backend = bk ?? sharedManager;
if (!hasVerifiedBacked && !m_options.NoBackendverification)
FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
IListResultFileset[] filesets = db.FilesetsWithBackupVersion.ToArray();
List<IListResultFileset> versionsToDelete = new List<IListResultFileset>();
versionsToDelete.AddRange(new SpecificVersionsRemover(this.m_options).GetFilesetsToDelete(filesets));
versionsToDelete.AddRange(new KeepTimeRemover(this.m_options).GetFilesetsToDelete(filesets));
versionsToDelete.AddRange(new RetentionPolicyRemover(this.m_options).GetFilesetsToDelete(filesets));
// When determining the number of full versions to keep, we need to ignore the versions already marked for removal.
versionsToDelete.AddRange(new KeepVersionsRemover(this.m_options).GetFilesetsToDelete(filesets.Except(versionsToDelete)));
if (!m_options.AllowFullRemoval && filesets.Length == versionsToDelete.Count)
{
Logging.Log.WriteInformationMessage(LOGTAG, "PreventingLastFilesetRemoval", "Preventing removal of last fileset, use --{0} to allow removal ...", "allow-full-removal");
versionsToDelete = versionsToDelete.OrderBy(x => x.Version).Skip(1).ToList();
}
if (versionsToDelete.Count > 0)
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count);
var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), transaction).ToArray();
foreach(var f in lst)
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, transaction);
if (!m_options.Dryrun)
{
transaction.Commit();
transaction = db.BeginTransaction();
}
foreach(var f in lst)
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
backend.WaitForComplete(db, transaction);
return;
}
if (!m_options.Dryrun)
backend.Delete(f.Key, f.Value);
else
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key);
}
if (sharedManager == null)
backend.WaitForComplete(db, transaction);
else
backend.WaitForEmpty(db, transaction);
var count = lst.Length;
if (!m_options.Dryrun)
{
if (count == 0)
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted");
else
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count);
}
else
{
if (count == 0)
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted");
else
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count);
if (count > 0 && m_options.Dryrun)
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files");
}
if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0))
{
m_result.CompactResults = new CompactResults(m_result);
new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, ref transaction, sharedManager);
}
m_result.SetResults(versionsToDelete.Select(v => new Tuple<long, DateTime>(v.Version, v.Time)), m_options.Dryrun);
}
}
}
public abstract class FilesetRemover
{
protected readonly Options Options;
protected FilesetRemover(Options options)
{
this.Options = options;
}
public abstract IEnumerable<IListResultFileset> GetFilesetsToDelete(IEnumerable<IListResultFileset> filesets);
}
/// <summary>
/// Remove versions specified by the --version option.
/// </summary>
public class SpecificVersionsRemover : FilesetRemover
{
public SpecificVersionsRemover(Options options) : base(options)
{
}
public override IEnumerable<IListResultFileset> GetFilesetsToDelete(IEnumerable<IListResultFileset> filesets)
{
ISet<long> versionsToDelete = new HashSet<long>(this.Options.Version ?? new long[0]);
return filesets.Where(x => versionsToDelete.Contains(x.Version));
}
}
/// <summary>
/// Keep backups that are newer than the date specified by the --keep-time option.
/// If none of the retained versions are full backups, then continue to keep versions
/// until we have a full backup.
/// </summary>
public class KeepTimeRemover : FilesetRemover
{
public KeepTimeRemover(Options options) : base(options)
{
}
public override IEnumerable<IListResultFileset> GetFilesetsToDelete(IEnumerable<IListResultFileset> filesets)
{
IListResultFileset[] sortedFilesets = filesets.OrderByDescending(x => x.Time).ToArray();
List<IListResultFileset> versionsToDelete = new List<IListResultFileset>();
DateTime earliestTime = this.Options.KeepTime;
if (earliestTime.Ticks > 0)
{
bool haveFullBackup = false;
versionsToDelete.AddRange(sortedFilesets.SkipWhile(x =>
{
bool keepBackup = (x.Time >= earliestTime) || !haveFullBackup;
haveFullBackup = haveFullBackup || (x.IsFullBackup == BackupType.FULL_BACKUP);
return keepBackup;
}));
}
return versionsToDelete;
}
}
/// <summary>
/// Keep a number of recent full backups as specified by the --keep-versions option.
/// Partial backups that are surrounded by full backups will also be removed.
/// </summary>
public class KeepVersionsRemover : FilesetRemover
{
public KeepVersionsRemover(Options options) : base(options)
{
}
public override IEnumerable<IListResultFileset> GetFilesetsToDelete(IEnumerable<IListResultFileset> filesets)
{
IListResultFileset[] sortedFilesets = filesets.OrderByDescending(x => x.Time).ToArray();
List<IListResultFileset> versionsToDelete = new List<IListResultFileset>();
// Check how many full backups will be remaining after the previous steps
// and remove oldest backups while there are still more backups than should be kept as specified via option
int fullVersionsToKeep = this.Options.KeepVersions;
if (fullVersionsToKeep > 0 && fullVersionsToKeep < sortedFilesets.Length)
{
int fullVersionsKept = 0;
ISet<IListResultFileset> intermediatePartials = new HashSet<IListResultFileset>();
// Enumerate the collection starting from the most recent full backup.
foreach (IListResultFileset fileset in sortedFilesets.SkipWhile(x => x.IsFullBackup == BackupType.PARTIAL_BACKUP))
{
if (fullVersionsKept >= fullVersionsToKeep)
{
// If we have enough full backups, delete all older backups.
versionsToDelete.Add(fileset);
}
else if (fileset.IsFullBackup == BackupType.FULL_BACKUP)
{
// We can delete partial backups that are surrounded by full backups.
versionsToDelete.AddRange(intermediatePartials);
intermediatePartials.Clear();
fullVersionsKept++;
}
else
{
intermediatePartials.Add(fileset);
}
}
}
return versionsToDelete;
}
}
/// <summary>
/// Remove backups according to the --retention-policy option.
/// Backups that are not within any of the specified time frames will will NOT be deleted.
/// Partial backups are not removed.
/// </summary>
public class RetentionPolicyRemover : FilesetRemover
{
private static readonly string LOGTAG_RETENTION = DeleteHandler.LOGTAG + ":RetentionPolicy";
public RetentionPolicyRemover(Options options) : base(options)
{
}
public override IEnumerable<IListResultFileset> GetFilesetsToDelete(IEnumerable<IListResultFileset> filesets)
{
IListResultFileset[] sortedFilesets = filesets.OrderByDescending(x => x.Time).ToArray();
List<IListResultFileset> versionsToDelete = new List<IListResultFileset>();
List<Options.RetentionPolicyValue> retentionPolicyOptionValues = this.Options.RetentionPolicy;
if (retentionPolicyOptionValues.Count == 0 || sortedFilesets.Length == 0)
{
return versionsToDelete;
}
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "StartCheck", "Start checking if backups can be removed");
// Work with a copy to not modify the enumeration that the caller passed
List<IListResultFileset> clonedBackupList = new List<IListResultFileset>(sortedFilesets);
// Most recent backup usually should never get deleted in this process, so exclude it for now,
// but keep a reference to potential delete it when allow-full-removal is set
IListResultFileset mostRecentBackup = clonedBackupList.ElementAt(0);
clonedBackupList.RemoveAt(0);
bool deleteMostRecentBackup = this.Options.AllowFullRemoval;
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "FramesAndIntervals", "Time frames and intervals pairs: {0}", string.Join(", ", retentionPolicyOptionValues));
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "BackupList", "Backups to consider: {0}", string.Join(", ", clonedBackupList.Select(x => x.Time)));
// Collect all potential backups in each time frame and thin out according to the specified interval,
// starting with the oldest backup in that time frame.
// The order in which the time frames values are checked has to be from the smallest to the largest.
DateTime now = DateTime.Now;
foreach (Options.RetentionPolicyValue singleRetentionPolicyOptionValue in retentionPolicyOptionValues.OrderBy(x => x.Timeframe))
{
// The timeframe in the retention policy option is only a timespan which has to be applied to the current DateTime to get the actual lower bound
DateTime timeFrame = (singleRetentionPolicyOptionValue.IsUnlimtedTimeframe()) ? DateTime.MinValue : (now - singleRetentionPolicyOptionValue.Timeframe);
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "NextTimeAndFrame", "Next time frame and interval pair: {0}", singleRetentionPolicyOptionValue);
List<IListResultFileset> backupsInTimeFrame = new List<IListResultFileset>();
while (clonedBackupList.Count > 0 && clonedBackupList[0].Time >= timeFrame)
{
backupsInTimeFrame.Insert(0, clonedBackupList[0]); // Insert at beginning to reverse order, which is necessary for next step
clonedBackupList.RemoveAt(0); // remove from here to not handle the same backup in two time frames
}
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "BackupsInFrame", "Backups in this time frame: {0}", string.Join(", ", backupsInTimeFrame.Select(x => x.Time)));
// Run through backups in this time frame
IListResultFileset lastKept = null;
foreach (IListResultFileset fileset in backupsInTimeFrame)
{
bool isFullBackup = fileset.IsFullBackup == BackupType.FULL_BACKUP;
// Keep this backup if
// - no backup has yet been added to the time frame (keeps at least the oldest backup in a time frame)
// - difference between last added backup and this backup is bigger than the specified interval
if (lastKept == null || singleRetentionPolicyOptionValue.IsKeepAllVersions() || (fileset.Time - lastKept.Time) >= singleRetentionPolicyOptionValue.Interval)
{
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "KeepBackups", $"Keeping {(isFullBackup ? "" : "partial")} backup: {fileset.Time}", Logging.LogMessageType.Profiling);
if (isFullBackup)
{
lastKept = fileset;
}
}
else
{
if (isFullBackup)
{
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "DeletingBackups", "Deleting backup: {0}", fileset.Time);
versionsToDelete.Add(fileset);
}
else
{
Logging.Log.WriteProfilingMessage(LOGTAG_RETENTION, "KeepBackups", $"Keeping partial backup: {fileset.Time}", Logging.LogMessageType.Profiling);
}
}
}
// Check if most recent backup is outside of this time frame (meaning older/smaller)
deleteMostRecentBackup &= (mostRecentBackup.Time < timeFrame);
}
// Delete all remaining backups
versionsToDelete.AddRange(clonedBackupList);
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "BackupsToDelete", "Backups outside of all time frames and thus getting deleted: {0}", string.Join(", ", clonedBackupList.Select(x => x.Time)));
// Delete most recent backup if allow-full-removal is set and the most current backup is outside of any time frame
if (deleteMostRecentBackup)
{
versionsToDelete.Add(mostRecentBackup);
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "DeleteMostRecent", "Deleting most recent backup: {0}", mostRecentBackup.Time);
}
Logging.Log.WriteInformationMessage(LOGTAG_RETENTION, "AllBackupsToDelete", "All backups to delete: {0}", string.Join(", ", versionsToDelete.Select(x => x.Time).OrderByDescending(x => x)));
return versionsToDelete;
}
}
}
|