Merge pull request #4691 from dkrahmer/DK__FixFileBackendPartialFiles

FileBackend file size verification

2 files changed, 108 insertions, 14 deletions

diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs
index 3b41484c9..4da500458 100644
--- a/Duplicati/Library/Backend/File/FileBackend.cs
+++ b/Duplicati/Library/Backend/File/FileBackend.cs
@@ -22,6 +22,7 @@ using Duplicati.Library.Common.IO;
 using Duplicati.Library.Interface;

 using System;

 using System.Collections.Generic;

+using System.IO;

 using System.Runtime.InteropServices;

 using System.Threading;

 using System.Threading.Tasks;

@@ -56,7 +57,7 @@ namespace Duplicati.Library.Backend
         {

             var uri = new Utility.Uri(url);

             m_path = uri.HostAndPath;

-            

+

             if (options.ContainsKey("auth-username"))

                 m_username = options["auth-username"];

             if (options.ContainsKey("auth-password"))

@@ -65,7 +66,7 @@ namespace Duplicati.Library.Backend
                 m_username = uri.Username;

             if (!string.IsNullOrEmpty(uri.Password))

                 m_password = uri.Password;

-            

+

             if (!System.IO.Path.IsPathRooted(m_path))

                 m_path = systemIO.PathGetFullPath(m_path);

 

@@ -81,7 +82,7 @@ namespace Duplicati.Library.Backend
                 if (!Platform.IsClientPosix)

                 {

                     System.IO.DriveInfo[] drives = System.IO.DriveInfo.GetDrives();

-                    

+

                     for (int i = 0; i < paths.Count; i++)

                     {

                         if (paths[i].StartsWith("*:", StringComparison.Ordinal))

@@ -195,10 +196,14 @@ namespace Duplicati.Library.Backend
             }

         }

 #else

-        public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)

+        public async Task PutAsync(string targetFilename, Stream sourceStream, CancellationToken cancelToken)

         {

-            using (System.IO.FileStream writestream = systemIO.FileCreate(GetRemoteName(remotename)))

-                await Utility.Utility.CopyStreamAsync(stream, writestream, true, cancelToken, m_copybuffer);

+            string targetFilePath = GetRemoteName(targetFilename);

+            long copiedBytes = 0;

+            using (var targetStream = systemIO.FileCreate(targetFilePath))

+                copiedBytes = await Utility.Utility.CopyStreamAsync(sourceStream, targetStream, true, cancelToken, m_copybuffer);

+

+            VerifyMatchingSize(targetFilePath, sourceStream, copiedBytes);

         }

 #endif

 

@@ -209,18 +214,22 @@ namespace Duplicati.Library.Backend
                 Utility.Utility.CopyStream(readstream, stream, true, m_copybuffer);

         }

 

-        public Task PutAsync(string remotename, string filename, CancellationToken cancelToken)

+        public Task PutAsync(string targetFilename, string sourceFilePath, CancellationToken cancelToken)

         {

-            string path = GetRemoteName(remotename);

+            string targetFilePath = GetRemoteName(targetFilename);

             if (m_moveFile)

             {

-                if (systemIO.FileExists(path))

-                    systemIO.FileDelete(path);

-                

-                systemIO.FileMove(filename, path);

+                if (systemIO.FileExists(targetFilePath))

+                    systemIO.FileDelete(targetFilePath);

+

+                systemIO.FileMove(sourceFilePath, targetFilePath);

             }

             else

-                systemIO.FileCopy(filename, path, true);

+            {

+                systemIO.FileCopy(sourceFilePath, targetFilePath, true);

+            }

+

+            VerifyMatchingSize(targetFilePath, sourceFilePath);

 

             return Task.FromResult(true);

         }

@@ -386,5 +395,50 @@ namespace Duplicati.Library.Backend
                 out ulong lpTotalNumberOfBytes,

                 out ulong lpTotalNumberOfFreeBytes);

         }

+

+        private static void VerifyMatchingSize(string targetFilePath, string sourceFilePath)

+        {

+            try

+            {

+                var targetFileInfo = new FileInfo(targetFilePath);

+                if (!targetFileInfo.Exists)

+                    throw new FileMissingException($"Target file does not exist. Target: {targetFilePath}");

+

+                var sourceFileInfo = new FileInfo(sourceFilePath);

+                if (!sourceFileInfo.Exists)

+                    throw new FileMissingException($"Source file does not exist. Source: {sourceFilePath}");

+

+                if (targetFileInfo.Length != sourceFileInfo.Length)

+                    throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from source file size ({sourceFileInfo.Length:n0}). Target: {targetFilePath}");

+            }

+            catch

+            {

+                try { System.IO.File.Delete(targetFilePath); } catch { }

+                throw;

+            }

+        }

+

+        private static void VerifyMatchingSize(string targetFilePath, Stream sourceStream, long? expectedLength)

+        {

+            try

+            {

+                var targetFileInfo = new FileInfo(targetFilePath);

+                if (!targetFileInfo.Exists)

+                    throw new FileMissingException($"Target file does not exist. Target: {targetFilePath}");

+

+                long? sourceStreamLength = Utility.Utility.GetStreamLength(sourceStream, out bool isStreamPostion);

+

+                if (sourceStreamLength.HasValue && targetFileInfo.Length != sourceStreamLength.Value)

+                    throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position)" : "")}. Target: {targetFilePath}");

+

+                if (expectedLength.HasValue && targetFileInfo.Length != expectedLength.Value)

+                    throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the expected length ({expectedLength.Value:n0}). Target: {targetFilePath}");

+            }

+            catch

+            {

+                try { System.IO.File.Delete(targetFilePath); } catch { }

+                throw;

+            }

+        }

     }

 }

diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs
index a41b2dc0f..1c7bb8e0e 100644
--- a/Duplicati/Library/Utility/Utility.cs
+++ b/Duplicati/Library/Utility/Utility.cs
@@ -25,6 +25,7 @@ using System.Text.RegularExpressions;
 using Duplicati.Library.Common.IO;

 using Duplicati.Library.Common;

 using System.Globalization;

+using Duplicati.Library.Interface;

 

 namespace Duplicati.Library.Utility

 {

@@ -139,11 +140,50 @@ namespace Duplicati.Library.Utility
                 await target.WriteAsync(buf, 0, read, cancelToken).ConfigureAwait(false);

                 total += read;

             }

-            

+

             return total;

         }

 

         /// <summary>

+        /// Get the length of a stream.

+        /// Attempt to use the stream's Position property if allowPositionFallback is <c>true</c> (only valid if stream is at the end).

+        /// </summary>

+        /// <param name="stream">Stream to get the length of.</param>

+        /// <param name="allowPositionFallback">Attempt to use the Position property if <c>true</c> and the Length property is not available (only valid if stream is at the end).</param>

+        /// <returns>Returns the stream's length, if available, or null if not supported by the stream.</returns>

+        public static long? GetStreamLength(Stream stream, bool allowPositionFallback = true)

+        {

+            return GetStreamLength(stream, out bool _, allowPositionFallback);

+        }

+

+        /// <summary>

+        /// Get the length of a stream.

+        /// Attempt to use the stream's Position property if allowPositionFallback is <c>true</c> (only valid if stream is at the end).

+        /// </summary>

+        /// <param name="stream">Stream to get the length of.</param>

+        /// <param name="isStreamPosition">Indicates if the Position value was used instead of Length.</param>

+        /// <param name="allowPositionFallback">Attempt to use the Position property if <c>true</c> and the Length property is not available (only valid if stream is at the end).</param>

+        /// <returns>Returns the stream's length, if available, or null if not supported by the stream.</returns>

+        public static long? GetStreamLength(Stream stream, out bool isStreamPosition, bool allowPositionFallback = true)

+        {

+            isStreamPosition = false;

+            long? streamLength = null;

+            try { streamLength = stream.Length; } catch { }

+            if (!streamLength.HasValue && allowPositionFallback)

+            {

+                try

+                {

+                    // Hack: This is a fall-back method to detect the source stream size, assuming the current position is the end of the stream.

+                    streamLength = stream.Position;

+                    isStreamPosition = true;

+                }

+                catch { } // 

+            }

+

+            return streamLength;

+        }

+

+        /// <summary>

         /// These are characters that must be escaped when using a globbing expression

         /// </summary>

         private static readonly string BADCHARS = @"\\|\+|\||\{|\[|\(|\)|\]|\}|\^|\$|\#|\.";