1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
#region Disclaimer / License
// Copyright (C) 2015, The Duplicati Team
// http://www.duplicati.com, info@duplicati.com
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
#endregion
using System;
using System.Collections.Generic;
using System.Text;
using Amazon.S3;
using Amazon.S3.Model;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Backend
{
/// <summary>
/// Helper class that fixes long list support and injects location headers, includes using directives etc.
/// </summary>
public class S3Wrapper : IDisposable
{
private const int ITEM_LIST_LIMIT = 1000;
protected string m_locationConstraint;
protected string m_storageClass;
protected AmazonS3Client m_client;
public S3Wrapper(string awsID, string awsKey, string locationConstraint, string servername, string storageClass, bool useSSL)
{
var cfg = new AmazonS3Config();
cfg.UseHttp = !useSSL;
cfg.ServiceURL = (useSSL ? "https://" : "http://") + servername;
//cfg.UserAgent = "Duplicati v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString() + " S3 client with AWS SDK v" + cfg.GetType().Assembly.GetName().Version.ToString();
cfg.BufferSize = (int)Duplicati.Library.Utility.Utility.DEFAULT_BUFFER_SIZE;
m_client = new Amazon.S3.AmazonS3Client(awsID, awsKey, cfg);
m_locationConstraint = locationConstraint;
m_storageClass = storageClass;
}
public void AddBucket(string bucketName)
{
PutBucketRequest request = new PutBucketRequest();
request.BucketName = bucketName;
if (!string.IsNullOrEmpty(m_locationConstraint))
request.BucketRegionName = m_locationConstraint;
m_client.PutBucket(request);
}
public virtual void GetFileStream(string bucketName, string keyName, System.IO.Stream target)
{
GetObjectRequest objectGetRequest = new GetObjectRequest();
objectGetRequest.BucketName = bucketName;
objectGetRequest.Key = keyName;
using(GetObjectResponse objectGetResponse = m_client.GetObject(objectGetRequest))
using(System.IO.Stream s = objectGetResponse.ResponseStream)
{
try { s.ReadTimeout = (int)TimeSpan.FromMinutes(1).TotalMilliseconds; }
catch { }
Utility.Utility.CopyStream(s, target);
}
}
public void GetFileObject(string bucketName, string keyName, string localfile)
{
using (System.IO.FileStream fs = System.IO.File.Open(localfile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
GetFileStream(bucketName, keyName, fs);
}
public void AddFileObject(string bucketName, string keyName, string localfile)
{
using (System.IO.FileStream fs = System.IO.File.Open(localfile, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
AddFileStream(bucketName, keyName, fs);
}
public virtual void AddFileStream(string bucketName, string keyName, System.IO.Stream source)
{
PutObjectRequest objectAddRequest = new PutObjectRequest();
objectAddRequest.BucketName = bucketName;
objectAddRequest.Key = keyName;
objectAddRequest.InputStream = source;
if (!string.IsNullOrWhiteSpace(m_storageClass))
objectAddRequest.StorageClass = new S3StorageClass(m_storageClass);
m_client.PutObject(objectAddRequest);
}
public void DeleteObject(string bucketName, string keyName)
{
DeleteObjectRequest objectDeleteRequest = new DeleteObjectRequest();
objectDeleteRequest.BucketName = bucketName;
objectDeleteRequest.Key = keyName;
m_client.DeleteObject(objectDeleteRequest);
}
public virtual List<IFileEntry> ListBucket(string bucketName, string prefix)
{
bool isTruncated = true;
string filename = null;
List<IFileEntry> files = new List<IFileEntry>();
//We truncate after ITEM_LIST_LIMIT elements, and then repeat
while (isTruncated)
{
ListObjectsRequest listRequest = new ListObjectsRequest();
listRequest.BucketName = bucketName;
if (!string.IsNullOrEmpty(filename))
listRequest.Marker = filename;
listRequest.MaxKeys = ITEM_LIST_LIMIT;
if (!string.IsNullOrEmpty(prefix))
listRequest.Prefix = prefix;
ListObjectsResponse listResponse = m_client.ListObjects(listRequest);
isTruncated = listResponse.IsTruncated;
filename = listResponse.NextMarker;
foreach (S3Object obj in listResponse.S3Objects)
{
files.Add(new FileEntry(
obj.Key,
obj.Size,
obj.LastModified,
obj.LastModified
));
}
}
//TODO: Figure out if this is the case with AWSSDK too
//Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results
Dictionary<string, string> tmp = new Dictionary<string, string>();
for (int i = 0; i < files.Count; i++)
if (tmp.ContainsKey(files[i].Name))
{
files.RemoveAt(i);
i--;
}
else
tmp.Add(files[i].Name, null);
return files;
}
public void RenameFile(string bucketName, string source, string target)
{
CopyObjectRequest copyObjectRequest = new CopyObjectRequest();
copyObjectRequest.SourceBucket = bucketName;
copyObjectRequest.SourceKey = source;
copyObjectRequest.DestinationBucket = bucketName;
copyObjectRequest.DestinationKey = target;
m_client.CopyObject(copyObjectRequest);
DeleteObject(bucketName, source);
}
#region IDisposable Members
public void Dispose()
{
if (m_client != null)
m_client.Dispose();
m_client = null;
}
#endregion
}
}
|
