From 77a46b34c02cc934e0c330d6abd16fd5644280dd Mon Sep 17 00:00:00 2001 From: Piegames <14054505+piegamesde@users.noreply.github.com> Date: Thu, 8 Mar 2018 17:55:34 +0100 Subject: Made python script run in parallel This made the process a lot faster for me 15h -> 7h. --- Tools/Commandline/ReEncrypt/ReEncrypt.py | 71 +++++++++++++++++--------------- Tools/Commandline/ReEncrypt/Readme | 6 +-- 2 files changed, 41 insertions(+), 36 deletions(-) (limited to 'Tools') diff --git a/Tools/Commandline/ReEncrypt/ReEncrypt.py b/Tools/Commandline/ReEncrypt/ReEncrypt.py index 389abf9a3..96fbc575a 100644 --- a/Tools/Commandline/ReEncrypt/ReEncrypt.py +++ b/Tools/Commandline/ReEncrypt/ReEncrypt.py @@ -28,6 +28,8 @@ from collections import OrderedDict from tempfile import mkstemp, mkdtemp, TemporaryFile, TemporaryDirectory, NamedTemporaryFile import gnupg import shutil +from joblib import Parallel, delayed +import multiprocessing def mainReEncrypt(options): # locate dlist @@ -44,48 +46,51 @@ def mainReEncrypt(options): # locate dlist dindex = [name for name in os.listdir(options['orig']['path']) if name.endswith(".dindex.%s" %(options['orig']['extension']))] + num_cores = multiprocessing.cpu_count() + Parallel(n_jobs=num_cores*2)(delayed(handleIndex)(options, dindex_enc) for dindex_enc in dindex) + +def handleIndex(options, dindex_enc): with NamedTemporaryFile() as temp_dindex, NamedTemporaryFile() as temp_dindex_reenc, TemporaryDirectory() as temp_path_zip: - for dindex_enc in dindex: - dindex_enc_fullpath = os.path.join(options['orig']['path'],dindex_enc) - dindex_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dindex_enc,options['orig']['extension'],options['new']['extension'])) - - decrypt(options['orig'],dindex_enc_fullpath,options['orig']['passwd'],temp_dindex.name) + dindex_enc_fullpath = os.path.join(options['orig']['path'],dindex_enc) + dindex_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dindex_enc,options['orig']['extension'],options['new']['extension'])) + + decrypt(options['orig'],dindex_enc_fullpath,options['orig']['passwd'],temp_dindex.name) - unzip(temp_dindex,temp_path_zip) + unzip(temp_dindex,temp_path_zip) - vol_path = os.path.join(temp_path_zip,'vol') + vol_path = os.path.join(temp_path_zip,'vol') - for dblock in os.listdir(vol_path): - data = [] - with open(os.path.join(vol_path, dblock)) as data_file: - data = json.load(data_file, object_pairs_hook=OrderedDict) - - expected_hash = data['volumehash'].encode('utf8') - expected_volumesize = data['volumesize'] - - if (options['verify_hash']): - actual_hash = computeHash(os.path.join(options['orig']['path'],dblock)) - actual_volumesize=os.stat(os.path.join(options['orig']['path'],dblock)).st_size - print('dblock: %s expected_hash: %s calc_hash: %s exact: %s' % (dblock,expected_hash.decode('utf8'),actual_hash.decode('utf8'),expected_hash==actual_hash)) - - with NamedTemporaryFile() as temp_dblock: - dblock_enc_fullpath = os.path.join(options['orig']['path'],dblock) - dblock_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension'])) - decrypt(options['orig'],dblock_enc_fullpath,options['orig']['passwd'],temp_dblock.name) - encrypt(options['new'],temp_dblock.name,options['new']['passwd'], dblock_reenc_fullpath) + for dblock in os.listdir(vol_path): + data = [] + with open(os.path.join(vol_path, dblock)) as data_file: + data = json.load(data_file, object_pairs_hook=OrderedDict) + + expected_hash = data['volumehash'].encode('utf8') + expected_volumesize = data['volumesize'] + + if (options['verify_hash']): + actual_hash = computeHash(os.path.join(options['orig']['path'],dblock)) + actual_volumesize=os.stat(os.path.join(options['orig']['path'],dblock)).st_size + print('dblock: %s expected_hash: %s calc_hash: %s exact: %s' % (dblock,expected_hash.decode('utf8'),actual_hash.decode('utf8'),expected_hash==actual_hash)) + + with NamedTemporaryFile() as temp_dblock: + dblock_enc_fullpath = os.path.join(options['orig']['path'],dblock) + dblock_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension'])) + decrypt(options['orig'],dblock_enc_fullpath,options['orig']['passwd'],temp_dblock.name) + encrypt(options['new'],temp_dblock.name,options['new']['passwd'], dblock_reenc_fullpath) new_hash = computeHash(dblock_reenc_fullpath) - data['volumehash'] = new_hash.decode('utf8') - data['volumesize'] = os.stat(os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))).st_size - print('dblock: %s old_hash: %s new_hash: %s' % (dblock,expected_hash.decode('utf8'),data['volumehash'])) + data['volumehash'] = new_hash.decode('utf8') + data['volumesize'] = os.stat(os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))).st_size + print('dblock: %s old_hash: %s new_hash: %s' % (dblock,expected_hash.decode('utf8'),data['volumehash'])) - with open(os.path.join(vol_path,dblock),'w') as data_file: - json.dump(data, data_file) + with open(os.path.join(vol_path,dblock),'w') as data_file: + json.dump(data, data_file) - os.rename(os.path.join(vol_path, dblock), os.path.join(vol_path, change_ext(dblock,options['orig']['extension'],options['new']['extension']))) + os.rename(os.path.join(vol_path, dblock), os.path.join(vol_path, change_ext(dblock,options['orig']['extension'],options['new']['extension']))) - make_zipfile(temp_dindex_reenc.name,temp_path_zip) - encrypt(options['new'],temp_dindex_reenc.name, options['new']['passwd'],dindex_reenc_fullpath) + make_zipfile(temp_dindex_reenc.name,temp_path_zip) + encrypt(options['new'],temp_dindex_reenc.name, options['new']['passwd'],dindex_reenc_fullpath) def change_ext(filename, ext_old, ext_new): return filename.replace(ext_old, ext_new) diff --git a/Tools/Commandline/ReEncrypt/Readme b/Tools/Commandline/ReEncrypt/Readme index 0184006c0..d05bb736c 100644 --- a/Tools/Commandline/ReEncrypt/Readme +++ b/Tools/Commandline/ReEncrypt/Readme @@ -6,12 +6,12 @@ Usage: 1) Confirm that your Duplicati data is on disk in .zip or .zip.aes or .zip.gpg format.. -2) Install Python 3 if it is not already installed. pyAesCrypt and gnupg packages need to be installed in python as well. +2) Install Python 3 if it is not already installed. pyAesCrypt, gnupg and joblib packages need to be installed in python as well. -3) Prepare a config file (example config.txt). +3) Prepare a config file (see example config.txt). 3) Run ReEncrypt.py -c config.txt. 4) Update your backup settings to the changed encryption settings. -5) Take a backup of your database and do a delete & recreate. +5) Take a backup of your database and do a delete & recreate. Don't forget to delete, a simple "repair" may delete the whole backup! -- cgit v1.2.3