Made python script run in parallel

This made the process a lot faster for me 15h -> 7h.

2 files changed, 41 insertions, 36 deletions

diff --git a/Tools/Commandline/ReEncrypt/ReEncrypt.py b/Tools/Commandline/ReEncrypt/ReEncrypt.py
index 389abf9a3..96fbc575a 100644
--- a/Tools/Commandline/ReEncrypt/ReEncrypt.py
+++ b/Tools/Commandline/ReEncrypt/ReEncrypt.py
@@ -28,6 +28,8 @@ from collections import OrderedDict
 from tempfile import mkstemp, mkdtemp, TemporaryFile, TemporaryDirectory, NamedTemporaryFile
 import gnupg
 import shutil
+from joblib import Parallel, delayed
+import multiprocessing
 
 def mainReEncrypt(options):
     # locate dlist
@@ -44,48 +46,51 @@ def mainReEncrypt(options):
     # locate dlist
     dindex = [name for name in os.listdir(options['orig']['path']) if name.endswith(".dindex.%s" %(options['orig']['extension']))]
 
+    num_cores = multiprocessing.cpu_count()
+    Parallel(n_jobs=num_cores*2)(delayed(handleIndex)(options, dindex_enc) for dindex_enc in dindex)
+
+def handleIndex(options, dindex_enc):
     with NamedTemporaryFile() as temp_dindex, NamedTemporaryFile() as temp_dindex_reenc, TemporaryDirectory() as temp_path_zip:
-        for dindex_enc in dindex:
-            dindex_enc_fullpath = os.path.join(options['orig']['path'],dindex_enc)
-            dindex_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dindex_enc,options['orig']['extension'],options['new']['extension']))
-            
-            decrypt(options['orig'],dindex_enc_fullpath,options['orig']['passwd'],temp_dindex.name)
+        dindex_enc_fullpath = os.path.join(options['orig']['path'],dindex_enc)
+        dindex_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dindex_enc,options['orig']['extension'],options['new']['extension']))
+        
+        decrypt(options['orig'],dindex_enc_fullpath,options['orig']['passwd'],temp_dindex.name)
             
-            unzip(temp_dindex,temp_path_zip)
+        unzip(temp_dindex,temp_path_zip)
 
-            vol_path = os.path.join(temp_path_zip,'vol')
+        vol_path = os.path.join(temp_path_zip,'vol')
             
-            for dblock in os.listdir(vol_path):
-                data = []
-                with open(os.path.join(vol_path, dblock)) as data_file:
-                    data = json.load(data_file, object_pairs_hook=OrderedDict)
-
-                expected_hash = data['volumehash'].encode('utf8')
-                expected_volumesize = data['volumesize']
-
-                if (options['verify_hash']):
-                    actual_hash = computeHash(os.path.join(options['orig']['path'],dblock))
-                    actual_volumesize=os.stat(os.path.join(options['orig']['path'],dblock)).st_size
-                    print('dblock: %s expected_hash: %s calc_hash: %s exact: %s' % (dblock,expected_hash.decode('utf8'),actual_hash.decode('utf8'),expected_hash==actual_hash))
-
-                with NamedTemporaryFile() as temp_dblock:
-                    dblock_enc_fullpath = os.path.join(options['orig']['path'],dblock)
-                    dblock_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))
-                    decrypt(options['orig'],dblock_enc_fullpath,options['orig']['passwd'],temp_dblock.name)
-                    encrypt(options['new'],temp_dblock.name,options['new']['passwd'], dblock_reenc_fullpath)
+        for dblock in os.listdir(vol_path):
+            data = []
+            with open(os.path.join(vol_path, dblock)) as data_file:
+                data = json.load(data_file, object_pairs_hook=OrderedDict)
+
+            expected_hash = data['volumehash'].encode('utf8')
+            expected_volumesize = data['volumesize']
+
+            if (options['verify_hash']):
+                actual_hash = computeHash(os.path.join(options['orig']['path'],dblock))
+                actual_volumesize=os.stat(os.path.join(options['orig']['path'],dblock)).st_size
+                print('dblock: %s expected_hash: %s calc_hash: %s exact: %s' % (dblock,expected_hash.decode('utf8'),actual_hash.decode('utf8'),expected_hash==actual_hash))
+
+            with NamedTemporaryFile() as temp_dblock:
+                dblock_enc_fullpath = os.path.join(options['orig']['path'],dblock)
+                dblock_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))
+                decrypt(options['orig'],dblock_enc_fullpath,options['orig']['passwd'],temp_dblock.name)
+                encrypt(options['new'],temp_dblock.name,options['new']['passwd'], dblock_reenc_fullpath)
                 new_hash = computeHash(dblock_reenc_fullpath)
                 
-                data['volumehash'] = new_hash.decode('utf8')
-                data['volumesize'] = os.stat(os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))).st_size
-                print('dblock: %s old_hash: %s new_hash: %s' % (dblock,expected_hash.decode('utf8'),data['volumehash']))
+            data['volumehash'] = new_hash.decode('utf8')
+            data['volumesize'] = os.stat(os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))).st_size
+            print('dblock: %s old_hash: %s new_hash: %s' % (dblock,expected_hash.decode('utf8'),data['volumehash']))
 
-                with open(os.path.join(vol_path,dblock),'w') as data_file:
-                    json.dump(data, data_file)
+            with open(os.path.join(vol_path,dblock),'w') as data_file:
+                json.dump(data, data_file)
 
-                os.rename(os.path.join(vol_path, dblock), os.path.join(vol_path, change_ext(dblock,options['orig']['extension'],options['new']['extension']))) 
+            os.rename(os.path.join(vol_path, dblock), os.path.join(vol_path, change_ext(dblock,options['orig']['extension'],options['new']['extension']))) 
      
-            make_zipfile(temp_dindex_reenc.name,temp_path_zip)
-            encrypt(options['new'],temp_dindex_reenc.name, options['new']['passwd'],dindex_reenc_fullpath)
+        make_zipfile(temp_dindex_reenc.name,temp_path_zip)
+        encrypt(options['new'],temp_dindex_reenc.name, options['new']['passwd'],dindex_reenc_fullpath)
 
 def change_ext(filename, ext_old, ext_new):
     return filename.replace(ext_old, ext_new)
diff --git a/Tools/Commandline/ReEncrypt/Readme b/Tools/Commandline/ReEncrypt/Readme
index 0184006c0..d05bb736c 100644
--- a/Tools/Commandline/ReEncrypt/Readme
+++ b/Tools/Commandline/ReEncrypt/Readme
@@ -6,12 +6,12 @@ Usage:
 
 1) Confirm that your Duplicati data is on disk in .zip or .zip.aes or .zip.gpg format..
 
-2) Install Python 3 if it is not already installed. pyAesCrypt and gnupg packages need to be installed in python as well.
+2) Install Python 3 if it is not already installed. pyAesCrypt, gnupg and joblib packages need to be installed in python as well.
 
-3) Prepare a config file (example config.txt). 
+3) Prepare a config file (see example config.txt). 
 
 3) Run ReEncrypt.py -c config.txt.
 
 4) Update your backup settings to the changed encryption settings. 
 
-5) Take a backup of your database and do a delete & recreate.
+5) Take a backup of your database and do a delete & recreate. Don't forget to delete, a simple "repair" may delete the whole backup!