BlenderKit: rerequests library

this library basically ensures that no server requests should fail if the token is after it's lifetime.
it refreshes token and re-tries the request in such cases.

9 files changed, 138 insertions, 57 deletions

diff --git a/blenderkit/bkit_oauth.py b/blenderkit/bkit_oauth.py
index f435d95d..57d3389f 100644
--- a/blenderkit/bkit_oauth.py
+++ b/blenderkit/bkit_oauth.py
@@ -74,7 +74,7 @@ def refresh_token(api_key_refresh, url):
     auth_token, refresh_token = authenticator.get_refreshed_token(api_key_refresh)
     if auth_token is not None and refresh_token is not None:
         tasks_queue.add_task((write_tokens, (auth_token, refresh_token)))
-
+    return auth_token, refresh_token
 
 def write_tokens(auth_token, refresh_token):
     utils.p('writing tokens')
@@ -85,7 +85,7 @@ def write_tokens(auth_token, refresh_token):
     props = utils.get_search_props()
     if props is not None:
         props.report = ''
-    ui.add_report('BlenderKit Login success')
+    ui.add_report('BlenderKit Re-Login success')
     search.get_profile()
     categories.fetch_categories_thread(auth_token)
 
diff --git a/blenderkit/categories.py b/blenderkit/categories.py
index 6407b050..1d411499 100644
--- a/blenderkit/categories.py
+++ b/blenderkit/categories.py
@@ -22,8 +22,9 @@ if "bpy" in locals():
     paths = reload(paths)
     utils = reload(utils)
     tasks_queue = reload(tasks_queue)
+    rerequests = reload(rerequests)
 else:
-    from blenderkit import paths, utils, tasks_queue
+    from blenderkit import paths, utils, tasks_queue, rerequests
 
 import requests
 import json
@@ -114,7 +115,7 @@ def fetch_categories(API_key):
     categories_filepath = os.path.join(tempdir, 'categories.json')
 
     try:
-        r = requests.get(url, headers=headers)
+        r = rerequests.get(url, headers=headers)
         rdata = r.json()
         categories = rdata['results']
         fix_category_counts(categories)
diff --git a/blenderkit/download.py b/blenderkit/download.py
index 366081ca..9f5f0314 100644
--- a/blenderkit/download.py
+++ b/blenderkit/download.py
@@ -25,8 +25,9 @@ if "bpy" in locals():
     ui = reload(ui)
     colors = reload(colors)
     tasks_queue = reload(tasks_queue)
+    rerequests = reload(rerequests)
 else:
-    from blenderkit import paths, append_link, utils, ui, colors, tasks_queue
+    from blenderkit import paths, append_link, utils, ui, colors, tasks_queue, rerequests
 
 import threading
 import time
@@ -266,7 +267,7 @@ def report_usages():
         scene['assets rated'][k] = scene['assets rated'].get(k, False)
     thread = threading.Thread(target=utils.requests_post_thread, args=(url, usage_report, headers))
     thread.start()
-    # r = requests.post(url, headers=headers, json=usage_report)
+    # r = rerequests.post(url, headers=headers, json=usage_report)
     mt = time.time() - mt
     print('report generation:                ', mt)
 
@@ -551,6 +552,7 @@ class Downloader(threading.Thread):
             tcom.downloaded = 100
             utils.p('not downloading, trying to append again')
             return;
+
         file_name = paths.get_download_filenames(asset_data)[0]  # prefer global dir if possible.
         # for k in asset_data:
         #    print(asset_data[k])
@@ -617,7 +619,6 @@ def download(asset_data, **kwargs):
         asset_data = copy.deepcopy(asset_data)
     else:
         asset_data = asset_data.to_dict()
-
     readthread = Downloader(asset_data, tcom, scene_id, api_key)
     readthread.start()
 
@@ -730,7 +731,7 @@ def get_download_url(asset_data, scene_id, api_key, tcom=None):
     }
     r = None
     try:
-        r = requests.get(asset_data['download_url'], params=data, headers=headers)
+        r = rerequests.get(asset_data['download_url'], params=data, headers=headers)
     except Exception as e:
         print(e)
         if tcom is not None:
@@ -755,10 +756,6 @@ def get_download_url(asset_data, scene_id, api_key, tcom=None):
         tasks_queue.add_task((ui.add_report, (r1, 5, colors.RED)))
         tcom.error = True
 
-    if r.status_code == 401:
-        tcom.report = 'Invalid API key'
-        tcom.error = True
-        return 'Invalid API key'
     elif r.status_code >= 500:
         tcom.report = 'Server error'
         tcom.error = True
@@ -833,7 +830,6 @@ class BlenderkitDownloadOperator(bpy.types.Operator):
     bl_label = "BlenderKit Asset Download"
     bl_options = {'REGISTER', 'UNDO', 'INTERNAL'}
 
-
     asset_type: EnumProperty(
         name="Type",
         items=asset_types,
diff --git a/blenderkit/ratings.py b/blenderkit/ratings.py
index ddf01e6c..7684d017 100644
--- a/blenderkit/ratings.py
+++ b/blenderkit/ratings.py
@@ -21,8 +21,9 @@ if "bpy" in locals():
 
     paths = reload(paths)
     utils = reload(utils)
+    rerequests = reload(rerequests)
 else:
-    from blenderkit import paths, utils
+    from blenderkit import paths, utils, rerequests
 
 import bpy
 import requests, threading
@@ -67,14 +68,14 @@ def uplaod_rating_thread(url, ratings, headers):
             }
 
             try:
-                r = requests.put(rating_url, data=data, verify=True, headers=headers)
+                r = rerequests.put(rating_url, data=data, verify=True, headers=headers)
 
             except requests.exceptions.RequestException as e:
                 print('ratings upload failed: %s' % str(e))
 
 
 def uplaod_review_thread(url, reviews, headers):
-    r = requests.put(url, data=reviews, verify=True, headers=headers)
+    r = rerequests.put(url, data=reviews, verify=True, headers=headers)
 
     # except requests.exceptions.RequestException as e:
     #     print('reviews upload failed: %s' % str(e))
diff --git a/blenderkit/rerequests.py b/blenderkit/rerequests.py
new file mode 100644
index 00000000..d58214bc
--- /dev/null
+++ b/blenderkit/rerequests.py
@@ -0,0 +1,95 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+if "bpy" in locals():
+    from importlib import reload
+
+    ui = reload(ui)
+    utils = reload(utils)
+    paths = reload(paths)
+    tasks_queue = reload(tasks_queue)
+    bkit_oauth = reload(bkit_oauth)
+else:
+    from blenderkit import ui, utils, paths, tasks_queue, bkit_oauth
+
+import requests
+import bpy
+
+
+def rerequest(method, url, **kwargs):
+    # first get any additional args from kwargs
+    immediate = False
+    if kwargs.get('immediate'):
+        immediate = kwargs['immediate']
+        kwargs.pop('immediate')
+    # first normal attempt
+    response = requests.request(method, url, **kwargs)
+
+    utils.p(url)
+    utils.p(response.status_code)
+
+    if response.status_code == 401:
+        try:
+            rdata = response.json()
+        except:
+            rdata = {}
+
+        tasks_queue.add_task((ui.add_report, (method + ' request Failed.' + str(rdata.get('detail')),)))
+
+        if rdata.get('detail') == 'Invalid token.':
+            user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
+            if user_preferences.api_key != '':
+                if user_preferences.enable_oauth:
+                    tasks_queue.add_task((ui.add_report, ('refreshing token.',)))
+                    refresh_url = paths.get_bkit_url()
+                    auth_token, refresh_token = bkit_oauth.refresh_token(user_preferences.api_key_refresh, refresh_url)
+
+                    # utils.p(auth_token, refresh_token)
+                    if auth_token is not None:
+                        if immediate == True:
+                            # this can write tokens occasionally into prefs. used e.g. in upload. Only possible
+                            #  in non-threaded tasks
+                            bpy.context.preferences.addons['blenderkit'].preferences.api_key = auth_token
+                            bpy.context.preferences.addons['blenderkit'].preferences.api_key_refresh = refresh_token
+
+                        kwargs['headers'] = utils.get_headers(auth_token)
+                        response = requests.request(method, url, **kwargs)
+                        utils.p('reresult', response.status_code)
+                        if response.status_code >= 400:
+                            utils.p('reresult', response.text)
+    return response
+
+
+def get(url, **kwargs):
+    response = rerequest('get', url, **kwargs)
+    return response
+
+
+def post(url, **kwargs):
+    response = rerequest('post', url, **kwargs)
+    return response
+
+
+def put(url, **kwargs):
+    response = rerequest('put', url, **kwargs)
+    return response
+
+
+def patch(url, **kwargs):
+    response = rerequest('patch', url, **kwargs)
+    return response
diff --git a/blenderkit/search.py b/blenderkit/search.py
index f19a019e..96c50a0e 100644
--- a/blenderkit/search.py
+++ b/blenderkit/search.py
@@ -27,8 +27,9 @@ if "bpy" in locals():
     bkit_oauth = reload(bkit_oauth)
     version_checker = reload(version_checker)
     tasks_queue = reload(tasks_queue)
+    rerequests = reload(rerequests)
 else:
-    from blenderkit import paths, utils, categories, ui, bkit_oauth, version_checker, tasks_queue
+    from blenderkit import paths, utils, categories, ui, bkit_oauth, version_checker, tasks_queue, rerequests
 
 import blenderkit
 from bpy.app.handlers import persistent
@@ -121,9 +122,11 @@ def timer_update():  # TODO might get moved to handle all blenderkit stuff.
     # causing a lot of throuble literally.
     if len(search_threads) == 0 or bpy.context.scene.blenderkitUI.dragging:
         return 1
-    for thread in search_threads:  # TODO this doesn't check all processess when removal... mostly 1 process will be running however.
+    for thread in search_threads:  # TODO this doesn't check all processess when one gets removed, but most time only
+        # one is running anyway
 
         if not thread[0].is_alive():
+            print('parsing')
             search_threads.remove(thread)  #
             icons_dir = thread[1]
             scene = bpy.context.scene
@@ -558,7 +561,7 @@ class ThumbDownloader(threading.Thread):
         return self._stop_event.is_set()
 
     def run(self):
-        r = requests.get(self.url, stream=False)
+        r = rerequests.get(self.url, stream=False)
         if r.status_code == 200:
             with open(self.path, 'wb') as f:
                 f.write(r.content)
@@ -581,7 +584,7 @@ def fetch_author(a_id, api_key):
     try:
         a_url = paths.get_api_url() + 'accounts/' + a_id + '/'
         headers = utils.get_headers(api_key)
-        r = requests.get(a_url, headers=headers)
+        r = rerequests.get(a_url, headers=headers)
         if r.status_code == 200:
             adata = r.json()
             if not hasattr(adata, 'id'):
@@ -591,13 +594,14 @@ def fetch_author(a_id, api_key):
             if adata.get('gravatarHash') is not None:
                 gravatar_path = paths.get_temp_dir(subdir='g/') + adata['gravatarHash'] + '.jpg'
                 url = "https://www.gravatar.com/avatar/" + adata['gravatarHash'] + '?d=404'
-                r = requests.get(url, stream=False)
+                r = rerequests.get(url, stream=False)
                 if r.status_code == 200:
                     with open(gravatar_path, 'wb') as f:
                         f.write(r.content)
                     adata['gravatarImg'] = gravatar_path
                 elif r.status_code == '404':
                     adata['gravatarHash'] = None
+                    utils.p('gravatar for author not available.')
     except Exception as e:
         utils.p(e)
     utils.p('finish fetch')
@@ -635,7 +639,7 @@ def write_profile(adata):
 def request_profile(api_key):
     a_url = paths.get_api_url() + 'me/'
     headers = utils.get_headers(api_key)
-    r = requests.get(a_url, headers=headers)
+    r = rerequests.get(a_url, headers=headers)
     adata = r.json()
     if adata.get('user') is None:
         utils.p(adata)
@@ -678,8 +682,7 @@ class Searcher(threading.Thread):
         return self._stop_event.is_set()
 
     def run(self):
-        maxthreads = 300
-        maximages = 50
+        maxthreads = 50
         query = self.query
         params = self.params
         global reports
@@ -731,7 +734,7 @@ class Searcher(threading.Thread):
 
         try:
             utils.p(urlquery)
-            r = requests.get(urlquery, headers=headers)
+            r = rerequests.get(urlquery, headers=headers)
             reports = ''
             # utils.p(r.text)
         except requests.exceptions.RequestException as e:
@@ -1115,7 +1118,7 @@ class SearchOperator(Operator):
                            description="get next page from previous search",
                            default=False)
 
-    keywords = StringProperty(
+    keywords: StringProperty(
         name="Keywords",
         description="Keywords",
         default="")
@@ -1132,9 +1135,8 @@ class SearchOperator(Operator):
         if self.keywords != '':
             sprops.search_keywords = self.keywords
 
-
         search(category=self.category, get_next=self.get_next, author_id=self.author_id)
-        #bpy.ops.view3d.blenderkit_asset_bar()
+        # bpy.ops.view3d.blenderkit_asset_bar()
 
         return {'FINISHED'}
 
@@ -1163,18 +1165,3 @@ def unregister_search():
 
     # bpy.app.timers.unregister(timer_update)
 
-
-'''
-search -
-build query
-START THREAD
-send query (bg already)
-get result - metadata, small thumbnails, big thumbnails paths (now generate this?)
-write metadata, possibly to
-download small thumbnails first
-start big thumbnails download. these don't have to be there on updates, if they aren't the Image in image editor doesn't get updated.
-parse metadata, save it in json in the temp dir which gets read on each update of the search.
-END THREAD
-when download is triggered, get also this metadata from json. E
-pass this single - item metadata in the download functions, threads.
-'''
diff --git a/blenderkit/upload.py b/blenderkit/upload.py
index c9e767a4..75f203df 100644
--- a/blenderkit/upload.py
+++ b/blenderkit/upload.py
@@ -31,9 +31,10 @@ if "bpy" in locals():
     ui = reload(ui)
     overrides = reload(overrides)
     colors = reload(colors)
+    rerequests = reload(rerequests)
 else:
     from blenderkit import asset_inspector, paths, utils, bg_blender, autothumb, version_checker, search, ui_panels, ui, \
-        overrides, colors
+        overrides, colors, rerequests
 
 import tempfile, os, subprocess, json, re
 
@@ -473,7 +474,7 @@ def verification_status_change(self, context, asset_id, state):
     url = paths.get_api_url() + 'assets/' + str(asset_id) + '/'
     headers = utils.get_headers(user_preferences.api_key)
     try:
-        r = requests.patch(url, json=upload_data, headers=headers, verify=True)  # files = files,
+        r = rerequests.patch(url, json=upload_data, headers=headers, verify=True)  # files = files,
         #print('changed status ')
         #print(r.text)
     except requests.exceptions.RequestException as e:
@@ -609,7 +610,7 @@ def start_upload(self, context, asset_type, reupload, upload_set):
     global reports
     if props.asset_base_id == '':
         try:
-            r = requests.post(url, json=json_metadata, headers=headers, verify=True)  # files = files,
+            r = rerequests.post(url, json=json_metadata, headers=headers, verify=True, immediate = True)  # files = files,
             ui.add_report('uploaded metadata')
             utils.p(r.text)
         except requests.exceptions.RequestException as e:
@@ -623,7 +624,7 @@ def start_upload(self, context, asset_type, reupload, upload_set):
         try:
             if upload_set != ['METADATA']:
                 json_metadata["verificationStatus"] = "uploading"
-            r = requests.put(url, json=json_metadata, headers=headers, verify=True)  # files = files,
+            r = rerequests.put(url, json=json_metadata, headers=headers, verify=True , immediate = True)  # files = files,
             ui.add_report('uploaded metadata')
             # parse the request
             # print('uploaded metadata')
diff --git a/blenderkit/upload_bg.py b/blenderkit/upload_bg.py
index d951eac2..a9c9f19b 100644
--- a/blenderkit/upload_bg.py
+++ b/blenderkit/upload_bg.py
@@ -24,8 +24,9 @@ if "bpy" in locals():
     append_link = reload(append_link)
     bg_blender = reload(bg_blender)
     utils = reload(utils)
+    rerequests = reload(rerequests)
 else:
-    from blenderkit import paths, append_link, bg_blender, utils
+    from blenderkit import paths, append_link, bg_blender, utils, rerequests
 
 import sys, json, os, time
 import requests
@@ -84,7 +85,7 @@ def upload_file(upload_data, f):
         'originalFilename': os.path.basename(f['file_path'])
     }
     upload_create_url = paths.get_api_url() + 'uploads/'
-    upload = requests.post(upload_create_url, json=upload_info, headers=headers, verify=True)
+    upload = rerequests.post(upload_create_url, json=upload_info, headers=headers, verify=True)
     upload = upload.json()
 
     chunk_size = 1024 * 256
@@ -109,7 +110,7 @@ def upload_file(upload_data, f):
 
             # confirm single file upload to bkit server
             upload_done_url = paths.get_api_url() + 'uploads_s3/' + upload['id'] + '/upload-file/'
-            upload_response = requests.post(upload_done_url, headers=headers, verify=True)
+            upload_response = rerequests.post(upload_done_url, headers=headers, verify=True)
 
     bg_blender.progress('finished uploading')
 
@@ -117,7 +118,6 @@ def upload_file(upload_data, f):
 
 
 def upload_files(upload_data, files):
-
     uploaded_all = True
     for f in files:
         uploaded = upload_file(upload_data, f)
@@ -180,10 +180,10 @@ if __name__ == "__main__":
         files = []
         if 'THUMBNAIL' in upload_set:
             files.append({
-            "type": "thumbnail",
-            "index": 0,
-            "file_path": export_data["thumbnail_path"]
-        })
+                "type": "thumbnail",
+                "index": 0,
+                "file_path": export_data["thumbnail_path"]
+            })
         if 'MAINFILE' in upload_set:
             files.append({
                 "type": "blend",
@@ -207,7 +207,7 @@ if __name__ == "__main__":
 
             url += upload_data["id"] + '/'
 
-            r = requests.patch(url, json=confirm_data, headers=headers, verify=True)  # files = files,
+            r = rerequests.patch(url, json=confirm_data, headers=headers, verify=True)  # files = files,
 
             bg_blender.progress('upload finished successfully')
         else:
diff --git a/blenderkit/utils.py b/blenderkit/utils.py
index 7c56ce8a..a6bb407d 100644
--- a/blenderkit/utils.py
+++ b/blenderkit/utils.py
@@ -385,7 +385,7 @@ def get_dimensions(obs):
 
 
 def requests_post_thread(url, json, headers):
-    r = requests.post(url, json=json, verify=True, headers=headers)
+    r = rerequests.post(url, json=json, verify=True, headers=headers)
 
 
 def get_headers(api_key):