BlenderKit: HDR resolutions

also  fix several issues with HDRs
 - categories upload (for all asset types was broken)
 - improve thumbnails color
 - tooltip drawing
 - resolution tag upload

12 files changed, 160 insertions, 68 deletions

diff --git a/blenderkit/__init__.py b/blenderkit/__init__.py
index d5ad436d..fa7ec290 100644
--- a/blenderkit/__init__.py
+++ b/blenderkit/__init__.py
@@ -916,8 +916,8 @@ class BlenderKitBrushSearchProps(PropertyGroup, BlenderKitCommonSearchProps):
 
 
 class BlenderKitHDRUploadProps(PropertyGroup, BlenderKitCommonUploadProps):
-    pass;
-
+    texture_resolution_max: IntProperty(name="Texture Resolution Max", description="texture resolution maximum",
+                                        default=0)
 
 class BlenderKitBrushUploadProps(PropertyGroup, BlenderKitCommonUploadProps):
     mode: EnumProperty(
diff --git a/blenderkit/append_link.py b/blenderkit/append_link.py
index c80027c5..b56897bb 100644
--- a/blenderkit/append_link.py
+++ b/blenderkit/append_link.py
@@ -118,14 +118,15 @@ def hdr_swap(name, hdr):
     :return: None
     '''
     w = bpy.context.scene.world
-    w.use_nodes = True
-    w.name = name
-    nt = w.node_tree
-    for n in nt.nodes:
-        if 'ShaderNodeTexEnvironment' == n.bl_rna.identifier:
-            env_node = n
-            env_node.image = hdr
-            return
+    if w:
+        w.use_nodes = True
+        w.name = name
+        nt = w.node_tree
+        for n in nt.nodes:
+            if 'ShaderNodeTexEnvironment' == n.bl_rna.identifier:
+                env_node = n
+                env_node.image = hdr
+                return
     new_hdr_world(name,hdr)
 
 
diff --git a/blenderkit/asset_inspector.py b/blenderkit/asset_inspector.py
index 0975acd8..ce4284ce 100644
--- a/blenderkit/asset_inspector.py
+++ b/blenderkit/asset_inspector.py
@@ -379,7 +379,7 @@ class AutoFillTags(bpy.types.Operator):
 
     @classmethod
     def poll(cls, context):
-        return bpy.context.view_layer.objects.active is not None
+        return utils.uploadable_asset_poll()
 
     def execute(self, context):
         get_autotags()
diff --git a/blenderkit/blendfiles/thumbnailer.blend b/blenderkit/blendfiles/thumbnailer.blend
index 0a05f76f..88cf8655 100644
--- a/blenderkit/blendfiles/thumbnailer.blend
+++ b/blenderkit/blendfiles/thumbnailer.blend
diff --git a/blenderkit/categories.py b/blenderkit/categories.py
index cb598709..81d9fd0b 100644
--- a/blenderkit/categories.py
+++ b/blenderkit/categories.py
@@ -146,8 +146,9 @@ def get_subcategory1_enums(self, context):
     items = []
     if self.category != '' and self.subcategory != '':
         asset_categories = get_category(wm['bkit_categories'], cat_path=(asset_type, self.category, self.subcategory, ))
-        for c in asset_categories['children']:
-            items.append((c['slug'], c['name'], c['description']))
+        if asset_categories:
+            for c in asset_categories['children']:
+                items.append((c['slug'], c['name'], c['description']))
     if len(items) == 0:
         items.append(('NONE', '', 'no categories on this level defined'))
     return items
diff --git a/blenderkit/image_utils.py b/blenderkit/image_utils.py
index 5e2b93bf..e0b49b10 100644
--- a/blenderkit/image_utils.py
+++ b/blenderkit/image_utils.py
@@ -13,6 +13,7 @@ def get_orig_render_settings():
         'quality': ims.quality,
         'color_mode': ims.color_mode,
         'compression': ims.compression,
+        'exr_codec': ims.exr_codec,
         'view_transform': vs.view_transform
     }
     return orig_settings
@@ -27,11 +28,12 @@ def set_orig_render_settings(orig_settings):
     ims.quality = orig_settings['quality']
     ims.color_mode = orig_settings['color_mode']
     ims.compression = orig_settings['compression']
+    ims.exr_codec = orig_settings['exr_codec']
 
     vs.view_transform = orig_settings['view_transform']
 
 
-def img_save_as(img, filepath='//', file_format='JPEG', quality=90, color_mode='RGB', compression=15, view_transform = 'Raw'):
+def img_save_as(img, filepath='//', file_format='JPEG', quality=90, color_mode='RGB', compression=15, view_transform = 'Raw', exr_codec = 'DWAA'):
     '''Uses Blender 'save render' to save images - BLender isn't really able so save images with other methods correctly.'''
 
     ors = get_orig_render_settings()
@@ -44,6 +46,7 @@ def img_save_as(img, filepath='//', file_format='JPEG', quality=90, color_mode='
     ims.quality = quality
     ims.color_mode = color_mode
     ims.compression = compression
+    ims.exr_codec = exr_codec
     vs.view_transform = view_transform
 
 
diff --git a/blenderkit/resolutions.py b/blenderkit/resolutions.py
index a3de18ee..2ad28b99 100644
--- a/blenderkit/resolutions.py
+++ b/blenderkit/resolutions.py
@@ -28,8 +28,9 @@ if "bpy" in locals():
     search = reload(search)
     rerequests = reload(rerequests)
     upload_bg = reload(upload_bg)
+    image_utils = reload(image_utils)
 else:
-    from blenderkit import paths, append_link, bg_blender, utils, download, search, rerequests, upload_bg
+    from blenderkit import paths, append_link, bg_blender, utils, download, search, rerequests, upload_bg, image_utils
 
 import sys, json, os, time
 import subprocess
@@ -306,13 +307,13 @@ def downscale(i):
         i.scale(sx, sy)
 
 
-def upload_resolutions(files, data):
+def upload_resolutions(files, asset_data):
     preferences = bpy.context.preferences.addons['blenderkit'].preferences
 
     upload_data = {
-        "name": data['asset_data']['name'],
+        "name": asset_data['name'],
         "token": preferences.api_key,
-        "id": data['asset_data']['id']
+        "id": asset_data['id']
     }
 
     uploaded = upload_bg.upload_files(upload_data, files)
@@ -435,6 +436,56 @@ def get_texture_filepath(tex_dir_path, image, resolution='blend'):
     return fpn
 
 
+def generate_lower_resolutions_hdr(asset_data, fpath):
+    '''generates lower resolutions for HDR images'''
+    hdr = bpy.data.images.load(fpath)
+    actres = max(hdr.size[0], hdr.size[1])
+    p2res = paths.round_to_closest_resolution(actres)
+    original_filesize = os.path.getsize(fpath) # for comparison on the original level
+    i = 0
+    finished = False
+    files = []
+    while not finished:
+        dirn = os.path.dirname(fpath)
+        fn_strip, ext = os.path.splitext(fpath)
+        ext = '.exr'
+        if i>0:
+            downscale(hdr)
+
+
+        hdr_resolution_filepath = fn_strip + paths.resolution_suffix[p2res] + ext
+        image_utils.img_save_as(hdr, filepath=hdr_resolution_filepath, file_format='OPEN_EXR', quality=20, color_mode='RGB', compression=15,
+                    view_transform='Raw', exr_codec = 'DWAA')
+
+        if os.path.exists(hdr_resolution_filepath):
+            reduced_filesize = os.path.getsize(hdr_resolution_filepath)
+
+        # compare file sizes
+        print(f'HDR size was reduced from {original_filesize} to {reduced_filesize}')
+        if reduced_filesize < original_filesize:
+            # this limits from uploaidng especially same-as-original resolution files in case when there is no advantage.
+            # usually however the advantage can be big also for same as original resolution
+            files.append({
+                "type": p2res,
+                "index": 0,
+                "file_path": hdr_resolution_filepath
+            })
+
+            print('prepared resolution file: ', p2res)
+
+        if rkeys.index(p2res) == 0:
+            finished = True
+        else:
+            p2res = rkeys[rkeys.index(p2res) - 1]
+        i+=1
+
+    print('uploading resolution files')
+    upload_resolutions(files, asset_data)
+
+    preferences = bpy.context.preferences.addons['blenderkit'].preferences
+    patch_asset_empty(asset_data['id'], preferences.api_key)
+
+
 def generate_lower_resolutions(data):
     asset_data = data['asset_data']
     actres = get_current_resolution()
@@ -517,7 +568,7 @@ def generate_lower_resolutions(data):
                 else:
                     p2res = rkeys[rkeys.index(p2res) - 1]
             print('uploading resolution files')
-            upload_resolutions(files, data)
+            upload_resolutions(files, data['asset_data'])
             preferences = bpy.context.preferences.addons['blenderkit'].preferences
             patch_asset_empty(data['asset_data']['id'], preferences.api_key)
         return
@@ -672,8 +723,9 @@ def load_assets_list(filepath):
 
 
 def check_needs_resolutions(a):
-    if a['verificationStatus'] == 'validated' and a['assetType'] in ('material', 'model', 'scene'):
+    if a['verificationStatus'] == 'validated' and a['assetType'] in ('material', 'model', 'scene', 'hdr'):
         # the search itself now picks the right assets so there's no need to filter more than asset types.
+        # TODO needs to check first if the upload date is older than resolution upload date, for that we need resolution upload date.
         for f in a['files']:
             if f['fileType'].find('resolution') > -1:
                 return False
@@ -698,7 +750,7 @@ def download_asset(asset_data, resolution='blend', unpack=False, api_key=''):
                                         resolution='blend')
     if has_url:
         fpath = download.download_file(asset_data)
-        if fpath and unpack:
+        if fpath and unpack and asset_data['assetType'] != 'hdr':
             send_to_bg(asset_data, fpath, command='unpack', wait=True)
         return fpath
 
@@ -719,13 +771,16 @@ def generate_resolution_thread(asset_data, api_key):
 
     fpath = download_asset(asset_data, unpack=True, api_key=api_key)
     if fpath:
-        print('send to bg ', fpath)
-        proc = send_to_bg(asset_data, fpath, command='generate_resolutions', wait=True);
+        if asset_data['assetType'] != 'hdr':
+            print('send to bg ', fpath)
+            proc = send_to_bg(asset_data, fpath, command='generate_resolutions', wait=True);
+        else:
+            generate_lower_resolutions_hdr(asset_data, fpath)
         # send_to_bg by now waits for end of the process.
         # time.sleep((5))
 
 
-def iterate_for_resolutions(filepath, process_count=12, api_key=''):
+def iterate_for_resolutions(filepath, process_count=12, api_key='', do_checks = True):
     ''' iterate through all assigned assets, check for those which need generation and send them to res gen'''
     assets = load_assets_list(filepath)
     print(len(assets))
@@ -734,10 +789,10 @@ def iterate_for_resolutions(filepath, process_count=12, api_key=''):
         asset_data = search.parse_result(asset_data)
         if asset_data is not None:
 
-            if check_needs_resolutions(asset_data):
+            if not do_checks or check_needs_resolutions(asset_data):
                 print('downloading and generating resolution for  %s' % asset_data['name'])
                 # this is just a quick hack for not using original dirs in blendrkit...
-                generate_resolution_thread(asset_data,api_key)
+                generate_resolution_thread(asset_data, api_key)
                 # thread = threading.Thread(target=generate_resolution_thread, args=(asset_data, api_key))
                 # thread.start()
                 #
diff --git a/blenderkit/search.py b/blenderkit/search.py
index f54d97d3..26fcef0e 100644
--- a/blenderkit/search.py
+++ b/blenderkit/search.py
@@ -461,7 +461,11 @@ def load_previews():
                     img.unpack(method='USE_ORIGINAL')
                 img.filepath = tpath
                 img.reload()
-            img.colorspace_settings.name = 'sRGB'
+            if r['assetType'] == 'hdr':
+                #to display hdr thumbnails correctly, we use non-color, otherwise looks shifted
+                img.colorspace_settings.name = 'Non-Color'
+            else:
+                img.colorspace_settings.name = 'sRGB'
 
             i += 1
     # print('previews loaded')
@@ -613,10 +617,12 @@ def generate_tooltip(mdata):
         t += 'texture size: %s\n' % fmt_length(mparams['textureSizeMeters'])
 
     if has(mparams, 'textureResolutionMax') and mparams['textureResolutionMax'] > 0:
-        if mparams['textureResolutionMin'] == mparams['textureResolutionMax']:
+        if not mparams.get('textureResolutionMin'):#for HDR's
+            t = writeblockm(t, mparams, key='textureResolutionMax', pretext='Resolution', width=col_w)
+        elif  mparams.get('textureResolutionMin') == mparams['textureResolutionMax']:
             t = writeblockm(t, mparams, key='textureResolutionMin', pretext='texture resolution', width=col_w)
         else:
-            t += 'tex resolution: %i - %i\n' % (mparams['textureResolutionMin'], mparams['textureResolutionMax'])
+            t += 'tex resolution: %i - %i\n' % (mparams.get('textureResolutionMin'), mparams['textureResolutionMax'])
 
     if has(mparams, 'thumbnailScale'):
         t = writeblockm(t, mparams, key='thumbnailScale', pretext='preview scale', width=col_w)
@@ -638,10 +644,11 @@ def generate_tooltip(mdata):
 
         t = writeblockm(t, mdata, key='isFree', width=col_w)
     else:
-        for f in fs:
-            if f['fileType'].find('resolution')>-1:
-                t+= 'Asset has lower resolutions available\n'
-                break;
+        if fs:
+            for f in fs:
+                if f['fileType'].find('resolution')>-1:
+                    t+= 'Asset has lower resolutions available\n'
+                    break;
     # generator is for both upload preview and search, this is only after search
     # if mdata.get('versionNumber'):
     #     # t = writeblockm(t, mdata, key='versionNumber', pretext='version', width = col_w)
@@ -1274,6 +1281,7 @@ def get_search_simple(parameters, filepath=None, page_size=100, max_results=1000
         requeststring += f'+{p}:{parameters[p]}'
 
     requeststring += '&page_size=' + str(page_size)
+    print(requeststring)
     response = rerequests.get(requeststring, headers=headers)  # , params = rparameters)
     # print(r.json())
     search_results = response.json()
diff --git a/blenderkit/ui.py b/blenderkit/ui.py
index c130a22e..02e3ef71 100644
--- a/blenderkit/ui.py
+++ b/blenderkit/ui.py
@@ -839,27 +839,32 @@ def draw_callback_2d_search(self, context):
                 if not r['thumbnail']:
                     tpath = paths.get_addon_thumbnail_path('thumbnail_not_available.jpg')
 
-                img = bpy.data.images.get(iname)
-                if img == None or img.filepath != tpath:
-                    # TODO replace it with a function
-                    if os.path.exists(tpath):
-
-                        if img is None:
-                            img = bpy.data.images.load(tpath)
-                            img.name = iname
-                        else:
-                            if img.filepath != tpath:
-                                # todo replace imgs reloads with a method that forces unpack for thumbs.
-                                if img.packed_file is not None:
-                                    img.unpack(method='USE_ORIGINAL')
-                                img.filepath = tpath
-                                img.reload()
-                                img.name = iname
-                    else:
-                        iname = utils.previmg_name(ui_props.active_index)
-                        img = bpy.data.images.get(iname)
-                    if img:
-                        img.colorspace_settings.name = 'sRGB'
+                # img = bpy.data.images.get(iname)
+                # if img == None or img.filepath != tpath:
+                #     # TODO replace it with a function
+                #     if os.path.exists(tpath):
+                #
+                #         if img is None:
+                #             img = bpy.data.images.load(tpath)
+                #             img.name = iname
+                #         else:
+                #             if img.filepath != tpath:
+                #                 # todo replace imgs reloads with a method that forces unpack for thumbs.
+                #                 if img.packed_file is not None:
+                #                     img.unpack(method='USE_ORIGINAL')
+                #                 img.filepath = tpath
+                #                 img.reload()
+                #                 img.name = iname
+                #     else:
+                #         iname = utils.previmg_name(ui_props.active_index)
+                #         img = bpy.data.images.get(iname)
+                #     if img:
+                #         img.colorspace_settings.name = 'sRGB'
+                if r['assetType'] == 'hdr':
+                    colorspace = 'Non-Color'
+                else:
+                    colorspace = 'sRGB'
+                img = utils.get_hidden_image(tpath, iname, colorspace = colorspace)
 
                 gimg = None
                 atip = ''
diff --git a/blenderkit/ui_panels.py b/blenderkit/ui_panels.py
index 6d89e110..9d8d6460 100644
--- a/blenderkit/ui_panels.py
+++ b/blenderkit/ui_panels.py
@@ -115,7 +115,6 @@ def draw_upload_common(layout, props, asset_type, context):
         op.process_type = 'UPLOAD'
         layout = layout.column()
         layout.enabled = False
-
     # if props.upload_state.find('Error') > -1:
     #     layout.label(text = props.upload_state)
 
@@ -887,7 +886,7 @@ class VIEW3D_PT_blenderkit_import_settings(Panel):
     def poll(cls, context):
         s = context.scene
         ui_props = s.blenderkitUI
-        return ui_props.down_up == 'SEARCH' and ui_props.asset_type in ['MATERIAL', 'MODEL']
+        return ui_props.down_up == 'SEARCH' and ui_props.asset_type in ['MATERIAL', 'MODEL', 'HDR']
 
     def draw(self, context):
         layout = self.layout
@@ -916,7 +915,8 @@ class VIEW3D_PT_blenderkit_import_settings(Panel):
             row = layout.row()
 
             row.prop(props, 'append_method', expand=True, icon_only=False)
-
+        if ui_props.asset_type == 'HDR':
+            props = s.blenderkit_HDR
 
         layout.prop(props, 'resolution')
         # layout.prop(props, 'unpack_files')
@@ -1188,7 +1188,7 @@ def draw_asset_context_menu(self, context, asset_data, from_panel=False):
         op.asset_id = asset_data['id']
         op.asset_type = asset_data['assetType']
 
-        if ui_props.asset_type in ('MODEL', 'MATERIAL') and \
+        if ui_props.asset_type in ('MODEL', 'MATERIAL', 'HDR') and \
                 utils.get_param(asset_data, 'textureResolutionMax') is not None and \
                 utils.get_param(asset_data, 'textureResolutionMax') > 512:
 
@@ -1225,6 +1225,7 @@ def draw_asset_context_menu(self, context, asset_data, from_panel=False):
 
             elif asset_data['assetBaseId'] in s['assets used'].keys():
                 # called from asset bar:
+                print('context menu')
                 op = col.operator('scene.blenderkit_download', text='Replace asset resolution')
 
                 op.asset_index = ui_props.active_index
@@ -1242,7 +1243,7 @@ def draw_asset_context_menu(self, context, asset_data, from_panel=False):
                         op.model_rotation = (0, 0, 0)
                 op.max_resolution = asset_data.get('max_resolution',
                                                    0)  # str(utils.get_param(asset_data, 'textureResolutionMax'))
-
+                print('should be drawn!')
             # print('operator res ', resolution)
             # op.resolution = resolution
 
diff --git a/blenderkit/upload.py b/blenderkit/upload.py
index 640e2085..fd17b7a5 100644
--- a/blenderkit/upload.py
+++ b/blenderkit/upload.py
@@ -452,6 +452,7 @@ def get_upload_data(self, context, asset_type):
         # mat analytics happen here, since they don't take up any time...
 
         upload_params = {
+            "textureResolutionMax" : props.texture_resolution_max
 
         }
 
@@ -484,12 +485,16 @@ def get_upload_data(self, context, asset_type):
     upload_data["name"] = props.name
     upload_data["description"] = props.description
     upload_data["tags"] = comma2array(props.tags)
+    #category is always only one value by a slug, that's why we go down to the lowest level and overwrite.
     if props.category == '':
         upload_data["category"] = asset_type.lower()
     else:
         upload_data["category"] = props.category
-    if props.subcategory != '':
+    if props.subcategory != 'NONE':
         upload_data["category"] = props.subcategory
+    if props.subcategory1 != 'NONE':
+        upload_data["category"] = props.subcategory1
+
     upload_data["license"] = props.license
     upload_data["isFree"] = props.is_free
     upload_data["isPrivate"] = props.is_private == 'PRIVATE'
@@ -973,7 +978,8 @@ def start_upload(self, context, asset_type, reupload, upload_set):
         props.id = ''
 
     export_data, upload_data = get_upload_data(self, context, asset_type)
-
+    # print(export_data)
+    # print(upload_data)
     # check if thumbnail exists, generate for HDR:
     if 'THUMBNAIL' in upload_set:
         if asset_type == 'HDR':
@@ -1061,7 +1067,7 @@ class UploadOperator(Operator):
 
     @classmethod
     def poll(cls, context):
-        return bpy.context.view_layer.objects.active is not None
+        return utils.uploadable_asset_poll()
 
     def execute(self, context):
         bpy.ops.object.blenderkit_auto_tags()
diff --git a/blenderkit/utils.py b/blenderkit/utils.py
index b3f65042..1bcb4c9b 100644
--- a/blenderkit/utils.py
+++ b/blenderkit/utils.py
@@ -290,6 +290,17 @@ def save_prefs(self, context):
             print(e)
 
 
+def uploadable_asset_poll():
+    '''returns true if active asset type can be uploaded'''
+    ui_props = bpy.context.scene.blenderkitUI
+    if ui_props.asset_type == 'MODEL':
+        return bpy.context.view_layer.objects.active is not None
+    if ui_props.asset_type == 'MATERIAL':
+        return bpy.context.view_layer.objects.active is not None and bpy.context.active_object.active_material is not None
+    if ui_props.asset_type == 'HDR':
+        return ui_props.hdr_upload_image is not None
+    return True
+
 def get_hidden_texture(tpath, bdata_name, force_reload=False):
     i = get_hidden_image(tpath, bdata_name, force_reload=force_reload)
     bdata_name = f".{bdata_name}"
@@ -301,15 +312,16 @@ def get_hidden_texture(tpath, bdata_name, force_reload=False):
     return t
 
 
-def get_hidden_image(tpath, bdata_name, force_reload=False):
-    hidden_name = '.%s' % bdata_name
+def get_hidden_image(tpath, bdata_name, force_reload=False, colorspace = 'sRGB'):
+    if bdata_name[0] == '.':
+        hidden_name = bdata_name
+    else:
+        hidden_name = '.%s' % bdata_name
     img = bpy.data.images.get(hidden_name)
 
     if tpath.startswith('//'):
         tpath = bpy.path.abspath(tpath)
 
-    gap = '\n\n\n'
-    en = '\n'
     if img == None or (img.filepath != tpath):
         if tpath.startswith('//'):
             tpath = bpy.path.abspath(tpath)
@@ -326,12 +338,12 @@ def get_hidden_image(tpath, bdata_name, force_reload=False):
 
                 img.filepath = tpath
                 img.reload()
-        img.colorspace_settings.name = 'sRGB'
+        img.colorspace_settings.name = colorspace
     elif force_reload:
         if img.packed_file is not None:
             img.unpack(method='USE_ORIGINAL')
         img.reload()
-        img.colorspace_settings.name = 'sRGB'
+        img.colorspace_settings.name = colorspace
     return img