# ##### BEGIN GPL LICENSE BLOCK ##### # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # ##### END GPL LICENSE BLOCK ##### from blenderkit import paths, append_link, bg_blender, utils, download, search, rerequests, upload_bg, image_utils import sys, json, os, time import subprocess import tempfile import bpy import requests import math import threading resolutions = { 'resolution_0_5K': 512, 'resolution_1K': 1024, 'resolution_2K': 2048, 'resolution_4K': 4096, 'resolution_8K': 8192, } rkeys = list(resolutions.keys()) resolution_props_to_server = { '512': 'resolution_0_5K', '1024': 'resolution_1K', '2048': 'resolution_2K', '4096': 'resolution_4K', '8192': 'resolution_8K', 'ORIGINAL': 'blend', } def get_current_resolution(): actres = 0 for i in bpy.data.images: if i.name != 'Render Result': actres = max(actres, i.size[0], i.size[1]) return actres def save_image_safely(teximage, filepath): ''' Blender makes it really hard to save images... Would be worth investigating PIL or similar instead Parameters ---------- teximage Returns ------- ''' JPEG_QUALITY = 98 rs = bpy.context.scene.render ims = rs.image_settings orig_file_format = ims.file_format orig_quality = ims.quality orig_color_mode = ims.color_mode orig_compression = ims.compression ims.file_format = teximage.file_format if teximage.file_format == 'PNG': ims.color_mode = 'RGBA' elif teximage.channels == 3: ims.color_mode = 'RGB' else: ims.color_mode = 'BW' # all pngs with max compression if ims.file_format == 'PNG': ims.compression = 100 # all jpgs brought to reasonable quality if ims.file_format == 'JPG': ims.quality = JPEG_QUALITY # it's actually very important not to try to change the image filepath and packed file filepath before saving, # blender tries to re-pack the image after writing to image.packed_image.filepath and reverts any changes. teximage.save_render(filepath=bpy.path.abspath(filepath), scene=bpy.context.scene) teximage.filepath = filepath for packed_file in teximage.packed_files: packed_file.filepath = filepath teximage.filepath_raw = filepath teximage.reload() ims.file_format = orig_file_format ims.quality = orig_quality ims.color_mode = orig_color_mode ims.compression = orig_compression def extxchange_to_resolution(filepath): base, ext = os.path.splitext(filepath) if ext in ('.png', '.PNG'): ext = 'jpg' def upload_resolutions(files, asset_data): preferences = bpy.context.preferences.addons['blenderkit'].preferences upload_data = { "name": asset_data['name'], "token": preferences.api_key, "id": asset_data['id'] } uploaded = upload_bg.upload_files(upload_data, files) if uploaded: bg_blender.progress('upload finished successfully') else: bg_blender.progress('upload failed.') def unpack_asset(data): utils.p('unpacking asset') asset_data = data['asset_data'] # utils.pprint(asset_data) blend_file_name = os.path.basename(bpy.data.filepath) ext = os.path.splitext(blend_file_name)[1] resolution = asset_data.get('resolution', 'blend') # TODO - passing resolution inside asset data might not be the best solution tex_dir_path = paths.get_texture_directory(asset_data, resolution=resolution) tex_dir_abs = bpy.path.abspath(tex_dir_path) if not os.path.exists(tex_dir_abs): try: os.mkdir(tex_dir_abs) except Exception as e: print(e) bpy.data.use_autopack = False for image in bpy.data.images: if image.name != 'Render Result': # suffix = paths.resolution_suffix(data['suffix']) fp = get_texture_filepath(tex_dir_path, image, resolution=resolution) utils.p('unpacking file', image.name) utils.p(image.filepath, fp) for pf in image.packed_files: pf.filepath = fp # bpy.path.abspath(fp) image.filepath = fp # bpy.path.abspath(fp) image.filepath_raw = fp # bpy.path.abspath(fp) # image.save() if len(image.packed_files) > 0: # image.unpack(method='REMOVE') image.unpack(method='WRITE_ORIGINAL') bpy.ops.wm.save_mainfile(compress=False) # now try to delete the .blend1 file try: os.remove(bpy.data.filepath + '1') except Exception as e: print(e) def patch_asset_empty(asset_id, api_key): ''' This function patches the asset for the purpose of it getting a reindex. Should be removed once this is fixed on the server and the server is able to reindex after uploads of resolutions Returns ------- ''' upload_data = { } url = paths.get_api_url() + 'assets/' + str(asset_id) + '/' headers = utils.get_headers(api_key) try: r = rerequests.patch(url, json=upload_data, headers=headers, verify=True) # files = files, except requests.exceptions.RequestException as e: print(e) return {'CANCELLED'} return {'FINISHED'} def reduce_all_images(target_scale=1024): for img in bpy.data.images: if img.name != 'Render Result': print('scaling ', img.name, img.size[0], img.size[1]) # make_possible_reductions_on_image(i) if max(img.size) > target_scale: ratio = float(target_scale) / float(max(img.size)) print(ratio) # i.save() fp = '//tempimagestorage' # print('generated filename',fp) # for pf in img.packed_files: # pf.filepath = fp # bpy.path.abspath(fp) img.filepath = fp img.filepath_raw = fp print(int(img.size[0] * ratio), int(img.size[1] * ratio)) img.scale(int(img.size[0] * ratio), int(img.size[1] * ratio)) img.update() # img.save() # img.reload() img.pack() def get_texture_filepath(tex_dir_path, image, resolution='blend'): image_file_name = bpy.path.basename(image.filepath) if image_file_name == '': image_file_name = image.name.split('.')[0] suffix = paths.resolution_suffix[resolution] fp = os.path.join(tex_dir_path, image_file_name) # check if there is allready an image with same name and thus also assigned path # (can happen easily with genearted tex sets and more materials) done = False fpn = fp i = 0 while not done: is_solo = True for image1 in bpy.data.images: if image != image1 and image1.filepath == fpn: is_solo = False fpleft, fpext = os.path.splitext(fp) fpn = fpleft + str(i).zfill(3) + fpext i += 1 if is_solo: done = True return fpn def generate_lower_resolutions_hdr(asset_data, fpath): '''generates lower resolutions for HDR images''' hdr = bpy.data.images.load(fpath) actres = max(hdr.size[0], hdr.size[1]) p2res = paths.round_to_closest_resolution(actres) original_filesize = os.path.getsize(fpath) # for comparison on the original level i = 0 finished = False files = [] while not finished: dirn = os.path.dirname(fpath) fn_strip, ext = os.path.splitext(fpath) ext = '.exr' if i>0: image_utils.downscale(hdr) hdr_resolution_filepath = fn_strip + paths.resolution_suffix[p2res] + ext image_utils.img_save_as(hdr, filepath=hdr_resolution_filepath, file_format='OPEN_EXR', quality=20, color_mode='RGB', compression=15, view_transform='Raw', exr_codec = 'DWAA') if os.path.exists(hdr_resolution_filepath): reduced_filesize = os.path.getsize(hdr_resolution_filepath) # compare file sizes print(f'HDR size was reduced from {original_filesize} to {reduced_filesize}') if reduced_filesize < original_filesize: # this limits from uploaidng especially same-as-original resolution files in case when there is no advantage. # usually however the advantage can be big also for same as original resolution files.append({ "type": p2res, "index": 0, "file_path": hdr_resolution_filepath }) print('prepared resolution file: ', p2res) if rkeys.index(p2res) == 0: finished = True else: p2res = rkeys[rkeys.index(p2res) - 1] i+=1 print('uploading resolution files') upload_resolutions(files, asset_data) preferences = bpy.context.preferences.addons['blenderkit'].preferences patch_asset_empty(asset_data['id'], preferences.api_key) def generate_lower_resolutions(data): asset_data = data['asset_data'] actres = get_current_resolution() # first let's skip procedural assets base_fpath = bpy.data.filepath s = bpy.context.scene print('current resolution of the asset ', actres) if actres > 0: p2res = paths.round_to_closest_resolution(actres) orig_res = p2res print(p2res) finished = False files = [] # now skip assets that have lowest possible resolution already if p2res != [0]: original_textures_filesize = 0 for i in bpy.data.images: abspath = bpy.path.abspath(i.filepath) if os.path.exists(abspath): original_textures_filesize += os.path.getsize(abspath) while not finished: blend_file_name = os.path.basename(base_fpath) dirn = os.path.dirname(base_fpath) fn_strip, ext = os.path.splitext(blend_file_name) fn = fn_strip + paths.resolution_suffix[p2res] + ext fpath = os.path.join(dirn, fn) tex_dir_path = paths.get_texture_directory(asset_data, resolution=p2res) tex_dir_abs = bpy.path.abspath(tex_dir_path) if not os.path.exists(tex_dir_abs): os.mkdir(tex_dir_abs) reduced_textures_filessize = 0 for i in bpy.data.images: if i.name != 'Render Result': print('scaling ', i.name, i.size[0], i.size[1]) fp = get_texture_filepath(tex_dir_path, i, resolution=p2res) if p2res == orig_res: # first, let's link the image back to the original one. i['blenderkit_original_path'] = i.filepath # first round also makes reductions on the image, while keeping resolution image_utils.make_possible_reductions_on_image(i, fp, do_reductions=True, do_downscale=False) else: # lower resolutions only downscale image_utils.make_possible_reductions_on_image(i, fp, do_reductions=False, do_downscale=True) abspath = bpy.path.abspath(i.filepath) if os.path.exists(abspath): reduced_textures_filessize += os.path.getsize(abspath) i.pack() # save print(fpath) # save the file bpy.ops.wm.save_as_mainfile(filepath=fpath, compress=True, copy=True) # compare file sizes print(f'textures size was reduced from {original_textures_filesize} to {reduced_textures_filessize}') if reduced_textures_filessize < original_textures_filesize: # this limits from uploaidng especially same-as-original resolution files in case when there is no advantage. # usually however the advantage can be big also for same as original resolution files.append({ "type": p2res, "index": 0, "file_path": fpath }) print('prepared resolution file: ', p2res) if rkeys.index(p2res) == 0: finished = True else: p2res = rkeys[rkeys.index(p2res) - 1] print('uploading resolution files') upload_resolutions(files, data['asset_data']) preferences = bpy.context.preferences.addons['blenderkit'].preferences patch_asset_empty(data['asset_data']['id'], preferences.api_key) return def regenerate_thumbnail_material(data): # this should re-generate material thumbnail and re-upload it. # first let's skip procedural assets base_fpath = bpy.data.filepath blend_file_name = os.path.basename(base_fpath) bpy.ops.mesh.primitive_cube_add() aob = bpy.context.active_object bpy.ops.object.material_slot_add() aob.material_slots[0].material = bpy.data.materials[0] props = aob.active_material.blenderkit props.thumbnail_generator_type = 'BALL' props.thumbnail_background = False props.thumbnail_resolution = '256' # layout.prop(props, 'thumbnail_generator_type') # layout.prop(props, 'thumbnail_scale') # layout.prop(props, 'thumbnail_background') # if props.thumbnail_background: # layout.prop(props, 'thumbnail_background_lightness') # layout.prop(props, 'thumbnail_resolution') # layout.prop(props, 'thumbnail_samples') # layout.prop(props, 'thumbnail_denoising') # layout.prop(props, 'adaptive_subdivision') # preferences = bpy.context.preferences.addons['blenderkit'].preferences # layout.prop(preferences, "thumbnail_use_gpu") # TODO: here it should call start_material_thumbnailer , but with the wait property on, so it can upload afterwards. bpy.ops.object.blenderkit_generate_material_thumbnail() time.sleep(130) # save # this does the actual job return def assets_db_path(): dpath = os.path.dirname(bpy.data.filepath) fpath = os.path.join(dpath, 'all_assets.json') return fpath def get_assets_search(): # bpy.app.debug_value = 2 results = [] preferences = bpy.context.preferences.addons['blenderkit'].preferences url = paths.get_api_url() + 'search/all' i = 0 while url is not None: headers = utils.get_headers(preferences.api_key) print('fetching assets from assets endpoint') print(url) retries = 0 while retries < 3: r = rerequests.get(url, headers=headers) try: adata = r.json() url = adata.get('next') print(i) i += 1 except Exception as e: print(e) print('failed to get next') if retries == 2: url = None if adata.get('results') != None: results.extend(adata['results']) retries = 3 print(f'fetched page {i}') retries += 1 fpath = assets_db_path() with open(fpath, 'w', encoding = 'utf-8') as s: json.dump(results, s, ensure_ascii=False, indent=4) def get_assets_for_resolutions(page_size=100, max_results=100000000): preferences = bpy.context.preferences.addons['blenderkit'].preferences dpath = os.path.dirname(bpy.data.filepath) filepath = os.path.join(dpath, 'assets_for_resolutions.json') params = { 'order': '-created', 'textureResolutionMax_gte': '100', # 'last_resolution_upload_lt':'2020-9-01' } search.get_search_simple(params, filepath=filepath, page_size=page_size, max_results=max_results, api_key=preferences.api_key) return filepath def get_materials_for_validation(page_size=100, max_results=100000000): preferences = bpy.context.preferences.addons['blenderkit'].preferences dpath = os.path.dirname(bpy.data.filepath) filepath = os.path.join(dpath, 'materials_for_validation.json') params = { 'order': '-created', 'asset_type': 'material', 'verification_status': 'uploaded' } search.get_search_simple(params, filepath=filepath, page_size=page_size, max_results=max_results, api_key=preferences.api_key) return filepath def load_assets_list(filepath): if os.path.exists(filepath): with open(filepath, 'r', encoding='utf-8') as s: assets = json.load(s) return assets def check_needs_resolutions(a): if a['verificationStatus'] == 'validated' and a['assetType'] in ('material', 'model', 'scene', 'hdr'): # the search itself now picks the right assets so there's no need to filter more than asset types. # TODO needs to check first if the upload date is older than resolution upload date, for that we need resolution upload date. for f in a['files']: if f['fileType'].find('resolution') > -1: return False return True return False def download_asset(asset_data, resolution='blend', unpack=False, api_key=''): ''' Download an asset non-threaded way. Parameters ---------- asset_data - search result from elastic or assets endpoints from API Returns ------- path to the resulting asset file or None if asset isn't accessible ''' has_url = download.get_download_url(asset_data, download.get_scene_id(), api_key, tcom=None, resolution='blend') if has_url: fpath = download.download_asset_file(asset_data, api_key = api_key) if fpath and unpack and asset_data['assetType'] != 'hdr': send_to_bg(asset_data, fpath, command='unpack', wait=True) return fpath return None def generate_resolution_thread(asset_data, api_key): ''' A thread that downloads file and only then starts an instance of Blender that generates the resolution Parameters ---------- asset_data Returns ------- ''' fpath = download_asset(asset_data, unpack=True, api_key=api_key) if fpath: if asset_data['assetType'] != 'hdr': print('send to bg ', fpath) proc = send_to_bg(asset_data, fpath, command='generate_resolutions', wait=True); else: generate_lower_resolutions_hdr(asset_data, fpath) # send_to_bg by now waits for end of the process. # time.sleep((5)) def iterate_for_resolutions(filepath, process_count=12, api_key='', do_checks = True): ''' iterate through all assigned assets, check for those which need generation and send them to res gen''' assets = load_assets_list(filepath) print(len(assets)) threads = [] for asset_data in assets: asset_data = search.parse_result(asset_data) if asset_data is not None: if not do_checks or check_needs_resolutions(asset_data): print('downloading and generating resolution for %s' % asset_data['name']) # this is just a quick hack for not using original dirs in blendrkit... generate_resolution_thread(asset_data, api_key) # thread = threading.Thread(target=generate_resolution_thread, args=(asset_data, api_key)) # thread.start() # # threads.append(thread) # print('processes ', len(threads)) # while len(threads) > process_count - 1: # for t in threads: # if not t.is_alive(): # threads.remove(t) # break; # else: # print(f'Failed to generate resolution:{asset_data["name"]}') else: print('not generated resolutions:', asset_data['name']) def send_to_bg(asset_data, fpath, command='generate_resolutions', wait=True): ''' Send varioust task to a new blender instance that runs and closes after finishing the task. This function waits until the process finishes. The function tries to set the same bpy.app.debug_value in the instance of Blender that is run. Parameters ---------- asset_data fpath - file that will be processed command - command which should be run in background. Returns ------- None ''' data = { 'fpath': fpath, 'debug_value': bpy.app.debug_value, 'asset_data': asset_data, 'command': command, } binary_path = bpy.app.binary_path tempdir = tempfile.mkdtemp() datafile = os.path.join(tempdir + 'resdata.json') script_path = os.path.dirname(os.path.realpath(__file__)) with open(datafile, 'w', encoding = 'utf-8') as s: json.dump(data, s, ensure_ascii=False, indent=4) print('opening Blender instance to do processing - ', command) if wait: proc = subprocess.run([ binary_path, "--background", "-noaudio", fpath, "--python", os.path.join(script_path, "resolutions_bg.py"), "--", datafile ], bufsize=1, stdout=sys.stdout, stdin=subprocess.PIPE, creationflags=utils.get_process_flags()) else: # TODO this should be fixed to allow multithreading. proc = subprocess.Popen([ binary_path, "--background", "-noaudio", fpath, "--python", os.path.join(script_path, "resolutions_bg.py"), "--", datafile ], bufsize=1, stdout=subprocess.PIPE, stdin=subprocess.PIPE, creationflags=utils.get_process_flags()) return proc def write_data_back(asset_data): '''ensures that the data in the resolution file is the same as in the database.''' pass; def run_bg(datafile): print('background file operation') with open(datafile, 'r',encoding='utf-8') as f: data = json.load(f) bpy.app.debug_value = data['debug_value'] write_data_back(data['asset_data']) if data['command'] == 'generate_resolutions': generate_lower_resolutions(data) elif data['command'] == 'unpack': unpack_asset(data) elif data['command'] == 'regen_thumbnail': regenerate_thumbnail_material(data) # load_assets_list() # generate_lower_resolutions() # class TestOperator(bpy.types.Operator): # """Tooltip""" # bl_idname = "object.test_anything" # bl_label = "Test Operator" # # @classmethod # def poll(cls, context): # return True # # def execute(self, context): # iterate_for_resolutions() # return {'FINISHED'} # # # def register(): # bpy.utils.register_class(TestOperator) # # # def unregister(): # bpy.utils.unregister_class(TestOperator)