diff options
Diffstat (limited to 'blenderkit/resolutions.py')
| -rw-r--r-- | blenderkit/resolutions.py | 716 |
1 files changed, 0 insertions, 716 deletions
diff --git a/blenderkit/resolutions.py b/blenderkit/resolutions.py deleted file mode 100644 index 23aa939d..00000000 --- a/blenderkit/resolutions.py +++ /dev/null @@ -1,716 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - - -from blenderkit import paths, append_link, bg_blender, utils, download, search, rerequests, upload_bg, image_utils - -import sys, json, os, time -import subprocess -import tempfile -import bpy -import requests -import math -import threading - -resolutions = { - 'resolution_0_5K': 512, - 'resolution_1K': 1024, - 'resolution_2K': 2048, - 'resolution_4K': 4096, - 'resolution_8K': 8192, -} -rkeys = list(resolutions.keys()) - -resolution_props_to_server = { - - '512': 'resolution_0_5K', - '1024': 'resolution_1K', - '2048': 'resolution_2K', - '4096': 'resolution_4K', - '8192': 'resolution_8K', - 'ORIGINAL': 'blend', -} - - -def get_current_resolution(): - actres = 0 - for i in bpy.data.images: - if i.name != 'Render Result': - actres = max(actres, i.size[0], i.size[1]) - return actres - - -def save_image_safely(teximage, filepath): - ''' - Blender makes it really hard to save images... - Would be worth investigating PIL or similar instead - Parameters - ---------- - teximage - - Returns - ------- - - ''' - JPEG_QUALITY = 98 - - rs = bpy.context.scene.render - ims = rs.image_settings - - orig_file_format = ims.file_format - orig_quality = ims.quality - orig_color_mode = ims.color_mode - orig_compression = ims.compression - - ims.file_format = teximage.file_format - if teximage.file_format == 'PNG': - ims.color_mode = 'RGBA' - elif teximage.channels == 3: - ims.color_mode = 'RGB' - else: - ims.color_mode = 'BW' - - # all pngs with max compression - if ims.file_format == 'PNG': - ims.compression = 100 - # all jpgs brought to reasonable quality - if ims.file_format == 'JPG': - ims.quality = JPEG_QUALITY - # it's actually very important not to try to change the image filepath and packed file filepath before saving, - # blender tries to re-pack the image after writing to image.packed_image.filepath and reverts any changes. - teximage.save_render(filepath=bpy.path.abspath(filepath), scene=bpy.context.scene) - - teximage.filepath = filepath - for packed_file in teximage.packed_files: - packed_file.filepath = filepath - teximage.filepath_raw = filepath - teximage.reload() - - ims.file_format = orig_file_format - ims.quality = orig_quality - ims.color_mode = orig_color_mode - ims.compression = orig_compression - - -def extxchange_to_resolution(filepath): - base, ext = os.path.splitext(filepath) - if ext in ('.png', '.PNG'): - ext = 'jpg' - - - - - - -def upload_resolutions(files, asset_data): - preferences = bpy.context.preferences.addons['blenderkit'].preferences - - upload_data = { - "name": asset_data['name'], - "token": preferences.api_key, - "id": asset_data['id'] - } - - uploaded = upload_bg.upload_files(upload_data, files) - - if uploaded: - bg_blender.progress('upload finished successfully') - else: - bg_blender.progress('upload failed.') - - -def unpack_asset(data): - utils.p('unpacking asset') - asset_data = data['asset_data'] - # utils.pprint(asset_data) - - blend_file_name = os.path.basename(bpy.data.filepath) - ext = os.path.splitext(blend_file_name)[1] - - resolution = asset_data.get('resolution', 'blend') - # TODO - passing resolution inside asset data might not be the best solution - tex_dir_path = paths.get_texture_directory(asset_data, resolution=resolution) - tex_dir_abs = bpy.path.abspath(tex_dir_path) - if not os.path.exists(tex_dir_abs): - try: - os.mkdir(tex_dir_abs) - except Exception as e: - print(e) - bpy.data.use_autopack = False - for image in bpy.data.images: - if image.name != 'Render Result': - # suffix = paths.resolution_suffix(data['suffix']) - fp = get_texture_filepath(tex_dir_path, image, resolution=resolution) - utils.p('unpacking file', image.name) - utils.p(image.filepath, fp) - - for pf in image.packed_files: - pf.filepath = fp # bpy.path.abspath(fp) - image.filepath = fp # bpy.path.abspath(fp) - image.filepath_raw = fp # bpy.path.abspath(fp) - # image.save() - if len(image.packed_files) > 0: - # image.unpack(method='REMOVE') - image.unpack(method='WRITE_ORIGINAL') - - #mark asset browser asset - data_block = None - if asset_data['assetType'] == 'model': - for ob in bpy.data.objects: - if ob.parent is None and ob in bpy.context.visible_objects: - ob.asset_mark() - # for c in bpy.data.collections: - # if c.get('asset_data') is not None: - # c.asset_mark() - # data_block = c - elif asset_data['assetType'] == 'material': - for m in bpy.data.materials: - m.asset_mark() - data_block = m - elif asset_data['assetType'] == 'scene': - bpy.context.scene.asset_mark() - elif asset_data['assetType'] =='brush': - for b in bpy.data.brushes: - if b.get('asset_data') is not None: - b.asset_mark() - data_block = b - if data_block is not None: - tags = data_block.asset_data.tags - for t in tags: - tags.remove(t) - tags.new('description: ' + asset_data['description']) - tags.new('tags: ' + ','.join(asset_data['tags'])) - # - # if this isn't here, blender crashes when saving file. - bpy.context.preferences.filepaths.file_preview_type = 'NONE' - - bpy.ops.wm.save_as_mainfile(filepath = bpy.data.filepath, compress=False) - # now try to delete the .blend1 file - try: - - os.remove(bpy.data.filepath + '1') - except Exception as e: - print(e) - - -def patch_asset_empty(asset_id, api_key): - ''' - This function patches the asset for the purpose of it getting a reindex. - Should be removed once this is fixed on the server and - the server is able to reindex after uploads of resolutions - Returns - ------- - ''' - upload_data = { - } - url = paths.get_api_url() + 'assets/' + str(asset_id) + '/' - headers = utils.get_headers(api_key) - try: - r = rerequests.patch(url, json=upload_data, headers=headers, verify=True) # files = files, - except requests.exceptions.RequestException as e: - print(e) - return {'CANCELLED'} - return {'FINISHED'} - - -def reduce_all_images(target_scale=1024): - for img in bpy.data.images: - if img.name != 'Render Result': - print('scaling ', img.name, img.size[0], img.size[1]) - # make_possible_reductions_on_image(i) - if max(img.size) > target_scale: - ratio = float(target_scale) / float(max(img.size)) - print(ratio) - # i.save() - fp = '//tempimagestorage' - # print('generated filename',fp) - # for pf in img.packed_files: - # pf.filepath = fp # bpy.path.abspath(fp) - - img.filepath = fp - img.filepath_raw = fp - print(int(img.size[0] * ratio), int(img.size[1] * ratio)) - img.scale(int(img.size[0] * ratio), int(img.size[1] * ratio)) - img.update() - # img.save() - # img.reload() - img.pack() - - -def get_texture_filepath(tex_dir_path, image, resolution='blend'): - image_file_name = bpy.path.basename(image.filepath) - if image_file_name == '': - image_file_name = image.name.split('.')[0] - - suffix = paths.resolution_suffix[resolution] - - fp = os.path.join(tex_dir_path, image_file_name) - # check if there is allready an image with same name and thus also assigned path - # (can happen easily with genearted tex sets and more materials) - done = False - fpn = fp - i = 0 - while not done: - is_solo = True - for image1 in bpy.data.images: - if image != image1 and image1.filepath == fpn: - is_solo = False - fpleft, fpext = os.path.splitext(fp) - fpn = fpleft + str(i).zfill(3) + fpext - i += 1 - if is_solo: - done = True - - return fpn - - -def generate_lower_resolutions_hdr(asset_data, fpath): - '''generates lower resolutions for HDR images''' - hdr = bpy.data.images.load(fpath) - actres = max(hdr.size[0], hdr.size[1]) - p2res = paths.round_to_closest_resolution(actres) - original_filesize = os.path.getsize(fpath) # for comparison on the original level - i = 0 - finished = False - files = [] - while not finished: - dirn = os.path.dirname(fpath) - fn_strip, ext = os.path.splitext(fpath) - ext = '.exr' - if i>0: - image_utils.downscale(hdr) - - - hdr_resolution_filepath = fn_strip + paths.resolution_suffix[p2res] + ext - image_utils.img_save_as(hdr, filepath=hdr_resolution_filepath, file_format='OPEN_EXR', quality=20, color_mode='RGB', compression=15, - view_transform='Raw', exr_codec = 'DWAA') - - if os.path.exists(hdr_resolution_filepath): - reduced_filesize = os.path.getsize(hdr_resolution_filepath) - - # compare file sizes - print(f'HDR size was reduced from {original_filesize} to {reduced_filesize}') - if reduced_filesize < original_filesize: - # this limits from uploaidng especially same-as-original resolution files in case when there is no advantage. - # usually however the advantage can be big also for same as original resolution - files.append({ - "type": p2res, - "index": 0, - "file_path": hdr_resolution_filepath - }) - - print('prepared resolution file: ', p2res) - - if rkeys.index(p2res) == 0: - finished = True - else: - p2res = rkeys[rkeys.index(p2res) - 1] - i+=1 - - print('uploading resolution files') - upload_resolutions(files, asset_data) - - preferences = bpy.context.preferences.addons['blenderkit'].preferences - patch_asset_empty(asset_data['id'], preferences.api_key) - - -def generate_lower_resolutions(data): - asset_data = data['asset_data'] - actres = get_current_resolution() - # first let's skip procedural assets - base_fpath = bpy.data.filepath - - s = bpy.context.scene - - print('current resolution of the asset ', actres) - if actres > 0: - p2res = paths.round_to_closest_resolution(actres) - orig_res = p2res - print(p2res) - finished = False - files = [] - # now skip assets that have lowest possible resolution already - if p2res != [0]: - original_textures_filesize = 0 - for i in bpy.data.images: - abspath = bpy.path.abspath(i.filepath) - if os.path.exists(abspath): - original_textures_filesize += os.path.getsize(abspath) - - while not finished: - - blend_file_name = os.path.basename(base_fpath) - - dirn = os.path.dirname(base_fpath) - fn_strip, ext = os.path.splitext(blend_file_name) - - fn = fn_strip + paths.resolution_suffix[p2res] + ext - fpath = os.path.join(dirn, fn) - - tex_dir_path = paths.get_texture_directory(asset_data, resolution=p2res) - - tex_dir_abs = bpy.path.abspath(tex_dir_path) - if not os.path.exists(tex_dir_abs): - os.mkdir(tex_dir_abs) - - reduced_textures_filessize = 0 - for i in bpy.data.images: - if i.name != 'Render Result': - - print('scaling ', i.name, i.size[0], i.size[1]) - fp = get_texture_filepath(tex_dir_path, i, resolution=p2res) - - if p2res == orig_res: - # first, let's link the image back to the original one. - i['blenderkit_original_path'] = i.filepath - # first round also makes reductions on the image, while keeping resolution - image_utils.make_possible_reductions_on_image(i, fp, do_reductions=True, do_downscale=False) - - else: - # lower resolutions only downscale - image_utils.make_possible_reductions_on_image(i, fp, do_reductions=False, do_downscale=True) - - abspath = bpy.path.abspath(i.filepath) - if os.path.exists(abspath): - reduced_textures_filessize += os.path.getsize(abspath) - - i.pack() - # save - print(fpath) - # if this isn't here, blender crashes. - bpy.context.preferences.filepaths.file_preview_type = 'NONE' - - # save the file - bpy.ops.wm.save_as_mainfile(filepath=fpath, compress=True, copy=True) - # compare file sizes - print(f'textures size was reduced from {original_textures_filesize} to {reduced_textures_filessize}') - if reduced_textures_filessize < original_textures_filesize: - # this limits from uploaidng especially same-as-original resolution files in case when there is no advantage. - # usually however the advantage can be big also for same as original resolution - files.append({ - "type": p2res, - "index": 0, - "file_path": fpath - }) - - print('prepared resolution file: ', p2res) - if rkeys.index(p2res) == 0: - finished = True - else: - p2res = rkeys[rkeys.index(p2res) - 1] - print('uploading resolution files') - upload_resolutions(files, data['asset_data']) - preferences = bpy.context.preferences.addons['blenderkit'].preferences - patch_asset_empty(data['asset_data']['id'], preferences.api_key) - return - - -def regenerate_thumbnail_material(data): - # this should re-generate material thumbnail and re-upload it. - # first let's skip procedural assets - base_fpath = bpy.data.filepath - blend_file_name = os.path.basename(base_fpath) - bpy.ops.mesh.primitive_cube_add() - aob = bpy.context.active_object - bpy.ops.object.material_slot_add() - aob.material_slots[0].material = bpy.data.materials[0] - props = aob.active_material.blenderkit - props.thumbnail_generator_type = 'BALL' - props.thumbnail_background = False - props.thumbnail_resolution = '256' - # layout.prop(props, 'thumbnail_generator_type') - # layout.prop(props, 'thumbnail_scale') - # layout.prop(props, 'thumbnail_background') - # if props.thumbnail_background: - # layout.prop(props, 'thumbnail_background_lightness') - # layout.prop(props, 'thumbnail_resolution') - # layout.prop(props, 'thumbnail_samples') - # layout.prop(props, 'thumbnail_denoising') - # layout.prop(props, 'adaptive_subdivision') - # preferences = bpy.context.preferences.addons['blenderkit'].preferences - # layout.prop(preferences, "thumbnail_use_gpu") - # TODO: here it should call start_material_thumbnailer , but with the wait property on, so it can upload afterwards. - bpy.ops.object.blenderkit_generate_material_thumbnail() - time.sleep(130) - # save - # this does the actual job - - return - - -def assets_db_path(): - dpath = os.path.dirname(bpy.data.filepath) - fpath = os.path.join(dpath, 'all_assets.json') - return fpath - - -def get_assets_search(): - # bpy.app.debug_value = 2 - - results = [] - preferences = bpy.context.preferences.addons['blenderkit'].preferences - url = paths.get_api_url() + 'search/all' - i = 0 - while url is not None: - headers = utils.get_headers(preferences.api_key) - print('fetching assets from assets endpoint') - print(url) - retries = 0 - while retries < 3: - r = rerequests.get(url, headers=headers) - - try: - adata = r.json() - url = adata.get('next') - print(i) - i += 1 - except Exception as e: - print(e) - print('failed to get next') - if retries == 2: - url = None - if adata.get('results') != None: - results.extend(adata['results']) - retries = 3 - print(f'fetched page {i}') - retries += 1 - - fpath = assets_db_path() - with open(fpath, 'w', encoding = 'utf-8') as s: - json.dump(results, s, ensure_ascii=False, indent=4) - - -def get_assets_for_resolutions(page_size=100, max_results=100000000): - preferences = bpy.context.preferences.addons['blenderkit'].preferences - - dpath = os.path.dirname(bpy.data.filepath) - filepath = os.path.join(dpath, 'assets_for_resolutions.json') - params = { - 'order': '-created', - 'textureResolutionMax_gte': '100', - # 'last_resolution_upload_lt':'2020-9-01' - } - search.get_search_simple(params, filepath=filepath, page_size=page_size, max_results=max_results, - api_key=preferences.api_key) - return filepath - - -def get_materials_for_validation(page_size=100, max_results=100000000): - preferences = bpy.context.preferences.addons['blenderkit'].preferences - dpath = os.path.dirname(bpy.data.filepath) - filepath = os.path.join(dpath, 'materials_for_validation.json') - params = { - 'order': '-created', - 'asset_type': 'material', - 'verification_status': 'uploaded' - } - search.get_search_simple(params, filepath=filepath, page_size=page_size, max_results=max_results, - api_key=preferences.api_key) - return filepath - - - - -def load_assets_list(filepath): - if os.path.exists(filepath): - with open(filepath, 'r', encoding='utf-8') as s: - assets = json.load(s) - return assets - - -def check_needs_resolutions(a): - if a['verificationStatus'] == 'validated' and a['assetType'] in ('material', 'model', 'scene', 'hdr'): - # the search itself now picks the right assets so there's no need to filter more than asset types. - # TODO needs to check first if the upload date is older than resolution upload date, for that we need resolution upload date. - for f in a['files']: - if f['fileType'].find('resolution') > -1: - return False - - return True - return False - - -def download_asset(asset_data, resolution='blend', unpack=False, api_key=''): - ''' - Download an asset non-threaded way. - Parameters - ---------- - asset_data - search result from elastic or assets endpoints from API - - Returns - ------- - path to the resulting asset file or None if asset isn't accessible - ''' - - has_url = download.get_download_url(asset_data, download.get_scene_id(), api_key, tcom=None, - resolution='blend') - if has_url: - fpath = download.download_asset_file(asset_data, api_key = api_key) - if fpath and unpack and asset_data['assetType'] != 'hdr': - send_to_bg(asset_data, fpath, command='unpack', wait=True) - return fpath - - return None - - -def generate_resolution_thread(asset_data, api_key): - ''' - A thread that downloads file and only then starts an instance of Blender that generates the resolution - Parameters - ---------- - asset_data - - Returns - ------- - - ''' - - fpath = download_asset(asset_data, unpack=True, api_key=api_key) - - if fpath: - if asset_data['assetType'] != 'hdr': - print('send to bg ', fpath) - proc = send_to_bg(asset_data, fpath, command='generate_resolutions', wait=True); - else: - generate_lower_resolutions_hdr(asset_data, fpath) - # send_to_bg by now waits for end of the process. - # time.sleep((5)) - - -def iterate_for_resolutions(filepath, process_count=12, api_key='', do_checks = True): - ''' iterate through all assigned assets, check for those which need generation and send them to res gen''' - assets = load_assets_list(filepath) - print(len(assets)) - threads = [] - for asset_data in assets: - asset_data = search.parse_result(asset_data) - if asset_data is not None: - - if not do_checks or check_needs_resolutions(asset_data): - print('downloading and generating resolution for %s' % asset_data['name']) - # this is just a quick hack for not using original dirs in blendrkit... - generate_resolution_thread(asset_data, api_key) - # thread = threading.Thread(target=generate_resolution_thread, args=(asset_data, api_key)) - # thread.start() - # - # threads.append(thread) - # print('processes ', len(threads)) - # while len(threads) > process_count - 1: - # for t in threads: - # if not t.is_alive(): - # threads.remove(t) - # break; - # else: - # print(f'Failed to generate resolution:{asset_data["name"]}') - else: - print('not generated resolutions:', asset_data['name']) - - -def send_to_bg(asset_data, fpath, command='generate_resolutions', wait=True): - ''' - Send varioust task to a new blender instance that runs and closes after finishing the task. - This function waits until the process finishes. - The function tries to set the same bpy.app.debug_value in the instance of Blender that is run. - Parameters - ---------- - asset_data - fpath - file that will be processed - command - command which should be run in background. - - Returns - ------- - None - ''' - data = { - 'fpath': fpath, - 'debug_value': bpy.app.debug_value, - 'asset_data': asset_data, - 'command': command, - } - binary_path = bpy.app.binary_path - tempdir = tempfile.mkdtemp() - datafile = os.path.join(tempdir + 'resdata.json') - script_path = os.path.dirname(os.path.realpath(__file__)) - with open(datafile, 'w', encoding = 'utf-8') as s: - json.dump(data, s, ensure_ascii=False, indent=4) - - print('opening Blender instance to do processing - ', command) - - if wait: - proc = subprocess.run([ - binary_path, - "--background", - "-noaudio", - fpath, - "--python", os.path.join(script_path, "resolutions_bg.py"), - "--", datafile - ], bufsize=1, stdout=sys.stdout, stdin=subprocess.PIPE, creationflags=utils.get_process_flags()) - - else: - # TODO this should be fixed to allow multithreading. - proc = subprocess.Popen([ - binary_path, - "--background", - "-noaudio", - fpath, - "--python", os.path.join(script_path, "resolutions_bg.py"), - "--", datafile - ], bufsize=1, stdout=subprocess.PIPE, stdin=subprocess.PIPE, creationflags=utils.get_process_flags()) - return proc - - -def write_data_back(asset_data): - '''ensures that the data in the resolution file is the same as in the database.''' - pass; - - -def run_bg(datafile): - print('background file operation') - with open(datafile, 'r',encoding='utf-8') as f: - data = json.load(f) - bpy.app.debug_value = data['debug_value'] - write_data_back(data['asset_data']) - if data['command'] == 'generate_resolutions': - generate_lower_resolutions(data) - elif data['command'] == 'unpack': - unpack_asset(data) - elif data['command'] == 'regen_thumbnail': - regenerate_thumbnail_material(data) - -# load_assets_list() -# generate_lower_resolutions() -# class TestOperator(bpy.types.Operator): -# """Tooltip""" -# bl_idname = "object.test_anything" -# bl_label = "Test Operator" -# -# @classmethod -# def poll(cls, context): -# return True -# -# def execute(self, context): -# iterate_for_resolutions() -# return {'FINISHED'} -# -# -# def register(): -# bpy.utils.register_class(TestOperator) -# -# -# def unregister(): -# bpy.utils.unregister_class(TestOperator) |