diff options
Diffstat (limited to 'io_blend_utils/blendfile_pack.py')
| -rwxr-xr-x | io_blend_utils/blendfile_pack.py | 601 |
1 files changed, 601 insertions, 0 deletions
diff --git a/io_blend_utils/blendfile_pack.py b/io_blend_utils/blendfile_pack.py new file mode 100755 index 00000000..225a941d --- /dev/null +++ b/io_blend_utils/blendfile_pack.py @@ -0,0 +1,601 @@ +#!/usr/bin/env python3 + +# ***** BEGIN GPL LICENSE BLOCK ***** +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# ***** END GPL LICENCE BLOCK ***** + +from blend import blendfile_path_walker + +TIMEIT = False + + +# ---------------------- +# debug low level output +# +# ... when internals _really_ fail & we want to know why +def _dbg(text): + import sys + from utils.system import colorize + if type(text) is bytes: + text = text.decode('utf-8') + sys.__stdout__.write(colorize(text, color='red') + "\n") + sys.__stdout__.flush() + + +def _relpath_remap( + path_src, + base_dir_src, + fp_basedir, + blendfile_src_dir_fakeroot, + ): + + import os + + if not os.path.isabs(path_src): + # Absolute win32 paths on a unix system + # cause bad issues! + if len(path_src) >= 2: + if path_src[0] != b'/'[0] and path_src[1] == b':'[0]: + pass + else: + raise Exception("Internal error 'path_src' -> %r must be absolute" % path_src) + + path_src = os.path.normpath(path_src) + path_dst = os.path.relpath(path_src, base_dir_src) + + if blendfile_src_dir_fakeroot is None: + # /foo/../bar.png --> /foo/__/bar.png + path_dst = path_dst.replace(b'..', b'__') + path_dst = os.path.normpath(path_dst) + else: + if b'..' in path_dst: + # remap, relative to project root + + # paths + path_dst = os.path.join(blendfile_src_dir_fakeroot, path_dst) + path_dst = os.path.normpath(path_dst) + # if there are paths outside the root still... + # This means they are outside the project directory, We dont support this, + # so name accordingly + if b'..' in path_dst: + # SHOULD NEVER HAPPEN + path_dst = path_dst.replace(b'..', b'__nonproject__') + path_dst = b'_' + path_dst + + # _dbg(b"FINAL A: " + path_dst) + path_dst_final = os.path.join(os.path.relpath(base_dir_src, fp_basedir), path_dst) + path_dst_final = os.path.normpath(path_dst_final) + # _dbg(b"FINAL B: " + path_dst_final) + + return path_dst, path_dst_final + + +def pack( + # store the blendfile relative to this directory, can be: + # os.path.dirname(blendfile_src) + # but in some cases we wan't to use a path higher up. + # base_dir_src, + blendfile_src, blendfile_dst, + mode='ZIP', + # optionally pass in the temp dir + base_dir_dst_temp=None, + paths_remap_relbase=None, + deps_remap=None, paths_remap=None, paths_uuid=None, + # load every libs dep, not just used deps. + all_deps=False, + compress_level=-1, + # yield reports + report=None, + + # The project path, eg: + # /home/me/myproject/mysession/path/to/blend/file.blend + # the path would be: b'path/to/blend' + # + # This is needed so we can choose to store paths + # relative to project or relative to the current file. + # + # When None, map _all_ paths are relative to the current blend. + # converting: '../../bar' --> '__/__/bar' + # so all paths are nested and not moved outside the session path. + blendfile_src_dir_fakeroot=None, + + # Read variations from json files. + use_variations=False, + + # do _everything_ except to write the paths. + # useful if we want to calculate deps to remap but postpone applying them. + readonly=False, + # dict of binary_edits: + # {file: [(ofs, bytes), ...], ...} + # ... where the file is the relative 'packed' location. + binary_edits=None, + + # Filename filter, allow to exclude files from the pack, + # function takes a string returns True if the files should be included. + filename_filter=None, + ): + """ + :param deps_remap: Store path deps_remap info as follows. + {"file.blend": {"path_new": "path_old", ...}, ...} + + :type deps_remap: dict or None + """ + + # Internal details: + # - we copy to a temp path before operating on the blend file + # so we can modify in-place. + # - temp files are only created once, (if we never touched them before), + # this way, for linked libraries - a single blend file may be used + # multiple times, each access will apply new edits on top of the old ones. + # - we track which libs we have touched (using 'lib_visit' arg), + # this means that the same libs wont be touched many times to modify the same data + # also prevents cyclic loops from crashing. + + import os + import sys + + if sys.stdout.isatty(): + from utils.system import colorize + else: + from utils.system import colorize_dummy as colorize + + # in case this is directly from the command line or user-input + blendfile_src = os.path.normpath(os.path.abspath(blendfile_src)) + blendfile_dst = os.path.normpath(os.path.abspath(blendfile_dst)) + + # first check args are OK + # fakeroot _cant_ start with a separator, since we prepend chars to it. + assert((blendfile_src_dir_fakeroot is None) or + (not blendfile_src_dir_fakeroot.startswith(os.sep.encode('ascii')))) + + path_temp_files = set() + path_copy_files = set() + + # path_temp_files --> original-location + path_temp_files_orig = {} + + TEMP_SUFFIX = b'@' + + if report is None: + def report(msg): + return msg + + yield report("%s: %r...\n" % (colorize("\nscanning deps", color='bright_green'), blendfile_src)) + + if TIMEIT: + import time + t = time.time() + + base_dir_src = os.path.dirname(blendfile_src) + base_dir_dst = os.path.dirname(blendfile_dst) + # _dbg(blendfile_src) + # _dbg(blendfile_dst) + + if base_dir_dst_temp is None: + base_dir_dst_temp = base_dir_dst + + if mode == 'ZIP': + base_dir_dst_temp = os.path.join(base_dir_dst_temp, b'__blendfile_temp__') + else: + base_dir_dst_temp = os.path.join(base_dir_dst_temp, b'__blendfile_pack__') + + def temp_remap_cb(filepath, rootdir): + """ + Create temp files in the destination path. + """ + filepath = blendfile_path_walker.utils.compatpath(filepath) + + if use_variations: + if blendfile_levels_dict_curr: + filepath = blendfile_levels_dict_curr.get(filepath, filepath) + + # ... + + # first remap this blend file to the location it will end up (so we can get images relative to _that_) + # TODO(cam) cache the results + fp_basedir_conv = _relpath_remap(os.path.join(rootdir, b'dummy'), base_dir_src, base_dir_src, blendfile_src_dir_fakeroot)[0] + fp_basedir_conv = os.path.join(base_dir_src, os.path.dirname(fp_basedir_conv)) + + # then get the file relative to the new location + filepath_tmp = _relpath_remap(filepath, base_dir_src, fp_basedir_conv, blendfile_src_dir_fakeroot)[0] + filepath_tmp = os.path.normpath(os.path.join(base_dir_dst_temp, filepath_tmp)) + TEMP_SUFFIX + + # only overwrite once (so we can write into a path already containing files) + if filepath_tmp not in path_temp_files: + if mode != 'NONE': + import shutil + os.makedirs(os.path.dirname(filepath_tmp), exist_ok=True) + shutil.copy(filepath, filepath_tmp) + path_temp_files.add(filepath_tmp) + path_temp_files_orig[filepath_tmp] = filepath + if mode != 'NONE': + return filepath_tmp + else: + return filepath + + # ----------------- + # Variation Support + # + # Use a json file to allow recursive-remapping of variations. + # + # file_a.blend + # file_a.json '{"variations": ["tree.blue.blend", ...]}' + # file_a.blend -> file_b.blend + # file_b.blend --> tree.blend + # + # the variation of `file_a.blend` causes `file_b.blend` + # to link in `tree.blue.blend` + + if use_variations: + blendfile_levels = [] + blendfile_levels_dict = [] + blendfile_levels_dict_curr = {} + + def blendfile_levels_rebuild(): + # after changing blend file configurations, + # re-create current variation lookup table + blendfile_levels_dict_curr.clear() + for d in blendfile_levels_dict: + if d is not None: + blendfile_levels_dict_curr.update(d) + + # use variations! + def blendfile_level_cb_enter(filepath): + import json + + filepath_json = os.path.splitext(filepath)[0] + b".json" + if os.path.exists(filepath_json): + with open(filepath_json, encoding='utf-8') as f_handle: + variations = [f.encode("utf-8") for f in json.load(f_handle).get("variations")] + # convert to absolute paths + basepath = os.path.dirname(filepath) + variations = { + # Reverse lookup, from non-variation to variation we specify in this file. + # {"/abs/path/foo.png": "/abs/path/foo.variation.png", ...} + # .. where the input _is_ the variation, + # we just make it absolute and use the non-variation as + # the key to the variation value. + b".".join(f.rsplit(b".", 2)[0::2]): f for f_ in variations + for f in (os.path.normpath(os.path.join(basepath, f_)),) + } + else: + variations = None + + blendfile_levels.append(filepath) + blendfile_levels_dict.append(variations) + + if variations: + blendfile_levels_rebuild() + + def blendfile_level_cb_exit(filepath): + blendfile_levels.pop() + blendfile_levels_dict.pop() + + if blendfile_levels_dict_curr: + blendfile_levels_rebuild() + else: + blendfile_level_cb_enter = blendfile_level_cb_exit = None + blendfile_levels_dict_curr = None + + lib_visit = {} + fp_blend_basename_last = b'' + + for fp, (rootdir, fp_blend_basename) in blendfile_path_walker.FilePath.visit_from_blend( + blendfile_src, + readonly=readonly, + temp_remap_cb=temp_remap_cb, + recursive=True, + recursive_all=all_deps, + lib_visit=lib_visit, + blendfile_level_cb=( + blendfile_level_cb_enter, + blendfile_level_cb_exit, + ) + ): + + # we could pass this in! + fp_blend = os.path.join(fp.basedir, fp_blend_basename) + + if fp_blend_basename_last != fp_blend_basename: + yield report(" %s: %s\n" % (colorize("blend", color='blue'), fp_blend)) + fp_blend_basename_last = fp_blend_basename + + if binary_edits is not None: + # TODO, temp_remap_cb makes paths, this isn't ideal, + # in this case we only want to remap! + if mode == 'NONE': + tmp = temp_remap_cb(fp_blend, base_dir_src) + tmp = os.path.relpath(tmp, base_dir_src) + else: + tmp = temp_remap_cb(fp_blend, base_dir_src) + tmp = os.path.relpath(tmp[:-len(TEMP_SUFFIX)], base_dir_dst_temp) + binary_edits_curr = binary_edits.setdefault(tmp, []) + del tmp + + # assume the path might be relative + path_src_orig = fp.filepath + path_rel = blendfile_path_walker.utils.compatpath(path_src_orig) + path_src = blendfile_path_walker.utils.abspath(path_rel, fp.basedir) + path_src = os.path.normpath(path_src) + + if filename_filter and not filename_filter(path_src): + yield report(" %s: %r\n" % (colorize("exclude", color='yellow'), path_src)) + continue + + # apply variation (if available) + if use_variations: + if blendfile_levels_dict_curr: + path_src_variation = blendfile_levels_dict_curr.get(path_src) + if path_src_variation is not None: + path_src = path_src_variation + path_rel = os.path.join(os.path.dirname(path_rel), os.path.basename(path_src)) + del path_src_variation + + # destination path realtive to the root + # assert(b'..' not in path_src) + assert(b'..' not in base_dir_src) + + # first remap this blend file to the location it will end up (so we can get images relative to _that_) + # TODO(cam) cache the results + fp_basedir_conv = _relpath_remap(fp_blend, base_dir_src, base_dir_src, blendfile_src_dir_fakeroot)[0] + fp_basedir_conv = os.path.join(base_dir_src, os.path.dirname(fp_basedir_conv)) + + # then get the file relative to the new location + path_dst, path_dst_final = _relpath_remap(path_src, base_dir_src, fp_basedir_conv, blendfile_src_dir_fakeroot) + + path_dst = os.path.join(base_dir_dst, path_dst) + + path_dst_final = b'//' + path_dst_final + + # Assign direct or add to edit-list (to apply later) + if not readonly: + fp.filepath = path_dst_final + if binary_edits is not None: + fp.filepath_assign_edits(path_dst_final, binary_edits_curr) + + # add to copy-list + # never copy libs (handled separately) + if not isinstance(fp, blendfile_path_walker.FPElem_block_path) or fp.userdata[0].code != b'LI': + path_copy_files.add((path_src, path_dst)) + + for file_list in ( + blendfile_path_walker.utils.find_sequence_paths(path_src) if fp.is_sequence else (), + fp.files_siblings(), + ): + + _src_dir = os.path.dirname(path_src) + _dst_dir = os.path.dirname(path_dst) + path_copy_files.update( + {(os.path.join(_src_dir, f), os.path.join(_dst_dir, f)) + for f in file_list + }) + del _src_dir, _dst_dir + + if deps_remap is not None: + # this needs to become JSON later... ugh, need to use strings + deps_remap.setdefault( + fp_blend_basename.decode('utf-8'), + {})[path_dst_final.decode('utf-8')] = path_src_orig.decode('utf-8') + + del lib_visit, fp_blend_basename_last + + if TIMEIT: + print(" Time: %.4f\n" % (time.time() - t)) + + yield report(("%s: %d files\n") % + (colorize("\narchiving", color='bright_green'), len(path_copy_files) + 1)) + + # handle deps_remap and file renaming + if deps_remap is not None: + blendfile_src_basename = os.path.basename(blendfile_src).decode('utf-8') + blendfile_dst_basename = os.path.basename(blendfile_dst).decode('utf-8') + + if blendfile_src_basename != blendfile_dst_basename: + if mode == 'FILE': + deps_remap[blendfile_dst_basename] = deps_remap[blendfile_src_basename] + del deps_remap[blendfile_src_basename] + del blendfile_src_basename, blendfile_dst_basename + + # store path mapping {dst: src} + if paths_remap is not None: + + if paths_remap_relbase is not None: + def relbase(fn): + return os.path.relpath(fn, paths_remap_relbase) + else: + def relbase(fn): + return fn + + for src, dst in path_copy_files: + # TODO. relative to project-basepath + paths_remap[os.path.relpath(dst, base_dir_dst).decode('utf-8')] = relbase(src).decode('utf-8') + # main file XXX, should have better way! + paths_remap[os.path.basename(blendfile_src).decode('utf-8')] = relbase(blendfile_src).decode('utf-8') + + # blend libs + for dst in path_temp_files: + src = path_temp_files_orig[dst] + k = os.path.relpath(dst[:-len(TEMP_SUFFIX)], base_dir_dst_temp).decode('utf-8') + paths_remap[k] = relbase(src).decode('utf-8') + del k + + del relbase + + if paths_uuid is not None: + from utils.system import uuid_from_file + + for src, dst in path_copy_files: + # reports are handled again, later on. + if os.path.exists(src): + paths_uuid[os.path.relpath(dst, base_dir_dst).decode('utf-8')] = uuid_from_file(src) + # XXX, better way to store temp target + blendfile_dst_tmp = temp_remap_cb(blendfile_src, base_dir_src) + paths_uuid[os.path.basename(blendfile_src).decode('utf-8')] = uuid_from_file(blendfile_dst_tmp) + + # blend libs + for dst in path_temp_files: + k = os.path.relpath(dst[:-len(TEMP_SUFFIX)], base_dir_dst_temp).decode('utf-8') + if k not in paths_uuid: + if mode == 'NONE': + dst = path_temp_files_orig[dst] + paths_uuid[k] = uuid_from_file(dst) + del k + + del blendfile_dst_tmp + del uuid_from_file + + # -------------------- + # Handle File Copy/Zip + + if mode == 'FILE': + import shutil + blendfile_dst_tmp = temp_remap_cb(blendfile_src, base_dir_src) + + shutil.move(blendfile_dst_tmp, blendfile_dst) + path_temp_files.remove(blendfile_dst_tmp) + + # strip TEMP_SUFFIX + for fn in path_temp_files: + shutil.move(fn, fn[:-len(TEMP_SUFFIX)]) + + for src, dst in path_copy_files: + assert(b'.blend' not in dst) + + # in rare cases a filepath could point to a directory + if (not os.path.exists(src)) or os.path.isdir(src): + yield report(" %s: %r\n" % (colorize("source missing", color='red'), src)) + else: + yield report(" %s: %r -> %r\n" % (colorize("copying", color='blue'), src, dst)) + shutil.copy(src, dst) + + yield report(" %s: %r\n" % (colorize("written", color='green'), blendfile_dst)) + + elif mode == 'ZIP': + import shutil + import zipfile + + # not awesome! + import zlib + assert(compress_level in range(-1, 10)) + _compress_level_orig = zlib.Z_DEFAULT_COMPRESSION + zlib.Z_DEFAULT_COMPRESSION = compress_level + _compress_mode = zipfile.ZIP_STORED if (compress_level == 0) else zipfile.ZIP_DEFLATED + if _compress_mode == zipfile.ZIP_STORED: + def is_compressed_filetype(fn): + return False + else: + from utils.system import is_compressed_filetype + + with zipfile.ZipFile(blendfile_dst.decode('utf-8'), 'w', _compress_mode) as zip_handle: + for fn in path_temp_files: + yield report(" %s: %r -> <archive>\n" % (colorize("copying", color='blue'), fn)) + zip_handle.write( + fn.decode('utf-8'), + arcname=os.path.relpath(fn[:-1], base_dir_dst_temp).decode('utf-8'), + ) + os.remove(fn) + + shutil.rmtree(base_dir_dst_temp) + + for src, dst in path_copy_files: + assert(not dst.endswith(b'.blend')) + + # in rare cases a filepath could point to a directory + if (not os.path.exists(src)) or os.path.isdir(src): + yield report(" %s: %r\n" % (colorize("source missing", color='red'), src)) + else: + yield report(" %s: %r -> <archive>\n" % (colorize("copying", color='blue'), src)) + zip_handle.write( + src.decode('utf-8'), + arcname=os.path.relpath(dst, base_dir_dst).decode('utf-8'), + compress_type=zipfile.ZIP_STORED if is_compressed_filetype(dst) else _compress_mode, + ) + + zlib.Z_DEFAULT_COMPRESSION = _compress_level_orig + del _compress_level_orig, _compress_mode + + yield report(" %s: %r\n" % (colorize("written", color='green'), blendfile_dst)) + elif mode == 'NONE': + pass + else: + raise Exception("%s not a known mode" % mode) + + +def create_argparse(): + import os + import argparse + + usage_text = ( + "Run this script to extract blend-files(s) to a destination path: " + + os.path.basename(__file__) + + " --input=FILE --output=FILE [options]") + + parser = argparse.ArgumentParser(description=usage_text) + + # for main_render() only, but validate args. + parser.add_argument( + "-i", "--input", dest="path_src", metavar='FILE', required=True, + help="Input blend file", + ) + parser.add_argument( + "-o", "--output", dest="path_dst", metavar='DIR', required=True, + help="Output file", + ) + parser.add_argument( + "-m", "--mode", dest="mode", metavar='MODE', required=False, + choices=('FILE', 'ZIP'), default='ZIP', + help="Type of archive to write into", + ) + parser.add_argument( + "-q", "--quiet", dest="use_quiet", action='store_true', required=False, + help="Suppress status output", + ) + parser.add_argument( + "-t", "--temp", dest="temp_path", metavar='DIR', required=False, + help="Override the default temp directory", + ) + + return parser + + +def main(): + import sys + + parser = create_argparse() + args = parser.parse_args(sys.argv[1:]) + + if args.use_quiet: + def report(msg): + pass + else: + def report(msg): + sys.stdout.write(msg) + sys.stdout.flush() + + for msg in pack( + args.path_src.encode('utf-8'), + args.path_dst.encode('utf-8'), + mode=args.mode, + base_dir_dst_temp=( + args.temp_path.encode('utf-8') + if args.temp_path else None), + ): + report(msg) + + +if __name__ == "__main__": + main() |