diff options
Diffstat (limited to 'power_sequencer/scripts')
18 files changed, 2055 insertions, 0 deletions
diff --git a/power_sequencer/scripts/BPSProxy/bpsproxy/__init__.py b/power_sequencer/scripts/BPSProxy/bpsproxy/__init__.py new file mode 100644 index 00000000..f14cfb6a --- /dev/null +++ b/power_sequencer/scripts/BPSProxy/bpsproxy/__init__.py @@ -0,0 +1,16 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# diff --git a/power_sequencer/scripts/BPSProxy/bpsproxy/__main__.py b/power_sequencer/scripts/BPSProxy/bpsproxy/__main__.py new file mode 100644 index 00000000..d8f25204 --- /dev/null +++ b/power_sequencer/scripts/BPSProxy/bpsproxy/__main__.py @@ -0,0 +1,171 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +""" +Tool to render video proxies using FFMPEG +Offers mp4 and webm options +""" +import argparse as ap +import glob as g +import logging as lg +import os.path as osp +import sys +from itertools import compress, starmap, tee + +from .call import call, call_makedirs +from .commands import get_commands, get_commands_vi +from .config import CONFIG as C +from .config import LOGGER, LOGLEV +from .utils import checktools, printw, printd, prints, ToolError + + +def find_files( + directory=".", ignored_directory=C["proxy_directory"], extensions=C["extensions"]["all"] +): + """ + Find files to process. + + Parameters + ---------- + directory: str + Working directory. + ignored_directory: str + Don't check for files in this directory. By default `BL_proxy`. + extensions: set(str) + Set of file extensions for filtering the directory tree. + + Returns + ------- + out: list(str) + List of file paths to be processed. + """ + if not osp.isdir(directory): + raise ValueError(("The given path '{}' is not a valid directory.".format(directory))) + xs = g.iglob("{}/**".format(osp.abspath(directory)), recursive=True) + xs = filter(lambda x: osp.isfile(x), xs) + xs = filter(lambda x: ignored_directory not in osp.dirname(x), xs) + xs = [x for x in xs if osp.splitext(x)[1].lower() in extensions] + return xs + + +def parse_arguments(cfg): + """ + Uses `argparse` to parse the command line arguments. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + + Returns + ------- + out: Namespace + Command line arguments. + """ + p = ap.ArgumentParser(description="Create proxies for Blender VSE using FFMPEG.") + p.add_argument( + "working_directory", + nargs="?", + default=".", + help="The directory containing media to create proxies for", + ) + p.add_argument( + "-p", + "--preset", + default="mp4", + choices=cfg["presets"], + help="a preset name for proxy encoding", + ) + p.add_argument( + "-s", + "--sizes", + nargs="+", + type=int, + default=[25], + choices=cfg["proxy_sizes"], + help="A list of sizes of the proxies to render, either 25, 50, or 100", + ) + p.add_argument( + "-v", "--verbose", action="count", default=0, help="Increase verbosity level (eg. -vvv)." + ) + p.add_argument( + "--dry-run", + action="store_true", + help=( + "Run the script without actual rendering or creating files and" + " folders. For DEBUGGING purposes" + ), + ) + + clargs = p.parse_args() + # normalize directory + clargs.working_directory = osp.abspath(clargs.working_directory) + # --dry-run implies maximum verbosity level + clargs.verbose = 99999 if clargs.dry_run else clargs.verbose + return clargs + + +def main(): + """ + Script entry point. + """ + tools = ["ffmpeg", "ffprobe"] + try: + # get command line arguments and set log level + clargs = parse_arguments(C) + lg.basicConfig(level=LOGLEV[min(clargs.verbose, len(LOGLEV) - 1)]) + + # log basic command line arguments + clargs.dry_run and LOGGER.info("DRY-RUN") + LOGGER.info("WORKING-DIRECTORY :: {}".format(clargs.working_directory)) + LOGGER.info("PRESET :: {}".format(clargs.preset)) + LOGGER.info("SIZES :: {}".format(clargs.sizes)) + + # check for external dependencies + checktools(tools) + + # find files to process + path_i = find_files(clargs.working_directory) + kwargs = {"path_i": path_i} + + printw(C, "Creating directories if necessary") + call_makedirs(C, clargs, **kwargs) + + printw(C, "Checking for existing proxies") + cmds = tee(get_commands(C, clargs, what="check", **kwargs)) + stdouts = call(C, clargs, cmds=cmds[0], check=False, shell=True, **kwargs) + checks = map(lambda s: s.strip().split(), stdouts) + checks = starmap(lambda fst, *tail: not all(fst == t for t in tail), checks) + kwargs["path_i"] = list(compress(kwargs["path_i"], checks)) + + if len(kwargs["path_i"]) != 0: + printw(C, "Processing", s="\n") + cmds = get_commands_vi(C, clargs, **kwargs) + call(C, clargs, cmds=cmds, **kwargs) + else: + printd(C, "All proxies exist or no files found, nothing to process", s="\n") + printd(C, "Done") + except (ToolError, ValueError) as e: + LOGGER.error(e) + prints(C, "Exiting") + except KeyboardInterrupt: + prints(C, "DirtyInterrupt. Exiting", s="\n\n") + sys.exit() + + +# this is so it can be ran as a module: `python3 -m bpsrender` (for testing) +if __name__ == "__main__": + main() diff --git a/power_sequencer/scripts/BPSProxy/bpsproxy/call.py b/power_sequencer/scripts/BPSProxy/bpsproxy/call.py new file mode 100644 index 00000000..dd74e3a8 --- /dev/null +++ b/power_sequencer/scripts/BPSProxy/bpsproxy/call.py @@ -0,0 +1,95 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +# import multiprocessing as mp +import os +import subprocess as sp +import sys + +from functools import partial +from itertools import chain, tee +from tqdm import tqdm +from .config import LOGGER +from .utils import get_dir, kickstart + +WINDOWS = ("win32", "cygwin") + + +def call_makedirs(cfg, clargs, **kwargs): + """ + Make BL_proxy directories if necessary. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments. + kwargs: dict + MANDATORY: path_i + Dictionary with additional information from previous step. + """ + path_i = kwargs["path_i"] + path_d = map(partial(get_dir, cfg, clargs, **kwargs), path_i) + path_d = tee(chain(*path_d)) + kickstart(map(lambda p: LOGGER.info("Directory @ {}".format(p)), path_d[0])) + if clargs.dry_run: + return + path_d = (os.makedirs(p, exist_ok=True) for p in path_d[1]) + kickstart(path_d) + + +def call(cfg, clargs, *, cmds, **kwargs): + """ + Generic subprocess calls. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments. + cmds: iter(tuple(str)) + kwargs: dict + MANDATORY: path_i + Dictionary with additional information from previous step. + + Returns + ------- + out: str + Stdout & Stderr gathered from subprocess call. + """ + kwargs_s = { + "stdout": sp.PIPE, + "stderr": sp.STDOUT, + "universal_newlines": True, + "check": kwargs.get("check", True), + "shell": kwargs.get("shell", False), + "creationflags": sp.CREATE_NEW_PROCESS_GROUP if sys.platform in WINDOWS else 0, + } + if kwargs_s["shell"]: + cmds = map(lambda cmd: (cmd[0], " ".join(cmd[1])), cmds) + cmds = tee(cmds) + kickstart(map(lambda cmd: LOGGER.debug("CALL :: {}".format(cmd[1])), cmds[0])) + if clargs.dry_run: + return [] + n = len(kwargs["path_i"]) + ps = tqdm( + map(lambda cmd: sp.run(cmd[1], **kwargs_s), cmds[1]), + total=n, + unit="file" if n == 1 else "files", + ) + return [p.stdout for p in ps] diff --git a/power_sequencer/scripts/BPSProxy/bpsproxy/commands.py b/power_sequencer/scripts/BPSProxy/bpsproxy/commands.py new file mode 100644 index 00000000..d481c58f --- /dev/null +++ b/power_sequencer/scripts/BPSProxy/bpsproxy/commands.py @@ -0,0 +1,190 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +import os.path as osp +import shlex as sl +from itertools import chain +from .utils import get_path + + +def get_commands_check(cfg, clargs, **kwargs): + """ + ffprobe subprocess command generation. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments. + cmds: iter(tuple(str)) + kwargs: dict + MANDATORY: path_i_1, path_o_1 + Dictionary with additional information from previous step. + + Returns + ------- + out: iter(tuple(str)) + Iterator containing commands. + """ + cmd = ( + "ffprobe -v error -select_streams v:0 -show_entries stream=nb_frames -of" + " default=noprint_wrappers=1:nokey=1 '{file}'" + ) + out = map(lambda s: kwargs["path_o_1"].format(size=s), clargs.sizes) + out = map(lambda f: cmd.format(file=f), out) + out = sl.split(cmd.format(file=kwargs["path_i_1"]) + " && " + " && ".join(out)) + return iter((out,)) + + +def get_commands_image_1(cfg, clargs, **kwargs): + """ + ffmpeg subprocess command generation for processing an image. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments. + cmds: iter(tuple(str)) + kwargs: dict + MANDATORY: path_i_1, path_o_1 + Dictionary with additional information from previous step. + + Returns + ------- + out: iter(tuple(str)) + Iterator containing commands. + """ + cmd = "ffmpeg -y -v quiet -stats -i '{path_i_1}' {common_all}" + common = "-f apng -filter:v scale=iw*{size}:ih*{size} '{path_o_1}'" + common_all = map(lambda s: kwargs["path_o_1"].format(size=s), clargs.sizes) + common_all = map( + lambda s: common.format(size=s[0] / 100.0, path_o_1=s[1]), zip(clargs.sizes, common_all) + ) + common_all = " ".join(common_all) + out = sl.split(cmd.format(path_i_1=kwargs["path_i_1"], common_all=common_all)) + return iter((out,)) + + +def get_commands_video_1(cfg, clargs, **kwargs): + """ + ffmpeg subprocess command generation for processing a video. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments. + cmds: iter(tuple(str)) + kwargs: dict + MANDATORY: path_i_1, path_o_1 + Dictionary with additional information from previous step. + + Returns + ------- + out: iter(tuple(str)) + Iterator containing commands. + """ + cmd = "ffmpeg -y -v quiet -stats -i '{path_i_1}' {common_all}" + common = ( + "-pix_fmt yuv420p" + " -g 1" + " -sn -an" + " -vf colormatrix=bt601:bt709" + " -vf scale=ceil(iw*{size}/2)*2:ceil(ih*{size}/2)*2" + " {preset}" + " '{path_o_1}'" + ) + common_all = map(lambda s: kwargs["path_o_1"].format(size=s), clargs.sizes) + common_all = map( + lambda s: common.format( + preset=cfg["presets"][clargs.preset], size=s[0] / 100.0, path_o_1=s[1] + ), + zip(clargs.sizes, common_all), + ) + common_all = " ".join(common_all) + out = sl.split(cmd.format(path_i_1=kwargs["path_i_1"], common_all=common_all)) + return iter((out,)) + + +def get_commands(cfg, clargs, *, what, **kwargs): + """ + Delegates the creation of commands lists to appropriate functions based on `what` parameter. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments. + cmds: iter(tuple(str)) + what: str + Determines the returned value (see: Returns[out]). + kwargs: dict + MANDATORY: path_i + Dictionary with additional information from previous step. + + Returns + ------- + out: iter(tuple(str, tuple(str))) + An iterator with the 1st element as a tag (the `what` parameter) and the 2nd + element as the iterator of the actual commands. + """ + get_commands_f = { + "video": get_commands_video_1, + "image": get_commands_image_1, + "check": get_commands_check, + } + ps = ( + kwargs["path_i"] + if what not in cfg["extensions"] + else filter( + lambda p: osp.splitext(p)[1].lower() in cfg["extensions"][what], kwargs["path_i"] + ) + ) + ps = map(lambda p: (p, get_path(cfg, clargs, p, **kwargs)), ps) + out = chain.from_iterable( + map(lambda p: get_commands_f[what](cfg, clargs, path_i_1=p[0], path_o_1=p[1], **kwargs), ps) + ) + return map(lambda c: (what, c), out) + + +def get_commands_vi(cfg, clargs, **kwargs): + """ + Delegates the creation of commands lists to appropriate functions for video/image processing. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments. + cmds: iter(tuple(str)) + kwargs: dict + MANDATORY: path_i_1, path_o_1 + Dictionary with additional information from previous step. + + Returns + ------- + out: iter(tuple(str, tuple(str))) + An iterator with the 1st element as a tag (the `what` parameter) and the 2nd + element as the iterator of the actual commands. + """ + ws = filter(lambda x: x is not "all", cfg["extensions"]) + return chain.from_iterable(map(lambda w: get_commands(cfg, clargs, what=w, **kwargs), ws)) diff --git a/power_sequencer/scripts/BPSProxy/bpsproxy/config.py b/power_sequencer/scripts/BPSProxy/bpsproxy/config.py new file mode 100644 index 00000000..eada47d5 --- /dev/null +++ b/power_sequencer/scripts/BPSProxy/bpsproxy/config.py @@ -0,0 +1,41 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +import multiprocessing as mp +from itertools import chain +import logging as lg + + +CONFIG = { + "logger": "BPS", + "proxy_directory": "BL_proxy", + "proxy_sizes": (25, 50, 100), + "extensions": { + "video": {".mp4", ".mkv", ".mov", ".flv", ".mts"}, + "image": {".png", ".jpg", ".jpeg"}, + }, + "presets": { + "webm": "-c:v libvpx -crf 25 -speed 16 -threads {}".format(str(mp.cpu_count())), + "mp4": "-c:v libx264 -crf 25 -preset faster", + "nvenc": "-c:v h264_nvenc -qp 25 -preset fast", + }, + "pre": {"work": "»", "done": "•", "skip": "~"}, +} +CONFIG["extensions"]["all"] = set(chain(*CONFIG["extensions"].values())) + +LOGGER = lg.getLogger(CONFIG["logger"]) +LOGLEV = [lg.INFO, lg.DEBUG] +LOGLEV = [None] + sorted(LOGLEV, reverse=True) diff --git a/power_sequencer/scripts/BPSProxy/bpsproxy/utils.py b/power_sequencer/scripts/BPSProxy/bpsproxy/utils.py new file mode 100644 index 00000000..832a0beb --- /dev/null +++ b/power_sequencer/scripts/BPSProxy/bpsproxy/utils.py @@ -0,0 +1,109 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +""" +Collection of utility functions, class-independent +""" +import os.path as osp +from collections import deque +from shutil import which + + +class ToolError(Exception): + """Raised if external dependencies aren't found on system. + """ + + pass + + +def checktools(tools): + tools = [(t, which(t) or "") for t in tools] + check = {"tools": tools, "test": all(map(lambda x: x[1], tools))} + if not check["test"]: + msg = ["BPSProxy couldn't find external dependencies:"] + msg += [ + "[{check}] {tool}: {path}".format( + check="v" if path is not "" else "X", tool=tool, path=path or "NOT FOUND" + ) + for tool, path in check["tools"] + ] + msg += [ + ( + "Check if you have them properly installed and available in the PATH" + " environemnt variable." + ) + ] + raise ToolError("\n".join(msg)) + + +def get_path_video(cfg, clargs, path, **kwargs): + return osp.join( + osp.dirname(path), cfg["proxy_directory"], osp.basename(path), "proxy_{size}.avi" + ) + + +def get_path_image(cfg, clargs, path, **kwargs): + return osp.join( + osp.dirname(path), + cfg["proxy_directory"], + "images", + "{size}", + "{file}_proxy.jpg".format(file=osp.basename(path)), + ) + + +def get_path(cfg, clargs, path, **kwargs): + get_path_f = {"video": get_path_video, "image": get_path_image} + what = what_vi(cfg, clargs, path, **kwargs) + return get_path_f[what](cfg, clargs, path, **kwargs) + + +def get_dir_video(cfg, clargs, path, **kwargs): + return iter((osp.join(osp.dirname(path), cfg["proxy_directory"], osp.basename(path)),)) + + +def get_dir_image(cfg, clargs, path, **kwargs): + ps = osp.join(osp.dirname(path), cfg["proxy_directory"], "images", "{size}") + return map(lambda s: ps.format(size=s), clargs.sizes) + + +def get_dir(cfg, clargs, path, **kwargs): + get_dir_f = {"video": get_dir_video, "image": get_dir_image} + what = what_vi(cfg, clargs, path, **kwargs) + return get_dir_f[what](cfg, clargs, path, **kwargs) + + +def what_vi(cfg, clargs, p, **kwargs): + return "video" if osp.splitext(p)[1].lower() in cfg["extensions"]["video"] else "image" + + +def kickstart(it): + deque(it, maxlen=0) + + +def printw(cfg, text, s="\n", e="...", p="", **kwargs): + p = p or cfg["pre"]["work"] + print("{s}{p} {}{e}".format(text, s=s, e=e, p=p), **kwargs) + + +def printd(cfg, text, s="", e=".", p="", **kwargs): + p = p or cfg["pre"]["done"] + printw(cfg, text, s=s, e=e, p=p, **kwargs) + + +def prints(cfg, text, s="", e=".", p="", **kwargs): + p = p or cfg["pre"]["skip"] + printw(cfg, text, s=s, e=e, p=p, **kwargs) diff --git a/power_sequencer/scripts/BPSProxy/setup.py b/power_sequencer/scripts/BPSProxy/setup.py new file mode 100644 index 00000000..22aacf60 --- /dev/null +++ b/power_sequencer/scripts/BPSProxy/setup.py @@ -0,0 +1,55 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +from setuptools import setup + + +def readme(): + with open("README.rst") as f: + return f.read() + + +setup( + name="bpsproxy", + version="0.1.3.post1", + description="Blender Power Sequencer proxy generator tool", + long_description=readme(), + classifiers=[ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3", + "Topic :: Multimedia :: Video", + "Topic :: Utilities", + ], + url="https://gitlab.com/razcore/bpsproxy", + keywords="blender proxy vse sequence editor productivity", + author="Răzvan C. Rădulescu", + author_email="razcore.art@gmail.com", + license="GPLv3", + packages=["bpsproxy"], + install_requires=["tqdm"], + zip_safe=False, + entry_points={"console_scripts": ["bpsproxy=bpsproxy.__main__:main"]}, + include_package_data=True, +) diff --git a/power_sequencer/scripts/BPSRender/bpsrender/__init__.py b/power_sequencer/scripts/BPSRender/bpsrender/__init__.py new file mode 100644 index 00000000..35a40273 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# + diff --git a/power_sequencer/scripts/BPSRender/bpsrender/__main__.py b/power_sequencer/scripts/BPSRender/bpsrender/__main__.py new file mode 100644 index 00000000..07c84fe2 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/__main__.py @@ -0,0 +1,147 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +""" +Renders videos edited in Blender 3D's Video Sequence Editor using multiple CPU +cores. Original script by Justin Warren: +https://github.com/sciactive/pulverize/blob/master/pulverize.py +Modified by sudopluto (Pranav Sharma), gdquest (Nathan Lovato) and +razcore (Razvan Radulescu) + +Under GPLv3 license +""" +import argparse as ap +import os.path as osp +import sys +from functools import partial + +from .calls import call +from .config import CONFIG as C +from .config import LOGGER +from .helpers import BSError, ToolError, checktools, kickstart, prints +from .setup import setup + +# https://github.com/mikeycal/the-video-editors-render-script-for-blender#configuring-the-script +# there seems no easy way to grab the ram usage in a mulitplatform way +# without writing platform dependent code, or by using a python module + +# Most popluar config is 4 cores, 8 GB ram, this is the default for the script +# https://store.steampowered.com/hwsurvey/ + + +def parse_arguments(cfg): + """ + Uses `argparse` to parse the command line arguments. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + + Returns + ------- + out: Namespace + Command line arguments (normalized). + """ + p = ap.ArgumentParser( + description="Multi-process Blender VSE rendering - will attempt to" + " create a folder called `render` inside of the folder" + " containing `blendfile`. Insider `render` another folder called" + " `parts` will be created for storing temporary files. These files" + " will be joined together as the last step to produce the final" + " render which will be stored inside `render` and it will have the" + " same name as `blendfile`" + ) + p.add_argument( + "-o", + "--output", + default=".", + help="Output folder (will contain a `bpsrender` temp folder for" "rendering parts).", + ) + p.add_argument( + "-w", + "--workers", + type=int, + default=cfg["cpu_count"], + help="Number of workers in the pool (for video rendering).", + ) + p.add_argument( + "-v", "--verbose", action="count", default=0, help="Increase verbosity level (eg. -vvv)." + ) + p.add_argument( + "--dry-run", + action="store_true", + help=( + "Run the script without actual rendering or creating files and" + " folders. For DEBUGGING purposes" + ), + ) + p.add_argument("-s", "--start", type=int, default=None, help="Start frame") + p.add_argument("-e", "--end", type=int, default=None, help="End frame") + p.add_argument( + "-m", "--mixdown-only", action="store_true", help="ONLY render the audio MIXDOWN" + ) + p.add_argument( + "-c", + "--concatenate-only", + action="store_true", + help="ONLY CONCATENATE the (already) available video chunks", + ) + p.add_argument( + "-d", + "--video-only", + action="store_true", + help="ONLY render the VIDEO (implies --concatenate-only).", + ) + p.add_argument( + "-j", + "--join-only", + action="store_true", + help="ONLY JOIN the mixdown with the video. This will produce the" " final render", + ) + p.add_argument("blendfile", help="Blender project file to render.") + + clargs = p.parse_args() + clargs.blendfile = osp.abspath(clargs.blendfile) + clargs.output = osp.abspath(clargs.output) + # --video-only implies --concatenate-only + clargs.concatenate_only = clargs.concatenate_only or clargs.video_only + # --dry-run implies maximum verbosity level + clargs.verbose = 99999 if clargs.dry_run else clargs.verbose + return clargs + + +def main(): + """ + Script entry point. + """ + tools = ["blender", "ffmpeg"] + try: + clargs = parse_arguments(C) + checktools(tools) + cmds, kwargs = setup(C, clargs) + kickstart(map(partial(call, C, clargs, **kwargs), cmds)) + except (BSError, ToolError) as e: + LOGGER.error(e) + except KeyboardInterrupt: + # TODO: add actual clean up code + prints(C, "DirtyInterrupt. Exiting", s="\n\n", e="...") + sys.exit() + + +# this is so it can be ran as a module: `python3 -m bpsrender` (for testing) +if __name__ == "__main__": + main() diff --git a/power_sequencer/scripts/BPSRender/bpsrender/bscripts/mixdown.py b/power_sequencer/scripts/BPSRender/bpsrender/bscripts/mixdown.py new file mode 100644 index 00000000..a6b885cc --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/bscripts/mixdown.py @@ -0,0 +1,30 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +import bpy +import os.path as osp +import sys + + +for strip in bpy.context.scene.sequence_editor.sequences_all: + if strip.type == "META": + continue + if strip.type != "SOUND": + strip.mute = True + +path = sys.argv[-1] +ext = osp.splitext(path)[1][1:].upper() +bpy.ops.sound.mixdown(filepath=path, check_existing=False, container=ext, codec=ext) diff --git a/power_sequencer/scripts/BPSRender/bpsrender/bscripts/probe.py b/power_sequencer/scripts/BPSRender/bpsrender/bscripts/probe.py new file mode 100644 index 00000000..92cffa60 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/bscripts/probe.py @@ -0,0 +1,30 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +import bpy + +EXT = { + "AVI_JPEG": ".avi", + "AVI_RAW": ".avi", + "FFMPEG": {"MKV": ".mkv", "OGG": ".ogv", "QUICKTIME": ".mov", "AVI": ".avi", "MPEG4": ".mp4"}, +} + +scene = bpy.context.scene + +ext = EXT.get(scene.render.image_settings.file_format, "UNDEFINED") +if scene.render.image_settings.file_format == "FFMPEG": + ext = ext[scene.render.ffmpeg.format] +print("\nBPS:{} {} {}\n".format(scene.frame_start, scene.frame_end, ext)) diff --git a/power_sequencer/scripts/BPSRender/bpsrender/bscripts/video.py b/power_sequencer/scripts/BPSRender/bpsrender/bscripts/video.py new file mode 100644 index 00000000..590e73eb --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/bscripts/video.py @@ -0,0 +1,19 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +import bpy + +bpy.context.scene.render.ffmpeg.audio_codec = "NONE" diff --git a/power_sequencer/scripts/BPSRender/bpsrender/calls.py b/power_sequencer/scripts/BPSRender/bpsrender/calls.py new file mode 100644 index 00000000..5a223dd6 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/calls.py @@ -0,0 +1,410 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +# IMPURE +import multiprocessing as mp +import os +import signal as sig +import subprocess as sp +from functools import partial, reduce +from itertools import chain, islice, starmap, tee +from multiprocessing import Queue + +from tqdm import tqdm + +from .config import LOGGER +from .helpers import BSError, checkblender, kickstart, printd, prints, printw + + +def chunk_frames(cfg, clargs, cmds, **kwargs): + """ + Recover the chunk start/end frames from the constructed commands for the + video step. This is necessary to preserve purity until later steps. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmds: iter(tuple) + Iterator of commands to be passed to `subprocess`. + kwargs: dict + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter(tuple) + Start/end pairs of frames corresponding to the chunk commands created at + the video step. + """ + out = map(lambda x: (x, islice(x, 1, None)), cmds) + out = map(lambda x: zip(*x), out) + out = map(lambda x: filter(lambda y: y[0] in ("-s", "-e"), x), out) + out = map(lambda x: map(lambda y: int(y[1]), x), out) + out = map(lambda x: reduce(lambda acc, y: acc + (y,), x, ()), out) + return out + + +def append_chunks_file(cfg, clargs, cmds, **kwargs): + """ + IMPURE + Helper function for creating the chunks file that will be used by `ffmpeg` + to concatenate the chunks into one video file. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmds: iter(tuple) + Iterator of commands to be passed to `subprocess`. + kwargs: dict + MANDATORY w_frame_start, w_frame_end, ext + Dictionary with additional information from the setup step. + """ + with open(kwargs["chunks_file_path"], "a") as f: + for fs, fe in chunk_frames(cfg, clargs, cmds, **kwargs): + f.write( + "file '{rcp}{fs}-{fe}{ext}'\n".format( + rcp=kwargs["render_chunk_path"].rstrip("#"), + fs="{fs:0{frame_pad}d}".format(fs=fs, **cfg), + fe="{fe:0{frame_pad}d}".format(fe=fe, **cfg), + **kwargs + ) + ) + + +def call_probe(cfg, clargs, cmds, **kwargs): + """ + IMPURE + Probe `clargs.blendfile` for frame start, frame end and extension (for + video only). + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmds: iter(tuple) + Iterator of commands to be passed to `subprocess`. + kwargs: dict + Dictionary with additional information from the setup step. + + Returns + ------- + out: dict + Dictionary with info extracted from `clargs.blendfile`, namely: start + frame, end frame and extension (only useful for video step). + """ + kwargs_p = {"stdout": sp.PIPE, "stderr": sp.STDOUT, "universal_newlines": True} + + printw(cfg, "Probing") + printw(cfg, "Input(blend) @ {}".format(clargs.blendfile), s="") + frame_start, frame_end, ext = (0, 0, "") + if not clargs.dry_run: + with sp.Popen(next(cmds), **kwargs_p) as cp: + try: + tmp = map(partial(checkblender, "PROBE", [cfg["probe_py"]], cp), cp.stdout) + tmp = filter(lambda x: x.startswith("BPS"), tmp) + tmp = map(lambda x: x[4:].strip().split(), tmp) + frame_start, frame_end, ext = chain(*tmp) + except BSError as e: + LOGGER.error(e) + except KeyboardInterrupt: + raise + finally: + cp.terminate() + returncode = cp.poll() + if returncode != 0: + raise sp.CalledProcessError(returncode, cp.args) + frame_start = frame_start if clargs.start is None else clargs.start + frame_end = frame_end if clargs.end is None else clargs.end + out = { + "frame_start": int(frame_start), + "frame_end": int(frame_end), + "frames_total": int(frame_end) - int(frame_start) + 1, + "ext": ext, + } + if out["ext"] == "UNDEFINED": + raise BSError("Video extension is {ext}. Stopping!".format(ext=ext)) + printd(cfg, "Probing done") + return out + + +def call_mixdown(cfg, clargs, cmds, **kwargs): + """ + IMPURE + Calls blender to render the audio mixdown. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmds: iter(tuple) + Iterator of commands to be passed to `subprocess`. + kwargs: dict + MANDATORY render_mixdown_path + Dictionary with additional information from the setup step. + """ + kwargs_p = {"stdout": sp.PIPE, "stderr": sp.STDOUT, "universal_newlines": True} + + printw(cfg, "Rendering mixdown") + printw(cfg, "Output @ {}".format(kwargs["render_mixdown_path"]), s="") + if not clargs.dry_run: + with sp.Popen(next(cmds), **kwargs_p) as cp: + try: + tmp = map(partial(checkblender, "MIXDOWN", [cfg["mixdown_py"]], cp), cp.stdout) + tmp = filter(lambda x: x.startswith("BPS"), tmp) + tmp = map(lambda x: x[4:].strip().split(), tmp) + kickstart(tmp) + except BSError as e: + LOGGER.error(e) + except KeyboardInterrupt: + raise + finally: + cp.terminate() + returncode = cp.poll() + if returncode != 0: + raise sp.CalledProcessError(returncode, cp.args) + printd(cfg, "Mixdown done") + + +def call_chunk(cfg, clargs, queue, cmd, **kwargs): + """ + IMPURE + Calls blender to render one chunk (which part is determined by `cmd`). + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmd: tuple + Tuple to be passed to `subprocess`. + kwargs: dict + Dictionary with additional information from the setup step. + """ + sig.signal(sig.SIGINT, sig.SIG_IGN) + kwargs_p = {"stdout": sp.PIPE, "stderr": sp.STDOUT, "universal_newlines": True} + + if not clargs.dry_run: + # can't use nice functional syntax if we want to simplify with `with` + with sp.Popen(cmd, **kwargs_p) as cp: + try: + tmp = map( + partial( + checkblender, + "VIDEO", + [cfg["video_py"], "The encoder timebase is not set"], + cp, + ), + cp.stdout, + ) + tmp = filter(lambda x: x.startswith("Append frame"), tmp) + tmp = map(lambda x: x.split()[-1], tmp) + tmp = map(int, tmp) + tmp = map(lambda x: True, tmp) + kickstart(map(queue.put, tmp)) + queue.put(False) + except BSError as e: + LOGGER.error(e) + + +def call_video(cfg, clargs, cmds, **kwargs): + """ + IMPURE + Multi-process call to blender for rendering the (video) chunks. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmds: iter(tuple) + Iterator of commands to be passed to `subprocess`. + kwargs: dict + Dictionary with additional information from the setup step. + """ + printw(cfg, "Rendering video (w/o audio)") + printw(cfg, "Output @ {}".format(kwargs["render_chunk_path"]), s="") + try: + not clargs.dry_run and os.remove(kwargs["chunks_file_path"]) + LOGGER.info("CALL-VIDEO: generating {}".format(kwargs["chunks_file_path"])) + except OSError as e: + LOGGER.info("CALL-VIDEO: skipping {}: {}".format(e.filename, e.strerror)) + + cmds, cmds_cf = tee(cmds) + (not clargs.dry_run and append_chunks_file(cfg, clargs, cmds_cf, **kwargs)) + # prepare queue/worker + queues = queues_close = (Queue(),) * clargs.workers + # prpare processes + proc = starmap( + lambda q, cmd: mp.Process(target=partial(call_chunk, cfg, clargs, **kwargs), args=(q, cmd)), + zip(queues, cmds), + ) + # split iterator in 2 for later joining the processes and sum + # one of them + proc, proc_close = tee(proc) + proc = map(lambda p: p.start(), proc) + try: + not clargs.dry_run and kickstart(proc) + + # communicate with processes through the queues and use tqdm to show a + # simple terminal progress bar baesd on video total frames + queues = map(lambda q: iter(q.get, False), queues) + queues = chain(*queues) + queues = tqdm(queues, total=kwargs["frame_end"] - kwargs["frame_start"] + 1, unit="frames") + not clargs.dry_run and kickstart(queues) + except KeyboardInterrupt: + proc_close = map(lambda x: x.terminate(), proc_close) + not clargs.dry_run and kickstart(proc_close) + raise + finally: + # close and join processes and queues + proc_close = map(lambda x: x.join(), proc_close) + not clargs.dry_run and kickstart(proc_close) + + queues_close = map(lambda q: (q, q.close()), queues_close) + queues_close = starmap(lambda q, _: q.join_thread(), queues_close) + not clargs.dry_run and kickstart(queues_close) + printd(cfg, "Video chunks rendering done") + + +def call_concatenate(cfg, clargs, cmds, **kwargs): + """ + IMPURE + Calls ffmpeg in order to concatenate the video chunks together. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmds: iter(tuple) + Iterator of commands to be passed to `subprocess`. + kwargs: dict + MANDATORY: render_video_path + Dictionary with additional information from the setup step. + + Note + ---- + It expects the video chunk files to already be available. + """ + kwargs_p = {"stdout": sp.DEVNULL, "stderr": sp.DEVNULL} + printw(cfg, "Concatenating (video) chunks") + printw(cfg, "Output @ {}".format(kwargs["render_video_path"]), s="") + if not clargs.dry_run: + with sp.Popen(next(cmds), **kwargs_p) as cp: + try: + returncode = cp.wait() + if returncode != 0: + raise sp.CalledProcessError(returncode, cp.args) + except KeyboardInterrupt: + raise + finally: + cp.terminate() + printd(cfg, "Concatenating done") + + +def call_join(cfg, clargs, cmds, **kwargs): + """ + IMPURE + Calls ffmpeg for joining the audio mixdown and the video. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmds: iter(tuple) + Iterator of commands to be passed to `subprocess`. + kwargs: dict + MANDATORY: render_audiovideo_path + Dictionary with additional information from the setup step. + + Note + ---- + It expects the audio mixdown and video files to already be available. + """ + kwargs_p = {"stdout": sp.DEVNULL, "stderr": sp.DEVNULL} + printw(cfg, "Joining audio/video") + printw(cfg, "Output @ {}".format(kwargs["render_audiovideo_path"]), s="") + if not clargs.dry_run: + with sp.Popen(next(cmds), **kwargs_p) as cp: + try: + returncode = cp.wait() + if returncode != 0: + raise sp.CalledProcessError(returncode, cp.args) + except KeyboardInterrupt: + raise + finally: + cp.terminate() + printd(cfg, "Joining done") + + +def call(cfg, clargs, cmds, **kwargs): + """ + IMPURE + Delegates work to appropriate `call_*` functions. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + cmds: iter(tuple) + Iterator of commands to be passed to `subprocess` + kwargs: dict + MANDATORY: render_audiovideo_path + Dictionary with additional information from the setup step. + + Returns + ------- + out: dict or None + It passes on the output from the `call_*` functions. See `call_*` for + specific details. + + Note + ---- + It tries to be smart and skip steps if child subprocesses give errors. + Example if `--join-only` is passed, but the audio mixdown or video file + aren't available on hard drive. + """ + calls = { + "probe": call_probe, + "mixdown": call_mixdown, + "video": call_video, + "concatenate": call_concatenate, + "join": call_join, + } + try: + out = calls[cmds[0]](cfg, clargs, cmds[1], **kwargs) + return out + except sp.CalledProcessError: + prints( + cfg, + ("WARNING:{}: Something went wrong when calling" " command - SKIPPING").format(cmds[0]), + ) diff --git a/power_sequencer/scripts/BPSRender/bpsrender/commands.py b/power_sequencer/scripts/BPSRender/bpsrender/commands.py new file mode 100644 index 00000000..dc669806 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/commands.py @@ -0,0 +1,341 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +import math as m +from collections import OrderedDict +from itertools import chain, islice + +from .config import LOGGER + + +def get_commands_probe(cfg, clargs, **kwargs): + """ + Create the command for probing the `clargs.blendfile`. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter(list) + An iterator for which each element is a list to be sent to functions like + `subprocess.run`. + """ + out = ( + "blender", + "--background", + clargs.blendfile, + "--python", + kwargs["probe_py_normalized"], + "--disable-autoexec", + ) + LOGGER.debug("CMD-PROBE: {cmd}".format(cmd=" ".join(out))) + return iter((out,)) + + +def get_commands_chunk(cfg, clargs, **kwargs): + """ + Create the command for rendering a (video) chunk from `clargs.blendfile`. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + MANDATORY render_chunk_path, w_frame_start, w_frame_end + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter(list) + An iterator for which each element is a list to be sent to functions like + `subprocess.run`. + """ + out = ( + "blender", + "--background", + clargs.blendfile, + "--python", + kwargs["video_py_normalized"], + "--disable-autoexec", + "--render-output", + kwargs["render_chunk_path"], + "-s", + str(kwargs["w_frame_start"]), + "-e", + str(kwargs["w_frame_end"]), + "--render-anim", + ) + LOGGER.debug( + "CMD-CHUNK({w_frame_start}-{w_frame_end}): {cmd}".format(cmd=" ".join(out), **kwargs) + ) + return iter((out,)) + + +def get_commands_video(cfg, clargs, **kwargs): + """ + Create the list of commands (one command per chunk) for rendering a video + from `clargs.blendfile`. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + MANDATORY chunk_file_path, frame_start, frame_end, frames_total + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter(tuple) + An iterator for which each element is a tuple to be sent to functions like + `subprocess.run`. + """ + LOGGER.debug("CMD-VIDEO:") + chunk_length = int(m.floor(kwargs["frames_total"] / clargs.workers)) + out = map(lambda w: (w, kwargs["frame_start"] + w * chunk_length), range(clargs.workers)) + out = map( + lambda x: ( + x[1], + x[1] + chunk_length - 1 if x[0] != clargs.workers - 1 else kwargs["frame_end"], + ), + out, + ) + out = map( + lambda x: get_commands( + cfg, clargs, "chunk", w_frame_start=x[0], w_frame_end=x[1], **kwargs + ), + out, + ) + out = map(lambda x: x[1], out) + out = chain(*out) + return tuple(out) + + +def get_commands_mixdown(cfg, clargs, **kwargs): + """ + Create the command to render the mixdown from `clargs.blendfile`. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + MANDATORY render_mixdown_path + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter(tuple) + An iterator for which each element is a tuple to be sent to functions like + `subprocess.run`. + """ + out = ( + "blender --background {blendfile} --python {mixdown_py_normalized}" + " --disable-autoexec -- {render_mixdown_path}".format(**cfg, **vars(clargs), **kwargs) + ) + out = ( + "blender", + "--background", + clargs.blendfile, + "--python", + kwargs["mixdown_py_normalized"], + "--disable-autoexec", + "--", + kwargs["render_mixdown_path"], + ) + LOGGER.debug("CMD-MIXDOWN: {cmd}".format(cmd=" ".join(out))) + return iter((out,)) + + +def get_commands_concatenate(cfg, clargs, **kwargs): + """ + Create the command to concatenate the available video chunks generated + beforehand. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + MANDATORY chunks_file_path, render_video_path + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter(tuple) + An iterator for which each element is a tuple to be sent to functions like + `subprocess.run`. + """ + out = ( + "ffmpeg", + "-stats", + "-f", + "concat", + "-safe", + "-0", + "-i", + kwargs["chunks_file_path"], + "-c", + "copy", + "-y", + kwargs["render_video_path"], + ) + LOGGER.debug("CMD-CONCATENATE: {cmd}".format(cmd=" ".join(out))) + return iter((out,)) + + +def get_commands_join(cfg, clargs, **kwargs): + """ + Create the command to join the available audio mixdown and video generated + beforehand. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + MANDATORY chunks_file_path, render_video_path + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter(tuple) + An iterator for which each element is a tuple to be sent to functions like + `subprocess.run`. + """ + out = ( + "ffmpeg", + "-stats", + "-i", + kwargs["render_video_path"], + "-i", + kwargs["render_mixdown_path"], + "-map", + "0:v:0", + "-c:v", + "copy", + "-map", + "1:a:0", + "-c:a", + "aac", + "-b:a", + "192k", + "-y", + kwargs["render_audiovideo_path"], + ) + LOGGER.debug("CMD-JOIN: {cmd}".format(cmd=" ".join(out))) + return iter((out,)) + + +def get_commands(cfg, clargs, what="", **kwargs): + """ + Delegates the creation of commands lists to appropriate functions based on + `what` parameter. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + what: str (default = '') + Determines the returned value (see: Returns[out]). + kwargs: dict + MANDATORY -- see individual functions for the list of mandatory keys + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter or (str, iter) + |- what == '' is True + An iterator with elements of the type (str) for determining the order in + which to call the functions in the setup step. + NOTE: it skipps the "internal use only" functions. + |- else + A tuple with the 1st element as a tag (the `what` parameter) and the 2nd + element as the iterator of the actual commands. + """ + get_commands_f = OrderedDict( + ( + # internal use only + ("probe", get_commands_probe), + ("chunk", get_commands_chunk), + # direct connection to command line arguments - in order of execution + ("mixdown", get_commands_mixdown), + ("video", get_commands_video), + ("concatenate", get_commands_concatenate), + ("join", get_commands_join), + ) + ) + + return ( + islice(get_commands_f, 2, None) + if what == "" + else (what, get_commands_f[what](cfg, clargs, **kwargs)) + ) + + +def get_commands_all(cfg, clargs, **kwargs): + """ + Prepare the list of commands to be executed depending on the command line + arguments. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + MANDATORY -- see individual functions for the list of mandatory keys + Dictionary with additional information from the setup step. + + Returns + ------- + out: iter((str, tuple)) + An iterator for which each element is a (str, iter(tuple)). The string + value is for tagging the iterator command list (2nd element) for filtering + later based on the given command line arguments. + """ + end = "_only" + out = filter(lambda x: x[0].endswith(end), vars(clargs).items()) + out = map(lambda x: (x[0][: -len(end)], x[1]), out) + order = list(get_commands(cfg, clargs)) + out = sorted(out, key=lambda x: order.index(x[0])) + out = ( + map(lambda k: k[0], out) + if all(map(lambda k: not k[1], out)) + else map(lambda k: k[0], filter(lambda k: k[1], out)) + ) + out = map(lambda k: get_commands(cfg, clargs, k, **kwargs), out) + return out diff --git a/power_sequencer/scripts/BPSRender/bpsrender/config.py b/power_sequencer/scripts/BPSRender/bpsrender/config.py new file mode 100644 index 00000000..b87e3ea3 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/config.py @@ -0,0 +1,37 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +import logging as lg +import multiprocessing as mp +import os.path as osp + +CONFIG = { + "logger": "BPS", + "cpu_count": min(int(mp.cpu_count() / 2), 6), + "bs_path": osp.join(osp.dirname(osp.abspath(__file__)), "bscripts"), + "frame_pad": 7, + "parts_folder": "bpsrender", + "chunks_file": "chunks.txt", + "video_file": "video{}", + "pre": {"work": "»", "done": "•", "skip": "~"}, + "probe_py": "probe.py", + "mixdown_py": "mixdown.py", + "video_py": "video.py", +} + +LOGGER = lg.getLogger(CONFIG["logger"]) +LOGLEV = [lg.INFO, lg.DEBUG] +LOGLEV = [None] + sorted(LOGLEV, reverse=True) diff --git a/power_sequencer/scripts/BPSRender/bpsrender/helpers.py b/power_sequencer/scripts/BPSRender/bpsrender/helpers.py new file mode 100644 index 00000000..9ebcf2b0 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/helpers.py @@ -0,0 +1,110 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +from collections import deque +from shutil import which + + +class BSError(Exception): + """ + Custom Exception raised if Blender is called with a python script argument + and gives error while trying to execute the script. + """ + + pass + + +class ToolError(Exception): + """Raised if external dependencies aren't found on system. + """ + + pass + + +def checktools(tools): + tools = [(t, which(t) or "") for t in tools] + check = {"tools": tools, "test": all(map(lambda x: x[1], tools))} + if not check["test"]: + msg = ["BPSRender couldn't find external dependencies:"] + msg += [ + "[{check}] {tool}: {path}".format( + check="v" if path is not "" else "X", tool=tool, path=path or "NOT FOUND" + ) + for tool, path in check["tools"] + ] + msg += [ + ( + "Check if you have them properly installed and available in the PATH" + " environemnt variable." + ), + "Exiting...", + ] + raise ToolError("\n".join(msg)) + + +def checkblender(what, search, cp, s): + """ + IMPURE + Check Blender output for python script execution error. + + Parameters + ---------- + what: str + A tag used in the exception message. + search: iter(str) + One or more string(s) to search for in Blender's output. + cp: Popen + Blender subprocess. + s: PIPE + Blender's output. + + Returns + ------- + out: PIPE + The same pipe `s` is returned so that it can be iterated over on later + steps. + """ + if not isinstance(search, list): + search = [search] + for search_item in search: + if search_item in s: + message = ( + "Script {what} was not properly executed in" " Blender".format(what=what), + "CMD: {cmd}".format(what=what, cmd=" ".join(cp.args)), + "DUMP:".format(what=what), + s, + ) + raise BSError("\n".join(message)) + return s + + +def printw(cfg, text, s="\n", e="...", p="", **kwargs): + p = p or cfg["pre"]["work"] + print("{s}{p} {}{e}".format(text, s=s, e=e, p=p), **kwargs) + + +def printd(cfg, text, s="", e=".", p="", **kwargs): + p = p or cfg["pre"]["done"] + printw(cfg, text, s=s, e=e, p=p, **kwargs) + + +def prints(cfg, text, s="", e=".", p="", **kwargs): + p = p or cfg["pre"]["skip"] + printw(cfg, text, s=s, e=e, p=p, **kwargs) + + +def kickstart(it): + deque(it, maxlen=0) diff --git a/power_sequencer/scripts/BPSRender/bpsrender/setup.py b/power_sequencer/scripts/BPSRender/bpsrender/setup.py new file mode 100644 index 00000000..aba30d07 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/bpsrender/setup.py @@ -0,0 +1,182 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +# IMPURE +import logging as lg +import os +import os.path as osp +from functools import reduce +from itertools import starmap + +from .calls import call +from .commands import get_commands, get_commands_all +from .config import LOGGER, LOGLEV +from .helpers import kickstart + + +def setup_bspy(cfg, clargs, **kwargs): + """ + Normalize the names of the script to be ran in Blender for certain steps. + Eg. the probe step depends on the script located in + `bpsrender/cfg['probe_py']`. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + + Returns + ------- + out: dict + Dictoinary to be used in call steps. + """ + out = filter(lambda x: x[0].endswith("_py"), cfg.items()) + out = starmap(lambda k, v: ("{}_normalized".format(k), osp.join(cfg["bs_path"], v)), out) + return dict(out) + + +def setup_probe(cfg, clargs, **kwargs): + """ + IMPURE + Call Blender and extract information that will be necessary for later + steps. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + MANDATORY -- see individual functions for the list of mandatory keys + Dictionary with additional information from the previous setup step. + + Returns + ------- + out: dict + Dictoinary to be used in call steps. + """ + return call(cfg, clargs, get_commands(cfg, clargs, "probe", **kwargs), **kwargs) + + +def setup_paths(cfg, clargs, **kwargs): + """ + Figure out appropriate path locations to store output for parts and final + render. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + MANDATORY -- see individual functions for the list of mandatory keys + Dictionary with additional information from the previous setup step. + + Returns + ------- + out: dict + Dictionary storing all relevant information pertaining to folder and file + paths. + + Note + ---- + It also creates the folder structure 'render/parts' where + `clargs.blendfile` is stored on disk. + """ + render_parts_path = osp.join(clargs.output, cfg["parts_folder"]) + name = osp.splitext(osp.basename(clargs.blendfile))[0] + render_mixdown_path = osp.join(render_parts_path, "{}_m.flac".format(name)) + render_chunk_path = osp.join(render_parts_path, "{}_c_{}".format(name, "#" * cfg["frame_pad"])) + render_video_path = osp.join(render_parts_path, "{}_v{}".format(name, kwargs["ext"])) + render_audiovideo_path = osp.join(clargs.output, "{}{}".format(name, kwargs["ext"])) + chunks_file_path = osp.join(render_parts_path, cfg["chunks_file"]) + + out = { + "render_path": clargs.output, + "render_parts_path": render_parts_path, + "chunks_file_path": chunks_file_path, + "render_chunk_path": render_chunk_path, + "render_video_path": render_video_path, + "render_mixdown_path": render_mixdown_path, + "render_audiovideo_path": render_audiovideo_path, + } + return out + + +def setup_folders_hdd(cfg, clargs, **kwargs): + """ + IMPURE + Prepares the folder structure `cfg['render']/cfg['parts']'`. + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + Dictionary with additional information from the previous setup step. + + Returns + ------- + out: (iter((str, iter(tuple))), dict) + 1st element: see commands.py:get_commands_all + 2nd elment: the keyword arguments used by calls.py:call + """ + # create folder structure if it doesn't exist already only if + # appropriate command line arguments are given + do_it = filter(lambda x: x[0].endswith("_only"), vars(clargs).items()) + do_it = all(map(lambda x: not x[1], do_it)) + do_it = not clargs.dry_run and clargs.video_only or clargs.mixdown_only or do_it + do_it and os.makedirs(kwargs["render_parts_path"], exist_ok=True) + return {} + + +def setup(cfg, clargs): + """ + IMPURE -- setup_paths + Prepares the folder structure 'render/parts', the appropriate command lists + to be called and the keyword arguments to be passed to call functions + (calls.py). + + Parameters + ---------- + cfg: dict + Configuration dictionary. + clargs: Namespace + Command line arguments (normalized). + kwargs: dict + Dictionary with additional information from the previous setup step. + + Returns + ------- + out: (iter((str, iter(tuple))), dict) + 1st element: see commands.py:get_commands_all + 2nd elment: the keyword arguments used by calls.py:call + """ + setups_f = (setup_bspy, setup_probe, setup_paths, setup_folders_hdd) + lg.basicConfig(level=LOGLEV[min(clargs.verbose, len(LOGLEV) - 1)]) + + kwargs = dict(reduce(lambda acc, sf: {**acc, **sf(cfg, clargs, **acc)}, setups_f, {})) + + LOGGER.info("Setup:") + kickstart(starmap(lambda k, v: LOGGER.info("{}: {}".format(k, v)), kwargs.items())) + return get_commands_all(cfg, clargs, **kwargs), kwargs diff --git a/power_sequencer/scripts/BPSRender/setup.py b/power_sequencer/scripts/BPSRender/setup.py new file mode 100644 index 00000000..4c4a74b9 --- /dev/null +++ b/power_sequencer/scripts/BPSRender/setup.py @@ -0,0 +1,55 @@ +# +# Copyright (C) 2016-2019 by Razvan Radulescu, Nathan Lovato, and contributors +# +# This file is part of Power Sequencer. +# +# Power Sequencer is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Power Sequencer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Power Sequencer. If +# not, see <https://www.gnu.org/licenses/>. +# +from setuptools import setup + + +def readme(): + with open("README.rst") as f: + return f.read() + + +setup( + name="bpsrender", + version="0.1.40.post1", + description="Blender Power Sequencer Renderer", + long_description=readme(), + classifiers=[ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Text Processing :: Linguistic", + "Topic :: Multimedia :: Video", + "Topic :: Utilities", + ], + url="https://gitlab.com/razcore/BPSRender", + keywords="blender render parallel multiprocess speedup utility" " productivty", + author="Răzvan C. Rădulescu", + author_email="razcore.art@gmail.com", + license="GPLv3", + packages=["bpsrender"], + install_requires=["tqdm"], + zip_safe=False, + entry_points={"console_scripts": ["bpsrender=bpsrender.__main__:main"]}, + include_package_data=True, +) |