diff options
| -rw-r--r-- | release/scripts/io/netrender/client.py | 7 | ||||
| -rw-r--r-- | release/scripts/io/netrender/master.py | 14 | ||||
| -rw-r--r-- | release/scripts/io/netrender/master_html.py | 106 | ||||
| -rw-r--r-- | release/scripts/io/netrender/model.py | 11 | ||||
| -rw-r--r-- | release/scripts/io/netrender/netrender.css | 4 | ||||
| -rw-r--r-- | release/scripts/io/netrender/slave.py | 45 | ||||
| -rw-r--r-- | release/scripts/io/netrender/ui.py | 22 | ||||
| -rw-r--r-- | release/scripts/io/netrender/utils.py | 14 |
8 files changed, 139 insertions, 84 deletions
diff --git a/release/scripts/io/netrender/client.py b/release/scripts/io/netrender/client.py index e49a1a28591..5c01884a20e 100644 --- a/release/scripts/io/netrender/client.py +++ b/release/scripts/io/netrender/client.py @@ -125,6 +125,10 @@ def clientSendJob(conn, scene, anim = False): file_path = bpy.utils.expandpath(image.filename) if os.path.exists(file_path): job.addFile(file_path) + + tex_path = os.path.splitext(file_path)[0] + ".tex" + if os.path.exists(tex_path): + job.addFile(tex_path) ########################### # FLUID + POINT CACHE @@ -144,6 +148,9 @@ def clientSendJob(conn, scene, anim = False): addPointCache(job, object, modifier.domain_settings.point_cache_low, default_path) if modifier.domain_settings.highres: addPointCache(job, object, modifier.domain_settings.point_cache_high, default_path) + elif modifier.type == "MULTIRES" and modifier.external: + file_path = bpy.utils.expandpath(modifier.filename) + job.addFile(file_path) # particles modifier are stupid and don't contain data # we have to go through the object property diff --git a/release/scripts/io/netrender/master.py b/release/scripts/io/netrender/master.py index 08763d2c211..324d046e00f 100644 --- a/release/scripts/io/netrender/master.py +++ b/release/scripts/io/netrender/master.py @@ -27,12 +27,16 @@ import netrender.balancing import netrender.master_html class MRenderFile(netrender.model.RenderFile): - def __init__(self, filepath, index, start, end): - super().__init__(filepath, index, start, end) + def __init__(self, filepath, index, start, end, signature): + super().__init__(filepath, index, start, end, signature) self.found = False def test(self): self.found = os.path.exists(self.filepath) + if self.found: + found_signature = hashFile(self.filepath) + self.found = self.signature == found_signature + return self.found @@ -74,7 +78,7 @@ class MRenderJob(netrender.model.RenderJob): # special server properties self.last_update = 0 self.save_path = "" - self.files = [MRenderFile(rfile.filepath, rfile.index, rfile.start, rfile.end) for rfile in job_info.files] + self.files = [MRenderFile(rfile.filepath, rfile.index, rfile.start, rfile.end, rfile.signature) for rfile in job_info.files] self.resolution = None @@ -716,7 +720,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler): buf = self.rfile.read(length) # add same temp file + renames as slave - + f = open(file_path, "wb") f.write(buf) f.close() @@ -875,7 +879,7 @@ class RenderMasterServer(socketserver.ThreadingMixIn, http.server.HTTPServer): self.job_id = 0 self.path = path + "master_" + str(os.getpid()) + os.sep - self.slave_timeout = 30 # 30 mins: need a parameter for that + self.slave_timeout = 5 # 5 mins: need a parameter for that self.balancer = netrender.balancing.Balancer() self.balancer.addRule(netrender.balancing.RatingUsageByCategory(self.getJobs)) diff --git a/release/scripts/io/netrender/master_html.py b/release/scripts/io/netrender/master_html.py index 7cf339541a6..c3695cd4f0f 100644 --- a/release/scripts/io/netrender/master_html.py +++ b/release/scripts/io/netrender/master_html.py @@ -106,53 +106,6 @@ def get(handler): handler.send_head(content = "text/html") head("NetRender") - output("<h2>Master</h2>") - - output("""<button title="remove all jobs" onclick="clear_jobs();">CLEAR JOB LIST</button>""") - - startTable(caption = "Rules", class_style = "rules") - - headerTable("type", "enabled", "description", "limit") - - for rule in handler.server.balancer.rules: - rowTable( - "rating", - checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))), - rule, - rule.str_limit() + - """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else " " - ) - - for rule in handler.server.balancer.priorities: - rowTable( - "priority", - checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))), - rule, - rule.str_limit() + - """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else " " - ) - - for rule in handler.server.balancer.exceptions: - rowTable( - "exception", - checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))), - rule, - rule.str_limit() + - """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else " " - ) - - endTable() - - output("<h2>Slaves</h2>") - - startTable() - headerTable("name", "address", "last seen", "stats", "job") - - for slave in handler.server.slaves: - rowTable(slave.name, slave.address[0], time.ctime(slave.last_seen), slave.stats, link(slave.job.name, "/html/job" + slave.job.id) if slave.job else "None") - - endTable() - output("<h2>Jobs</h2>") startTable() @@ -204,6 +157,53 @@ def get(handler): ) endTable() + + output("<h2>Slaves</h2>") + + startTable() + headerTable("name", "address", "last seen", "stats", "job") + + for slave in handler.server.slaves: + rowTable(slave.name, slave.address[0], time.ctime(slave.last_seen), slave.stats, link(slave.job.name, "/html/job" + slave.job.id) if slave.job else "None") + + endTable() + + output("<h2>Configuration</h2>") + + output("""<button title="remove all jobs" onclick="clear_jobs();">CLEAR JOB LIST</button>""") + + startTable(caption = "Rules", class_style = "rules") + + headerTable("type", "enabled", "description", "limit") + + for rule in handler.server.balancer.rules: + rowTable( + "rating", + checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))), + rule, + rule.str_limit() + + """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else " " + ) + + for rule in handler.server.balancer.priorities: + rowTable( + "priority", + checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))), + rule, + rule.str_limit() + + """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else " " + ) + + for rule in handler.server.balancer.exceptions: + rowTable( + "exception", + checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))), + rule, + rule.str_limit() + + """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else " " + ) + + endTable() output("</body></html>") @@ -235,13 +235,17 @@ def get(handler): tot_cache = 0 tot_fluid = 0 + rowTable(job.files[0].filepath) + rowTable("Other Files", class_style = "toggle", extra = "onclick='toggleDisplay(".other", "none", "table-row")'") + for file in job.files: if file.filepath.endswith(".bphys"): tot_cache += 1 elif file.filepath.endswith(".bobj.gz") or file.filepath.endswith(".bvel.gz"): tot_fluid += 1 else: - rowTable(file.filepath) + if file != job.files[0]: + rowTable(file.filepath, class_style = "other") if tot_cache > 0: rowTable("%i physic cache files" % tot_cache, class_style = "toggle", extra = "onclick='toggleDisplay(".cache", "none", "table-row")'") @@ -257,9 +261,9 @@ def get(handler): endTable() - output("<h2>Blacklist</h2>") - if job.blacklist: + output("<h2>Blacklist</h2>") + startTable() headerTable("name", "address") @@ -268,8 +272,6 @@ def get(handler): rowTable(slave.name, slave.address[0]) endTable() - else: - output("<i>Empty</i>") output("<h2>Frames</h2>") diff --git a/release/scripts/io/netrender/model.py b/release/scripts/io/netrender/model.py index 8b0f50ba848..a2912c78c56 100644 --- a/release/scripts/io/netrender/model.py +++ b/release/scripts/io/netrender/model.py @@ -103,8 +103,9 @@ JOB_TYPES = { } class RenderFile: - def __init__(self, filepath = "", index = 0, start = -1, end = -1): + def __init__(self, filepath = "", index = 0, start = -1, end = -1, signature=0): self.filepath = filepath + self.signature = signature self.index = index self.start = start self.end = end @@ -114,7 +115,8 @@ class RenderFile: "filepath": self.filepath, "index": self.index, "start": self.start, - "end": self.end + "end": self.end, + "signature": self.signature } @staticmethod @@ -122,7 +124,7 @@ class RenderFile: if not data: return None - rfile = RenderFile(data["filepath"], data["index"], data["start"], data["end"]) + rfile = RenderFile(data["filepath"], data["index"], data["start"], data["end"], data["signature"]) return rfile @@ -153,7 +155,8 @@ class RenderJob: self.blacklist = job_info.blacklist def addFile(self, file_path, start=-1, end=-1): - self.files.append(RenderFile(file_path, len(self.files), start, end)) + signature = hashFile(file_path) + self.files.append(RenderFile(file_path, len(self.files), start, end, signature)) def addFrame(self, frame_number, command = ""): frame = RenderFrame(frame_number, command) diff --git a/release/scripts/io/netrender/netrender.css b/release/scripts/io/netrender/netrender.css index cc8a93bb9a7..0c54690e002 100644 --- a/release/scripts/io/netrender/netrender.css +++ b/release/scripts/io/netrender/netrender.css @@ -68,6 +68,10 @@ button { display: none; } +.other { + display: none; +} + .rules { width: 60em; text-align: left; diff --git a/release/scripts/io/netrender/slave.py b/release/scripts/io/netrender/slave.py index abed0b44f1e..43420c1b5b6 100644 --- a/release/scripts/io/netrender/slave.py +++ b/release/scripts/io/netrender/slave.py @@ -64,12 +64,21 @@ def testCancel(conn, job_id, frame_number): else: return False -def testFile(conn, job_id, slave_id, file_index, JOB_PREFIX, file_path, main_path = None): - job_full_path = prefixPath(JOB_PREFIX, file_path, main_path) - - if not os.path.exists(job_full_path): +def testFile(conn, job_id, slave_id, rfile, JOB_PREFIX, main_path = None): + job_full_path = prefixPath(JOB_PREFIX, rfile.filepath, main_path) + + found = os.path.exists(job_full_path) + + if found: + found_signature = hashFile(job_full_path) + found = found_signature == rfile.signature + + if not found: + print("Found file %s at %s but signature mismatch!" % (rfile.filepath, job_full_path)) + + if not found: temp_path = JOB_PREFIX + "slave.temp.blend" - conn.request("GET", fileURL(job_id, file_index), headers={"slave-id":slave_id}) + conn.request("GET", fileURL(job_id, rfile.index), headers={"slave-id":slave_id}) response = conn.getresponse() if response.status != http.client.OK: @@ -126,14 +135,14 @@ def render_slave(engine, netsettings, threads): job_path = job.files[0].filepath # path of main file main_path, main_file = os.path.split(job_path) - job_full_path = testFile(conn, job.id, slave_id, 0, JOB_PREFIX, job_path) + job_full_path = testFile(conn, job.id, slave_id, job.files[0], JOB_PREFIX) print("Fullpath", job_full_path) print("File:", main_file, "and %i other files" % (len(job.files) - 1,)) engine.update_stats("", "Render File "+ main_file+ " for job "+ job.id) for rfile in job.files[1:]: print("\t", rfile.filepath) - testFile(conn, job.id, slave_id, rfile.index, JOB_PREFIX, rfile.filepath, main_path) + testFile(conn, job.id, slave_id, rfile, JOB_PREFIX, main_path) # announce log to master logfile = netrender.model.LogFile(job.id, slave_id, [frame.number for frame in job.frames]) @@ -178,6 +187,10 @@ def render_slave(engine, netsettings, threads): # (only need to update on one frame, they are linked conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers) response = conn.getresponse() + + # Also output on console + if netsettings.slave_thumb: + print(str(stdout, encoding='utf8'), end="") stdout = bytes() @@ -194,6 +207,17 @@ def render_slave(engine, netsettings, threads): process.terminate() continue # to next frame + # flush the rest of the logs + if stdout: + # Also output on console + if netsettings.slave_thumb: + print(str(stdout, encoding='utf8'), end="") + + # (only need to update on one frame, they are linked + conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers) + if conn.getresponse().status == http.client.NO_CONTENT: + continue + total_t = time.time() - start_t avg_t = total_t / len(job.frames) @@ -202,13 +226,6 @@ def render_slave(engine, netsettings, threads): print("status", status) - # flush the rest of the logs - if stdout: - # (only need to update on one frame, they are linked - conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers) - if conn.getresponse().status == http.client.NO_CONTENT: - continue - headers = {"job-id":job.id, "slave-id":slave_id, "job-time":str(avg_t)} diff --git a/release/scripts/io/netrender/ui.py b/release/scripts/io/netrender/ui.py index e09d673aae2..cef2c542b9c 100644 --- a/release/scripts/io/netrender/ui.py +++ b/release/scripts/io/netrender/ui.py @@ -76,7 +76,7 @@ def verify_address(netsettings): else: netsettings.server_address = "[default]" -class RenderButtonsPanel(bpy.types.Panel): +class RenderButtonsPanel(): bl_space_type = "PROPERTIES" bl_region_type = "WINDOW" bl_context = "render" @@ -88,7 +88,7 @@ class RenderButtonsPanel(bpy.types.Panel): # Setting panel, use in the scene for now. @rnaType -class RENDER_PT_network_settings(RenderButtonsPanel): +class RENDER_PT_network_settings(bpy.types.Panel, RenderButtonsPanel): bl_label = "Network Settings" COMPAT_ENGINES = {'NET_RENDER'} @@ -123,7 +123,7 @@ class RENDER_PT_network_settings(RenderButtonsPanel): layout.operator("render.netclientweb", icon='QUESTION') @rnaType -class RENDER_PT_network_slave_settings(RenderButtonsPanel): +class RENDER_PT_network_slave_settings(bpy.types.Panel, RenderButtonsPanel): bl_label = "Slave Settings" COMPAT_ENGINES = {'NET_RENDER'} @@ -141,13 +141,14 @@ class RENDER_PT_network_slave_settings(RenderButtonsPanel): layout.prop(netsettings, "slave_clear") layout.prop(netsettings, "slave_thumb") + layout.prop(netsettings, "slave_outputlog") layout.label(text="Threads:") layout.prop(rd, "threads_mode", expand=True) sub = layout.column() sub.enabled = rd.threads_mode == 'FIXED' sub.prop(rd, "threads") @rnaType -class RENDER_PT_network_master_settings(RenderButtonsPanel): +class RENDER_PT_network_master_settings(bpy.types.Panel, RenderButtonsPanel): bl_label = "Master Settings" COMPAT_ENGINES = {'NET_RENDER'} @@ -166,7 +167,7 @@ class RENDER_PT_network_master_settings(RenderButtonsPanel): layout.prop(netsettings, "master_clear") @rnaType -class RENDER_PT_network_job(RenderButtonsPanel): +class RENDER_PT_network_job(bpy.types.Panel, RenderButtonsPanel): bl_label = "Job Settings" COMPAT_ENGINES = {'NET_RENDER'} @@ -206,7 +207,7 @@ class RENDER_PT_network_job(RenderButtonsPanel): row.prop(netsettings, "chunks") @rnaType -class RENDER_PT_network_slaves(RenderButtonsPanel): +class RENDER_PT_network_slaves(bpy.types.Panel, RenderButtonsPanel): bl_label = "Slaves Status" COMPAT_ENGINES = {'NET_RENDER'} @@ -245,7 +246,7 @@ class RENDER_PT_network_slaves(RenderButtonsPanel): layout.label(text="Stats: " + slave.stats) @rnaType -class RENDER_PT_network_slaves_blacklist(RenderButtonsPanel): +class RENDER_PT_network_slaves_blacklist(bpy.types.Panel, RenderButtonsPanel): bl_label = "Slaves Blacklist" COMPAT_ENGINES = {'NET_RENDER'} @@ -283,7 +284,7 @@ class RENDER_PT_network_slaves_blacklist(RenderButtonsPanel): layout.label(text="Stats: " + slave.stats) @rnaType -class RENDER_PT_network_jobs(RenderButtonsPanel): +class RENDER_PT_network_jobs(bpy.types.Panel, RenderButtonsPanel): bl_label = "Jobs" COMPAT_ENGINES = {'NET_RENDER'} @@ -365,6 +366,11 @@ NetRenderSettings.BoolProperty( attr="slave_thumb", description="Generate thumbnails on slaves instead of master", default = False) +NetRenderSettings.BoolProperty( attr="slave_outputlog", + name="Output render log on console", + description="Output render text log to console as well as sending it to the master", + default = True) + NetRenderSettings.BoolProperty( attr="master_clear", name="Clear on exit", description="delete saved files on exit", diff --git a/release/scripts/io/netrender/utils.py b/release/scripts/io/netrender/utils.py index 37787732f09..acd45178c2f 100644 --- a/release/scripts/io/netrender/utils.py +++ b/release/scripts/io/netrender/utils.py @@ -19,7 +19,7 @@ import sys, os import re import http, http.client, http.server, urllib, socket -import subprocess, shutil, time, hashlib +import subprocess, shutil, time, hashlib, zlib import netrender.model @@ -154,6 +154,18 @@ def renderURL(job_id, frame_number): def cancelURL(job_id): return "/cancel_%s" % (job_id) +def hashFile(path): + f = open(path, "rb") + value = hashData(f.read()) + f.close() + return value + +def hashData(data): + m = hashlib.md5() + m.update(data) + return m.hexdigest() + + def prefixPath(prefix_directory, file_path, prefix_path): if os.path.isabs(file_path): # if an absolute path, make sure path exists, if it doesn't, use relative local path |