From bd7cfbc3ccf3587f2da944ce8c8dde3a23f55d76 Mon Sep 17 00:00:00 2001 From: Martin Poirier Date: Sat, 14 Aug 2010 16:30:41 +0000 Subject: netrender fixes: slave use output log option correctly use os.path.join more consistently (reported by Dalai Felinto) unify frame range (was 4 digits on master, 6 elsewhere, now 6 everywhere). Fix occasional broken pipe between master and slave (didn't affect result, just polluted log). --- release/scripts/io/netrender/client.py | 10 ++++++++-- release/scripts/io/netrender/master.py | 21 ++++++++++----------- release/scripts/io/netrender/operators.py | 7 ++++++- release/scripts/io/netrender/slave.py | 30 +++++++++++++++++------------- release/scripts/io/netrender/utils.py | 13 +++++++++---- 5 files changed, 50 insertions(+), 31 deletions(-) diff --git a/release/scripts/io/netrender/client.py b/release/scripts/io/netrender/client.py index 4a116bb982a..a1ad4d3e91c 100644 --- a/release/scripts/io/netrender/client.py +++ b/release/scripts/io/netrender/client.py @@ -171,6 +171,7 @@ def clientSendJob(conn, scene, anim = False): # try to send path first conn.request("POST", "/job", repr(job.serialize())) response = conn.getresponse() + response.read() job_id = response.getheader("job-id") @@ -181,6 +182,7 @@ def clientSendJob(conn, scene, anim = False): conn.request("PUT", fileURL(job_id, rfile.index), f) f.close() response = conn.getresponse() + response.read() # server will reply with ACCEPTED until all files are found @@ -236,6 +238,7 @@ class NetworkRenderEngine(bpy.types.RenderEngine): requestResult(conn, job_id, scene.frame_current) response = conn.getresponse() + response.read() if response.status == http.client.NO_CONTENT: new_job = True @@ -244,16 +247,19 @@ class NetworkRenderEngine(bpy.types.RenderEngine): requestResult(conn, job_id, scene.frame_current) response = conn.getresponse() + response.read() while response.status == http.client.ACCEPTED and not self.test_break(): time.sleep(1) requestResult(conn, job_id, scene.frame_current) response = conn.getresponse() + response.read() # cancel new jobs (animate on network) on break if self.test_break() and new_job: conn.request("POST", cancelURL(job_id)) response = conn.getresponse() + response.read() print( response.status, response.reason ) netsettings.job_id = 0 @@ -265,7 +271,7 @@ class NetworkRenderEngine(bpy.types.RenderEngine): x= int(r.resolution_x*r.resolution_percentage*0.01) y= int(r.resolution_y*r.resolution_percentage*0.01) - f = open(netsettings.path + "output.exr", "wb") + f = open(os.path.join(netsettings.path, "output.exr"), "wb") buf = response.read(1024) while buf: @@ -275,7 +281,7 @@ class NetworkRenderEngine(bpy.types.RenderEngine): f.close() result = self.begin_result(0, 0, x, y) - result.load_from_file(netsettings.path + "output.exr") + result.load_from_file(os.path.join(netsettings.path, "output.exr")) self.end_result(result) conn.close() diff --git a/release/scripts/io/netrender/master.py b/release/scripts/io/netrender/master.py index f227f61a536..6deb925420b 100644 --- a/release/scripts/io/netrender/master.py +++ b/release/scripts/io/netrender/master.py @@ -89,7 +89,7 @@ class MRenderJob(netrender.model.RenderJob): def save(self): if self.save_path: - f = open(self.save_path + "job.txt", "w") + f = open(os.path.join(self.save_path, "job.txt"), "w") f.write(repr(self.serialize())) f.close() @@ -134,8 +134,8 @@ class MRenderJob(netrender.model.RenderJob): self.status = JOB_QUEUED def addLog(self, frames): - log_name = "_".join(("%04d" % f for f in frames)) + ".log" - log_path = self.save_path + log_name + log_name = "_".join(("%06d" % f for f in frames)) + ".log" + log_path = os.path.join(self.save_path, log_name) for number in frames: frame = self[number] @@ -260,7 +260,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler): elif frame.status == DONE: self.server.stats("", "Sending result to client") - filename = job.save_path + "%04d" % frame_number + ".exr" + filename = os.path.join(job.save_path, "%06d.exr" % frame_number) f = open(filename, 'rb') self.send_head(content = "image/x-exr") @@ -294,7 +294,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler): if frame.status in (QUEUED, DISPATCHED): self.send_head(http.client.ACCEPTED) elif frame.status == DONE: - filename = job.save_path + "%04d" % frame_number + ".exr" + filename = os.path.join(job.save_path, "%06d.exr" % frame_number) thumbname = thumbnail(filename) @@ -716,7 +716,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler): if file_index > 0: file_path = prefixPath(job.save_path, render_file.filepath, main_path) else: - file_path = job.save_path + main_name + file_path = os.path.join(job.save_path, main_name) buf = self.rfile.read(length) @@ -772,7 +772,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler): if job_result == DONE: length = int(self.headers['content-length']) buf = self.rfile.read(length) - f = open(job.save_path + "%04d" % job_frame + ".exr", 'wb') + f = open(os.path.join(job.save_path, "%06d.exr" % job_frame), 'wb') f.write(buf) f.close() @@ -822,13 +822,12 @@ class RenderHandler(http.server.BaseHTTPRequestHandler): if job.type == netrender.model.JOB_BLENDER: length = int(self.headers['content-length']) buf = self.rfile.read(length) - f = open(job.save_path + "%04d" % job_frame + ".jpg", 'wb') + f = open(os.path.join(job.save_path, "%06d.jpg" % job_frame), 'wb') f.write(buf) f.close() del buf - self.send_head() else: # frame not found self.send_head(http.client.NO_CONTENT) else: # job not found @@ -880,7 +879,7 @@ class RenderMasterServer(socketserver.ThreadingMixIn, http.server.HTTPServer): self.job_id = 0 if subdir: - self.path = path + "master_" + str(os.getpid()) + os.sep + self.path = os.path.join(path, "master_" + str(os.getpid())) else: self.path = path @@ -1007,7 +1006,7 @@ class RenderMasterServer(socketserver.ThreadingMixIn, http.server.HTTPServer): self.jobs_map[job.id] = job # create job directory - job.save_path = self.path + "job_" + job.id + os.sep + job.save_path = os.path.join(self.path, "job_" + job.id) if not os.path.exists(job.save_path): os.mkdir(job.save_path) diff --git a/release/scripts/io/netrender/operators.py b/release/scripts/io/netrender/operators.py index 55c2d11e28e..c9645d0d1ee 100644 --- a/release/scripts/io/netrender/operators.py +++ b/release/scripts/io/netrender/operators.py @@ -205,6 +205,7 @@ class RENDER_OT_netclientstatus(bpy.types.Operator): conn.request("GET", "/status") response = conn.getresponse() + response.read() print( response.status, response.reason ) jobs = (netrender.model.RenderJob.materialize(j) for j in eval(str(response.read(), encoding='utf8'))) @@ -306,6 +307,7 @@ class RENDER_OT_netclientslaves(bpy.types.Operator): conn.request("GET", "/slaves") response = conn.getresponse() + response.read() print( response.status, response.reason ) slaves = (netrender.model.RenderSlave.materialize(s) for s in eval(str(response.read(), encoding='utf8'))) @@ -354,6 +356,7 @@ class RENDER_OT_netclientcancel(bpy.types.Operator): conn.request("POST", cancelURL(job.id)) response = conn.getresponse() + response.read() print( response.status, response.reason ) netsettings.jobs.remove(netsettings.active_job_index) @@ -380,6 +383,7 @@ class RENDER_OT_netclientcancelall(bpy.types.Operator): conn.request("POST", "/clear") response = conn.getresponse() + response.read() print( response.status, response.reason ) while(len(netsettings.jobs) > 0): @@ -412,6 +416,7 @@ class netclientdownload(bpy.types.Operator): for frame in job.frames: client.requestResult(conn, job.id, frame.number) response = conn.getresponse() + response.read() if response.status != http.client.OK: print("missing", frame.number) @@ -419,7 +424,7 @@ class netclientdownload(bpy.types.Operator): print("got back", frame.number) - f = open(netsettings.path + "%06d" % frame.number + ".exr", "wb") + f = open(os.path.join(netsettings.path, "%06d.exr" % frame.number), "wb") buf = response.read(1024) while buf: diff --git a/release/scripts/io/netrender/slave.py b/release/scripts/io/netrender/slave.py index fbaf2254504..7528ea1053a 100644 --- a/release/scripts/io/netrender/slave.py +++ b/release/scripts/io/netrender/slave.py @@ -59,8 +59,8 @@ def slave_Info(): def testCancel(conn, job_id, frame_number): conn.request("HEAD", "/status", headers={"job-id":job_id, "job-frame": str(frame_number)}) - # cancelled if job isn't found anymore - if conn.getresponse().status == http.client.NO_CONTENT: + # canceled if job isn't found anymore + if responseStatus(conn) == http.client.NO_CONTENT: return True else: return False @@ -81,7 +81,7 @@ def testFile(conn, job_id, slave_id, rfile, JOB_PREFIX, main_path = None): if not found: # Force prefix path if not found job_full_path = prefixPath(JOB_PREFIX, rfile.filepath, main_path, force = True) - temp_path = JOB_PREFIX + "slave.temp" + temp_path = os.path.join(JOB_PREFIX, "slave.temp") conn.request("GET", fileURL(job_id, rfile.index), headers={"slave-id":slave_id}) response = conn.getresponse() @@ -113,10 +113,11 @@ def render_slave(engine, netsettings, threads): if conn: conn.request("POST", "/slave", repr(slave_Info().serialize())) response = conn.getresponse() + response.read() slave_id = response.getheader("slave-id") - NODE_PREFIX = netsettings.path + "slave_" + slave_id + os.sep + NODE_PREFIX = os.path.join(netsettings.path, "slave_" + slave_id) if not os.path.exists(NODE_PREFIX): os.mkdir(NODE_PREFIX) @@ -132,7 +133,7 @@ def render_slave(engine, netsettings, threads): job = netrender.model.RenderJob.materialize(eval(str(response.read(), encoding='utf8'))) engine.update_stats("", "Network render processing job from master") - JOB_PREFIX = NODE_PREFIX + "job_" + job.id + os.sep + JOB_PREFIX = os.path.join(NODE_PREFIX, "job_" + job.id) if not os.path.exists(JOB_PREFIX): os.mkdir(JOB_PREFIX) @@ -157,6 +158,7 @@ def render_slave(engine, netsettings, threads): logfile = netrender.model.LogFile(job.id, slave_id, [frame.number for frame in job.frames]) conn.request("POST", "/log", bytes(repr(logfile.serialize()), encoding='utf8')) response = conn.getresponse() + response.read() first_frame = job.frames[0].number @@ -172,7 +174,7 @@ def render_slave(engine, netsettings, threads): frame_args += ["-f", str(frame.number)] val = SetErrorMode() - process = subprocess.Popen([BLENDER_PATH, "-b", "-noaudio", job_full_path, "-t", str(threads), "-o", JOB_PREFIX + "######", "-E", "BLENDER_RENDER", "-F", "MULTILAYER"] + frame_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + process = subprocess.Popen([BLENDER_PATH, "-b", "-noaudio", job_full_path, "-t", str(threads), "-o", os.path.join(JOB_PREFIX, "######"), "-E", "BLENDER_RENDER", "-F", "MULTILAYER"] + frame_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) RestoreErrorMode(val) elif job.type == netrender.model.JOB_PROCESS: command = job.frames[0].command @@ -196,9 +198,10 @@ def render_slave(engine, netsettings, threads): # (only need to update on one frame, they are linked conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers) response = conn.getresponse() + response.read() # Also output on console - if netsettings.slave_thumb: + if netsettings.slave_outputlog: print(str(stdout, encoding='utf8'), end="") stdout = bytes() @@ -227,7 +230,7 @@ def render_slave(engine, netsettings, threads): # (only need to update on one frame, they are linked conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers) - if conn.getresponse().status == http.client.NO_CONTENT: + if responseStatus(conn) == http.client.NO_CONTENT: continue total_t = time.time() - start_t @@ -248,7 +251,7 @@ def render_slave(engine, netsettings, threads): if job.type == netrender.model.JOB_BLENDER: # send image back to server - filename = JOB_PREFIX + "%06d" % frame.number + ".exr" + filename = os.path.join(JOB_PREFIX, "%06d.exr" % frame.number) # thumbnail first if netsettings.slave_thumb: @@ -257,17 +260,18 @@ def render_slave(engine, netsettings, threads): f = open(thumbname, 'rb') conn.request("PUT", "/thumb", f, headers=headers) f.close() - conn.getresponse() + responseStatus(conn) + f = open(filename, 'rb') conn.request("PUT", "/render", f, headers=headers) f.close() - if conn.getresponse().status == http.client.NO_CONTENT: + if responseStatus(conn) == http.client.NO_CONTENT: continue elif job.type == netrender.model.JOB_PROCESS: conn.request("PUT", "/render", headers=headers) - if conn.getresponse().status == http.client.NO_CONTENT: + if responseStatus(conn) == http.client.NO_CONTENT: continue else: headers["job-result"] = str(ERROR) @@ -275,7 +279,7 @@ def render_slave(engine, netsettings, threads): headers["job-frame"] = str(frame.number) # send error result back to server conn.request("PUT", "/render", headers=headers) - if conn.getresponse().status == http.client.NO_CONTENT: + if responseStatus(conn) == http.client.NO_CONTENT: continue engine.update_stats("", "Network render connected to master, waiting for jobs") diff --git a/release/scripts/io/netrender/utils.py b/release/scripts/io/netrender/utils.py index 31123ce26da..81617ac0d30 100644 --- a/release/scripts/io/netrender/utils.py +++ b/release/scripts/io/netrender/utils.py @@ -57,6 +57,11 @@ FRAME_STATUS_TEXT = { ERROR: "Error" } +def responseStatus(conn): + response = conn.getresponse() + response.read() + return response.status + def reporting(report, message, errorType = None): if errorType: t = 'ERROR' @@ -171,16 +176,16 @@ def prefixPath(prefix_directory, file_path, prefix_path, force = False): if prefix_path and p.startswith(prefix_path): if len(prefix_path) < len(p): - directory = prefix_directory + p[len(prefix_path)+1:] + os.sep # +1 to remove separator + directory = os.path.join(prefix_directory, p[len(prefix_path)+1:]) # +1 to remove separator if not os.path.exists(directory): os.mkdir(directory) else: directory = prefix_directory - full_path = directory + n + full_path = os.path.join(directory, n) else: - full_path = prefix_directory + n + full_path = os.path.join(prefix_directory, n) else: - full_path = prefix_directory + file_path + full_path = (prefix_directory, file_path) return full_path -- cgit v1.2.3