support for multiple file: linked libraries

4 files changed, 172 insertions, 109 deletions

diff --git a/release/io/netrender/client.py b/release/io/netrender/client.py
index bc4d363c996..59cb19c2075 100644
--- a/release/io/netrender/client.py
+++ b/release/io/netrender/client.py
@@ -69,7 +69,7 @@ class NetworkRenderEngine(bpy.types.RenderEngine):
 			x= int(r.resolution_x*r.resolution_percentage*0.01)
 			y= int(r.resolution_y*r.resolution_percentage*0.01)
 			
-			f = open(netsetting.path + "output.exr", "wb")
+			f = open(netsettings.path + "output.exr", "wb")
 			buf = response.read(1024)
 			
 			while buf:
diff --git a/release/io/netrender/master.py b/release/io/netrender/master.py
index 6f97685935d..6f360a4e7a9 100644
--- a/release/io/netrender/master.py
+++ b/release/io/netrender/master.py
@@ -46,8 +46,6 @@ class MRenderJob(netrender.model.RenderJob):
 		self.id = job_id
 		self.name = name
 		self.files = files
-		self.render_files = [MRenderFile(path) for path in files]
-		self.status = JOB_WAITING
 		self.frames = []
 		self.chunks = chunks
 		self.priority = priority
@@ -55,8 +53,19 @@ class MRenderJob(netrender.model.RenderJob):
 		self.blacklist = blacklist
 		self.last_dispatched = time.time()
 	
+		# special server properties
+		self.save_path = ""
+		self.files_map = {path: MRenderFile(path) for path in files}
+		self.status = JOB_WAITING
+	
+	def save(self):
+		if self.save_path:
+			f = open(self.save_path + "job.txt", "w")
+			f.write(repr(self.serialize()))
+			f.close()
+	
 	def testStart(self):
-		for f in self.render_files:
+		for f in self.files_map.values():
 			if not f.test():
 				return False
 		
@@ -184,7 +193,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
 						self.send_head(http.client.ACCEPTED)
 					elif frame.status == DONE:
 						self.server.stats("", "Sending result back to client")
-						f = open(self.server.path + job_id + "%04d" % job_frame + ".exr", 'rb')
+						f = open(job.save_path + "%04d" % job_frame + ".exr", 'rb')
 						
 						self.send_head()
 						
@@ -215,7 +224,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
 						self.send_head(http.client.PROCESSING)
 					else:
 						self.server.stats("", "Sending log back to client")
-						f = open(self.server.path + job_id + "%04d" % job_frame + ".log", 'rb')
+						f = open(job.save_path + "%04d" % job_frame + ".log", 'rb')
 						
 						self.send_head()
 						
@@ -270,8 +279,6 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
 			
 			print("slave-id", slave_id)
 			
-			self.server.getSlave(slave_id)
-			
 			slave = self.server.updateSlave(slave_id)
 			
 			if slave: # only if slave id is valid
@@ -297,22 +304,35 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
 				self.send_head(http.client.NO_CONTENT)
 		# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 		elif self.path == "file":
-			job_id = self.headers['job-id']
-			print("file:", job_id, "\n")
+			slave_id = self.headers['slave-id']
 			
-			job = self.server.getJobByID(job_id)
+			slave = self.server.updateSlave(slave_id)
 			
-			if job:
-				self.send_head(headers={"job-id": job.id})
-				
-				self.server.stats("", "Sending file to render node")
-				f = open(self.server.path + job.id + ".blend", 'rb')
+			if slave: # only if slave id is valid
+				job_id = self.headers['job-id']
+				job_file = self.headers['job-file']
+				print("job:", job_id, "\n")
+				print("file:", job_file, "\n")
 				
-				shutil.copyfileobj(f, self.wfile)
+				job = self.server.getJobByID(job_id)
 				
-				f.close()
-			else:
-				# no such job id
+				if job:
+					render_file = job.files_map.get(job_file, None)
+					
+					if render_file:
+						self.server.stats("", "Sending file to render node")
+						f = open(render_file.path, 'rb')
+						
+						shutil.copyfileobj(f, self.wfile)
+						
+						f.close()
+					else:
+						# no such file
+						self.send_head(http.client.NO_CONTENT)
+				else:
+					# no such job id
+					self.send_head(http.client.NO_CONTENT)
+			else: # invalid slave id
 				self.send_head(http.client.NO_CONTENT)
 		# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 		elif self.path == "slave":
@@ -345,14 +365,15 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
 			
 			job_id = self.server.nextJobID()
 			
-			print("chunks", job_info.chunks)
+			print(job_info.files)
 			
 			job = MRenderJob(job_id, job_info.name, job_info.files, chunks = job_info.chunks, priority = job_info.priority, blacklist = job_info.blacklist)
-			self.server.addJob(job)
 			
 			for frame in job_info.frames:
 				frame = job.addFrame(frame.number)
-				
+			
+			self.server.addJob(job)
+			
 			headers={"job-id": job_id}
 			
 			if job.testStart():
@@ -411,86 +432,100 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
 			self.server.stats("", "Receiving job")
 			
 			length = int(self.headers['content-length'])
-			job_frame_string = self.headers['job-frame']
-			job_name = self.headers.get('job-name', "")
-			job_chunks = int(self.headers.get('job-chunks', "1"))
-			blacklist = self.headers.get('slave-blacklist', '').split()
-			
-			job_id = self.server.nextJobID()
-			
-			buf = self.rfile.read(length)
-			
-			job_path = job_id + ".blend"
+			job_id = self.headers['job-id']
+			job_file = self.headers['job-file']
 			
-			f = open(self.server.path + job_path, "wb")
-			f.write(buf)
-			f.close()
-			del buf
+			job = self.server.getJobByID(job_id)
 			
-			job = MRenderJob(job_id, job_name, job_path, chunks = job_chunks, blacklist = blacklist)
-			self.server.addJob(job)
+			if job:
 				
-			if ":" in job_frame_string:
-				frame_start, frame_end = [int(x) for x in job_frame_string.split(":")]
+				render_file = job.files_map.get(job_file, None)
 				
-				for job_frame in range(frame_start, frame_end + 1):
-					frame = job.addFrame(job_frame)
-			else:
-				job_frame = int(job_frame_string)
-				frame = job.addFrame(job_frame)
-	
-			job.start()
-			
-			self.send_head(headers={"job-id": job_id})
+				if render_file:
+					main_file = job.files[0]
+					
+					main_path, main_name = os.path.split(main_file)
+					
+					if job_file != main_file:
+						file_path = prefixPath(job.save_path, job_file, main_path)
+					else:
+						file_path = job.save_path + main_name
+					
+					buf = self.rfile.read(length)
+					
+					f = open(file_path, "wb")
+					f.write(buf)
+					f.close()
+					del buf
+					
+					render_file.path = file_path # set the new path
+					
+					if job.testStart():
+						self.send_head(headers=headers)
+					else:
+						self.send_head(http.client.ACCEPTED, headers=headers)
+				else: # invalid file
+					self.send_head(http.client.NO_CONTENT)
+			else: # job not found
+				self.send_head(http.client.NO_CONTENT)
 		# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 		elif self.path == "render":
 			print("writing result file")
 			self.server.stats("", "Receiving render result")
 			
 			job_id = self.headers['job-id']
-			job_frame = int(self.headers['job-frame'])
-			job_result = int(self.headers['job-result'])
-			job_time = float(self.headers['job-time'])
 			
-			if job_result == DONE:
-				length = int(self.headers['content-length'])
-				buf = self.rfile.read(length)
-				f = open(self.server.path + job_id + "%04d" % job_frame + ".exr", 'wb')
-				f.write(buf)
-				f.close()
-				
-				del buf
-				
 			job = self.server.getJobByID(job_id)
-			frame = job[job_frame]
-			frame.status = job_result
-			frame.time = job_time
-	
-			self.server.updateSlave(self.headers['slave-id'])
 			
-			self.send_head()
+			if job:
+				job_frame = int(self.headers['job-frame'])
+				job_result = int(self.headers['job-result'])
+				job_time = float(self.headers['job-time'])
+				
+				if job_result == DONE:
+					length = int(self.headers['content-length'])
+					buf = self.rfile.read(length)
+					f = open(job.save_path + "%04d" % job_frame + ".exr", 'wb')
+					f.write(buf)
+					f.close()
+					
+					del buf
+					
+				job = self.server.getJobByID(job_id)
+				frame = job[job_frame]
+				frame.status = job_result
+				frame.time = job_time
+		
+				self.server.updateSlave(self.headers['slave-id'])
+				
+				self.send_head()
+			else: # job not found
+				self.send_head(http.client.NO_CONTENT)
 		# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 		elif self.path == "log":
 			print("writing log file")
 			self.server.stats("", "Receiving log file")
 			
-			length = int(self.headers['content-length'])
 			job_id = self.headers['job-id']
-			job_frame = int(self.headers['job-frame'])
 			
-			print("log length:", length)
+			job = self.server.getJobByID(job_id)
 			
-			buf = self.rfile.read(length)
-			f = open(self.server.path + job_id + "%04d" % job_frame + ".log", 'wb')
-			f.write(buf)
-			f.close()
+			if job:
+				length = int(self.headers['content-length'])
+				job_frame = int(self.headers['job-frame'])
 				
-			del buf
-			
-			self.server.updateSlave(self.headers['slave-id'])
-			
-			self.send_head()
-
+				buf = self.rfile.read(length)
+				f = open(job.save_path + "%04d" % job_frame + ".log", 'wb')
+				f.write(buf)
+				f.close()
+					
+				del buf
+				
+				self.server.updateSlave(self.headers['slave-id'])
+				
+				self.send_head()
+			else: # job not found
+				self.send_head(http.client.NO_CONTENT)
 
 class RenderMasterServer(http.server.HTTPServer):
 	def __init__(self, address, handler_class, path):
@@ -500,7 +535,10 @@ class RenderMasterServer(http.server.HTTPServer):
 		self.slaves = []
 		self.slaves_map = {}
 		self.job_id = 0
-		self.path = path
+		self.path = path + "master_" + str(os.getpid()) + os.sep
+		
+		if not os.path.exists(self.path):
+			os.mkdir(self.path)
 	
 	def nextJobID(self):
 		self.job_id += 1
@@ -539,6 +577,13 @@ class RenderMasterServer(http.server.HTTPServer):
 	def addJob(self, job):
 		self.jobs.append(job)
 		self.jobs_map[job.id] = job
+		
+		# create job directory
+		job.save_path = self.path + "job_" + job.id + os.sep
+		if not os.path.exists(job.save_path):
+			os.mkdir(job.save_path)
+			
+		job.save()
 	
 	def getJobByID(self, id):
 		return self.jobs_map.get(id, None)
diff --git a/release/io/netrender/slave.py b/release/io/netrender/slave.py
index 64801743d70..3d7153a0c54 100644
--- a/release/io/netrender/slave.py
+++ b/release/io/netrender/slave.py
@@ -27,24 +27,10 @@ def testCancel(conn, job_id):
 			return False
 
 def testFile(conn, JOB_PREFIX, file_path, main_path = None):
-	if os.path.isabs(file_path):
-		# if an absolute path, make sure path exists, if it doesn't, use relative local path
-		job_full_path = file_path
-		if not os.path.exists(job_full_path):
-			p, n = os.path.split(job_full_path)
-			
-			if main_path and p.startswith(main_path):
-				directory = JOB_PREFIX + p[len(main_path):]
-				job_full_path = directory + n
-				if not os.path.exists(directory):
-					os.mkdir(directory)
-			else:
-				job_full_path = JOB_PREFIX + n
-	else:
-		job_full_path = JOB_PREFIX + file_path
+	job_full_path = prefixPath(JOB_PREFIX, file_path, main_path)
 	
 	if not os.path.exists(job_full_path):
-		conn.request("GET", "file", headers={"job-id": job.id, "slave-id":slave_id})
+		conn.request("GET", "file", headers={"job-id": job.id, "slave-id":slave_id, "job-file":file_path})
 		response = conn.getresponse()
 		
 		if response.status != http.client.OK:
@@ -76,7 +62,7 @@ def render_slave(engine, scene):
 		
 		slave_id = response.getheader("slave-id")
 		
-		NODE_PREFIX = netsettings.path + "node_" + slave_id + os.sep
+		NODE_PREFIX = netsettings.path + "slave_" + slave_id + os.sep
 		if not os.path.exists(NODE_PREFIX):
 			os.mkdir(NODE_PREFIX)
 	
diff --git a/release/io/netrender/utils.py b/release/io/netrender/utils.py
index d64b6fcda1e..30d96b2f92f 100644
--- a/release/io/netrender/utils.py
+++ b/release/io/netrender/utils.py
@@ -53,11 +53,22 @@ def clientSendJob(conn, scene, anim = False, chunks = 5):
 	filename = bpy.data.filename
 	job.files.append(filename)
 	
-	name = netsettings.job_name
-	if name == "[default]":
-		path, name = os.path.split(filename)
+	job_name = netsettings.job_name
+	path, name = os.path.split(filename)
+	if job_name == "[default]":
+		job_name = name
 	
-	job.name = name
+	for lib in bpy.data.libraries:
+		lib_path = lib.filename
+		
+		if lib_path.startswith("//"):
+			lib_path = path + os.sep + lib_path[2:]
+			
+		job.files.append(lib_path)
+	
+	print(job.files)
+	
+	job.name = job_name
 	
 	for slave in scene.network_render.slaves_blacklist:
 		job.blacklist.append(slave.id)
@@ -71,12 +82,13 @@ def clientSendJob(conn, scene, anim = False, chunks = 5):
 	
 	job_id = response.getheader("job-id")
 	
-	# if not found, send whole file
-	if response.status == http.client.NOT_FOUND:
-		f = open(bpy.data.filename, "rb")
-		conn.request("PUT", "file", f, headers={"job-id": job_id})
-		f.close()
-		response = conn.getresponse()
+	# if not ACCEPTED (but not processed), send files
+	if response.status == http.client.ACCEPTED:
+		for filepath in job.files:
+			f = open(filepath, "rb")
+			conn.request("PUT", "file", f, headers={"job-id": job_id, "job-file": filepath})
+			f.close()
+			response = conn.getresponse()
 	
 	# server will reply with NOT_FOUD until all files are found
 	
@@ -84,3 +96,23 @@ def clientSendJob(conn, scene, anim = False, chunks = 5):
 
 def clientRequestResult(conn, scene, job_id):
 	conn.request("GET", "render", headers={"job-id": job_id, "job-frame":str(scene.current_frame)})
+
+
+def prefixPath(prefix_directory, file_path, prefix_path):
+	if os.path.isabs(file_path):
+		# if an absolute path, make sure path exists, if it doesn't, use relative local path
+		full_path = file_path
+		if not os.path.exists(full_path):
+			p, n = os.path.split(full_path)
+			
+			if main_path and p.startswith(main_path):
+				directory = prefix_directory + p[len(main_path):]
+				full_path = directory + n
+				if not os.path.exists(directory):
+					os.mkdir(directory)
+			else:
+				full_path = prefix_directory + n
+	else:
+		full_path = prefix_directory + file_path
+	
+	return full_path
\ No newline at end of file