Cycles OSL: shader script node

Documentation here:
http://wiki.blender.org/index.php/Doc:2.6/Manual/Render/Cycles/Nodes/OSL
http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.65/Cycles

These changes require an OSL build from this repository:
https://github.com/DingTo/OpenShadingLanguage

The lib/ OSL has not been updated yet, so you might want to keep OSL disabled
until that is done.

Still todo:
* Auto update for external .osl files not working currently, press update manually
* Node could indicate better when a refresh is needed
* Attributes like UV or generated coordinates may be missing when requested from
  an OSL shader, need a way to request them to be loaded by cycles
* Expose string, enum and other non-socket parameters
* Scons build support

Thanks to Thomas, Lukas and Dalai for the implementation.

1 files changed, 232 insertions, 98 deletions

diff --git a/intern/cycles/render/osl.cpp b/intern/cycles/render/osl.cpp
index 4856a8d4e0c..a8a40a4e596 100644
--- a/intern/cycles/render/osl.cpp
+++ b/intern/cycles/render/osl.cpp
@@ -31,6 +31,7 @@
 #include "osl_shader.h"
 
 #include "util_foreach.h"
+#include "util_md5.h"
 #include "util_path.h"
 #include "util_progress.h"
 
@@ -46,36 +47,8 @@ OSLShaderManager::OSLShaderManager()
 {
 	services = new OSLRenderServices();
 
-	/* if we let OSL create it, it leaks */
-	ts = TextureSystem::create(true);
-	ts->attribute("automip",  1);
-	ts->attribute("autotile", 64);
-
-	ss = OSL::ShadingSystem::create(services, ts, &errhandler);
-	ss->attribute("lockgeom", 1);
-	ss->attribute("commonspace", "world");
-	ss->attribute("optimize", 2);
-	//ss->attribute("debug", 1);
-	//ss->attribute("statistics:level", 1);
-	ss->attribute("searchpath:shader", path_get("shader").c_str());
-
-	/* our own ray types */
-	static const char *raytypes[] = {
-		"camera",		/* PATH_RAY_CAMERA */
-		"reflection",	/* PATH_RAY_REFLECT */
-		"refraction",	/* PATH_RAY_TRANSMIT */
-		"diffuse",		/* PATH_RAY_DIFFUSE */
-		"glossy",		/* PATH_RAY_GLOSSY */
-		"singular",		/* PATH_RAY_SINGULAR */
-		"transparent",	/* PATH_RAY_TRANSPARENT */
-		"shadow",		/* PATH_RAY_SHADOW_OPAQUE */
-		"shadow",		/* PATH_RAY_SHADOW_TRANSPARENT */
-	};
-
-	const int nraytypes = sizeof(raytypes)/sizeof(raytypes[0]);
-	ss->attribute("raytypes", TypeDesc(TypeDesc::STRING, nraytypes), raytypes);
-
-	OSLShader::register_closures(ss);
+	shading_system_init();
+	texture_system_init();
 }
 
 OSLShaderManager::~OSLShaderManager()
@@ -87,13 +60,6 @@ OSLShaderManager::~OSLShaderManager()
 
 void OSLShaderManager::device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress& progress)
 {
-	/* test if we need to update */
-	bool need_update = false;
-
-	foreach(Shader *shader, scene->shaders)
-		if(shader->need_update)
-			need_update = true;
-	
 	if(!need_update)
 		return;
 
@@ -113,7 +79,7 @@ void OSLShaderManager::device_update(Device *device, DeviceScene *dscene, Scene
 		if(shader->sample_as_light && shader->has_surface_emission)
 			scene->light_manager->need_update = true;
 
-		OSLCompiler compiler((void*)ss);
+		OSLCompiler compiler((void*)this, (void*)ss);
 		compiler.background = (shader == scene->shaders[scene->default_background]);
 		compiler.compile(og, shader);
 	}
@@ -129,6 +95,8 @@ void OSLShaderManager::device_update(Device *device, DeviceScene *dscene, Scene
 
 	foreach(Shader *shader, scene->shaders)
 		shader->need_update = false;
+
+	need_update = false;
 	
 	/* set texture system */
 	scene->image_manager->set_osl_texture_system((void*)ts);
@@ -154,10 +122,165 @@ void OSLShaderManager::device_free(Device *device, DeviceScene *dscene)
 	og->background_state.reset();
 }
 
+void OSLShaderManager::texture_system_init()
+{
+	/* if we let OSL create it, it leaks */
+	ts = TextureSystem::create(true);
+	ts->attribute("automip",  1);
+	ts->attribute("autotile", 64);
+
+	/* effectively unlimited for now, until we support proper mipmap lookups */
+	ts->attribute("max_memory_MB", 16384);
+}
+
+void OSLShaderManager::shading_system_init()
+{
+	ss = OSL::ShadingSystem::create(services, ts, &errhandler);
+	ss->attribute("lockgeom", 1);
+	ss->attribute("commonspace", "world");
+	ss->attribute("optimize", 2);
+	//ss->attribute("debug", 1);
+	//ss->attribute("statistics:level", 1);
+	ss->attribute("searchpath:shader", path_get("shader"));
+
+	/* our own ray types */
+	static const char *raytypes[] = {
+		"camera",		/* PATH_RAY_CAMERA */
+		"reflection",	/* PATH_RAY_REFLECT */
+		"refraction",	/* PATH_RAY_TRANSMIT */
+		"diffuse",		/* PATH_RAY_DIFFUSE */
+		"glossy",		/* PATH_RAY_GLOSSY */
+		"singular",		/* PATH_RAY_SINGULAR */
+		"transparent",	/* PATH_RAY_TRANSPARENT */
+		"shadow",		/* PATH_RAY_SHADOW_OPAQUE */
+		"shadow",		/* PATH_RAY_SHADOW_TRANSPARENT */
+	};
+
+	const int nraytypes = sizeof(raytypes)/sizeof(raytypes[0]);
+	ss->attribute("raytypes", TypeDesc(TypeDesc::STRING, nraytypes), raytypes);
+
+	OSLShader::register_closures(ss);
+
+	loaded_shaders.clear();
+}
+
+bool OSLShaderManager::osl_compile(const string& inputfile, const string& outputfile)
+{
+	vector<string> options;
+	string stdosl_path;
+
+	/* specify output file name */
+	options.push_back("-o");
+	options.push_back(outputfile);
+
+	/* specify standard include path */
+	options.push_back("-I" + path_get("shader"));
+	stdosl_path = path_get("shader/stdosl.h");
+
+	/* compile */
+	OSL::OSLCompiler *compiler = OSL::OSLCompiler::create();
+	bool ok = compiler->compile(inputfile, options, stdosl_path);
+	delete compiler;
+
+	return ok;
+}
+
+bool OSLShaderManager::osl_query(OSL::OSLQuery& query, const string& filepath)
+{
+	string searchpath = path_user_get("shaders");
+	return query.open(filepath, searchpath);
+}
+
+static string shader_filepath_hash(const string& filepath, uint64_t modified_time)
+{
+	/* compute a hash from filepath and modified time to detect changes */
+	MD5Hash md5;
+	md5.append((const uint8_t*)filepath.c_str(), filepath.size());
+	md5.append((const uint8_t*)&modified_time, sizeof(modified_time));
+
+	return md5.get_hex();
+}
+
+const char *OSLShaderManager::shader_test_loaded(const string& hash)
+{
+	set<string>::iterator it = loaded_shaders.find(hash);
+	return (it == loaded_shaders.end())? NULL: it->c_str();
+}
+
+const char *OSLShaderManager::shader_load_filepath(string filepath)
+{
+	size_t len = filepath.size();
+	string extension = filepath.substr(len - 4);
+	uint64_t modified_time = path_modified_time(filepath);
+
+	if(extension == ".osl") {
+		/* .OSL File */
+		string osopath = filepath.substr(0, len - 4) + ".oso";
+		uint64_t oso_modified_time = path_modified_time(osopath);
+
+		/* test if we have loaded the corresponding .OSO already */
+		if(oso_modified_time != 0) {
+			const char *hash = shader_test_loaded(shader_filepath_hash(osopath, oso_modified_time));
+
+			if(hash)
+				return hash;
+		}
+
+		/* autocompile .OSL to .OSO if needed */
+		if(oso_modified_time == 0 || (oso_modified_time < modified_time)) {
+			OSLShaderManager::osl_compile(filepath, osopath);
+			modified_time = path_modified_time(osopath);
+		}
+		else
+			modified_time = oso_modified_time;
+
+		filepath = osopath;
+	}
+	else {
+		if(extension == ".oso") {
+			/* .OSO File, nothing to do */
+		}
+		else if(path_dirname(filepath) == "") {
+			/* .OSO File in search path */
+			filepath = path_join(path_user_get("shaders"), filepath + ".oso");
+		}
+		else {
+			/* unknown file */
+			return NULL;
+		}
+
+		/* test if we have loaded this .OSO already */
+		const char *hash = shader_test_loaded(shader_filepath_hash(filepath, modified_time));
+
+		if(hash)
+			return hash;
+	}
+
+	/* read oso bytecode from file */
+	string bytecode_hash = shader_filepath_hash(filepath, modified_time);
+	string bytecode;
+
+	if(!path_read_text(filepath, bytecode)) {
+		fprintf(stderr, "Cycles shader graph: failed to read file %s\n", filepath.c_str());
+		loaded_shaders.insert(bytecode_hash); /* to avoid repeat tries */
+		return NULL;
+	}
+
+	return shader_load_bytecode(bytecode_hash, bytecode);
+}
+
+const char *OSLShaderManager::shader_load_bytecode(const string& hash, const string& bytecode)
+{
+	ss->LoadMemoryShader(hash.c_str(), bytecode.c_str());
+
+	return loaded_shaders.insert(hash).first->c_str();
+}
+
 /* Graph Compiler */
 
-OSLCompiler::OSLCompiler(void *shadingsys_)
+OSLCompiler::OSLCompiler(void *manager_, void *shadingsys_)
 {
+	manager = manager_;
 	shadingsys = shadingsys_;
 	current_type = SHADER_TYPE_SURFACE;
 	current_shader = NULL;
@@ -238,10 +361,18 @@ bool OSLCompiler::node_skip_input(ShaderNode *node, ShaderInput *input)
 	return false;
 }
 
-void OSLCompiler::add(ShaderNode *node, const char *name)
+void OSLCompiler::add(ShaderNode *node, const char *name, bool isfilepath)
 {
 	OSL::ShadingSystem *ss = (OSL::ShadingSystem*)shadingsys;
 
+	/* load filepath */
+	if(isfilepath) {
+		name = ((OSLShaderManager*)manager)->shader_load_filepath(name);
+
+		if(name == NULL)
+			return;
+	}
+
 	/* pass in fixed parameter values */
 	foreach(ShaderInput *input, node->inputs) {
 		if(!input->link) {
@@ -510,82 +641,85 @@ void OSLCompiler::compile_type(Shader *shader, ShaderGraph *graph, ShaderType ty
 
 void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
 {
-	OSL::ShadingSystem *ss = (OSL::ShadingSystem*)shadingsys;
-	ShaderGraph *graph = shader->graph;
-	ShaderNode *output = (graph)? graph->output(): NULL;
+	if(shader->need_update) {
+		OSL::ShadingSystem *ss = (OSL::ShadingSystem*)shadingsys;
+		ShaderGraph *graph = shader->graph;
+		ShaderNode *output = (graph)? graph->output(): NULL;
 
-	/* copy graph for shader with bump mapping */
-	if(output->input("Surface")->link && output->input("Displacement")->link)
-		if(!shader->graph_bump)
-			shader->graph_bump = shader->graph->copy();
+		/* copy graph for shader with bump mapping */
+		if(output->input("Surface")->link && output->input("Displacement")->link)
+			if(!shader->graph_bump)
+				shader->graph_bump = shader->graph->copy();
 
-	/* finalize */
-	shader->graph->finalize(false, true);
-	if(shader->graph_bump)
-		shader->graph_bump->finalize(true, true);
+		/* finalize */
+		shader->graph->finalize(false, true);
+		if(shader->graph_bump)
+			shader->graph_bump->finalize(true, true);
 
-	current_shader = shader;
+		current_shader = shader;
 
-	shader->has_surface = false;
-	shader->has_surface_emission = false;
-	shader->has_surface_transparent = false;
-	shader->has_volume = false;
-	shader->has_displacement = false;
+		shader->has_surface = false;
+		shader->has_surface_emission = false;
+		shader->has_surface_transparent = false;
+		shader->has_volume = false;
+		shader->has_displacement = false;
 
-	/* generate surface shader */
-	if(shader->used && graph && output->input("Surface")->link) {
-		compile_type(shader, shader->graph, SHADER_TYPE_SURFACE);
-		og->surface_state.push_back(ss->state());
+		/* generate surface shader */
+		if(shader->used && graph && output->input("Surface")->link) {
+			compile_type(shader, shader->graph, SHADER_TYPE_SURFACE);
+			shader->osl_surface_ref = ss->state();
+
+			if(shader->graph_bump) {
+				ss->clear_state();
+				compile_type(shader, shader->graph_bump, SHADER_TYPE_SURFACE);
+			}
 
-		if(shader->graph_bump) {
+			shader->osl_surface_bump_ref = ss->state();
 			ss->clear_state();
-			compile_type(shader, shader->graph_bump, SHADER_TYPE_SURFACE);
-			og->surface_state.push_back(ss->state());
-		}
-		else
-			og->surface_state.push_back(ss->state());
 
-		ss->clear_state();
+			shader->has_surface = true;
+		}
+		else {
+			shader->osl_surface_ref = OSL::ShadingAttribStateRef();
+			shader->osl_surface_bump_ref = OSL::ShadingAttribStateRef();
+		}
 
-		shader->has_surface = true;
-	}
-	else {
-		og->surface_state.push_back(OSL::ShadingAttribStateRef());
-		og->surface_state.push_back(OSL::ShadingAttribStateRef());
-	}
+		/* generate volume shader */
+		if(shader->used && graph && output->input("Volume")->link) {
+			compile_type(shader, shader->graph, SHADER_TYPE_VOLUME);
+			shader->has_volume = true;
 
-	/* generate volume shader */
-	if(shader->used && graph && output->input("Volume")->link) {
-		compile_type(shader, shader->graph, SHADER_TYPE_VOLUME);
-		shader->has_volume = true;
+			shader->osl_volume_ref = ss->state();
+			ss->clear_state();
+		}
+		else
+			shader->osl_volume_ref = OSL::ShadingAttribStateRef();
 
-		og->volume_state.push_back(ss->state());
-		og->volume_state.push_back(ss->state());
-		ss->clear_state();
-	}
-	else {
-		og->volume_state.push_back(OSL::ShadingAttribStateRef());
-		og->volume_state.push_back(OSL::ShadingAttribStateRef());
+		/* generate displacement shader */
+		if(shader->used && graph && output->input("Displacement")->link) {
+			compile_type(shader, shader->graph, SHADER_TYPE_DISPLACEMENT);
+			shader->has_displacement = true;
+			shader->osl_displacement_ref = ss->state();
+			ss->clear_state();
+		}
+		else
+			shader->osl_displacement_ref = OSL::ShadingAttribStateRef();
 	}
 
-	/* generate displacement shader */
-	if(shader->used && graph && output->input("Displacement")->link) {
-		compile_type(shader, shader->graph, SHADER_TYPE_DISPLACEMENT);
-		shader->has_displacement = true;
+	/* push state to array for lookup */
+	og->surface_state.push_back(shader->osl_surface_ref);
+	og->surface_state.push_back(shader->osl_surface_bump_ref);
 
-		og->displacement_state.push_back(ss->state());
-		og->displacement_state.push_back(ss->state());
-		ss->clear_state();
-	}
-	else {
-		og->displacement_state.push_back(OSL::ShadingAttribStateRef());
-		og->displacement_state.push_back(OSL::ShadingAttribStateRef());
-	}
+	og->volume_state.push_back(shader->osl_volume_ref);
+	og->volume_state.push_back(shader->osl_volume_ref);
+
+	og->displacement_state.push_back(shader->osl_displacement_ref);
+	og->displacement_state.push_back(shader->osl_displacement_ref);
 }
 
 #else
 
-void OSLCompiler::add(ShaderNode *node, const char *name)
+void OSLCompiler::add(ShaderNode *node, const char *name, bool isfilepath)
 {
 }