Cycles: add option to cache BVH's between subsequent renders, storing the BVH on

disk to be reused by the next render.

This is useful for rendering animations where only the camera or materials change.
Note that saving the BVH to disk only to be removed for the next frame is slower
if this is not the case and the meshes do actually change.

For a render, it will save bvh files to the cache user directory, and remove all
cache files from other renders. The files are named using a MD5 hash based on the
mesh, to verify if the meshes are still the same.

9 files changed, 135 insertions, 28 deletions

diff --git a/intern/cycles/blender/addon/properties.py b/intern/cycles/blender/addon/properties.py
index 0b096c529b8..0a3cffd5071 100644
--- a/intern/cycles/blender/addon/properties.py
+++ b/intern/cycles/blender/addon/properties.py
@@ -103,6 +103,8 @@ class CyclesRenderSettings(bpy.types.PropertyGroup):
             items=enums.bvh_types, default="DYNAMIC_BVH")
         cls.debug_use_spatial_splits = BoolProperty(name="Use Spatial Splits", description="Use BVH spatial splits: longer builder time, faster render",
             default=False)
+        cls.use_cache = BoolProperty(name="Cache BVH", description="Cache last built BVH to disk for faster re-render if no geometry changed",
+            default=False)
 
     @classmethod
     def unregister(cls):
diff --git a/intern/cycles/blender/addon/ui.py b/intern/cycles/blender/addon/ui.py
index ea23e2b56a5..d3c06084ad3 100644
--- a/intern/cycles/blender/addon/ui.py
+++ b/intern/cycles/blender/addon/ui.py
@@ -147,6 +147,7 @@ class CyclesRender_PT_performance(CyclesButtonsPanel, Panel):
         sub.label(text="Acceleration structure:")
         sub.prop(cscene, "debug_bvh_type", text="")
         sub.prop(cscene, "debug_use_spatial_splits")
+        sub.prop(cscene, "use_cache")
 
 class CyclesRender_PT_layers(CyclesButtonsPanel, Panel):
     bl_label = "Layers"
diff --git a/intern/cycles/blender/blender_sync.cpp b/intern/cycles/blender/blender_sync.cpp
index c00320f0094..29ab0ebef1f 100644
--- a/intern/cycles/blender/blender_sync.cpp
+++ b/intern/cycles/blender/blender_sync.cpp
@@ -236,6 +236,7 @@ SceneParams BlenderSync::get_scene_params(BL::Scene b_scene, bool background)
 		params.bvh_type = (SceneParams::BVHType)RNA_enum_get(&cscene, "debug_bvh_type");
 
 	params.use_bvh_spatial_split = RNA_boolean_get(&cscene, "debug_use_spatial_splits");
+	params.use_bvh_cache = (background)? RNA_boolean_get(&cscene, "use_cache"): false;
 
 	return params;
 }
diff --git a/intern/cycles/bvh/bvh.cpp b/intern/cycles/bvh/bvh.cpp
index cd3ad709812..c9bfa964332 100644
--- a/intern/cycles/bvh/bvh.cpp
+++ b/intern/cycles/bvh/bvh.cpp
@@ -75,12 +75,18 @@ bool BVH::cache_read(CacheData& key)
 	foreach(Object *ob, objects) {
 		key.add(ob->mesh->verts);
 		key.add(ob->mesh->triangles);
+		key.add(&ob->bounds, sizeof(ob->bounds));
+		key.add(&ob->visibility, sizeof(ob->visibility));
+		key.add(&ob->mesh->transform_applied, sizeof(bool));
 	}
 
 	CacheData value;
 
 	if(Cache::global.lookup(key, value)) {
+		cache_filename = key.get_filename();
+
 		value.read(pack.root_index);
+		value.read(pack.SAH);
 
 		value.read(pack.nodes);
 		value.read(pack.object_node);
@@ -101,6 +107,7 @@ void BVH::cache_write(CacheData& key)
 	CacheData value;
 
 	value.add(pack.root_index);
+	value.add(pack.SAH);
 
 	value.add(pack.nodes);
 	value.add(pack.object_node);
@@ -111,6 +118,26 @@ void BVH::cache_write(CacheData& key)
 	value.add(pack.is_leaf);
 
 	Cache::global.insert(key, value);
+
+	cache_filename = key.get_filename();
+}
+
+void BVH::clear_cache_except()
+{
+	set<string> except;
+
+	if(!cache_filename.empty())
+		except.insert(cache_filename);
+
+	foreach(Object *ob, objects) {
+		Mesh *mesh = ob->mesh;
+		BVH *bvh = mesh->bvh;
+
+		if(bvh && !bvh->cache_filename.empty())
+			except.insert(bvh->cache_filename);
+	}
+
+	Cache::global.clear_except("bvh", except);
 }
 
 /* Building */
@@ -177,6 +204,10 @@ void BVH::build(Progress& progress)
 	if(params.use_cache) {
 		progress.set_substatus("Writing BVH cache");
 		cache_write(key);
+
+		/* clear other bvh files from cache */
+		if(params.top_level)
+			clear_cache_except();
 	}
 }
 
diff --git a/intern/cycles/bvh/bvh.h b/intern/cycles/bvh/bvh.h
index e502af72335..30ae7dac106 100644
--- a/intern/cycles/bvh/bvh.h
+++ b/intern/cycles/bvh/bvh.h
@@ -20,6 +20,7 @@
 
 #include "bvh_params.h"
 
+#include "util_string.h"
 #include "util_types.h"
 #include "util_vector.h"
 
@@ -83,6 +84,7 @@ public:
 	PackedBVH pack;
 	BVHParams params;
 	vector<Object*> objects;
+	string cache_filename;
 
 	static BVH *create(const BVHParams& params, const vector<Object*>& objects);
 	virtual ~BVH() {}
@@ -90,6 +92,8 @@ public:
 	void build(Progress& progress);
 	void refit(Progress& progress);
 
+	void clear_cache_except();
+
 protected:
 	BVH(const BVHParams& params, const vector<Object*>& objects);
 
diff --git a/intern/cycles/bvh/bvh_params.h b/intern/cycles/bvh/bvh_params.h
index b38e40cfbda..38093438500 100644
--- a/intern/cycles/bvh/bvh_params.h
+++ b/intern/cycles/bvh/bvh_params.h
@@ -26,7 +26,7 @@ class BVHParams
 {
 public:
 	/* spatial split area threshold */
-	bool use_spatial_split;
+	int use_spatial_split;
 	float spatial_split_alpha;
 
 	/* SAH costs */
@@ -38,13 +38,15 @@ public:
 	int max_leaf_size;
 
 	/* object or mesh level bvh */
-	bool top_level;
+	int top_level;
 
 	/* disk cache */
-	bool use_cache;
+	int use_cache;
 
 	/* QBVH */
-	bool use_qbvh;
+	int use_qbvh;
+
+	int pad;
 
 	/* fixed parameters */
 	enum {
@@ -67,6 +69,7 @@ public:
 		top_level = false;
 		use_cache = false;
 		use_qbvh = false;
+		pad = false;
 	}
 
 	/* SAH costs */
diff --git a/intern/cycles/render/mesh.cpp b/intern/cycles/render/mesh.cpp
index 5d65ce69a00..cd533f24058 100644
--- a/intern/cycles/render/mesh.cpp
+++ b/intern/cycles/render/mesh.cpp
@@ -586,6 +586,7 @@ void MeshManager::device_update_bvh(Device *device, DeviceScene *dscene, Scene *
 	bparams.top_level = true;
 	bparams.use_qbvh = scene->params.use_qbvh;
 	bparams.use_spatial_split = scene->params.use_bvh_spatial_split;
+	bparams.use_cache = scene->params.use_bvh_cache;
 
 	delete bvh;
 	bvh = BVH::create(bparams, scene->objects);
diff --git a/intern/cycles/util/util_cache.cpp b/intern/cycles/util/util_cache.cpp
index 49a0f62cae8..44d784ba741 100644
--- a/intern/cycles/util/util_cache.cpp
+++ b/intern/cycles/util/util_cache.cpp
@@ -19,11 +19,18 @@
 #include <stdio.h>
 
 #include "util_cache.h"
+#include "util_debug.h"
 #include "util_foreach.h"
+#include "util_map.h"
 #include "util_md5.h"
 #include "util_path.h"
 #include "util_types.h"
 
+#define BOOST_FILESYSTEM_VERSION 2
+
+#include <boost/filesystem.hpp> 
+#include <boost/algorithm/string.hpp>
+
 CCL_NAMESPACE_BEGIN
 
 /* CacheData */
@@ -32,6 +39,7 @@ CacheData::CacheData(const string& name_)
 {
 	name = name_;
 	f = NULL;
+	have_filename = false;
 }
 
 CacheData::~CacheData()
@@ -40,24 +48,35 @@ CacheData::~CacheData()
 		fclose(f);
 }
 
+const string& CacheData::get_filename()
+{
+	if(!have_filename) {
+		MD5Hash hash;
+
+		foreach(const CacheBuffer& buffer, buffers)
+			if(buffer.size)
+				hash.append((uint8_t*)buffer.data, buffer.size);
+		
+		filename = name + "_" + hash.get_hex();
+		have_filename = true;
+	}
+
+	return filename;
+}
+
 /* Cache */
 
 Cache Cache::global;
 
-string Cache::data_filename(const CacheData& key)
+string Cache::data_filename(CacheData& key)
 {
-	MD5Hash hash;
-
-	foreach(const CacheBuffer& buffer, key.buffers)
-		hash.append((uint8_t*)buffer.data, buffer.size);
-	
-	string fname = key.name + "_" + hash.get_hex();
-	return path_get("cache/" + fname);
+	return path_user_get(path_join("cache", key.get_filename()));
 }
 
-void Cache::insert(const CacheData& key, const CacheData& value)
+void Cache::insert(CacheData& key, CacheData& value)
 {
 	string filename = data_filename(key);
+	path_create_directories(filename);
 	FILE *f = fopen(filename.c_str(), "wb");
 
 	if(!f) {
@@ -65,17 +84,18 @@ void Cache::insert(const CacheData& key, const CacheData& value)
 		return;
 	}
 
-	foreach(const CacheBuffer& buffer, value.buffers) {
+	foreach(CacheBuffer& buffer, value.buffers) {
 		if(!fwrite(&buffer.size, sizeof(buffer.size), 1, f))
 			fprintf(stderr, "Failed to write to file %s.\n", filename.c_str());
-		if(!fwrite(buffer.data, buffer.size, 1, f))
-			fprintf(stderr, "Failed to write to file %s.\n", filename.c_str());
+		if(buffer.size)
+			if(!fwrite(buffer.data, buffer.size, 1, f))
+				fprintf(stderr, "Failed to write to file %s.\n", filename.c_str());
 	}
 	
 	fclose(f);
 }
 
-bool Cache::lookup(const CacheData& key, CacheData& value)
+bool Cache::lookup(CacheData& key, CacheData& value)
 {
 	string filename = data_filename(key);
 	FILE *f = fopen(filename.c_str(), "rb");
@@ -89,5 +109,22 @@ bool Cache::lookup(const CacheData& key, CacheData& value)
 	return true;
 }
 
+void Cache::clear_except(const string& name, const set<string>& except)
+{
+	string dir = path_user_get("cache");
+
+	if(boost::filesystem::exists(dir)) {
+		boost::filesystem::directory_iterator it(dir), it_end;
+
+		for(; it != it_end; it++) {
+			string filename = it->path().filename();
+
+			if(boost::starts_with(filename, name))
+				if(except.find(filename) == except.end())
+					boost::filesystem::remove(it->path());
+		}
+	}
+}
+
 CCL_NAMESPACE_END
 
diff --git a/intern/cycles/util/util_cache.h b/intern/cycles/util/util_cache.h
index 6e3c7c47e39..e8f111a5397 100644
--- a/intern/cycles/util/util_cache.h
+++ b/intern/cycles/util/util_cache.h
@@ -32,6 +32,7 @@
  * different scenes where it may be hard to detect duplicate work.
  */
 
+#include "util_set.h"
 #include "util_string.h"
 #include "util_vector.h"
 
@@ -50,25 +51,25 @@ class CacheData {
 public:
 	vector<CacheBuffer> buffers;
 	string name;
+	string filename;
+	bool have_filename;
 	FILE *f;
 
 	CacheData(const string& name = "");
 	~CacheData();
 
+	const string& get_filename();
+
 	template<typename T> void add(const vector<T>& data)
 	{
-		if(data.size()) {
-			CacheBuffer buffer(&data[0], data.size()*sizeof(T));
-			buffers.push_back(buffer);
-		}
+		CacheBuffer buffer(data.size()? &data[0]: NULL, data.size()*sizeof(T));
+		buffers.push_back(buffer);
 	}
 
 	template<typename T> void add(const array<T>& data)
 	{
-		if(data.size()) {
-			CacheBuffer buffer(&data[0], data.size()*sizeof(T));
-			buffers.push_back(buffer);
-		}
+		CacheBuffer buffer(data.size()? &data[0]: NULL, data.size()*sizeof(T));
+		buffers.push_back(buffer);
 	}
 
 	void add(void *data, size_t size)
@@ -85,6 +86,12 @@ public:
 		buffers.push_back(buffer);
 	}
 
+	void add(float& data)
+	{
+		CacheBuffer buffer(&data, sizeof(float));
+		buffers.push_back(buffer);
+	}
+
 	void add(size_t& data)
 	{
 		CacheBuffer buffer(&data, sizeof(size_t));
@@ -113,12 +120,30 @@ public:
 
 	void read(int& data)
 	{
+		size_t size;
+
+		if(!fread(&size, sizeof(size), 1, f))
+			fprintf(stderr, "Failed to read int size from cache.\n");
 		if(!fread(&data, sizeof(data), 1, f))
 			fprintf(stderr, "Failed to read int from cache.\n");
 	}
 
+	void read(float& data)
+	{
+		size_t size;
+
+		if(!fread(&size, sizeof(size), 1, f))
+			fprintf(stderr, "Failed to read float size from cache.\n");
+		if(!fread(&data, sizeof(data), 1, f))
+			fprintf(stderr, "Failed to read float from cache.\n");
+	}
+
 	void read(size_t& data)
 	{
+		size_t size;
+
+		if(!fread(&size, sizeof(size), 1, f))
+			fprintf(stderr, "Failed to read size_t size from cache.\n");
 		if(!fread(&data, sizeof(data), 1, f))
 			fprintf(stderr, "Failed to read size_t from cache.\n");
 	}
@@ -128,11 +153,13 @@ class Cache {
 public:
 	static Cache global;
 
-	void insert(const CacheData& key, const CacheData& value);
-	bool lookup(const CacheData& key, CacheData& value);
+	void insert(CacheData& key, CacheData& value);
+	bool lookup(CacheData& key, CacheData& value);
+
+	void clear_except(const string& name, const set<string>& except);
 
 protected:
-	string data_filename(const CacheData& key);
+	string data_filename(CacheData& key);
 };
 
 CCL_NAMESPACE_END