Work in progress commit on saving OpenEXR with all render-layers and

passes in single file. Code is currently disabled, commit is mainly to
have a nicer method of excluding OpenEXR dependency from render module.
This should compile with disabled WITH_OPENEXR too.

Reason why EXR is great to include by default in Blender is its feature
to store unlimited layers and channels, and write this tile based. I
need the feature for saving memory; while rendering tiles, all full-size
buffers for all layers and passes are kept in memory now, which can go
into 100s of MB easily.
The code I commit now doesn't allocate these buffers while rendering, but
saves the tiles to disk. In the end is it read back. Overhead for large
renders (like 300 meg buffers) is 10-15 seconds, not bad.

Two more interesting aspects:
- Blender can save such multi-layer files in the temp directory, storing
it with .blend file name and scene name. That way, on each restart of Blender,
or on switching scenes, these buffers can be read. So you always see what was
rendered last. Also great for compositing work.
- This can also become an output image type for rendering. There's plenty of
cases where you want specific layers or passes saved to disk for later use.

Anyhoo, finishing it is another days of work, and I got more urgent stuff
now!

3 files changed, 234 insertions, 60 deletions

diff --git a/source/blender/imbuf/intern/openexr/openexr_api.cpp b/source/blender/imbuf/intern/openexr/openexr_api.cpp
index 13ac8b14edf..8c0bbb66911 100644
--- a/source/blender/imbuf/intern/openexr/openexr_api.cpp
+++ b/source/blender/imbuf/intern/openexr/openexr_api.cpp
@@ -42,6 +42,9 @@ extern "C"
 #include "IMB_imbuf_types.h"
 #include "IMB_imbuf.h"
 #include "IMB_allocimbuf.h"
+
+#define WITH_OPENEXR
+#include "openexr_multi.h"
 }
 
 #include <iostream>
@@ -333,103 +336,213 @@ short imb_save_openexr(struct ImBuf *ibuf, char *name, int flags)
 
 /* ********************* Tile file support ************************************ */
 
-typedef struct ExrTileHandle {
-	TiledOutputFile *file;
+typedef struct ExrHandle {
+	InputFile *ifile;
+	TiledOutputFile *tofile;
+	OutputFile *ofile;
 	int tilex, tiley;
 	int width, height;
 	ListBase channels;
-} ExrTileHandle;
+} ExrHandle;
 
-typedef struct ExrTileChannel {
-	struct ExrTileChannel *next, *prev;
-	char *name;
+#define CHANMAXNAME 64
+typedef struct ExrChannel {
+	struct ExrChannel *next, *prev;
+	char name[2*CHANMAXNAME + 1];
 	int xstride, ystride;
 	float *rect;
-} ExrTileChannel;
+} ExrChannel;
 
 /* not threaded! write one tiled file at a time */
-void *imb_exrtile_get_handle(void)
+void *IMB_exr_get_handle(void)
 {
-	static ExrTileHandle data;
+	static ExrHandle data;
+	
+	memset(&data, sizeof(ExrHandle), 0);
 	
-	data.channels.first= data.channels.last= NULL;
 	return &data;
 }
 
-void imb_exrtile_add_channel(void *handle, char *channame)
+/* still clumsy name handling, layers/channels can be ordered as list in list later */
+void IMB_exr_add_channel(void *handle, const char *layname, const char *channame)
 {
-	ExrTileHandle *data= (ExrTileHandle *)handle;
-	ExrTileChannel *echan;
+	ExrHandle *data= (ExrHandle *)handle;
+	ExrChannel *echan;
 	
-	echan= (ExrTileChannel *)MEM_callocN(sizeof(ExrTileChannel), "exr tile channel");
-	echan->name= channame;
+	echan= (ExrChannel *)MEM_callocN(sizeof(ExrChannel), "exr tile channel");
+	
+	if(layname) {
+		char lay[CHANMAXNAME], chan[CHANMAXNAME];
+		strncpy(lay, layname, CHANMAXNAME-1);
+		strncpy(chan, channame, CHANMAXNAME-1);
+
+		sprintf(echan->name, "%s.%s", lay, chan);
+	}
+	else
+		strncpy(echan->name, channame, 2*CHANMAXNAME);
+	printf("added channel %s\n", echan->name);
 	BLI_addtail(&data->channels, echan);
 }
 
-void imb_exrtile_begin_write(void *handle, char *filename, int width, int height, int tilex, int tiley)
+void IMB_exr_begin_write(void *handle, char *filename, int width, int height)
 {
-	ExrTileHandle *data= (ExrTileHandle *)handle;
+	ExrHandle *data= (ExrHandle *)handle;
 	Header header (width, height);
-	ExrTileChannel *echan;
+	ExrChannel *echan;
+	
+	data->width= width;
+	data->height= height;
+	
+	for(echan= (ExrChannel *)data->channels.first; echan; echan= echan->next)
+		header.channels().insert (echan->name, Channel (FLOAT));
+	
+	header.insert ("comments", StringAttribute ("Blender MultiChannel"));
+	
+	data->ofile = new OutputFile(filename, header);
+}
+
+void IMB_exrtile_begin_write(void *handle, char *filename, int width, int height, int tilex, int tiley)
+{
+	ExrHandle *data= (ExrHandle *)handle;
+	Header header (width, height);
+	ExrChannel *echan;
 	
 	data->tilex= tilex;
 	data->tiley= tiley;
 	data->width= width;
 	data->height= height;
 	
-	for(echan= (ExrTileChannel *)data->channels.first; echan; echan= echan->next)
+	for(echan= (ExrChannel *)data->channels.first; echan; echan= echan->next)
 		header.channels().insert (echan->name, Channel (FLOAT));
 	
 	header.setTileDescription (TileDescription (tilex, tiley, ONE_LEVEL));
+	header.lineOrder() = RANDOM_Y,
+	header.compression() = NO_COMPRESSION;
 	
-	header.insert ("comments", StringAttribute ("Blender RenderResult"));
+	header.insert ("comments", StringAttribute ("Blender MultiChannel"));
 	
-	data->file = new TiledOutputFile(filename, header);
+	data->tofile = new TiledOutputFile(filename, header);
 }
 
+int IMB_exr_begin_read(void *handle, char *filename, int *width, int *height)
+{
+	ExrHandle *data= (ExrHandle *)handle;
+	
+	data->ifile = new InputFile(filename);
+	if(data->ifile) {
+		Box2i dw = data->ifile->header().dataWindow();
+		data->width= *width  = dw.max.x - dw.min.x + 1;
+		data->height= *height = dw.max.y - dw.min.y + 1;
+		
+		const ChannelList &channels = data->ifile->header().channels();
+		
+		for (ChannelList::ConstIterator i = channels.begin(); i != channels.end(); ++i)
+			IMB_exr_add_channel(data, NULL, i.name());
+		
+		return 1;
+	}
+	return 0;
+}
 
-void imb_exrtile_set_channel(void *handle, char *channame, int xstride, int ystride, float *rect)
+/* still clumsy name handling, layers/channels can be ordered as list in list later */
+void IMB_exr_set_channel(void *handle, char *layname, char *channame, int xstride, int ystride, float *rect)
 {
-	ExrTileHandle *data= (ExrTileHandle *)handle;
-	ExrTileChannel *echan;
+	ExrHandle *data= (ExrHandle *)handle;
+	ExrChannel *echan;
+	char name[2*CHANMAXNAME + 1];
 	
-	for(echan= (ExrTileChannel *)data->channels.first; echan; echan= echan->next)
-		if(strcmp(echan->name, channame)==0)
+	if(layname) {
+		char lay[CHANMAXNAME], chan[CHANMAXNAME];
+		strncpy(lay, layname, CHANMAXNAME-1);
+		strncpy(chan, channame, CHANMAXNAME-1);
+		
+		sprintf(name, "%s.%s", lay, chan);
+	}
+	else
+		strncpy(name, channame, 2*CHANMAXNAME);
+	
+	
+	for(echan= (ExrChannel *)data->channels.first; echan; echan= echan->next)
+		if(strcmp(echan->name, name)==0)
 			break;
+	
 	if(echan) {
 		echan->xstride= xstride;
 		echan->ystride= ystride;
 		echan->rect= rect;
 	}
 	else
-		printf("imb_exrtile_set_channel error\n");
+		printf("IMB_exrtile_set_channel error %s\n", name);
 }
 
 
-void imb_exrtile_write_channels(void *handle, int partx, int party)
+void IMB_exrtile_write_channels(void *handle, int partx, int party)
 {
-	ExrTileHandle *data= (ExrTileHandle *)handle;
+	ExrHandle *data= (ExrHandle *)handle;
 	FrameBuffer frameBuffer;
-	ExrTileChannel *echan;
+	ExrChannel *echan;
 	
-	for(echan= (ExrTileChannel *)data->channels.first; echan; echan= echan->next) {
+	for(echan= (ExrChannel *)data->channels.first; echan; echan= echan->next) {
 		float *rect= echan->rect - echan->xstride*partx - echan->ystride*party;
 
 		frameBuffer.insert (echan->name, Slice (FLOAT,  (char *)rect, 
 							echan->xstride*sizeof(float), echan->ystride*sizeof(float)));
 	}
 	
-	data->file->setFrameBuffer (frameBuffer);
+	data->tofile->setFrameBuffer (frameBuffer);
 	printf("write tile %d %d\n", partx/data->tilex, party/data->tiley);
-	data->file->writeTile (partx/data->tilex, party/data->tiley);	
+	data->tofile->writeTile (partx/data->tilex, party/data->tiley);	
 	
 }
 
-void imb_exrtile_close(void *handle)
+void IMB_exr_write_channels(void *handle)
 {
-	ExrTileHandle *data= (ExrTileHandle *)handle;
+	ExrHandle *data= (ExrHandle *)handle;
+	FrameBuffer frameBuffer;
+	ExrChannel *echan;
 	
-	delete data->file;
+	for(echan= (ExrChannel *)data->channels.first; echan; echan= echan->next)
+		frameBuffer.insert (echan->name, Slice (FLOAT,  (char *)echan->rect, 
+												echan->xstride*sizeof(float), echan->ystride*sizeof(float)));
+	
+	data->ofile->setFrameBuffer (frameBuffer);
+	data->ofile->writePixels (data->height);	
+	
+}
+
+void IMB_exr_read_channels(void *handle)
+{
+	ExrHandle *data= (ExrHandle *)handle;
+	FrameBuffer frameBuffer;
+	ExrChannel *echan;
+	
+	for(echan= (ExrChannel *)data->channels.first; echan; echan= echan->next) {
+		/* no datawindow correction needed */
+		if(echan->rect)
+			frameBuffer.insert (echan->name, Slice (FLOAT,  (char *)echan->rect, 
+												echan->xstride*sizeof(float), echan->ystride*sizeof(float)));
+	}
+	
+	data->ifile->setFrameBuffer (frameBuffer);
+	data->ifile->readPixels (0, data->height-1);	
+}
+
+
+void IMB_exr_close(void *handle)
+{
+	ExrHandle *data= (ExrHandle *)handle;
+	ExrChannel *echan;
+	
+	if(data->ifile)
+		delete data->ifile;
+	else if(data->ofile)
+		delete data->ofile;
+	else if(data->tofile)
+		delete data->tofile;
+	
+	data->ifile= NULL;
+	data->ofile= NULL;
+	data->tofile= NULL;
 	
 	BLI_freelistN(&data->channels);
 }
@@ -473,11 +586,9 @@ static int exr_has_zbuffer(InputFile *file)
 static int exr_is_renderresult(InputFile *file)
 {
 	const StringAttribute *comments= file->header().findTypedAttribute<StringAttribute>("comments");
-	if(comments) {
-	
-		if(comments->value() == "Blender RenderResult")
+	if(comments)
+		if(comments->value() == "Blender MultiChannel")
 			return 1;
-	}
 	return 0;
 }
 
@@ -529,7 +640,8 @@ struct ImBuf *imb_load_openexr(unsigned char *mem, int size, int flags)
 																		/* 1.0 is fill value */
 				frameBuffer.insert ("A", Slice (FLOAT,  (char *) (first+3), xstride, ystride, 1, 1, 1.0f));
 
-				if(exr_has_zbuffer(file)) {
+				if(exr_has_zbuffer(file)) 
+				{
 					float *firstz;
 					
 					addzbuffloatImBuf(ibuf);
diff --git a/source/blender/imbuf/intern/openexr/openexr_api.h b/source/blender/imbuf/intern/openexr/openexr_api.h
index a82713b6595..7572dbcb292 100644
--- a/source/blender/imbuf/intern/openexr/openexr_api.h
+++ b/source/blender/imbuf/intern/openexr/openexr_api.h
@@ -1,15 +1,12 @@
 /**
  * $Id$ 
  *
- * ***** BEGIN GPL/BL DUAL LICENSE BLOCK *****
+ * ***** BEGIN GPL LICENSE BLOCK *****
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version. The Blender
- * Foundation also sells licenses for use in proprietary software under
- * the Blender License.  See http://www.blender.org/BL/ for information
- * about this.
+ * of the License, or (at your option) any later version. 
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -20,14 +17,14 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  *
- * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
+ * The Original Code is Copyright (C) 2005 Blender Foundation
  * All rights reserved.
  *
  * The Original Code is: all of this file.
  *
- * Contributor(s): Austin Benesh.
+ * Contributor(s): Austin Benesh. Ton Roosendaal.
  *
- * ***** END GPL/BL DUAL LICENSE BLOCK *****
+ * ***** END GPL LICENSE BLOCK *****
  */
 
 #ifndef _OPENEXR_API_H
@@ -37,8 +34,8 @@
 extern "C" {
 #endif
 
-#define OPENEXR_FLOATRGB 0x1
-#define OPENEXR_ZBUF 0x2
+#define OPENEXR_FLOATRGB	0x1
+#define OPENEXR_ZBUF		0x2
   
 #include <stdio.h>
   
@@ -53,15 +50,6 @@ short	imb_save_openexr			(struct ImBuf *ibuf, char *name, int flags);
 
 struct ImBuf *imb_load_openexr		(unsigned char *mem, int size, int flags);
 
-
-void *	imb_exrtile_get_handle		(void);
-void	imb_exrtile_add_channel		(void *handle, char *channame);
-void	imb_exrtile_begin_write		(void *handle, char *filename, int width, int height, int tilex, int tiley);
-void	imb_exrtile_set_channel		(void *handle, char *channame, int xstride, int ystride, float *rect);
-void	imb_exrtile_write_channels	(void *handle, int partx, int party);
-void	imb_exrtile_close			(void *handle);
-
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/blender/imbuf/intern/openexr/openexr_multi.h b/source/blender/imbuf/intern/openexr/openexr_multi.h
new file mode 100644
index 00000000000..ff71aab5f3c
--- /dev/null
+++ b/source/blender/imbuf/intern/openexr/openexr_multi.h
@@ -0,0 +1,74 @@
+/**
+ * $Id$ 
+ *
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version. 
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The Original Code is Copyright (C) 2006 Blender Foundation
+ * All rights reserved.
+ *
+ * The Original Code is: all of this file.
+ *
+ * Contributor(s): Ton Roosendaal.
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+#ifndef _OPENEXR_MULTI_H
+#define _OPENEXR_MULTI_H
+
+/* experiment with more advanced exr api */
+
+#ifdef WITH_OPENEXR
+void *	IMB_exr_get_handle			(void);
+void	IMB_exr_add_channel			(void *handle, const char *layname, const char *channame);
+
+int		IMB_exr_begin_read			(void *handle, char *filename, int *width, int *height);
+void	IMB_exr_begin_write			(void *handle, char *filename, int width, int height);
+void	IMB_exrtile_begin_write		(void *handle, char *filename, int width, int height, int tilex, int tiley);
+
+void	IMB_exr_set_channel			(void *handle, char *layname, char *channame, int xstride, int ystride, float *rect);
+
+void	IMB_exr_read_channels		(void *handle);
+void	IMB_exr_write_channels		(void *handle);
+void	IMB_exrtile_write_channels	(void *handle, int partx, int party);
+
+void	IMB_exr_close				(void *handle);
+
+#else
+
+/* ugly... but we only use it on pipeline.c, render module, now */
+
+void *	IMB_exr_get_handle			(void) {return NULL;}
+void	IMB_exr_add_channel			(void *handle, const char *layname, const char *channame) {}
+
+int		IMB_exr_begin_read			(void *handle, char *filename, int *width, int *height) {return 0;}
+void	IMB_exr_begin_write			(void *handle, char *filename, int width, int height) {}
+void	IMB_exrtile_begin_write		(void *handle, char *filename, int width, int height, int tilex, int tiley) {}
+
+void	IMB_exr_set_channel			(void *handle, char *layname, char *channame, int xstride, int ystride, float *rect) {}
+
+void	IMB_exr_read_channels		(void *handle) {}
+void	IMB_exr_write_channels		(void *handle) {}
+void	IMB_exrtile_write_channels	(void *handle, int partx, int party) {}
+
+void	IMB_exr_close				(void *handle) {}
+
+#endif
+
+
+
+#endif /* __OPENEXR_MULTI_H */