Refactor of proxies build operators

Split proxy build operator into three parts:
- Prepare context (IMB_anim_index_rebuild_context) which prepares all
  needed data and stores it in an anonymous structure used by specific
  builder lately.
- Build proxies/timecodes into temporary files (IMB_anim_index_rebuild)
  This function will build all selected proxies/timecodes into a temporary
  files so old proxies will be still available during building.
- Finish building proxies (IMB_anim_index_rebuild_finish) which copies
  temporary files over old proxies filed and releases all resources used
  by a context.

Context creation and finishing building happens in a main thread so
it's easy and safe to close all opened handles of proxies files and
refresh cache after rebuilding is finished.

This should finally fix #30315: Temporary proxy files are not erased and old proxys are not updated if the proxy is built more then once (windows)

3 files changed, 240 insertions, 139 deletions

diff --git a/source/blender/imbuf/IMB_imbuf.h b/source/blender/imbuf/IMB_imbuf.h
index 5f38142726f..dfd53a145f6 100644
--- a/source/blender/imbuf/IMB_imbuf.h
+++ b/source/blender/imbuf/IMB_imbuf.h
@@ -217,13 +217,19 @@ void IMB_anim_set_index_dir(struct anim * anim, const char * dir);
 int IMB_anim_index_get_frame_index(struct anim * anim, IMB_Timecode_Type tc,
                                    int position);
 
+struct IndexBuildContext;
+
+/* prepare context for proxies/imecodes builder */
+struct IndexBuildContext *IMB_anim_index_rebuild_context(struct anim *anim, IMB_Timecode_Type tcs_in_use,
+                                                         IMB_Proxy_Size proxy_sizes_in_use, int quality);
+
 /* will rebuild all used indices and proxies at once */
-void IMB_anim_index_rebuild(struct anim * anim, 
-                            IMB_Timecode_Type build_tcs,
-                            IMB_Proxy_Size build_preview_sizes,
-                            int build_quality,
+void IMB_anim_index_rebuild(struct IndexBuildContext *context,
                             short *stop, short *do_update, float *progress);
 
+/* finish rebuilding proxises/timecodes and free temporary contexts used */
+void IMB_anim_index_rebuild_finish(struct IndexBuildContext *context, short stop);
+
 /**
  * Return the length (in frames) of the given @a anim.
  */
@@ -243,6 +249,7 @@ int IMB_anim_get_fps(struct anim * anim,
  */
 struct anim *IMB_open_anim(const char *name, int ib_flags, int streamindex);
 void IMB_close_anim(struct anim *anim);
+void IMB_close_anim_proxies(struct anim *anim);
 
 
 /**
diff --git a/source/blender/imbuf/intern/anim_movie.c b/source/blender/imbuf/intern/anim_movie.c
index a0903379528..8722ca8e90a 100644
--- a/source/blender/imbuf/intern/anim_movie.c
+++ b/source/blender/imbuf/intern/anim_movie.c
@@ -241,6 +241,10 @@ void IMB_close_anim(struct anim * anim)
 	IMB_free_anim(anim);
 }
 
+void IMB_close_anim_proxies(struct anim *anim)
+{
+	IMB_free_indices(anim);
+}
 
 struct anim * IMB_open_anim( const char * name, int ib_flags, int streamindex)
 {
diff --git a/source/blender/imbuf/intern/indexer.c b/source/blender/imbuf/intern/indexer.c
index ca2c4719917..cc95cc0fbbe 100644
--- a/source/blender/imbuf/intern/indexer.c
+++ b/source/blender/imbuf/intern/indexer.c
@@ -695,102 +695,112 @@ static void free_proxy_output_ffmpeg(struct proxy_output_ctx * ctx,
 	MEM_freeN(ctx);
 }
 
+typedef struct IndexBuildContext {
+	int anim_type;
+} IndexBuildContext;
 
-static int index_rebuild_ffmpeg(struct anim * anim, 
-				IMB_Timecode_Type tcs_in_use,
-				IMB_Proxy_Size proxy_sizes_in_use,
-				int quality,
-				short *stop, short *do_update, 
-				float *progress)
-{
-	int i, videoStream;
-	unsigned long long seek_pos = 0;
-	unsigned long long last_seek_pos = 0;
-	unsigned long long seek_pos_dts = 0;
-	unsigned long long seek_pos_pts = 0;
-	unsigned long long last_seek_pos_dts = 0;
-	unsigned long long start_pts = 0;
-	double frame_rate;
-	double pts_time_base;
-	int frameno = 0, frameno_gapless = 0;
-	int start_pts_set = FALSE;
+typedef struct FFmpegIndexBuilderContext {
+	int anim_type;
 
 	AVFormatContext *iFormatCtx;
 	AVCodecContext *iCodecCtx;
 	AVCodec *iCodec;
 	AVStream *iStream;
-	AVFrame* in_frame = 0;
-	AVPacket next_packet;
-	int streamcount;
+	int videoStream;
+
+	int num_proxy_sizes;
+	int num_indexers;
 
 	struct proxy_output_ctx * proxy_ctx[IMB_PROXY_MAX_SLOT];
 	anim_index_builder * indexer [IMB_TC_MAX_SLOT];
 
+	IMB_Timecode_Type tcs_in_use;
+	IMB_Proxy_Size proxy_sizes_in_use;
+} FFmpegIndexBuilderContext;
+
+typedef struct FallbackIndexBuilderContext {
+	int anim_type;
+
+	struct anim *anim;
+	AviMovie *proxy_ctx[IMB_PROXY_MAX_SLOT];
+	IMB_Proxy_Size proxy_sizes_in_use;
+} FallbackIndexBuilderContext;
+
+static IndexBuildContext *index_ffmpeg_create_context(struct anim *anim, IMB_Timecode_Type tcs_in_use,
+                                                      IMB_Proxy_Size proxy_sizes_in_use, int quality)
+{
+	FFmpegIndexBuilderContext *context = MEM_callocN(sizeof(FFmpegIndexBuilderContext), "FFmpeg index builder context");
 	int num_proxy_sizes = IMB_PROXY_MAX_SLOT;
 	int num_indexers = IMB_TC_MAX_SLOT;
-	uint64_t stream_size;
+	int i, streamcount;
 
-	memset(proxy_ctx, 0, sizeof(proxy_ctx));
-	memset(indexer, 0, sizeof(indexer));
+	context->tcs_in_use = tcs_in_use;
+	context->proxy_sizes_in_use = proxy_sizes_in_use;
+	context->num_proxy_sizes = IMB_PROXY_MAX_SLOT;
+	context->num_indexers = IMB_TC_MAX_SLOT;
 
-	if(av_open_input_file(&iFormatCtx, anim->name, NULL, 0, NULL) != 0) {
-		return 0;
+	memset(context->proxy_ctx, 0, sizeof(context->proxy_ctx));
+	memset(context->indexer, 0, sizeof(context->indexer));
+
+	if(av_open_input_file(&context->iFormatCtx, anim->name, NULL, 0, NULL) != 0) {
+		MEM_freeN(context);
+		return NULL;
 	}
 
-	if (av_find_stream_info(iFormatCtx) < 0) {
-		av_close_input_file(iFormatCtx);
-		return 0;
+	if (av_find_stream_info(context->iFormatCtx) < 0) {
+		av_close_input_file(context->iFormatCtx);
+		MEM_freeN(context);
+		return NULL;
 	}
 
 	streamcount = anim->streamindex;
 
 	/* Find the video stream */
-	videoStream = -1;
-	for (i = 0; i < iFormatCtx->nb_streams; i++)
-		if(iFormatCtx->streams[i]->codec->codec_type
+	context->videoStream = -1;
+	for (i = 0; i < context->iFormatCtx->nb_streams; i++)
+		if(context->iFormatCtx->streams[i]->codec->codec_type
 		   == AVMEDIA_TYPE_VIDEO) {
 			if (streamcount > 0) {
 				streamcount--;
 				continue;
 			}
-			videoStream = i;
+			context->videoStream = i;
 			break;
 		}
 
-	if (videoStream == -1) {
-		av_close_input_file(iFormatCtx);
-		return 0;
+	if (context->videoStream == -1) {
+		av_close_input_file(context->iFormatCtx);
+		MEM_freeN(context);
+		return NULL;
 	}
 
-	iStream = iFormatCtx->streams[videoStream];
-	iCodecCtx = iStream->codec;
-
-	iCodec = avcodec_find_decoder(iCodecCtx->codec_id);
-	
-	if (iCodec == NULL) {
-		av_close_input_file(iFormatCtx);
-		return 0;
-	}
+	context->iStream = context->iFormatCtx->streams[context->videoStream];
+	context->iCodecCtx = context->iStream->codec;
 
-	iCodecCtx->workaround_bugs = 1;
+	context->iCodec = avcodec_find_decoder(context->iCodecCtx->codec_id);
 
-	if (avcodec_open(iCodecCtx, iCodec) < 0) {
-		av_close_input_file(iFormatCtx);
-		return 0;
+	if (context->iCodec == NULL) {
+		av_close_input_file(context->iFormatCtx);
+		MEM_freeN(context);
+		return NULL;
 	}
 
-	in_frame = avcodec_alloc_frame();
+	context->iCodecCtx->workaround_bugs = 1;
 
-	stream_size = avio_size(iFormatCtx->pb);
+	if (avcodec_open(context->iCodecCtx, context->iCodec) < 0) {
+		av_close_input_file(context->iFormatCtx);
+		MEM_freeN(context);
+		return NULL;
+	}
 
 	for (i = 0; i < num_proxy_sizes; i++) {
 		if (proxy_sizes_in_use & proxy_sizes[i]) {
-			proxy_ctx[i] = alloc_proxy_output_ffmpeg(
-				anim, iStream, proxy_sizes[i],
-				iCodecCtx->width * proxy_fac[i],
-				iCodecCtx->height * proxy_fac[i],
+			context->proxy_ctx[i] = alloc_proxy_output_ffmpeg(
+				anim, context->iStream, proxy_sizes[i],
+				context->iCodecCtx->width * proxy_fac[i],
+				context->iCodecCtx->height * proxy_fac[i],
 				quality);
-			if (!proxy_ctx[i]) {
+			if (!context->proxy_ctx[i]) {
 				proxy_sizes_in_use &= ~proxy_sizes[i];
 			}
 		}
@@ -802,17 +812,61 @@ static int index_rebuild_ffmpeg(struct anim * anim,
 
 			get_tc_filename(anim, tc_types[i], fname);
 
-			indexer[i] = IMB_index_builder_create(fname);
-			if (!indexer[i]) {
+			context->indexer[i] = IMB_index_builder_create(fname);
+			if (!context->indexer[i]) {
 				tcs_in_use &= ~tc_types[i];
 			}
 		}
 	}
 
-	frame_rate = av_q2d(iStream->r_frame_rate);
-	pts_time_base = av_q2d(iStream->time_base);
+	return (IndexBuildContext *)context;
+}
+
+static void index_rebuild_ffmpeg_finish(FFmpegIndexBuilderContext *context, int stop)
+{
+	int i;
+
+	for (i = 0; i < context->num_indexers; i++) {
+		if (context->tcs_in_use & tc_types[i]) {
+			IMB_index_builder_finish(context->indexer[i], stop);
+		}
+	}
+
+	for (i = 0; i < context->num_proxy_sizes; i++) {
+		if (context->proxy_sizes_in_use & proxy_sizes[i]) {
+			free_proxy_output_ffmpeg(context->proxy_ctx[i], stop);
+		}
+	}
+
+	MEM_freeN(context);
+}
 
-	while(av_read_frame(iFormatCtx, &next_packet) >= 0) {
+static int index_rebuild_ffmpeg(FFmpegIndexBuilderContext *context,
+                                short *stop, short *do_update, float *progress)
+{
+	int i;
+	unsigned long long seek_pos = 0;
+	unsigned long long last_seek_pos = 0;
+	unsigned long long seek_pos_dts = 0;
+	unsigned long long seek_pos_pts = 0;
+	unsigned long long last_seek_pos_dts = 0;
+	unsigned long long start_pts = 0;
+	double frame_rate;
+	double pts_time_base;
+	int frameno = 0, frameno_gapless = 0;
+	int start_pts_set = FALSE;
+	AVFrame* in_frame = 0;
+	AVPacket next_packet;
+	uint64_t stream_size;
+
+	in_frame = avcodec_alloc_frame();
+
+	stream_size = avio_size(context->iFormatCtx->pb);
+
+	frame_rate = av_q2d(context->iStream->r_frame_rate);
+	pts_time_base = av_q2d(context->iStream->time_base);
+
+	while(av_read_frame(context->iFormatCtx, &next_packet) >= 0) {
 		int frame_finished = 0;
 		float next_progress =  (float)((int)floor(((double) next_packet.pos) * 100 /
 		                                   ((double) stream_size)+0.5)) / 100;
@@ -827,7 +881,7 @@ static int index_rebuild_ffmpeg(struct anim * anim,
 			break;
 		}
 
-		if (next_packet.stream_index == videoStream) {
+		if (next_packet.stream_index == context->videoStream) {
 			if (next_packet.flags & AV_PKT_FLAG_KEY) {
 				last_seek_pos = seek_pos;
 				last_seek_pos_dts = seek_pos_dts;
@@ -837,7 +891,7 @@ static int index_rebuild_ffmpeg(struct anim * anim,
 			}
 
 			avcodec_decode_video2(
-				iCodecCtx, in_frame, &frame_finished, 
+				context->iCodecCtx, in_frame, &frame_finished,
 				&next_packet);
 		}
 
@@ -845,11 +899,11 @@ static int index_rebuild_ffmpeg(struct anim * anim,
 			unsigned long long s_pos = seek_pos;
 			unsigned long long s_dts = seek_pos_dts;
 			unsigned long long pts 
-				= av_get_pts_from_frame(iFormatCtx, in_frame);
+				= av_get_pts_from_frame(context->iFormatCtx, in_frame);
 
-			for (i = 0; i < num_proxy_sizes; i++) {
+			for (i = 0; i < context->num_proxy_sizes; i++) {
 				add_to_proxy_output_ffmpeg(
-					proxy_ctx[i], in_frame);
+					context->proxy_ctx[i], in_frame);
 			}
 
 			if (!start_pts_set) {
@@ -872,15 +926,15 @@ static int index_rebuild_ffmpeg(struct anim * anim,
 				s_dts = last_seek_pos_dts;
 			}
 
-			for (i = 0; i < num_indexers; i++) {
-				if (tcs_in_use & tc_types[i]) {
+			for (i = 0; i < context->num_indexers; i++) {
+				if (context->tcs_in_use & tc_types[i]) {
 					int tc_frameno = frameno;
 
 					if(tc_types[i] == IMB_TC_RECORD_RUN_NO_GAPS)
 						tc_frameno = frameno_gapless;
 
 					IMB_index_builder_proc_frame(
-						indexer[i], 
+						context->indexer[i],
 						next_packet.data, 
 						next_packet.size,
 						tc_frameno,
@@ -893,18 +947,6 @@ static int index_rebuild_ffmpeg(struct anim * anim,
 		av_free_packet(&next_packet);
 	}
 
-	for (i = 0; i < num_indexers; i++) {
-		if (tcs_in_use & tc_types[i]) {
-			IMB_index_builder_finish(indexer[i], *stop);
-		}
-	}
-
-	for (i = 0; i < num_proxy_sizes; i++) {
-		if (proxy_sizes_in_use & proxy_sizes[i]) {
-			free_proxy_output_ffmpeg(proxy_ctx[i], *stop);
-		}
-	}
-
 	av_free(in_frame);
 
 	return 1;
@@ -955,20 +997,11 @@ static AviMovie * alloc_proxy_output_avi(
 	return avi;
 }
 
-static void index_rebuild_fallback(struct anim * anim,
-				   IMB_Timecode_Type UNUSED(tcs_in_use),
-				   IMB_Proxy_Size proxy_sizes_in_use,
-				   int quality,
-				   short *stop, short *do_update, 
-				   float *progress)
+static IndexBuildContext *index_fallback_create_context(struct anim *anim, IMB_Timecode_Type UNUSED(tcs_in_use),
+                                                        IMB_Proxy_Size proxy_sizes_in_use, int quality)
 {
-	int cnt = IMB_anim_get_duration(anim, IMB_TC_NONE);
-	int i, pos;
-	AviMovie * proxy_ctx[IMB_PROXY_MAX_SLOT];
-	char fname[FILE_MAX];
-	char fname_tmp[FILE_MAX];
-	
-	memset(proxy_ctx, 0, sizeof(proxy_ctx));
+	FallbackIndexBuilderContext *context;
+	int i;
 
 	/* since timecode indices only work with ffmpeg right now,
 	   don't know a sensible fallback here...
@@ -976,28 +1009,67 @@ static void index_rebuild_fallback(struct anim * anim,
 	   so no proxies, no game to play...
 	*/
 	if (proxy_sizes_in_use == IMB_PROXY_NONE) {
-		return;
+		return NULL;
 	}
 
+	context = MEM_callocN(sizeof(FallbackIndexBuilderContext), "fallback index builder context");
+
+	context->anim = anim;
+	context->proxy_sizes_in_use = proxy_sizes_in_use;
+
+	memset(context->proxy_ctx, 0, sizeof(context->proxy_ctx));
+
 	for (i = 0; i < IMB_PROXY_MAX_SLOT; i++) {
-		if (proxy_sizes_in_use & proxy_sizes[i]) {
+		if (context->proxy_sizes_in_use & proxy_sizes[i]) {
 			char fname[FILE_MAX];
 
 			get_proxy_filename(anim, proxy_sizes[i], fname, TRUE);
 			BLI_make_existing_file(fname);
 
-			proxy_ctx[i] = alloc_proxy_output_avi(
-				anim, fname,
-				anim->x * proxy_fac[i],
-				anim->y * proxy_fac[i],
-				quality);
+			context->proxy_ctx[i] = alloc_proxy_output_avi(anim, fname,
+				anim->x * proxy_fac[i], anim->y * proxy_fac[i], quality);
 		}
 	}
 
+	return (IndexBuildContext *)context;
+}
+
+static void index_rebuild_fallback_finish(FallbackIndexBuilderContext *context, int stop)
+{
+	struct anim *anim = context->anim;
+	char fname[FILE_MAX];
+	char fname_tmp[FILE_MAX];
+	int i;
+
+	for (i = 0; i < IMB_PROXY_MAX_SLOT; i++) {
+		if (context->proxy_sizes_in_use & proxy_sizes[i]) {
+			AVI_close_compress(context->proxy_ctx[i]);
+			MEM_freeN(context->proxy_ctx[i]);
+
+			get_proxy_filename(anim, proxy_sizes[i], fname_tmp, TRUE);
+			get_proxy_filename(anim, proxy_sizes[i], fname, FALSE);
+
+			if (stop) {
+				unlink(fname_tmp);
+			} else {
+				unlink(fname);
+				rename(fname_tmp, fname);
+			}
+		}
+	}
+}
+
+static void index_rebuild_fallback(FallbackIndexBuilderContext *context,
+                                   short *stop, short *do_update, float *progress)
+{
+	int cnt = IMB_anim_get_duration(context->anim, IMB_TC_NONE);
+	int i, pos;
+	struct anim *anim = context->anim;
+
 	for (pos = 0; pos < cnt; pos++) {
-		struct ImBuf * ibuf = IMB_anim_absolute(
-			anim, pos, IMB_TC_NONE, IMB_PROXY_NONE);
-		int next_progress = (int) ((double) pos / (double) cnt);
+		struct ImBuf *ibuf = IMB_anim_absolute(anim, pos, IMB_TC_NONE, IMB_PROXY_NONE);
+		struct ImBuf *tmp_ibuf = IMB_dupImBuf(ibuf);
+		float next_progress = (float) pos / (float) cnt;
 
 		if (*progress != next_progress) {
 			*progress = next_progress;
@@ -1008,19 +1080,20 @@ static void index_rebuild_fallback(struct anim * anim,
 			break;
 		}
 
-		IMB_flipy(ibuf);
+		IMB_flipy(tmp_ibuf);
 
 		for (i = 0; i < IMB_PROXY_MAX_SLOT; i++) {
-			if (proxy_sizes_in_use & proxy_sizes[i]) {
+			if (context->proxy_sizes_in_use & proxy_sizes[i]) {
 				int x = anim->x * proxy_fac[i];
 				int y = anim->y * proxy_fac[i];
 
-				struct ImBuf * s_ibuf = IMB_scalefastImBuf(
-					ibuf, x, y);
+				struct ImBuf * s_ibuf = IMB_dupImBuf(tmp_ibuf);
+
+				IMB_scalefastImBuf(s_ibuf, x, y);
 
 				IMB_convert_rgba_to_abgr(s_ibuf);
 	
-				AVI_write_frame (proxy_ctx[i], pos, 
+				AVI_write_frame (context->proxy_ctx[i], pos, 
 						 AVI_FORMAT_RGB32, 
 						 s_ibuf->rect, x * y * 4);
 
@@ -1030,25 +1103,9 @@ static void index_rebuild_fallback(struct anim * anim,
 				IMB_freeImBuf(s_ibuf);
 			}
 		}
-	}
-
-	for (i = 0; i < IMB_PROXY_MAX_SLOT; i++) {
-		if (proxy_sizes_in_use & proxy_sizes[i]) {
-			AVI_close_compress (proxy_ctx[i]);
-			MEM_freeN (proxy_ctx[i]);
 
-			get_proxy_filename(anim, proxy_sizes[i], 
-					   fname_tmp, TRUE);
-			get_proxy_filename(anim, proxy_sizes[i], 
-					   fname, FALSE);
-
-			if (*stop) {
-				unlink(fname_tmp);
-			} else {
-				unlink(fname);
-				rename(fname_tmp, fname);
-			}
-		}
+		IMB_freeImBuf(tmp_ibuf);
+		IMB_freeImBuf(ibuf);
 	}
 }
 
@@ -1056,25 +1113,58 @@ static void index_rebuild_fallback(struct anim * anim,
    - public API
    ---------------------------------------------------------------------- */
 
-void IMB_anim_index_rebuild(struct anim * anim, IMB_Timecode_Type tcs_in_use,
-			    IMB_Proxy_Size proxy_sizes_in_use,
-			    int quality,
-			    short *stop, short *do_update, float *progress)
+IndexBuildContext *IMB_anim_index_rebuild_context(struct anim *anim, IMB_Timecode_Type tcs_in_use,
+                                                         IMB_Proxy_Size proxy_sizes_in_use, int quality)
 {
+	IndexBuildContext *context = NULL;
+
 	switch (anim->curtype) {
 #ifdef WITH_FFMPEG
 	case ANIM_FFMPEG:
-		index_rebuild_ffmpeg(anim, tcs_in_use, proxy_sizes_in_use,
-				     quality, stop, do_update, progress);
+		context = index_ffmpeg_create_context(anim, tcs_in_use, proxy_sizes_in_use, quality);
+		break;
+#endif
+	default:
+		context = index_fallback_create_context(anim, tcs_in_use, proxy_sizes_in_use, quality);
+		break;
+	}
+
+	if (context)
+		context->anim_type = anim->curtype;
+
+	return context;
+}
+
+void IMB_anim_index_rebuild(struct IndexBuildContext *context,
+                            short *stop, short *do_update, float *progress)
+{
+	switch (context->anim_type) {
+#ifdef WITH_FFMPEG
+	case ANIM_FFMPEG:
+		index_rebuild_ffmpeg((FFmpegIndexBuilderContext*)context, stop, do_update, progress);
+		break;
+#endif
+	default:
+		index_rebuild_fallback((FallbackIndexBuilderContext*)context, stop, do_update, progress);
+		break;
+	}
+}
+
+void IMB_anim_index_rebuild_finish(IndexBuildContext *context, short stop)
+{
+	switch (context->anim_type) {
+#ifdef WITH_FFMPEG
+	case ANIM_FFMPEG:
+		index_rebuild_ffmpeg_finish((FFmpegIndexBuilderContext*)context, stop);
 		break;
 #endif
 	default:
-		index_rebuild_fallback(anim, tcs_in_use, proxy_sizes_in_use,
-				       quality, stop, do_update, progress);
+		index_rebuild_fallback_finish((FallbackIndexBuilderContext*)context, stop);
 		break;
 	}
 }
 
+
 void IMB_free_indices(struct anim * anim)
 {
 	int i;