17 files changed, 596 insertions, 19 deletions

diff --git a/intern/memutil/MEM_CacheLimiter.h b/intern/memutil/MEM_CacheLimiter.h
index 4e776929731..dec4d0b1c30 100644
--- a/intern/memutil/MEM_CacheLimiter.h
+++ b/intern/memutil/MEM_CacheLimiter.h
@@ -161,6 +161,13 @@ public:
 		delete handle;
 	}
 
+	size_t get_memory_in_use() {
+		if (getDataSize)
+			return total_size();
+		else
+			return MEM_get_memory_in_use();
+	}
+
 	void enforce_limits() {
 		size_t max = MEM_CacheLimiter_get_maximum();
 		size_t mem_in_use, cur_size;
@@ -169,12 +176,7 @@ public:
 			return;
 		}
 
-		if (getDataSize) {
-			mem_in_use = total_size();
-		}
-		else {
-			mem_in_use = MEM_get_memory_in_use();
-		}
+		mem_in_use = get_memory_in_use();
 
 		if (mem_in_use <= max) {
 			return;
diff --git a/intern/memutil/MEM_CacheLimiterC-Api.h b/intern/memutil/MEM_CacheLimiterC-Api.h
index c05c9d61ea2..7579dbdd4d1 100644
--- a/intern/memutil/MEM_CacheLimiterC-Api.h
+++ b/intern/memutil/MEM_CacheLimiterC-Api.h
@@ -49,7 +49,7 @@ typedef int (*MEM_CacheLimiter_ItemPriority_Func) (void*, int);
 
 #ifndef __MEM_CACHELIMITER_H__
 void MEM_CacheLimiter_set_maximum(size_t m);
-int MEM_CacheLimiter_get_maximum(void);
+size_t MEM_CacheLimiter_get_maximum(void);
 #endif /* __MEM_CACHELIMITER_H__ */
 
 /**
@@ -145,6 +145,8 @@ void *MEM_CacheLimiter_get(MEM_CacheLimiterHandleC *handle);
 void MEM_CacheLimiter_ItemPriority_Func_set(MEM_CacheLimiterC *This,
                                             MEM_CacheLimiter_ItemPriority_Func item_priority_func);
 
+size_t MEM_CacheLimiter_get_memory_in_use(MEM_CacheLimiterC *This);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/intern/memutil/intern/MEM_CacheLimiterC-Api.cpp b/intern/memutil/intern/MEM_CacheLimiterC-Api.cpp
index f946c970711..0e11fbed4e7 100644
--- a/intern/memutil/intern/MEM_CacheLimiterC-Api.cpp
+++ b/intern/memutil/intern/MEM_CacheLimiterC-Api.cpp
@@ -202,3 +202,8 @@ void MEM_CacheLimiter_ItemPriority_Func_set(MEM_CacheLimiterC *This,
 {
 	cast(This)->get_cache()->set_item_priority_func(item_priority_func);
 }
+
+size_t MEM_CacheLimiter_get_memory_in_use(MEM_CacheLimiterC *This)
+{
+	return cast(This)->get_cache()->get_memory_in_use();
+}
diff --git a/source/blender/blenkernel/BKE_movieclip.h b/source/blender/blenkernel/BKE_movieclip.h
index c8c94b2898b..f97b5b1f3a1 100644
--- a/source/blender/blenkernel/BKE_movieclip.h
+++ b/source/blender/blenkernel/BKE_movieclip.h
@@ -70,7 +70,11 @@ void BKE_movieclip_build_proxy_frame_for_ibuf(struct MovieClip *clip, struct ImB
 float BKE_movieclip_remap_scene_to_clip_frame(struct MovieClip *clip, float framenr);
 float BKE_movieclip_remap_clip_to_scene_frame(struct MovieClip *clip, float framenr);
 
-void BKE_movieclip_filename_for_frame(struct MovieClip *clip, int framenr, char *name);
+void BKE_movieclip_filename_for_frame(struct MovieClip *clip, struct MovieClipUser *user, char *name);
+struct ImBuf *BKE_movieclip_anim_ibuf_for_frame(struct MovieClip *clip, struct MovieClipUser *user);
+
+int BKE_movieclip_has_cached_frame(struct MovieClip *clip, struct MovieClipUser *user);
+int BKE_movieclip_put_frame_if_possible(struct MovieClip *clip, struct MovieClipUser *user, struct ImBuf *ibuf);
 
 /* cacheing flags */
 #define MOVIECLIP_CACHE_SKIP        (1 << 0)
diff --git a/source/blender/blenkernel/intern/movieclip.c b/source/blender/blenkernel/intern/movieclip.c
index 821c8fe3bda..49a64d8e478 100644
--- a/source/blender/blenkernel/intern/movieclip.c
+++ b/source/blender/blenkernel/intern/movieclip.c
@@ -457,7 +457,29 @@ static ImBuf *get_imbuf_cache(MovieClip *clip, MovieClipUser *user, int flag)
 	return NULL;
 }
 
-static void put_imbuf_cache(MovieClip *clip, MovieClipUser *user, ImBuf *ibuf, int flag)
+static int has_imbuf_cache(MovieClip *clip, MovieClipUser *user, int flag)
+{
+	if (clip->cache) {
+		MovieClipImBufCacheKey key;
+
+		key.framenr = user->framenr;
+
+		if (flag & MCLIP_USE_PROXY) {
+			key.proxy = rendersize_to_proxy(user, flag);
+			key.render_flag = user->render_flag;
+		}
+		else {
+			key.proxy = IMB_PROXY_NONE;
+			key.render_flag = 0;
+		}
+
+		return IMB_moviecache_has_frame(clip->cache->moviecache, &key);
+	}
+
+	return FALSE;
+}
+
+static bool put_imbuf_cache(MovieClip *clip, MovieClipUser *user, ImBuf *ibuf, int flag, bool destructive)
 {
 	MovieClipImBufCacheKey key;
 
@@ -489,7 +511,13 @@ static void put_imbuf_cache(MovieClip *clip, MovieClipUser *user, ImBuf *ibuf, i
 		key.render_flag = 0;
 	}
 
-	IMB_moviecache_put(clip->cache->moviecache, &key, ibuf);
+	if (destructive) {
+		IMB_moviecache_put(clip->cache->moviecache, &key, ibuf);
+		return true;
+	}
+	else {
+		return IMB_moviecache_put_if_possible(clip->cache->moviecache, &key, ibuf);
+	}
 }
 
 /*********************** common functions *************************/
@@ -814,7 +842,7 @@ static ImBuf *movieclip_get_postprocessed_ibuf(MovieClip *clip, MovieClipUser *u
 		}
 
 		if (ibuf && (cache_flag & MOVIECLIP_CACHE_SKIP) == 0)
-			put_imbuf_cache(clip, user, ibuf, flag);
+			put_imbuf_cache(clip, user, ibuf, flag, true);
 	}
 
 	if (ibuf) {
@@ -1111,6 +1139,7 @@ void BKE_movieclip_reload(MovieClip *clip)
 	free_buffers(clip);
 
 	clip->tracking.stabilization.ok = FALSE;
+	clip->prefetch_ok = FALSE;
 
 	/* update clip source */
 	detect_clip_source(clip);
@@ -1420,13 +1449,58 @@ float BKE_movieclip_remap_clip_to_scene_frame(MovieClip *clip, float framenr)
 	return framenr + (float) clip->start_frame - 1.0f;
 }
 
-void BKE_movieclip_filename_for_frame(MovieClip *clip, int framenr, char *name)
+void BKE_movieclip_filename_for_frame(MovieClip *clip, MovieClipUser *user, char *name)
 {
-	if (clip->source != MCLIP_SRC_MOVIE) {
-		get_sequence_fname(clip, framenr, name);
+	if (clip->source == MCLIP_SRC_SEQUENCE) {
+		int use_proxy;
+
+		use_proxy = (clip->flag & MCLIP_USE_PROXY) && user->render_size != MCLIP_PROXY_RENDER_SIZE_FULL;
+
+		if (use_proxy) {
+			int undistort = user->render_flag & MCLIP_PROXY_RENDER_UNDISTORT;
+			get_proxy_fname(clip, user->render_size, undistort, user->framenr, name);
+		}
+		else {
+			get_sequence_fname(clip, user->framenr, name);
+		}
 	}
 	else {
 		BLI_strncpy(name, clip->name, FILE_MAX);
 		BLI_path_abs(name, ID_BLEND_PATH(G.main, &clip->id));
 	}
 }
+
+ImBuf *BKE_movieclip_anim_ibuf_for_frame(MovieClip *clip, MovieClipUser *user)
+{
+	ImBuf *ibuf = NULL;
+
+	if (clip->source == MCLIP_SRC_MOVIE) {
+		BLI_lock_thread(LOCK_MOVIECLIP);
+		ibuf = movieclip_load_movie_file(clip, user, user->framenr, clip->flag);
+		BLI_unlock_thread(LOCK_MOVIECLIP);
+	}
+
+	return ibuf;
+}
+
+int BKE_movieclip_has_cached_frame(MovieClip *clip, MovieClipUser *user)
+{
+	int has_frame = FALSE;
+
+	BLI_lock_thread(LOCK_MOVIECLIP);
+	has_frame = has_imbuf_cache(clip, user, clip->flag);
+	BLI_unlock_thread(LOCK_MOVIECLIP);
+
+	return has_frame;
+}
+
+int BKE_movieclip_put_frame_if_possible(MovieClip *clip, MovieClipUser *user, ImBuf *ibuf)
+{
+	bool result;
+
+	BLI_lock_thread(LOCK_MOVIECLIP);
+	result = put_imbuf_cache(clip, user, ibuf, clip->flag, false);
+	BLI_unlock_thread(LOCK_MOVIECLIP);
+
+	return result;
+}
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index 1f399f10766..9d847c390ad 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -6579,6 +6579,8 @@ static void direct_link_movieclip(FileData *fd, MovieClip *clip)
 	clip->tracking.dopesheet.channels.first = clip->tracking.dopesheet.channels.last = NULL;
 	clip->tracking.dopesheet.coverage_segments.first = clip->tracking.dopesheet.coverage_segments.last = NULL;
 
+	clip->prefetch_ok = FALSE;
+
 	link_list(fd, &tracking->objects);
 	
 	for (object = tracking->objects.first; object; object = object->next) {
diff --git a/source/blender/editors/space_clip/clip_draw.c b/source/blender/editors/space_clip/clip_draw.c
index 4678351c0ac..2d3dc9127c3 100644
--- a/source/blender/editors/space_clip/clip_draw.c
+++ b/source/blender/editors/space_clip/clip_draw.c
@@ -1484,6 +1484,8 @@ void clip_draw_main(const bContext *C, SpaceClip *sc, ARegion *ar)
 	if (ibuf) {
 		draw_movieclip_buffer(C, sc, ar, ibuf, width, height, zoomx, zoomy);
 		IMB_freeImBuf(ibuf);
+
+		clip_start_prefetch_job(C);
 	}
 	else {
 		ED_region_grid_draw(ar, zoomx, zoomy);
diff --git a/source/blender/editors/space_clip/clip_editor.c b/source/blender/editors/space_clip/clip_editor.c
index e2102920298..1ee8d21624e 100644
--- a/source/blender/editors/space_clip/clip_editor.c
+++ b/source/blender/editors/space_clip/clip_editor.c
@@ -30,6 +30,15 @@
  */
 
 #include <stddef.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifndef WIN32
+#  include <unistd.h>
+#else
+#  include <io.h>
+#endif
 
 #include "MEM_guardedalloc.h"
 
@@ -37,10 +46,13 @@
 #include "DNA_object_types.h"	/* SELECT */
 
 #include "BLI_utildefines.h"
+#include "BLI_fileops.h"
 #include "BLI_math.h"
 #include "BLI_string.h"
 #include "BLI_rect.h"
+#include "BLI_threads.h"
 
+#include "BKE_global.h"
 #include "BKE_main.h"
 #include "BKE_mask.h"
 #include "BKE_movieclip.h"
@@ -724,3 +736,384 @@ void ED_space_clip_free_texture_buffer(SpaceClip *sc)
 		MEM_freeN(context);
 	}
 }
+
+/* ******** pre-fetching functions ******** */
+
+typedef struct PrefetchJob {
+	MovieClip *clip;
+	int start_frame, end_frame;
+	short render_size, render_flag;
+} PrefetchJob;
+
+typedef struct PrefetchQueue {
+	int current_frame, start_frame, end_frame;
+	short render_size, render_flag;
+
+	SpinLock spin;
+
+	short *stop;
+	short *do_update;
+	float *progress;
+} PrefetchQueue;
+
+typedef struct PrefetchThread {
+	MovieClip *clip;
+	PrefetchQueue *queue;
+} PrefetchThread;
+
+/* check whether pre-fetching is allowed */
+static bool check_prefetch_allowed(void)
+{
+	wmWindowManager *wm;
+
+	/* if there's any job started, better to leave all CPU and
+	 * HDD bandwidth to it
+	 *
+	 * also, display transform could be needed during playback,
+	 * so better to avoid prefetching in this case and reserve
+	 * all the power for display transform
+	 */
+	for (wm = G.main->wm.first; wm; wm = wm->id.next) {
+		if (WM_jobs_has_running_except(wm, WM_JOB_TYPE_CLIP_PREFETCH))
+			return false;
+
+		if (ED_screen_animation_playing(wm))
+			return false;
+	}
+
+	return true;
+}
+
+/* read file for specified frame number to the memory */
+static unsigned char *prefetch_read_file_to_memory(MovieClip *clip, int current_frame, short render_size,
+                                                   short render_flag , size_t *size_r)
+{
+	MovieClipUser user = {0};
+	char name[FILE_MAX];
+	size_t size;
+	int file;
+	unsigned char *mem;
+
+	user.framenr = current_frame;
+	user.render_size = render_size;
+	user.render_flag = render_flag;
+
+	BKE_movieclip_filename_for_frame(clip, &user, name);
+
+	file = open(name, O_BINARY | O_RDONLY, 0);
+	if (file < 0) {
+		return NULL;
+	}
+
+	size = BLI_file_descriptor_size(file);
+	if (size < 1) {
+		close(file);
+		return NULL;
+	}
+
+	mem = MEM_mallocN(size, "movieclip prefetch memory file");
+
+	if (read(file, mem, size) != size) {
+		close(file);
+		MEM_freeN(mem);
+		return NULL;
+	}
+
+	*size_r = size;
+
+	close(file);
+
+	return mem;
+}
+
+/* find first uncached frame within prefetching frame range */
+static int prefetch_find_uncached_frame(MovieClip *clip, int from_frame, int end_frame,
+                                        short render_size, short render_flag)
+{
+	int current_frame;
+
+	for (current_frame = from_frame; current_frame <= end_frame; current_frame++) {
+		MovieClipUser user = {0};
+
+		user.framenr = current_frame;
+		user.render_size = render_size;
+		user.render_flag = render_flag;
+
+		if (!BKE_movieclip_has_cached_frame(clip, &user))
+			break;
+	}
+
+	return current_frame;
+}
+
+/* get memory buffer for first uncached frame within prefetch frame range */
+static unsigned char *prefetch_thread_next_frame(PrefetchQueue *queue, MovieClip *clip,
+                                                 size_t *size_r, int *current_frame_r)
+{
+	unsigned char *mem = NULL;
+
+	BLI_spin_lock(&queue->spin);
+	if (!*queue->stop && queue->current_frame <= queue->end_frame && check_prefetch_allowed()) {
+		int current_frame;
+		current_frame = prefetch_find_uncached_frame(clip, queue->current_frame + 1, queue->end_frame,
+		                                             queue->render_size, queue->render_flag);
+
+		if (current_frame <= queue->end_frame) {
+			mem = prefetch_read_file_to_memory(clip, current_frame, queue->render_size,
+			                                   queue->render_flag, size_r);
+
+			*current_frame_r = current_frame;
+
+			queue->current_frame = current_frame;
+
+			*queue->do_update = 1;
+			*queue->progress = (float)(queue->current_frame - queue->start_frame) /
+				(queue->end_frame - queue->start_frame);
+		}
+	}
+	BLI_spin_unlock(&queue->spin);
+
+	return mem;
+}
+
+static void *do_prefetch_thread(void *data_v)
+{
+	PrefetchThread *data = (PrefetchThread *) data_v;
+	unsigned char *mem;
+	size_t size;
+	int current_frame;
+
+	while ((mem = prefetch_thread_next_frame(data->queue, data->clip, &size, &current_frame))) {
+		ImBuf *ibuf;
+		MovieClipUser user = {0};
+		int flag = IB_rect | IB_alphamode_detect;
+		int result;
+
+		user.framenr = current_frame;
+		user.render_size = data->queue->render_size;
+		user.render_flag = data->queue->render_flag;
+
+		ibuf = IMB_ibImageFromMemory(mem, size, flag, NULL, "prefetch frame");
+
+		result = BKE_movieclip_put_frame_if_possible(data->clip, &user, ibuf);
+
+		IMB_freeImBuf(ibuf);
+
+		MEM_freeN(mem);
+
+		if (!result) {
+			/* no more space in the cache, stop reading frames */
+			*data->queue->stop = 1;
+			break;
+		}
+	}
+
+	return NULL;
+}
+
+static void start_prefetch_threads(MovieClip *clip, int start_frame, int end_frame, short render_size,
+                                   short render_flag, short *stop, short *do_update, float *progress)
+{
+	ListBase threads;
+	PrefetchQueue queue;
+	PrefetchThread *handles;
+	int tot_thread = BLI_system_thread_count();
+	int i;
+
+	/* reserve one thread for the interface */
+	if (tot_thread > 1)
+		tot_thread--;
+
+	/* initialize queue */
+	BLI_spin_init(&queue.spin);
+
+	queue.current_frame = start_frame;
+	queue.start_frame = start_frame;
+	queue.end_frame = end_frame;
+	queue.render_size = render_size;
+	queue.render_flag = render_flag;
+
+	queue.stop = stop;
+	queue.do_update = do_update;
+	queue.progress = progress;
+
+	/* fill in thread handles */
+	handles = MEM_callocN(sizeof(PrefetchThread) * tot_thread, "prefetch threaded handles");
+
+	if (tot_thread > 1)
+		BLI_init_threads(&threads, do_prefetch_thread, tot_thread);
+
+	for (i = 0; i < tot_thread; i++) {
+		PrefetchThread *handle = &handles[i];
+
+		handle->clip = clip;
+		handle->queue = &queue;
+
+		if (tot_thread > 1)
+			BLI_insert_thread(&threads, handle);
+	}
+
+	/* run the threads */
+	if (tot_thread > 1)
+		BLI_end_threads(&threads);
+	else
+		do_prefetch_thread(handles);
+
+	MEM_freeN(handles);
+}
+
+static void do_prefetch_movie(MovieClip *clip, int start_frame, int end_frame, short render_size,
+                              short render_flag, short *stop, short *do_update, float *progress)
+{
+	int current_frame;
+
+	for (current_frame = start_frame; current_frame <= end_frame; current_frame++) {
+		MovieClipUser user = {0};
+		ImBuf *ibuf;
+
+		if (!check_prefetch_allowed() || *stop)
+			break;
+
+		user.framenr = current_frame;
+		user.render_size = render_size;
+		user.render_flag = render_flag;
+
+		if (!BKE_movieclip_has_cached_frame(clip, &user)) {
+			ibuf = BKE_movieclip_anim_ibuf_for_frame(clip, &user);
+
+			if (ibuf) {
+				int result;
+
+				result = BKE_movieclip_put_frame_if_possible(clip, &user, ibuf);
+
+				if (!result) {
+					/* no more space in the cache, we could stop prefetching here */
+					*stop = 1;
+				}
+
+				IMB_freeImBuf(ibuf);
+			}
+			else {
+				/* error reading frame, fair enough stop attempting further reading */
+				*stop = 1;
+			}
+		}
+
+		*do_update = 1;
+		*progress = (float)(current_frame - start_frame) / (end_frame - start_frame);
+	}
+}
+
+static void prefetch_startjob(void *pjv, short *stop, short *do_update, float *progress)
+{
+	PrefetchJob *pj = pjv;
+
+	if (pj->clip->source == MCLIP_SRC_SEQUENCE) {
+		/* read sequence files in multiple threads */
+		start_prefetch_threads(pj->clip, pj->start_frame, pj->end_frame,
+		                       pj->render_size, pj->render_flag,
+		                       stop, do_update, progress);
+	}
+	else if (pj->clip->source == MCLIP_SRC_MOVIE) {
+		/* read movie in a single thread */
+		do_prefetch_movie(pj->clip, pj->start_frame, pj->end_frame,
+		                  pj->render_size, pj->render_flag,
+		                  stop, do_update, progress);
+	}
+	else {
+		BLI_assert(!"Unknown movie clip source when prefetching frames");
+	}
+}
+
+static void prefetch_freejob(void *pjv)
+{
+	PrefetchJob *pj = pjv;
+
+	MEM_freeN(pj);
+}
+
+static int prefetch_get_final_frame(const bContext *C)
+{
+	Scene *scene = CTX_data_scene(C);
+	SpaceClip *sc = CTX_wm_space_clip(C);
+	MovieClip *clip = ED_space_clip_get_clip(sc);
+	int end_frame;
+
+	/* check whether all the frames from prefetch range are cached */
+	end_frame = min_ii(sc->user.framenr + U.prefetchframes - 1, EFRA);
+
+	if (clip->len)
+		end_frame = min_ii(end_frame, clip->len);
+
+	return end_frame;
+}
+
+/* returns true if early out is possible */
+static bool prefetch_check_early_out(const bContext *C)
+{
+	SpaceClip *sc = CTX_wm_space_clip(C);
+	MovieClip *clip = ED_space_clip_get_clip(sc);
+	int first_uncached_frame, end_frame;
+	int clip_len;
+
+	if (clip->prefetch_ok)
+		return true;
+
+	/* prefetch is disabled in user preferences */
+	if (U.prefetchframes == 0)
+		return true;
+
+	clip_len = BKE_movieclip_get_duration(clip);
+
+	/* check whether all the frames from prefetch range are cached */
+	end_frame = prefetch_get_final_frame(C);
+
+	first_uncached_frame =
+		prefetch_find_uncached_frame(clip, sc->user.framenr, end_frame,
+		                             sc->user.render_size, sc->user.render_flag);
+
+	if (first_uncached_frame > end_frame || first_uncached_frame == clip_len)
+		return true;
+
+	return false;
+}
+
+void clip_start_prefetch_job(const bContext *C)
+{
+	wmJob *wm_job;
+	PrefetchJob *pj;
+	SpaceClip *sc = CTX_wm_space_clip(C);
+	MovieClip *clip = ED_space_clip_get_clip(sc);
+
+	if (prefetch_check_early_out(C))
+		return;
+
+	wm_job = WM_jobs_get(CTX_wm_manager(C), CTX_wm_window(C), CTX_wm_area(C), "Prefetching",
+	                     WM_JOB_PROGRESS, WM_JOB_TYPE_CLIP_PREFETCH);
+
+	if (WM_jobs_is_running(wm_job)) {
+		/* if job is already running, it'll call clip editor redraw when
+		 * it's finished, so cache line is nicely updated
+		 * this will also trigger call of this function, which will ensure
+		 * all needed frames are prefetched
+		 */
+		return;
+	}
+
+	clip->prefetch_ok = true;
+
+	/* create new job */
+	pj = MEM_callocN(sizeof(PrefetchJob), "prefetch job");
+	pj->clip = ED_space_clip_get_clip(sc);
+	pj->start_frame = sc->user.framenr;
+	pj->end_frame = prefetch_get_final_frame(C);
+	pj->render_size = sc->user.render_size;
+	pj->render_flag = sc->user.render_flag;
+
+	WM_jobs_customdata_set(wm_job, pj, prefetch_freejob);
+	WM_jobs_timer(wm_job, 0.2, NC_MOVIECLIP, 0);
+	WM_jobs_callbacks(wm_job, prefetch_startjob, NULL, NULL, NULL);
+
+	/* and finally start the job */
+	WM_jobs_start(CTX_wm_manager(C), wm_job);
+}
diff --git a/source/blender/editors/space_clip/clip_intern.h b/source/blender/editors/space_clip/clip_intern.h
index cd7da4229d2..99222ec63ac 100644
--- a/source/blender/editors/space_clip/clip_intern.h
+++ b/source/blender/editors/space_clip/clip_intern.h
@@ -74,6 +74,9 @@ void clip_draw_main(const struct bContext *C, struct SpaceClip *sc, struct ARegi
 void clip_draw_grease_pencil(struct bContext *C, int onlyv2d);
 void clip_draw_curfra_label(const int framenr, const float x, const float y);
 
+/* clip_editor.c */
+void clip_start_prefetch_job(const struct bContext *C);
+
 /* clip_graph_draw.c */
 void clip_draw_graph(struct SpaceClip *sc, struct ARegion *ar, struct Scene *scene);
 
diff --git a/source/blender/editors/space_clip/clip_ops.c b/source/blender/editors/space_clip/clip_ops.c
index 914eb9526a8..8e03691e64f 100644
--- a/source/blender/editors/space_clip/clip_ops.c
+++ b/source/blender/editors/space_clip/clip_ops.c
@@ -1064,11 +1064,14 @@ static unsigned char *proxy_thread_next_frame(ProxyQueue *queue, MovieClip *clip
 
 	BLI_spin_lock(&queue->spin);
 	if (!*queue->stop && queue->cfra <= queue->efra) {
+		MovieClipUser user = {0};
 		char name[FILE_MAX];
 		size_t size;
 		int file;
 
-		BKE_movieclip_filename_for_frame(clip, queue->cfra, name);
+		user.framenr = queue->cfra;
+
+		BKE_movieclip_filename_for_frame(clip, &user, name);
 
 		file = open(name, O_BINARY | O_RDONLY, 0);
 		if (file < 0) {
diff --git a/source/blender/editors/space_clip/space_clip.c b/source/blender/editors/space_clip/space_clip.c
index 4f9757a6640..ced19020034 100644
--- a/source/blender/editors/space_clip/space_clip.c
+++ b/source/blender/editors/space_clip/space_clip.c
@@ -236,6 +236,16 @@ static void clip_stabilization_tag_refresh(ScrArea *sa)
 	}
 }
 
+static void clip_prefetch_tag_refresh(ScrArea *sa)
+{
+	SpaceClip *sc = (SpaceClip *) sa->spacedata.first;
+	MovieClip *clip = ED_space_clip_get_clip(sc);
+
+	if (clip) {
+		clip->prefetch_ok = FALSE;
+	}
+}
+
 /* ******************** default callbacks for clip space ***************** */
 
 static SpaceLink *clip_new(const bContext *C)
@@ -351,6 +361,7 @@ static void clip_listener(ScrArea *sa, wmNotifier *wmn)
 			switch (wmn->data) {
 				case ND_FRAME:
 					clip_scopes_tag_refresh(sa);
+					clip_prefetch_tag_refresh(sa);
 					/* no break! */
 
 				case ND_FRAME_RANGE:
@@ -359,11 +370,19 @@ static void clip_listener(ScrArea *sa, wmNotifier *wmn)
 			}
 			break;
 		case NC_MOVIECLIP:
+			if (wmn->data == 0 && wmn->action == 0) {
+				/* a nit funky, happens from prefetch job to update
+				 * cache line and job progress
+				 */
+				ED_area_tag_redraw(sa);
+			}
+
 			switch (wmn->data) {
 				case ND_DISPLAY:
 				case ND_SELECT:
 					clip_scopes_tag_refresh(sa);
 					ED_area_tag_redraw(sa);
+					clip_prefetch_tag_refresh(sa);
 					break;
 			}
 			switch (wmn->action) {
@@ -407,6 +426,7 @@ static void clip_listener(ScrArea *sa, wmNotifier *wmn)
 		case NC_SCREEN:
 			switch (wmn->data) {
 				case ND_ANIMPLAY:
+					clip_prefetch_tag_refresh(sa);
 					ED_area_tag_redraw(sa);
 					break;
 			}
@@ -415,6 +435,7 @@ static void clip_listener(ScrArea *sa, wmNotifier *wmn)
 			if (wmn->data == ND_SPACE_CLIP) {
 				clip_scopes_tag_refresh(sa);
 				clip_stabilization_tag_refresh(sa);
+				clip_prefetch_tag_refresh(sa);
 				ED_area_tag_redraw(sa);
 			}
 			break;
@@ -424,6 +445,10 @@ static void clip_listener(ScrArea *sa, wmNotifier *wmn)
 				ED_area_tag_redraw(sa);
 			}
 			break;
+		case NC_WM:
+			if (wmn->data == ND_FILEREAD)
+				clip_prefetch_tag_refresh(sa);
+			break;
 	}
 }
 
diff --git a/source/blender/imbuf/IMB_moviecache.h b/source/blender/imbuf/IMB_moviecache.h
index 4588c2bcee5..1c569712968 100644
--- a/source/blender/imbuf/IMB_moviecache.h
+++ b/source/blender/imbuf/IMB_moviecache.h
@@ -58,7 +58,9 @@ void IMB_moviecache_set_priority_callback(struct MovieCache *cache, MovieCacheGe
                                           MovieCachePriorityDeleterFP prioritydeleterfp);
 
 void IMB_moviecache_put(struct MovieCache *cache, void *userkey, struct ImBuf *ibuf);
+int IMB_moviecache_put_if_possible(struct MovieCache *cache, void *userkey, struct ImBuf *ibuf);
 struct ImBuf *IMB_moviecache_get(struct MovieCache *cache, void *userkey);
+int IMB_moviecache_has_frame(struct MovieCache *cache, void *userkey);
 void IMB_moviecache_free(struct MovieCache *cache);
 
 void IMB_moviecache_cleanup(struct MovieCache *cache, int (cleanup_check_cb) (void *userkey, void *userdata), void *userdata);
diff --git a/source/blender/imbuf/intern/moviecache.c b/source/blender/imbuf/intern/moviecache.c
index a168c9c3051..94fe1f44d91 100644
--- a/source/blender/imbuf/intern/moviecache.c
+++ b/source/blender/imbuf/intern/moviecache.c
@@ -306,7 +306,7 @@ void IMB_moviecache_set_priority_callback(struct MovieCache *cache, MovieCacheGe
 	cache->prioritydeleterfp = prioritydeleterfp;
 }
 
-void IMB_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
+static void do_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf, int need_lock)
 {
 	MovieCacheKey *key;
 	MovieCacheItem *item;
@@ -341,7 +341,8 @@ void IMB_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
 		memcpy(cache->last_userkey, userkey, cache->keysize);
 	}
 
-	BLI_mutex_lock(&limitor_lock);
+	if (need_lock)
+		BLI_mutex_lock(&limitor_lock);
 
 	item->c_handle = MEM_CacheLimiter_insert(limitor, item);
 
@@ -349,7 +350,8 @@ void IMB_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
 	MEM_CacheLimiter_enforce_limits(limitor);
 	MEM_CacheLimiter_unref(item->c_handle);
 
-	BLI_mutex_unlock(&limitor_lock);
+	if (need_lock)
+		BLI_mutex_unlock(&limitor_lock);
 
 	/* cache limiter can't remove unused keys which points to destoryed values */
 	check_unused_keys(cache);
@@ -360,6 +362,32 @@ void IMB_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
 	}
 }
 
+void IMB_moviecache_put(MovieCache *cache, void *userkey, ImBuf *ibuf)
+{
+	do_moviecache_put(cache, userkey, ibuf, TRUE);
+}
+
+int IMB_moviecache_put_if_possible(MovieCache *cache, void *userkey, ImBuf *ibuf)
+{
+	size_t mem_in_use, mem_limit, elem_size;
+	int result = FALSE;
+
+	elem_size = IMB_get_size_in_memory(ibuf);
+	mem_limit = MEM_CacheLimiter_get_maximum();
+
+	BLI_mutex_lock(&limitor_lock);
+	mem_in_use = MEM_CacheLimiter_get_memory_in_use(limitor);
+
+	if (mem_in_use + elem_size <= mem_limit) {
+		do_moviecache_put(cache, userkey, ibuf, FALSE);
+		result = TRUE;
+	}
+
+	BLI_mutex_unlock(&limitor_lock);
+
+	return result;
+}
+
 ImBuf *IMB_moviecache_get(MovieCache *cache, void *userkey)
 {
 	MovieCacheKey key;
@@ -384,6 +412,18 @@ ImBuf *IMB_moviecache_get(MovieCache *cache, void *userkey)
 	return NULL;
 }
 
+int IMB_moviecache_has_frame(MovieCache *cache, void *userkey)
+{
+	MovieCacheKey key;
+	MovieCacheItem *item;
+
+	key.cache_owner = cache;
+	key.userkey = userkey;
+	item = (MovieCacheItem *)BLI_ghash_lookup(cache->hash, &key);
+
+	return item != NULL;
+}
+
 void IMB_moviecache_free(MovieCache *cache)
 {
 	PRINT("%s: cache '%s' free\n", __func__, cache->name);
diff --git a/source/blender/makesdna/DNA_movieclip_types.h b/source/blender/makesdna/DNA_movieclip_types.h
index 499f1c50155..119d2cdfdf7 100644
--- a/source/blender/makesdna/DNA_movieclip_types.h
+++ b/source/blender/makesdna/DNA_movieclip_types.h
@@ -98,6 +98,11 @@ typedef struct MovieClip {
 
 	/* color management */
 	ColorManagedColorspaceSettings colorspace_settings;
+
+	/* runtime prefetching stuff */
+	char prefetch_ok;
+
+	char pad[7];
 } MovieClip;
 
 typedef struct MovieClipScopes {
diff --git a/source/blender/makesrna/intern/rna_userdef.c b/source/blender/makesrna/intern/rna_userdef.c
index 0023f164edc..58685229b44 100644
--- a/source/blender/makesrna/intern/rna_userdef.c
+++ b/source/blender/makesrna/intern/rna_userdef.c
@@ -3426,7 +3426,7 @@ static void rna_def_userdef_system(BlenderRNA *brna)
 	prop = RNA_def_property(srna, "prefetch_frames", PROP_INT, PROP_NONE);
 	RNA_def_property_int_sdna(prop, NULL, "prefetchframes");
 	RNA_def_property_range(prop, 0, 500);
-	RNA_def_property_ui_text(prop, "Prefetch Frames", "Number of frames to render ahead during playback (sequencer only)");
+	RNA_def_property_ui_text(prop, "Prefetch Frames", "Number of frames to render ahead during playback");
 
 	prop = RNA_def_property(srna, "memory_cache_limit", PROP_INT, PROP_NONE);
 	RNA_def_property_int_sdna(prop, NULL, "memcachelimit");
diff --git a/source/blender/windowmanager/WM_api.h b/source/blender/windowmanager/WM_api.h
index 4a9c82f62ab..2fd80d17bb7 100644
--- a/source/blender/windowmanager/WM_api.h
+++ b/source/blender/windowmanager/WM_api.h
@@ -356,6 +356,7 @@ enum {
 	WM_JOB_TYPE_CLIP_BUILD_PROXY,
 	WM_JOB_TYPE_CLIP_TRACK_MARKERS,
 	WM_JOB_TYPE_CLIP_SOLVE_CAMERA,
+	WM_JOB_TYPE_CLIP_PREFETCH,
 	WM_JOB_TYPE_SEQ_BUILD_PROXY,
 	/* add as needed, screencast, seq proxy build
 	 * if having hard coded values is a problem */
@@ -385,6 +386,7 @@ void		WM_jobs_kill_all_except(struct wmWindowManager *wm, void *owner);
 void		WM_jobs_kill_type(struct wmWindowManager *wm, int job_type);
 
 int			WM_jobs_has_running(struct wmWindowManager *wm);
+int			WM_jobs_has_running_except(struct wmWindowManager *wm, int job_type);
 
 			/* clipboard */
 char       *WM_clipboard_text_get(int selection);
diff --git a/source/blender/windowmanager/intern/wm_jobs.c b/source/blender/windowmanager/intern/wm_jobs.c
index c637b77738e..e5963840261 100644
--- a/source/blender/windowmanager/intern/wm_jobs.c
+++ b/source/blender/windowmanager/intern/wm_jobs.c
@@ -591,3 +591,16 @@ int WM_jobs_has_running(wmWindowManager *wm)
 
 	return FALSE;
 }
+
+int WM_jobs_has_running_except(wmWindowManager *wm, int job_type)
+{
+	wmJob *wm_job;
+
+	for (wm_job = wm->jobs.first; wm_job; wm_job = wm_job->next) {
+		if (wm_job->running && wm_job->job_type != job_type) {
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}