threads: Fix the shared global state with TLS

See `global.c` for a description of what we're doing.

When libgit2 is built with GIT_THREADS support, the threading system
must be explicitly initialized with `git_threads_init()`.

8 files changed, 233 insertions, 63 deletions

diff --git a/src/common.h b/src/common.h
index f7a41d47e..727a08e77 100644
--- a/src/common.h
+++ b/src/common.h
@@ -8,7 +8,6 @@
 #define INCLUDE_common_h__
 
 #include "git2/common.h"
-#include "git2/thread-utils.h"
 #include "cc-compat.h"
 
 #include <assert.h>
diff --git a/src/errors.c b/src/errors.c
index 18afff3b5..81770e786 100644
--- a/src/errors.c
+++ b/src/errors.c
@@ -5,13 +5,9 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 #include "common.h"
-#include "git2/thread-utils.h" /* for GIT_TLS */
-#include "thread-utils.h" /* for GIT_TLS */
-
+#include "global.h"
 #include <stdarg.h>
 
-static GIT_TLS char g_last_error[1024];
-
 static struct {
 	int num;
 	const char *str;
@@ -59,19 +55,26 @@ const char *git_strerror(int num)
 	return "Unknown error";
 }
 
+#define ERROR_MAX_LEN 1024
+
 void git___rethrow(const char *msg, ...)
 {
-	char new_error[1024];
+	char new_error[ERROR_MAX_LEN];
+	char *last_error;
 	char *old_error = NULL;
 
 	va_list va;
 
+	last_error = GIT_GLOBAL->error.last;
+
 	va_start(va, msg);
-	vsnprintf(new_error, sizeof(new_error), msg, va);
+	vsnprintf(new_error, ERROR_MAX_LEN, msg, va);
 	va_end(va);
 
-	old_error = git__strdup(g_last_error);
-	snprintf(g_last_error, sizeof(g_last_error), "%s \n	- %s", new_error, old_error);
+	old_error = git__strdup(last_error);
+
+	snprintf(last_error, ERROR_MAX_LEN, "%s \n	- %s", new_error, old_error);
+
 	git__free(old_error);
 }
 
@@ -80,19 +83,22 @@ void git___throw(const char *msg, ...)
 	va_list va;
 
 	va_start(va, msg);
-	vsnprintf(g_last_error, sizeof(g_last_error), msg, va);
+	vsnprintf(GIT_GLOBAL->error.last, ERROR_MAX_LEN, msg, va);
 	va_end(va);
 }
 
 const char *git_lasterror(void)
 {
-	if (!g_last_error[0])
+	char *last_error = GIT_GLOBAL->error.last;
+
+	if (!last_error[0])
 		return NULL;
 
-	return g_last_error;
+	return last_error;
 }
 
 void git_clearerror(void)
 {
-	g_last_error[0] = '\0';
+	char *last_error = GIT_GLOBAL->error.last;
+	last_error[0] = '\0';
 }
diff --git a/src/global.c b/src/global.c
new file mode 100644
index 000000000..8ef286ef0
--- /dev/null
+++ b/src/global.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2009-2011 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "global.h"
+#include "git2/threads.h" 
+#include "thread-utils.h"
+
+/**
+ * Handle the global state with TLS
+ *
+ * If libgit2 is built with GIT_THREADS enabled,
+ * the `git_threads_init()` function must be called
+ * before calling any other function of the library.
+ *
+ * This function allocates a TLS index (using pthreads
+ * or the native Win32 API) to store the global state
+ * on a per-thread basis.
+ *
+ * Any internal method that requires global state will
+ * then call `git__global_state()` which returns a pointer
+ * to the global state structure; this pointer is lazily
+ * allocated on each thread.
+ *
+ * Before shutting down the library, the
+ * `git_threads_shutdown` method must be called to free
+ * the previously reserved TLS index.
+ *
+ * If libgit2 is built without threading support, the
+ * `git__global_statestate()` call returns a pointer to a single,
+ * statically allocated global state. The `git_thread_`
+ * functions are not available in that case.
+ */
+
+#if defined(GIT_THREADS) && defined(GIT_WIN32)
+
+static DWORD _tls_index;
+static int _tls_init = 0;
+
+void git_threads_init(void)
+{
+	if (_tls_init)
+		return;
+
+	_tls_index = TlsAlloc();
+	_tls_init = 1;
+}
+
+void git_threads_shutdown(void)
+{
+	TlsFree(_tls_index);
+	_tls_init = 0;
+}
+
+git_global_st *git__global_state(void)
+{
+	void *ptr;
+
+	if ((ptr = TlsGetValue(_tls_index)) != NULL)
+		return ptr;
+
+	ptr = malloc(sizeof(git_global_st));
+	if (!ptr)
+		return NULL;
+
+	memset(ptr, 0x0, sizeof(git_global_st));
+	TlsSetValue(_tls_index, ptr);
+	return ptr;
+}
+
+#elif defined(GIT_THREADS) && defined(_POSIX_THREADS)
+
+static pthread_key_t _tls_key;
+static int _tls_init = 0;
+
+static void cb__free_status(void *st)
+{
+	free(st);
+}
+
+void git_threads_init(void)
+{
+	if (_tls_init)
+		return;
+
+	pthread_key_create(&_tls_key, &cb__free_status);
+	_tls_init = 1;
+}
+
+void git_threads_shutdown(void)
+{
+	pthread_key_delete(_tls_key);
+	_tls_init = 0;
+}
+
+git_global_st *git__global_state(void)
+{
+	void *ptr;
+
+	if ((ptr = pthread_getspecific(_tls_key)) != NULL)
+		return ptr;
+
+	ptr = malloc(sizeof(git_global_st));
+	if (!ptr)
+		return NULL;
+
+	memset(ptr, 0x0, sizeof(git_global_st));
+	pthread_setspecific(_tls_key, ptr);
+	return ptr;
+}
+
+#else
+
+static git_global_st __state;
+
+void git_threads_init(void)
+{
+	/* noop */ 
+}
+
+void git_threads_shutdown(void)
+{
+	/* noop */
+}
+
+git_global_st *git__global_state(void)
+{
+	return &__state;
+}
+
+#endif /* GIT_THREADS */
diff --git a/src/global.h b/src/global.h
new file mode 100644
index 000000000..641f47cbc
--- /dev/null
+++ b/src/global.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2009-2011 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_global_h__
+#define INCLUDE_global_h__
+
+#include "mwindow.h"
+
+typedef struct {
+	struct {
+		char last[1024];
+	} error;
+
+	git_mwindow_ctl mem_ctl;
+} git_global_st;
+
+git_global_st *git__global_state(void);
+
+#define GIT_GLOBAL (git__global_state())
+
+#endif
diff --git a/src/mwindow.c b/src/mwindow.c
index 126268fd9..8dc4573b4 100644
--- a/src/mwindow.c
+++ b/src/mwindow.c
@@ -10,6 +10,7 @@
 #include "vector.h"
 #include "fileops.h"
 #include "map.h"
+#include "global.h"
 
 #define DEFAULT_WINDOW_SIZE \
 	(sizeof(void*) >= 8 \
@@ -20,21 +21,15 @@
 	((1024 * 1024) * (sizeof(void*) >= 8 ? 8192ULL : 256UL))
 
 /*
- * We need this because each process is only allowed a specific amount
- * of memory. Making it writable should generate one instance per
- * process, but we still need to set a couple of variables.
+ * These are the global options for mmmap limits.
+ * TODO: allow the user to change these
  */
-
-static git_mwindow_ctl ctl = {
-	0,
-	0,
+static struct {
+	size_t window_size;
+	size_t mapped_limit;
+} _mw_options = {
 	DEFAULT_WINDOW_SIZE,
 	DEFAULT_MAPPED_LIMIT,
-	0,
-	0,
-	0,
-	0,
-	{0, 0, 0, 0, 0}
 };
 
 /*
@@ -43,28 +38,29 @@ static git_mwindow_ctl ctl = {
  */
 void git_mwindow_free_all(git_mwindow_file *mwf)
 {
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
 	unsigned int i;
 	/*
 	 * Remove these windows from the global list
 	 */
-	for (i = 0; i < ctl.windowfiles.length; ++i){
-		if (git_vector_get(&ctl.windowfiles, i) == mwf) {
-			git_vector_remove(&ctl.windowfiles, i);
+	for (i = 0; i < ctl->windowfiles.length; ++i){
+		if (git_vector_get(&ctl->windowfiles, i) == mwf) {
+			git_vector_remove(&ctl->windowfiles, i);
 			break;
 		}
 	}
 
-	if (ctl.windowfiles.length == 0) {
-		git_vector_free(&ctl.windowfiles);
-		ctl.windowfiles.contents = NULL;
+	if (ctl->windowfiles.length == 0) {
+		git_vector_free(&ctl->windowfiles);
+		ctl->windowfiles.contents = NULL;
 	}
 
 	while (mwf->windows) {
 		git_mwindow *w = mwf->windows;
 		assert(w->inuse_cnt == 0);
 
-		ctl.mapped -= w->window_map.len;
-		ctl.open_windows--;
+		ctl->mapped -= w->window_map.len;
+		ctl->open_windows--;
 
 		git_futils_mmap_free(&w->window_map);
 
@@ -115,6 +111,7 @@ void git_mwindow_scan_lru(
  */
 static int git_mwindow_close_lru(git_mwindow_file *mwf)
 {
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
 	unsigned int i;
 	git_mwindow *lru_w = NULL, *lru_l = NULL, **list = &mwf->windows;
 
@@ -122,16 +119,16 @@ static int git_mwindow_close_lru(git_mwindow_file *mwf)
 	if(mwf->windows)
 		git_mwindow_scan_lru(mwf, &lru_w, &lru_l);
 
-	for (i = 0; i < ctl.windowfiles.length; ++i) {
+	for (i = 0; i < ctl->windowfiles.length; ++i) {
 		git_mwindow *last = lru_w;
-		git_mwindow_file *cur = git_vector_get(&ctl.windowfiles, i);
+		git_mwindow_file *cur = git_vector_get(&ctl->windowfiles, i);
 		git_mwindow_scan_lru(cur, &lru_w, &lru_l);
 		if (lru_w != last)
 			list = &cur->windows;
 	}
 
 	if (lru_w) {
-		ctl.mapped -= lru_w->window_map.len;
+		ctl->mapped -= lru_w->window_map.len;
 		git_futils_mmap_free(&lru_w->window_map);
 
 		if (lru_l)
@@ -140,7 +137,7 @@ static int git_mwindow_close_lru(git_mwindow_file *mwf)
 			*list = lru_w->next;
 
 		git__free(lru_w);
-		ctl.open_windows--;
+		ctl->open_windows--;
 
 		return GIT_SUCCESS;
 	}
@@ -148,9 +145,14 @@ static int git_mwindow_close_lru(git_mwindow_file *mwf)
 	return git__throw(GIT_ERROR, "Failed to close memory window. Couln't find LRU");
 }
 
-static git_mwindow *new_window(git_mwindow_file *mwf, git_file fd, git_off_t size, git_off_t offset)
+static git_mwindow *new_window(
+	git_mwindow_file *mwf,
+	git_file fd,
+	git_off_t size,
+	git_off_t offset)
 {
-	size_t walign = ctl.window_size / 2;
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
+	size_t walign = _mw_options.window_size / 2;
 	git_off_t len;
 	git_mwindow *w;
 
@@ -162,16 +164,16 @@ static git_mwindow *new_window(git_mwindow_file *mwf, git_file fd, git_off_t siz
 	w->offset = (offset / walign) * walign;
 
 	len = size - w->offset;
-	if (len > (git_off_t)ctl.window_size)
-		len = (git_off_t)ctl.window_size;
+	if (len > (git_off_t)_mw_options.window_size)
+		len = (git_off_t)_mw_options.window_size;
 
-	ctl.mapped += (size_t)len;
+	ctl->mapped += (size_t)len;
 
-	while(ctl.mapped_limit < ctl.mapped &&
-			git_mwindow_close_lru(mwf) == GIT_SUCCESS) {}
+	while (_mw_options.mapped_limit < ctl->mapped &&
+			git_mwindow_close_lru(mwf) == GIT_SUCCESS) /* nop */;
 
 	/*
-	 * We treat ctl.mapped_limit as a soft limit. If we can't find a
+	 * We treat _mw_options.mapped_limit as a soft limit. If we can't find a
 	 * window to close and are above the limit, we still mmap the new
 	 * window.
 	 */
@@ -179,14 +181,14 @@ static git_mwindow *new_window(git_mwindow_file *mwf, git_file fd, git_off_t siz
 	if (git_futils_mmap_ro(&w->window_map, fd, w->offset, (size_t)len) < GIT_SUCCESS)
 		goto cleanup;
 
-	ctl.mmap_calls++;
-	ctl.open_windows++;
+	ctl->mmap_calls++;
+	ctl->open_windows++;
 
-	if (ctl.mapped > ctl.peak_mapped)
-		ctl.peak_mapped = ctl.mapped;
+	if (ctl->mapped > ctl->peak_mapped)
+		ctl->peak_mapped = ctl->mapped;
 
-	if (ctl.open_windows > ctl.peak_open_windows)
-		ctl.peak_open_windows = ctl.open_windows;
+	if (ctl->open_windows > ctl->peak_open_windows)
+		ctl->peak_open_windows = ctl->open_windows;
 
 	return w;
 
@@ -199,9 +201,14 @@ cleanup:
  * Open a new window, closing the least recenty used until we have
  * enough space. Don't forget to add it to your list
  */
-unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor,
-								git_off_t offset, int extra, unsigned int *left)
+unsigned char *git_mwindow_open(
+	git_mwindow_file *mwf,
+	git_mwindow **cursor,
+	git_off_t offset,
+	int extra,
+	unsigned int *left)
 {
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
 	git_mwindow *w = *cursor;
 
 	if (!w || !git_mwindow_contains(w, offset + extra)) {
@@ -229,7 +236,7 @@ unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor,
 
 	/* If we changed w, store it in the cursor */
 	if (w != *cursor) {
-		w->last_used = ctl.used_ctr++;
+		w->last_used = ctl->used_ctr++;
 		w->inuse_cnt++;
 		*cursor = w;
 	}
@@ -245,13 +252,14 @@ unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor,
 
 int git_mwindow_file_register(git_mwindow_file *mwf)
 {
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
 	int error;
 
-	if (ctl.windowfiles.length == 0 &&
-		(error = git_vector_init(&ctl.windowfiles, 8, NULL)) < GIT_SUCCESS)
+	if (ctl->windowfiles.length == 0 &&
+		(error = git_vector_init(&ctl->windowfiles, 8, NULL)) < GIT_SUCCESS)
 		return error;
 
-	return git_vector_insert(&ctl.windowfiles, mwf);
+	return git_vector_insert(&ctl->windowfiles, mwf);
 }
 
 void git_mwindow_close(git_mwindow **window)
diff --git a/src/mwindow.h b/src/mwindow.h
index ec75f901f..11c3aa840 100644
--- a/src/mwindow.h
+++ b/src/mwindow.h
@@ -10,7 +10,6 @@
 
 #include "map.h"
 #include "vector.h"
-#include "fileops.h"
 
 typedef struct git_mwindow {
 	struct git_mwindow *next;
@@ -29,8 +28,6 @@ typedef struct git_mwindow_file {
 typedef struct git_mwindow_ctl {
 	size_t mapped;
 	unsigned int open_windows;
-	size_t window_size; /* needs default value */
-	size_t mapped_limit; /* needs default value */
 	unsigned int mmap_calls;
 	unsigned int peak_open_windows;
 	size_t peak_mapped;
diff --git a/src/pack.c b/src/pack.c
index 429bb5e0f..ae954b988 100644
--- a/src/pack.c
+++ b/src/pack.c
@@ -5,11 +5,13 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
-#include "mwindow.h"
+#include "common.h"
 #include "odb.h"
 #include "pack.h"
 #include "delta-apply.h"
 #include "sha1_lookup.h"
+#include "mwindow.h"
+#include "fileops.h"
 
 #include "git2/oid.h"
 #include "git2/zlib.h"
diff --git a/src/thread-utils.h b/src/thread-utils.h
index 3361ed8bc..c5554799c 100644
--- a/src/thread-utils.h
+++ b/src/thread-utils.h
@@ -7,7 +7,7 @@
 #ifndef INCLUDE_thread_utils_h__
 #define INCLUDE_thread_utils_h__
 
-
+#include "common.h"
 
 /* Common operations even if threading has been disabled */
 typedef struct {