Prevent overlay from disappearing on D3D DLL reloading.

This is confirmed to fix the missing overlay in Guild Wars 2.

When D3D was used by a process thus loading the DLL, but then unloaded we did not inject on subsequent loading of the D3D DLL (in the same exe). Guild Wars 2 did un- and reload the DLL when switching from their launcher to the game.
This commit introduces HardHook::reset() and checks for inactive hardhooks for a newly loaded DLL.

The changes were submitted on sourceforge
https://sourceforge.net/p/mumble/bugs/909/#bf9e

4 files changed, 33 insertions, 1 deletions

diff --git a/overlay/HardHook.cpp b/overlay/HardHook.cpp
index e6e59c016..65b849dc2 100644
--- a/overlay/HardHook.cpp
+++ b/overlay/HardHook.cpp
@@ -282,6 +282,10 @@ void HardHook::setupInterface(IUnknown *unkn, LONG funcoffset, voidFunc replacem
 	setup(reinterpret_cast<voidFunc>(ptr[funcoffset]), replacement);
 }
 
+void HardHook::reset() {
+	baseptr = 0;
+}
+
 /**
  * @brief Injects redirection code into the target function.
  *
diff --git a/overlay/HardHook.h b/overlay/HardHook.h
index 19b1937d9..8e472f76d 100644
--- a/overlay/HardHook.h
+++ b/overlay/HardHook.h
@@ -67,6 +67,7 @@ struct HardHook {
 	void *cloneCode(void **orig);
 	void setup(voidFunc func, voidFunc replacement);
 	void setupInterface(IUnknown *intf, LONG funcoffset, voidFunc replacement);
+	void reset();
 	void inject(bool force = false);
 	void restore(bool force = false);
 	void print();
diff --git a/overlay/d3d9.cpp b/overlay/d3d9.cpp
index 72edfb9eb..b8ea153e6 100644
--- a/overlay/d3d9.cpp
+++ b/overlay/d3d9.cpp
@@ -734,6 +734,19 @@ void checkD3D9Hook(bool preonly) {
 				bHooked = false;
 			}
 		}
+	} else {
+		if (bHooked) {
+			fods("D3D9: DLL has been unloaded");
+			hhCreateDevice.reset();
+			hhCreateDeviceEx.reset();
+			hhReset.reset();
+			hhAddRef.reset();
+			hhRelease.reset();
+			hhPresent.reset();
+			hhPresentEx.reset();
+			hhSwapPresent.reset();
+			bHooked = false;
+		}
 	}
 
 	bChaining = false;
diff --git a/overlay/lib.cpp b/overlay/lib.cpp
index 1ec581e71..fb323d277 100644
--- a/overlay/lib.cpp
+++ b/overlay/lib.cpp
@@ -407,8 +407,15 @@ static HMODULE WINAPI MyLoadLibrary(const char *lpFileName) {
 	LoadLibraryAType oLoadLibrary = (LoadLibraryAType) hhLoad.call;
 	hhLoad.restore();
 
+	if (! bBlackListed) {
+		// Before loading DLL, check if existing hooks were lost
+		checkD3D9Hook();
+		checkDXGIHook();
+		checkOpenGLHook();
+	}
+
 	HMODULE h = oLoadLibrary(lpFileName);
-//	ods("Library %s loaded to %p", lpFileName, h);
+	ods("Library %s loaded to %p", lpFileName, h);
 
 	if (! bBlackListed) {
 		checkD3D9Hook();
@@ -425,6 +432,13 @@ static HMODULE WINAPI MyLoadLibraryW(const wchar_t *lpFileName) {
 	LoadLibraryWType oLoadLibrary = (LoadLibraryWType) hhLoadW.call;
 	hhLoadW.restore();
 
+	if (! bBlackListed) {
+		// Before loading DLL, check if existing hooks were lost
+		checkD3D9Hook();
+		checkDXGIHook();
+		checkOpenGLHook();
+	}
+
 	HMODULE h = oLoadLibrary(lpFileName);
 	ods("Library %ls wloaded to %p", lpFileName, h);