overlay: Use size_t as correct type for memory addresses

Use correct types for handling pointer size/memory addresses.

Make use of boost optional as GetFnOffsetInModules return type. The
unsigned return type does not allow for negative error values anymore,
which we did not make use of anyway, and optional is more explicit.

Replaces workaround/error detection from the commits
114495e59f002531ac9bb02070a1bcbd6652669e
a3e7958f1605339560679cbbd3a27de4fd12066c

Fixes #1924

5 files changed, 46 insertions, 74 deletions

diff --git a/overlay/d3d10.cpp b/overlay/d3d10.cpp
index 31b3041f5..0d7b71aaf 100644
--- a/overlay/d3d10.cpp
+++ b/overlay/d3d10.cpp
@@ -716,13 +716,13 @@ void PrepareDXGI10(IDXGIAdapter1 *pAdapter, bool initializeDXGIData) {
 			void ***vtbl = (void ***) pSwapChain;
 
 			void *pPresent = (*vtbl)[8];
-			int offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D10", "Present");
-			if (offset >= 0) {
+			boost::optional<size_t> offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D10", "Present");
+			if (offset) {
 				if (initializeDXGIData) {
-					dxgi->offsetPresent = offset;
+					dxgi->offsetPresent = *offset;
 					ods("D3D10: Successfully found Present offset: %ls: %d", dxgi->wcFileName, dxgi->offsetPresent);
 				} else {
-					if (dxgi->offsetPresent == offset) {
+					if (dxgi->offsetPresent == *offset) {
 						ods("D3D10: Successfully verified Present offset: %ls: %d", dxgi->wcFileName, dxgi->offsetPresent);
 					} else {
 						ods("D3D10: Failed to verify Present offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->offsetPresent);
@@ -732,12 +732,12 @@ void PrepareDXGI10(IDXGIAdapter1 *pAdapter, bool initializeDXGIData) {
 
 			void *pResize = (*vtbl)[13];
 			offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pResize), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D10", "ResizeBuffers");
-			if (offset >= 0) {
+			if (offset) {
 				if (initializeDXGIData) {
-					dxgi->offsetResize = offset;
+					dxgi->offsetResize = *offset;
 					ods("D3D10: Successfully found ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->offsetResize);
 				} else {
-					if (dxgi->offsetResize == offset) {
+					if (dxgi->offsetResize == *offset) {
 						ods("D3D10: Successfully verified ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->offsetResize);
 					} else {
 						ods("D3D10: Failed to verify ResizeBuffers offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->offsetResize);
@@ -749,15 +749,15 @@ void PrepareDXGI10(IDXGIAdapter1 *pAdapter, bool initializeDXGIData) {
 
 			void *pAddRef = (*vtbl)[1];
 			offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pAddRef), d3d10->wcFileName, ARRAY_NUM_ELEMENTS(d3d10->wcFileName), "D3D10", "AddRef");
-			if (offset >= 0) {
-				d3d10->offsetAddRef = offset;
+			if (offset) {
+				d3d10->offsetAddRef = *offset;
 				ods("D3D10: Successfully found AddRef offset: %ls: %d", d3d10->wcFileName, d3d10->offsetAddRef);
 			}
 
 			void *pRelease = (*vtbl)[2];
 			offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pRelease), d3d10->wcFileName, ARRAY_NUM_ELEMENTS(d3d10->wcFileName), "D3D10", "Release");
-			if (offset >= 0) {
-				d3d10->offsetRelease = offset;
+			if (offset) {
+				d3d10->offsetRelease = *offset;
 				ods("D3D10: Successfully found Release offset: %ls: %d", d3d10->wcFileName, d3d10->offsetRelease);
 			}
 		}
diff --git a/overlay/d3d11.cpp b/overlay/d3d11.cpp
index 37ca0c115..038027c61 100644
--- a/overlay/d3d11.cpp
+++ b/overlay/d3d11.cpp
@@ -729,13 +729,13 @@ void PrepareDXGI11(IDXGIAdapter1* pAdapter, bool initializeDXGIData) {
 				void ***vtbl = (void ***) pSwapChain;
 
 				void *pPresent = (*vtbl)[8];
-				int offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D11", "Present");
-				if (offset >= 0) {
+				boost::optional<size_t> offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D11", "Present");
+				if (offset) {
 					if (initializeDXGIData) {
-						dxgi->offsetPresent = offset;
+						dxgi->offsetPresent = *offset;
 						ods("D3D11: Successfully found Present offset: %ls: %d", dxgi->wcFileName, dxgi->offsetPresent);
 					} else {
-						if (dxgi->offsetPresent == offset) {
+						if (dxgi->offsetPresent == *offset) {
 							ods("D3D11: Successfully verified Present offset: %ls: %d", dxgi->wcFileName, dxgi->offsetPresent);
 						} else {
 							ods("D3D11: Failed to verify Present offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->offsetPresent);
@@ -745,12 +745,12 @@ void PrepareDXGI11(IDXGIAdapter1* pAdapter, bool initializeDXGIData) {
 
 				void *pResize = (*vtbl)[13];
 				offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pResize), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D11", "ResizeBuffers");
-				if (offset >= 0) {
+				if (offset) {
 					if (initializeDXGIData) {
-						dxgi->offsetResize = offset;
+						dxgi->offsetResize = *offset;
 						ods("D3D11: Successfully found ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->offsetResize);
 					} else {
-						if (dxgi->offsetResize == offset) {
+						if (dxgi->offsetResize == *offset) {
 							ods("D3D11: Successfully verified ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->offsetResize);
 						} else {
 							ods("D3D11: Failed to verify ResizeBuffers offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->offsetResize);
@@ -762,15 +762,15 @@ void PrepareDXGI11(IDXGIAdapter1* pAdapter, bool initializeDXGIData) {
 
 				void *pAddRef = (*vtbl)[1];
 				offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pAddRef), d3d11->wcFileName, ARRAY_NUM_ELEMENTS(d3d11->wcFileName), "D3D11", "AddRef");
-				if (offset >= 0) {
-					d3d11->offsetAddRef = offset;
+				if (offset) {
+					d3d11->offsetAddRef = *offset;
 					ods("D3D11: Successfully found AddRef offset: %ls: %d", d3d11->wcFileName, d3d11->offsetAddRef);
 				}
 
 				void *pRelease = (*vtbl)[2];
 				offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pRelease), d3d11->wcFileName, ARRAY_NUM_ELEMENTS(d3d11->wcFileName), "D3D11", "Release");
-				if (offset >= 0) {
-					d3d11->offsetRelease = offset;
+				if (offset) {
+					d3d11->offsetRelease = *offset;
 					ods("D3D11: Successfully found Release offset: %ls: %d", d3d11->wcFileName, d3d11->offsetRelease);
 				}
 			}
diff --git a/overlay/d3d9.cpp b/overlay/d3d9.cpp
index b563dbe6e..eaac9d15e 100644
--- a/overlay/d3d9.cpp
+++ b/overlay/d3d9.cpp
@@ -7,8 +7,6 @@
 #include <d3d9.h>
 #include <time.h>
 
-#undef max // for std::numeric_limits<T>::max()
-
 Direct3D9Data *d3dd = NULL;
 
 typedef IDirect3D9* (WINAPI *pDirect3DCreate9)(UINT SDKVersion) ;
@@ -1090,19 +1088,10 @@ extern "C" __declspec(dllexport) void __cdecl PrepareD3D9() {
 					if (!IsFnInModule(reinterpret_cast<voidFunc>(pCreate), d3dd->wcFileName, "D3D9", "CreateDevice")) {
 						ods("D3D9: CreateDevice is not in D3D9 library");
 					} else {
-						unsigned char *fn = reinterpret_cast<unsigned char *>(pCreate);
-						unsigned char *base = reinterpret_cast<unsigned char *>(hD3D);
-						unsigned long off = static_cast<unsigned long>(fn - base);
-
-						// XXX: convert the offset to use something other than int.
-						// Issue mumble-voip/mumble#1924.
-						if (off > static_cast<unsigned long>(std::numeric_limits<int>::max())) {
-							ods("D3D9: Internal overlay error: CreateDevice offset is > 2GB, does not fit the current data structure.");
-						} else {
-							d3dd->offsetCreate = static_cast<int>(off);
-							ods("D3D9: Successfully found prepatch offset: %p %p %p: %d", hD3D, d3dcreate9, pCreate, d3dd->offsetCreate);
-						}
-					}
+						size_t fn = reinterpret_cast<size_t>(pCreate);
+						size_t base = reinterpret_cast<size_t>(hD3D);
+						d3dd->offsetCreate = fn - base;
+						ods("D3D9: Successfully found prepatch offset: %p %p %p: %d", hD3D, d3dcreate9, pCreate, d3dd->offsetCreate);					}
 					id3d9->Release();
 				}
 			}
@@ -1125,18 +1114,10 @@ extern "C" __declspec(dllexport) void __cdecl PrepareD3D9() {
 						if (!IsFnInModule(reinterpret_cast<voidFunc>(pCreateEx), d3dd->wcFileName, "D3D9", "CreateDeviceEx")) {
 							ods("D3D9: CreateDeviceEx is not in D3D9 library");
 						} else {
-							unsigned char *fn = reinterpret_cast<unsigned char *>(pCreateEx);
-							unsigned char *base = reinterpret_cast<unsigned char *>(hD3D);
-							unsigned long off = static_cast<unsigned long>(fn - base);
-
-							// XXX: convert the offset to use something other than int.
-							// Issue mumble-voip/mumble#1924.
-							if (off > static_cast<unsigned long>(std::numeric_limits<int>::max())) {
-								ods("D3D9: Internal overlay error: CreateDeviceEx offset is > 2GB, does not fit the current data structure.");
-							} else {
-								d3dd->offsetCreateEx = static_cast<int>(off);
-								ods("D3D9: Successfully found prepatch ex offset: %p %p %p: %d", hD3D, d3dcreate9ex, pCreateEx, d3dd->offsetCreateEx);
-							}
+							size_t fn = reinterpret_cast<size_t>(pCreateEx);
+							size_t base = reinterpret_cast<size_t>(hD3D);
+							d3dd->offsetCreateEx = fn - base;
+							ods("D3D9: Successfully found prepatch ex offset: %p %p %p: %d", hD3D, d3dcreate9ex, pCreateEx, d3dd->offsetCreateEx);
 						}
 
 						id3d9->Release();
diff --git a/overlay/lib.cpp b/overlay/lib.cpp
index a8900ece1..1304ff701 100644
--- a/overlay/lib.cpp
+++ b/overlay/lib.cpp
@@ -8,8 +8,6 @@
 
 #include "overlay_exe/overlay_exe.h"
 
-#undef max // for std::numeric_limits<T>::max()
-
 static HANDLE hMapObject = NULL;
 static HANDLE hHookMutex = NULL;
 static HHOOK hhookWnd = 0;
@@ -706,13 +704,13 @@ bool IsFnInModule(voidFunc fnptr, wchar_t *refmodulepath, const std::string &log
 	return false;
 }
 
-int GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int refmodulepathLen, const std::string &logPrefix, const std::string &fnName) {
+boost::optional<size_t> GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int refmodulepathLen, const std::string &logPrefix, const std::string &fnName) {
 
 	HMODULE hModule = NULL;
 
 	if (! GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCTSTR>(fnptr), &hModule)) {
 		ods((logPrefix + ": Failed to get module for " + fnName).c_str());
-		return -1;
+		return boost::none;
 	}
 
 	const bool bInit = refmodulepath[0] == '\0';
@@ -723,20 +721,12 @@ int GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int ref
 		GetModuleFileNameW(hModule, modulename, ARRAY_NUM_ELEMENTS(modulename));
 		if (_wcsicmp(modulename, refmodulepath) != 0) {
 			ods((logPrefix + ": " + fnName + " functions module path does not match previously found. Now: '%ls', Previously: '%ls'").c_str(), modulename, refmodulepath);
-			return -2;
+			return boost::none;
 		}
 	}
 
-	unsigned char *fn = reinterpret_cast<unsigned char *>(fnptr);
-	unsigned char *base = reinterpret_cast<unsigned char *>(hModule);
-	unsigned long off = static_cast<unsigned long>(fn - base);
-
-	// XXX: convert this function to use something other than int.
-	// Issue mumble-voip/mumble#1924.
-	if (off > static_cast<unsigned long>(std::numeric_limits<int>::max())) {
-		ods("Internal overlay error: GetFnOffsetInModule() offset greater than return type can hold.");
-		return -1;
-	}
+	size_t fn = reinterpret_cast<size_t>(fnptr);
+	size_t base = reinterpret_cast<size_t>(hModule);
 
-	return static_cast<int>(off);
+	return fn - base;
 }
diff --git a/overlay/lib.h b/overlay/lib.h
index a6584e917..fbb82088a 100644
--- a/overlay/lib.h
+++ b/overlay/lib.h
@@ -19,6 +19,7 @@
 #include <map>
 #include <vector>
 #include <string>
+#include <boost/optional.hpp>
 #include "overlay.h"
 #include "HardHook.h"
 #include "ods.h"
@@ -40,29 +41,29 @@ const int PROCNAMEFILEPATH_EXTENDED_BUFFER_BUFLEN = PROCNAMEFILEPATH_BUFLEN + PR
 struct Direct3D9Data {
 	/// Filepath of the module the offsets are for.
 	wchar_t wcFileName[MODULEFILEPATH_BUFLEN];
-	int offsetCreate;
-	int offsetCreateEx;
+	size_t offsetCreate;
+	size_t offsetCreateEx;
 };
 
 struct DXGIData {
 	/// Filepath of the module the offsets are for.
 	wchar_t wcFileName[MODULEFILEPATH_BUFLEN];
-	int offsetPresent;
-	int offsetResize;
+	size_t offsetPresent;
+	size_t offsetResize;
 };
 
 struct D3D10Data {
 	/// Filepath of the module the offsets are for.
 	wchar_t wcFileName[MODULEFILEPATH_BUFLEN];
-	int offsetAddRef;
-	int offsetRelease;
+	size_t offsetAddRef;
+	size_t offsetRelease;
 };
 
 struct D3D11Data {
 	/// Filepath of the module the offsets are for.
 	wchar_t wcFileName[MODULEFILEPATH_BUFLEN];
-	int offsetAddRef;
-	int offsetRelease;
+	size_t offsetAddRef;
+	size_t offsetRelease;
 };
 
 struct SharedData {
@@ -141,6 +142,6 @@ extern bool IsFnInModule(voidFunc fnptr, wchar_t *refmodulepath, const std::stri
 /// Checks fnptr is in a loaded module with module path refmodulepath.
 ///
 /// @return Offset as int or < 0 on failure.
-extern int GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int refmodulepathLen, const std::string &logPrefix, const std::string &fnName);
+extern boost::optional<size_t> GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int refmodulepathLen, const std::string &logPrefix, const std::string &fnName);
 
 #endif