plugins: detect Win32 process architecture by reading NT header(s)

This is better than our current method because:
1. We don't need an extra handle (IsWow64Process() requires PROCESS_QUERY_INFORMATION or PROCESS_QUERY_LIMITED_INFORMATION).
2. It's more reliable: our current method consists in checking the result of IsWow64Process() (true if the process is 32 bit on 64 bit Windows) and the size of void * (4 bytes if 32 bit, 8 bytes if 64 bit).
It works correctly with a 32 bit plugin on 32 bit Windows and with a 64 bit plugin on 64 bit Windows; it doesn't work correctly when the plugin is 32 bit on 64 bit Windows: 64 bit processes are detected as 32 bit.
3. We can use the same method on Linux (the NT image is loaded in memory by Wine).

2 files changed, 79 insertions, 51 deletions

diff --git a/plugins/mumble_plugin_main.h b/plugins/mumble_plugin_main.h
index dc770b55a..831d576cf 100644
--- a/plugins/mumble_plugin_main.h
+++ b/plugins/mumble_plugin_main.h
@@ -15,6 +15,7 @@
 #define MUMBLE_PLUGIN_MAIN_H_
 
 #include "mumble_plugin.h"
+#include "mumble_plugin_win32_internals.h"
 
 static bool is64Bit;
 static procid_t pPid;
@@ -32,6 +33,17 @@ static inline bool peekProc(const procptr_t &addr, T &dest) {
 	return peekProc(addr, &dest, sizeof(T));
 }
 
+template<class T>
+static inline T peekProc(const procptr_t &addr) {
+	T ret;
+
+	if (!peekProc(addr, &ret, sizeof(T))) {
+		memset(&ret, 0, sizeof(ret));
+	}
+
+	return ret;
+}
+
 static inline procptr_t peekProcPtr(const procptr_t &addr) {
 	procptr_t v = 0;
 
@@ -42,6 +54,37 @@ static inline procptr_t peekProcPtr(const procptr_t &addr) {
 	return v;
 }
 
+// This function returns:
+// -1 in case of failure.
+// 0 if the process is 32-bit.
+// 1 if the process is 64-bit.
+#ifdef WIN32
+static inline int8_t isProcess64Bit(const procptr_t &baseAddress) {
+#else
+// We use a different name because the function is called by the Linux version
+// of isProcess64Bit() in case the process is running through Wine.
+static inline int8_t isWin32Process64Bit(const procptr_t &baseAddress) {
+#endif
+	const auto dos = peekProc<ImageDosHeader>(baseAddress);
+	if (!(dos.magic[0] == 'M' && dos.magic[1] == 'Z')) {
+		// Invalid DOS signature
+		return -1;
+	}
+
+	const auto nt = peekProc<ImageNtHeadersNoOptional>(baseAddress + dos.addressOfNtHeader);
+	if (!(nt.signature[0] == 'P' && nt.signature[1] == 'E' && nt.signature[2] == '\0' && nt.signature[3] == '\0')) {
+		// Invalid NT signature
+		return -1;
+	}
+
+	switch (nt.fileHeader.machine) {
+		case 0x14c: // IMAGE_FILE_MACHINE_I386
+			return 0;
+		default:
+			return 1;
+	}
+}
+
 #ifdef WIN32
 # include "../mumble_plugin_win32.h"
 #else
diff --git a/plugins/mumble_plugin_win32.h b/plugins/mumble_plugin_win32.h
index 653ac16c9..5029a47b9 100644
--- a/plugins/mumble_plugin_win32.h
+++ b/plugins/mumble_plugin_win32.h
@@ -48,31 +48,6 @@ static inline procid_t getProcess(const wchar_t *exename) {
 	return pid;
 }
 
-static inline int checkProcessIs64Bit(const procid_t &pid) {
-	// This function returns 0 if the process is 32-bit and 1 if it's 64-bit.
-	// In case of failure, it returns -1.
-
-	HANDLE handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid);
-	if (!handle) {
-		return -1;
-	}
-
-	// IsWow64Process() returns true if the process is 32-bit, running on a 64-bit system.
-	// It returns false in case the process is 64-bit, running on a 64-bit system.
-	// It also returns false if the process is 32-bit, running on a 32-bit system.
-
-	BOOL isWow64Process;
-	if (!IsWow64Process(handle, &isWow64Process)) {
-		CloseHandle(handle);
-		return -1;
-	}
-	
-	CloseHandle(handle);
-
-	bool is32 = isWow64Process || sizeof(void*) == 4;
-	return !is32;
-}
-
 static inline procptr_t getModuleAddr(const procid_t &pid, const wchar_t *modname) {
 	MODULEENTRY32 me;
 	procptr_t ret = 0;
@@ -100,57 +75,67 @@ static inline bool peekProc(const procptr_t &addr, void *dest, const size_t &len
 	return (ok && (r == len));
 }
 
-static bool inline initialize(const std::multimap<std::wstring, unsigned long long int> &pids, const wchar_t *procname, const wchar_t *modname = nullptr) {
+static void generic_unlock() {
+	if (hProcess) {
+		CloseHandle(hProcess);
+		hProcess = nullptr;
+	}
+
+	pPid = 0;
+	pModule = 0;
+}
+
+static bool initialize(const std::multimap<std::wstring, unsigned long long int> &pids, const wchar_t *procname, const wchar_t *modname = nullptr) {
 	hProcess = nullptr;
 	pModule = 0;
 
 	if (!pids.empty()) {
-		std::multimap<std::wstring, unsigned long long int>::const_iterator iter = pids.find(std::wstring(procname));
+		auto iter = pids.find(std::wstring(procname));
 
-		if (iter != pids.end())
+		if (iter != pids.end()) {
 			pPid = static_cast<procid_t>(iter->second);
-		else
+		} else {
 			pPid = 0;
+		}
 	} else {
 		pPid = getProcess(procname);
 	}
 
-	if (!pPid)
+	if (!pPid) {
 		return false;
+	}
 
-	const int result = checkProcessIs64Bit(pPid);
-	if (result == -1) {
+	hProcess = OpenProcess(PROCESS_VM_READ, false, pPid);
+	if (!hProcess) {
 		pPid = 0;
 		return false;
 	}
 
-	// We compare to 1 to prevent the following warning:
-	// C4800: 'BOOL': forcing value to bool 'true' or 'false' (performance warning)
-	is64Bit = (result == 1);
-
-	pModule = getModuleAddr(modname ? modname : procname);
+	pModule = getModuleAddr(procname);
 	if (!pModule) {
-		pPid = 0;
+		generic_unlock();
 		return false;
 	}
 
-	hProcess = OpenProcess(PROCESS_VM_READ, false, pPid);
-	if (!hProcess) {
-		pPid = 0;
-		pModule = 0;
+	const int8_t result = isProcess64Bit(pModule);
+	if (result == -1) {
+		generic_unlock();
 		return false;
 	}
 
-	return true;
-}
-
-static void generic_unlock() {
-	if (hProcess) {
-		CloseHandle(hProcess);
-		hProcess = nullptr;
-		pModule = 0;
-		pPid = 0;
+	if (modname) {
+		pModule = getModuleAddr(modname);
+		if (!pModule) {
+			generic_unlock();
+			return false;
+		}
 	}
+
+	// We compare to 1 to prevent the following warning:
+	// C4800: 'BOOL': forcing value to bool 'true' or 'false' (performance warning)
+	is64Bit = (result == 1);
+
+	return true;
 }
 
 #endif