diff options
author | Jan Klass <kissaki0@gmail.com> | 2013-11-27 22:38:51 +0400 |
---|---|---|
committer | Jan Klass <kissaki0@gmail.com> | 2013-11-27 22:38:51 +0400 |
commit | 5f98a65698bbe8e4361bd5c882c79ff6e9d0c74a (patch) | |
tree | 10077f721ccbc4270f47e969e55ed659f6b1f8b7 /overlay | |
parent | d322ad16c20aa57793c1fbbe59bef3018867e7f0 (diff) | |
parent | 71e6ab11335569cf8847d3f1e6327ea70291769c (diff) |
Merge pull request #1075 from mumble-voip/olay-changes
Overlay: Rework Code.
Diffstat (limited to 'overlay')
-rw-r--r-- | overlay/HardHook.cpp | 30 | ||||
-rw-r--r-- | overlay/d3d10.cpp | 394 | ||||
-rw-r--r-- | overlay/d3d9.cpp | 379 | ||||
-rw-r--r-- | overlay/dxgi.cpp | 214 | ||||
-rw-r--r-- | overlay/lib.cpp | 166 | ||||
-rw-r--r-- | overlay/lib.h | 58 | ||||
-rw-r--r-- | overlay/opengl.cpp | 11 | ||||
-rw-r--r-- | overlay/overlay.pro | 8 |
8 files changed, 799 insertions, 461 deletions
diff --git a/overlay/HardHook.cpp b/overlay/HardHook.cpp index 29a2af923..8757e8c6c 100644 --- a/overlay/HardHook.cpp +++ b/overlay/HardHook.cpp @@ -131,9 +131,6 @@ void *HardHook::cloneCode(void **porig) { } unsigned char *o = (unsigned char *) *porig; - unsigned char *n = (unsigned char *) pCode; - n += uiCode; - unsigned int idx = 0; DWORD origProtect; if (!VirtualProtect(o, CODEPROTECTSIZE, PAGE_EXECUTE_READ, &origProtect)) { @@ -161,15 +158,16 @@ void *HardHook::cloneCode(void **porig) { } } + unsigned char *n = (unsigned char *) pCode; + n += uiCode; + unsigned int idx = 0; + do { unsigned char opcode = o[idx]; unsigned char a = o[idx+1]; unsigned char b = o[idx+2]; unsigned int extra = 0; - n[idx] = opcode; - ++idx; - switch (opcode) { case 0x50: // PUSH case 0x51: @@ -210,8 +208,8 @@ void *HardHook::cloneCode(void **porig) { break; } - fods("HardHook: CloneCode failed; Unknown opcode at %d: %2x %2x %2x %2x %2x %2x %2x %2x %2x %2x %2x %2x", - idx-1, o[0], o[1], o[2], o[3], o[4], o[5], o[6], o[7], o[8], o[9], o[10], o[11]); + fods("HardHook: CloneCode failed; Unknown opcode %02x at %d: %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x", + opcode, idx, o[0], o[1], o[2], o[3], o[4], o[5], o[6], o[7], o[8], o[9], o[10], o[11]); DWORD tempProtect; VirtualProtect(o, CODEPROTECTSIZE, origProtect, &tempProtect); return NULL; @@ -219,6 +217,9 @@ void *HardHook::cloneCode(void **porig) { } } + n[idx] = opcode; + ++idx; + for (unsigned int i = 0; i < extra; ++i) n[idx+i] = o[idx+i]; idx += extra; @@ -228,10 +229,11 @@ void *HardHook::cloneCode(void **porig) { DWORD tempProtect; VirtualProtect(o, CODEPROTECTSIZE, origProtect, &tempProtect); - // Add a relative jmp back to the original code + // Add a relative jmp back to the original code, to after the copied code n[idx++] = 0xe9; int *iptr = reinterpret_cast<int *>(&n[idx]); - int offs = o - n - 5; + const int JMP_OP_SIZE = 5; + int offs = o - n - JMP_OP_SIZE; *iptr = offs; idx += 4; @@ -269,14 +271,16 @@ void HardHook::setup(voidFunc func, voidFunc replacement) { bTrampoline = true; } else { // Could not create a trampoline. Use alternative method instead. + // This alternative method is dependant on the replacement code + // restoring before calling the original. Otherwise we get a jump recursion bTrampoline = false; call = func; } DWORD origProtect; if (VirtualProtect(fptr, CODEPROTECTSIZE, PAGE_EXECUTE_READ, &origProtect)) { - unsigned char **iptr = reinterpret_cast<unsigned char **>(&replace[1]); replace[0] = 0x68; // PUSH immediate 1 Byte + unsigned char **iptr = reinterpret_cast<unsigned char **>(&replace[1]); *iptr = nptr; // (imm. value = nptr) 4 Byte replace[5] = 0xc3; // RETN 1 Byte @@ -286,9 +290,6 @@ void HardHook::setup(voidFunc func, voidFunc replacement) { baseptr = fptr; - //TODO: Why do we want to force, even without a trampoline?!? We may - // break the x86 instructions. - // This is only safe as long as we always restore before calling .call. inject(true); DWORD tempProtect; @@ -397,7 +398,6 @@ void HardHook::check() { // If they match the original code, inject our hook. if (memcmp(baseptr, orig, CODEREPLACESIZE) == 0) { fods("HardHook: Reinjecting hook into function %p", baseptr); - //TODO: Why do we want to force, even without a trampoline?!? inject(true); } else { fods("HardHook: Function %p replaced by third party. Lost injected hook."); diff --git a/overlay/d3d10.cpp b/overlay/d3d10.cpp index 2e4672ca8..b21f699d0 100644 --- a/overlay/d3d10.cpp +++ b/overlay/d3d10.cpp @@ -34,7 +34,7 @@ #include <d3dx10.h> #include <time.h> -DXGIData *dxgi = NULL; +D3D10Data *d3d10 = NULL; static bool bHooked = false; static HardHook hhPresent; @@ -49,8 +49,6 @@ typedef HRESULT(__stdcall *D3D10CreateStateBlockType)(ID3D10Device *, D3D10_STAT typedef HRESULT(__stdcall *D3D10StateBlockMaskEnableAllType)(D3D10_STATE_BLOCK_MASK *); typedef HRESULT(__stdcall *D3D10CreateEffectFromMemoryType)(void *, SIZE_T, UINT, ID3D10Device *, ID3D10EffectPool *, ID3D10Effect **); -typedef HRESULT(__stdcall *PresentType)(IDXGISwapChain *, UINT, UINT); -typedef HRESULT(__stdcall *ResizeBuffersType)(IDXGISwapChain *, UINT, UINT, UINT, DXGI_FORMAT, UINT); typedef ULONG(__stdcall *AddRefType)(ID3D10Device *); typedef ULONG(__stdcall *ReleaseType)(ID3D10Device *); @@ -63,7 +61,6 @@ struct SimpleVertex { class D10State: protected Pipe { public: - LONG lHighMark; LONG initRefCount; @@ -103,8 +100,10 @@ class D10State: protected Pipe { virtual void newTexture(unsigned int w, unsigned int h); }; -map<IDXGISwapChain *, D10State *> chains; -map<ID3D10Device *, D10State *> devices; +typedef map<IDXGISwapChain *, D10State *> SwapchainMap; +SwapchainMap chains; +typedef map<ID3D10Device *, D10State *> DeviceMap; +DeviceMap devices; D10State::D10State(IDXGISwapChain *pSwapChain, ID3D10Device *pDevice) { this->pSwapChain = pSwapChain; @@ -228,7 +227,7 @@ void D10State::newTexture(unsigned int w, unsigned int h) { desc.CPUAccessFlags = D3D10_CPU_ACCESS_WRITE; hr = pDevice->CreateTexture2D(&desc, NULL, &pTexture); - if (! SUCCEEDED(hr)) { + if (FAILED(hr)) { pTexture = NULL; ods("D3D10: Failed to create texture."); return; @@ -242,7 +241,7 @@ void D10State::newTexture(unsigned int w, unsigned int h) { srvDesc.Texture2D.MipLevels = desc.MipLevels; hr = pDevice->CreateShaderResourceView(pTexture, &srvDesc, &pSRView); - if (! SUCCEEDED(hr)) { + if (FAILED(hr)) { pSRView = NULL; pTexture->Release(); pTexture = NULL; @@ -268,7 +267,7 @@ void D10State::init() { pOrigStateBlock->Capture(); - ID3D10Texture2D* pBackBuffer = NULL; + ID3D10Texture2D *pBackBuffer = NULL; hr = pSwapChain->GetBuffer(0, __uuidof(*pBackBuffer), (LPVOID*)&pBackBuffer); if (FAILED(hr)) ods("D3D10: pSwapChain->GetBuffer failure!"); @@ -427,7 +426,7 @@ void D10State::draw() { pDevice->IASetVertexBuffers(0, 1, &pVertexBuffer, &stride, &offset); hr = pDiffuseTexture->SetResource(pSRView); - if (! SUCCEEDED(hr)) + if (FAILED(hr)) ods("D3D10: Failed to set resource"); for (UINT p = 0; p < techDesc.Passes; ++p) { @@ -440,25 +439,22 @@ void D10State::draw() { dwMyThread = 0; } - - -static HRESULT __stdcall myPresent(IDXGISwapChain *pSwapChain, UINT SyncInterval, UINT Flags) { - // Present is called for each frame. Thus, we do not want to always log here. - #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT - ods("D3D10: Call to Present; Drawing and then chaining the call"); - #endif +// D3D10 specific logic for the Present function. +HRESULT presentD3D10(IDXGISwapChain *pSwapChain) { ID3D10Device *pDevice = NULL; HRESULT hr = pSwapChain->GetDevice(__uuidof(ID3D10Device), (void **) &pDevice); if (SUCCEEDED(hr) && pDevice) { - D10State *ds = chains[pSwapChain]; + SwapchainMap::iterator it = chains.find(pSwapChain); + D10State *ds = it != chains.end() ? it->second : NULL; + if (ds && ds->pDevice != pDevice) { ods("D3D10: SwapChain device changed"); devices.erase(ds->pDevice); delete ds; ds = NULL; } - if (! ds) { + if (ds == NULL) { ods("D3D10: New state"); ds = new D10State(pSwapChain, pDevice); chains[pSwapChain] = ds; @@ -469,38 +465,29 @@ static HRESULT __stdcall myPresent(IDXGISwapChain *pSwapChain, UINT SyncInterval ds->draw(); pDevice->Release(); } else { + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + // DXGI is used for multiple D3D versions. Thus, it is possible a device + // associated with the DXGISwapChain may very well not be a D3D10 one, + // in which case we can safely ignore it. ods("D3D10: Could not draw because ID3D10Device could not be retrieved."); + #endif } - //TODO: Move logic to HardHook. - // Call base without active hook in case of no trampoline. - PresentType oPresent = (PresentType) hhPresent.call; - hhPresent.restore(); - hr = oPresent(pSwapChain, SyncInterval, Flags); - hhPresent.inject(); return hr; } -static HRESULT __stdcall myResize(IDXGISwapChain *pSwapChain, UINT BufferCount, UINT Width, UINT Height, DXGI_FORMAT NewFormat, UINT SwapChainFlags) { +void resizeD3D10(IDXGISwapChain *pSwapChain) { // Remove the D10State from our "cache" (= Invalidate) - D10State *ds = chains[pSwapChain]; - if (ds) { + SwapchainMap::iterator it = chains.find(pSwapChain); + if (it != chains.end()) { + D10State *ds = it->second; devices.erase(ds->pDevice); - chains.erase(pSwapChain); + chains.erase(it); delete ds; } - - //TODO: Move logic to HardHook. - // Call base without active hook in case of no trampoline. - ResizeBuffersType oResize = (ResizeBuffersType) hhResize.call; - hhResize.restore(); - HRESULT hr = oResize(pSwapChain, BufferCount, Width, Height, NewFormat, SwapChainFlags); - hhResize.inject(); - return hr; } static ULONG __stdcall myAddRef(ID3D10Device *pDevice) { - //TODO: Move logic to HardHook. // Call base without active hook in case of no trampoline. AddRefType oAddRef = (AddRefType) hhAddRef.call; @@ -545,214 +532,183 @@ static void HookAddRelease(voidFunc vfAdd, voidFunc vfRelease) { hhRelease.setup(vfRelease, reinterpret_cast<voidFunc>(myRelease)); } -static void HookPresentRaw(voidFunc vfPresent) { - hhPresent.setup(vfPresent, reinterpret_cast<voidFunc>(myPresent)); -} - -static void HookResizeRaw(voidFunc vfResize) { - ods("D3D10: Injecting ResizeBuffers Raw"); - hhResize.setup(vfResize, reinterpret_cast<voidFunc>(myResize)); -} +static void hookD3D10(HMODULE hD3D10, bool preonly); -void checkDXGIHook(bool preonly) { +void checkDXGI10Hook(bool preonly) { static bool bCheckHookActive = false; if (bCheckHookActive) { - ods("D3D10: Recursion in checkDXGIHook"); + ods("D3D10: Recursion in checkDXGI10Hook"); return; } - if (! dxgi->iOffsetPresent || ! dxgi->iOffsetResize) + if (d3d10->iOffsetAddRef == 0 || d3d10->iOffsetRelease == 0) { return; + } bCheckHookActive = true; - HMODULE hDXGI = GetModuleHandleW(L"DXGI.DLL"); HMODULE hD3D10 = GetModuleHandleW(L"D3D10CORE.DLL"); - if (hDXGI && hD3D10) { + if (hD3D10) { if (! bHooked) { - const int procnamesize = 2048; - wchar_t procname[procnamesize]; - GetModuleFileNameW(NULL, procname, procnamesize); - ods("D3D10: checkDXGIHook in unhooked D3D App %ls", procname); - - // Add a ref to ourselves; we do NOT want to get unloaded directly from this process. - GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, reinterpret_cast<char *>(&checkDXGIHook), &hSelf); - - bHooked = true; - - // Can we use the prepatch data? - GetModuleFileNameW(hDXGI, procname, procnamesize); - if (_wcsicmp(dxgi->wcDXGIFileName, procname) == 0) { - unsigned char *raw = (unsigned char *) hDXGI; - HookPresentRaw((voidFunc)(raw + dxgi->iOffsetPresent)); - HookResizeRaw((voidFunc)(raw + dxgi->iOffsetResize)); - - GetModuleFileNameW(hD3D10, procname, 2048); - if (_wcsicmp(dxgi->wcD3D10FileName, procname) == 0) { - unsigned char *raw = (unsigned char *) hD3D10; - HookAddRelease((voidFunc)(raw + dxgi->iOffsetAddRef), (voidFunc)(raw + dxgi->iOffsetRelease)); - } - } else if (! preonly) { - ods("D3D10: Interface changed, can't rawpatch"); - } else { - bHooked = false; - } + hookD3D10(hD3D10, preonly); } + } else { + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + if (hDXGI) { + ods("D3D10: No DXGI.DLL found as loaded. No hooking at this point."); + } else { + ods("D3D10: No D3D10CORE.DLL found as loaded. No hooking at this point."); + } + #endif } bCheckHookActive = false; } -extern "C" __declspec(dllexport) void __cdecl PrepareDXGI() { - // This function is called by the Mumble client in Mumble's scope - // mainly to extract the offsets of various functions in the IDXGISwapChain - // and IDXGIObject interfaces that need to be hooked in target - // applications. The data is stored in the dxgi shared memory structure. +/// @param hD3D10 must be a valid module handle +void hookD3D10(HMODULE hD3D10, bool preonly) { + + // Add a ref to ourselves; we do NOT want to get unloaded directly from this process. + HMODULE hTempSelf = NULL; + GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, reinterpret_cast<char *>(&hookD3D10), &hTempSelf); + + bHooked = true; - if (! dxgi) + wchar_t modulename[MODULEFILEPATH_BUFLEN]; + GetModuleFileNameW(hD3D10, modulename, ARRAY_NUM_ELEMENTS(modulename)); + + if (_wcsicmp(d3d10->wcFileName, modulename) == 0) { + unsigned char *raw = (unsigned char *) hD3D10; + HookAddRelease((voidFunc)(raw + d3d10->iOffsetAddRef), (voidFunc)(raw + d3d10->iOffsetRelease)); + } else if (! preonly) { + ods("D3D10: Interface changed, can't rawpatch. Current: %ls ; Previously: %ls", modulename, d3d10->wcFileName); + } else { + bHooked = false; + } +} + +/// Prepares DXGI and D3D10 data by trying to determining the module filepath +/// and function offsets in memory. +/// (This data can later be used for hooking / code injection. +/// +/// Adjusts the data behind the global variables dxgi and d3d10. +void PrepareDXGI10(IDXGIAdapter1 *pAdapter, bool initializeDXGIData) { + + if (!dxgi || !d3d10 || !pAdapter) return; - ods("D3D10: Preparing static data for DXGI Injection"); - - dxgi->wcDXGIFileName[0] = 0; - dxgi->wcD3D10FileName[0] = 0; - dxgi->iOffsetPresent = 0; - dxgi->iOffsetResize = 0; - dxgi->iOffsetAddRef = 0; - dxgi->iOffsetRelease = 0; - - OSVERSIONINFOEXW ovi; - memset(&ovi, 0, sizeof(ovi)); - ovi.dwOSVersionInfoSize = sizeof(ovi); - GetVersionExW(reinterpret_cast<OSVERSIONINFOW *>(&ovi)); - // Make sure this is vista or greater as quite a number of <=WinXP users have fake DX10 libs installed - if ((ovi.dwMajorVersion >= 7) || ((ovi.dwMajorVersion == 6) && (ovi.dwBuildNumber >= 6001))) { - HMODULE hD3D10 = LoadLibrary("D3D10.DLL"); - HMODULE hDXGI = LoadLibrary("DXGI.DLL"); - - if (hDXGI != NULL && hD3D10 != NULL) { - CreateDXGIFactoryType pCreateDXGIFactory = reinterpret_cast<CreateDXGIFactoryType>(GetProcAddress(hDXGI, "CreateDXGIFactory")); - ods("D3D10: Got CreateDXGIFactory at %p", pCreateDXGIFactory); - if (pCreateDXGIFactory) { - IDXGIFactory * pFactory; - HRESULT hr = pCreateDXGIFactory(__uuidof(IDXGIFactory), (void**)(&pFactory)); - if (FAILED(hr)) - ods("D3D10: Call to pCreateDXGIFactory failed!"); - if (pFactory) { - HWND hwnd = CreateWindowW(L"STATIC", L"Mumble DXGI Window", WS_OVERLAPPEDWINDOW, - CW_USEDEFAULT, CW_USEDEFAULT, 640, 480, 0, - NULL, NULL, 0); - - IDXGIAdapter *pAdapter = NULL; - pFactory->EnumAdapters(0, &pAdapter); - - D3D10CreateDeviceAndSwapChainType pD3D10CreateDeviceAndSwapChain = reinterpret_cast<D3D10CreateDeviceAndSwapChainType>(GetProcAddress(hD3D10, "D3D10CreateDeviceAndSwapChain")); - - DXGI_SWAP_CHAIN_DESC desc; - ZeroMemory(&desc, sizeof(desc)); - - RECT rcWnd; - GetClientRect(hwnd, &rcWnd); - desc.BufferDesc.Width = rcWnd.right - rcWnd.left; - desc.BufferDesc.Height = rcWnd.bottom - rcWnd.top; - - ods("D3D10: Got ClientRect W %d H %d", desc.BufferDesc.Width, desc.BufferDesc.Height); - - desc.BufferDesc.RefreshRate.Numerator = 60; - desc.BufferDesc.RefreshRate.Denominator = 1; - desc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB; - desc.BufferDesc.ScanlineOrdering = DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED; - desc.BufferDesc.Scaling = DXGI_MODE_SCALING_CENTERED; - - desc.SampleDesc.Count = 1; - desc.SampleDesc.Quality = 0; - - desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; - - desc.BufferCount = 2; - - desc.OutputWindow = hwnd; - - desc.Windowed = true; - - desc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD; - - IDXGISwapChain *pSwapChain = NULL; - ID3D10Device *pDevice = NULL; - hr = pD3D10CreateDeviceAndSwapChain(pAdapter, D3D10_DRIVER_TYPE_HARDWARE, NULL, 0, D3D10_SDK_VERSION, &desc, &pSwapChain, &pDevice); - if (FAILED(hr)) - ods("D3D10: pD3D10CreateDeviceAndSwapChain failure!"); - - if (pDevice && pSwapChain) { - HMODULE hRef; - // For VC++ the vtable is located at the base addr. of the object and each function entry is a single pointer. Since p.e. the base classes - // of IDXGISwapChain have a total of 8 functions the 8+Xth entry points to the Xth added function in the derived interface. - void ***vtbl = (void ***) pSwapChain; - void *pPresent = (*vtbl)[8]; - if (! GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (char *) pPresent, &hRef)) { - ods("D3D10: Failed to get module for Present"); - } else { - GetModuleFileNameW(hRef, dxgi->wcDXGIFileName, 2048); - unsigned char *b = (unsigned char *) pPresent; - unsigned char *a = (unsigned char *) hRef; - dxgi->iOffsetPresent = b-a; - ods("D3D10: Successfully found Present offset: %ls: %d", dxgi->wcDXGIFileName, dxgi->iOffsetPresent); - } - - void *pResize = (*vtbl)[13]; - if (! GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (char *) pResize, &hRef)) { - ods("D3D10: Failed to get module for ResizeBuffers"); - } else { - wchar_t buff[2048]; - GetModuleFileNameW(hRef, buff, 2048); - if (wcscmp(buff, dxgi->wcDXGIFileName) == 0) { - unsigned char *b = (unsigned char *) pResize; - unsigned char *a = (unsigned char *) hRef; - dxgi->iOffsetResize = b-a; - ods("D3D10: Successfully found ResizeBuffers offset: %ls: %d", dxgi->wcDXGIFileName, dxgi->iOffsetResize); - } - } - - vtbl = (void ***) pDevice; - - void *pAddRef = (*vtbl)[1]; - if (! GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (char *) pAddRef, &hRef)) { - ods("D3D10: Failed to get module for AddRef"); - } else { - GetModuleFileNameW(hRef, dxgi->wcD3D10FileName, 2048); - unsigned char *b = (unsigned char *) pAddRef; - unsigned char *a = (unsigned char *) hRef; - dxgi->iOffsetAddRef = b-a; - ods("D3D10: Successfully found AddRef offset: %ls: %d", dxgi->wcD3D10FileName, dxgi->iOffsetAddRef); - } - - void *pRelease = (*vtbl)[2]; - if (! GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (char *) pRelease, &hRef)) { - ods("D3D10: Failed to get module for Release"); - } else { - wchar_t buff[2048]; - GetModuleFileNameW(hRef, buff, 2048); - if (wcscmp(buff, dxgi->wcD3D10FileName) == 0) { - unsigned char *b = (unsigned char *) pRelease; - unsigned char *a = (unsigned char *) hRef; - dxgi->iOffsetRelease = b-a; - ods("D3D10: Successfully found Release offset: %ls: %d", dxgi->wcD3D10FileName, dxgi->iOffsetRelease); - } - } - } + ods("D3D10: Preparing static data for DXGI and D3D10 Injection"); - if (pDevice) - pDevice->Release(); - if (pSwapChain) - pSwapChain->Release(); - DestroyWindow(hwnd); + d3d10->wcFileName[0] = 0; + d3d10->iOffsetAddRef = 0; + d3d10->iOffsetRelease = 0; - pFactory->Release(); + HMODULE hD3D10 = LoadLibrary("D3D10.DLL"); + + if (hD3D10 != NULL) { + + HWND hwnd = CreateWindowW(L"STATIC", L"Mumble DXGI Window", WS_OVERLAPPEDWINDOW, + CW_USEDEFAULT, CW_USEDEFAULT, 640, 480, 0, + NULL, NULL, 0); + + D3D10CreateDeviceAndSwapChainType pD3D10CreateDeviceAndSwapChain = reinterpret_cast<D3D10CreateDeviceAndSwapChainType>(GetProcAddress(hD3D10, "D3D10CreateDeviceAndSwapChain")); + + DXGI_SWAP_CHAIN_DESC desc; + ZeroMemory(&desc, sizeof(desc)); + + RECT rcWnd; + GetClientRect(hwnd, &rcWnd); + desc.BufferDesc.Width = rcWnd.right - rcWnd.left; + desc.BufferDesc.Height = rcWnd.bottom - rcWnd.top; + + ods("D3D10: Got ClientRect W %d H %d", desc.BufferDesc.Width, desc.BufferDesc.Height); + + desc.BufferDesc.RefreshRate.Numerator = 60; + desc.BufferDesc.RefreshRate.Denominator = 1; + desc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB; + desc.BufferDesc.ScanlineOrdering = DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED; + desc.BufferDesc.Scaling = DXGI_MODE_SCALING_CENTERED; + + desc.SampleDesc.Count = 1; + desc.SampleDesc.Quality = 0; + + desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; + + desc.BufferCount = 2; + + desc.OutputWindow = hwnd; + + desc.Windowed = true; + + desc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD; + + IDXGISwapChain *pSwapChain = NULL; + ID3D10Device *pDevice = NULL; + HRESULT hr = pD3D10CreateDeviceAndSwapChain(pAdapter, D3D10_DRIVER_TYPE_HARDWARE, NULL, 0, D3D10_SDK_VERSION, &desc, &pSwapChain, &pDevice); + if (FAILED(hr)) + ods("D3D10: pD3D10CreateDeviceAndSwapChain failure!"); + + if (pDevice && pSwapChain) { + + // For VC++ the vtable is located at the base addr. of the object and each function entry is a single pointer. Since p.e. the base classes + // of IDXGISwapChain have a total of 8 functions the 8+Xth entry points to the Xth added function in the derived interface. + + void ***vtbl = (void ***) pSwapChain; + + void *pPresent = (*vtbl)[8]; + int offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D10", "Present"); + if (offset >= 0) { + if (initializeDXGIData) { + dxgi->iOffsetPresent = offset; + ods("D3D10: Successfully found Present offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetPresent); + } else { + if (dxgi->iOffsetPresent == offset) { + ods("D3D10: Successfully verified Present offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetPresent); + } else { + ods("D3D10: Failed to verify Present offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->iOffsetPresent); + } + } + } + + void *pResize = (*vtbl)[13]; + offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pResize), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D10", "ResizeBuffers"); + if (offset >= 0) { + if (initializeDXGIData) { + dxgi->iOffsetResize = offset; + ods("D3D10: Successfully found ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetResize); + } else { + if (dxgi->iOffsetResize == offset) { + ods("D3D10: Successfully verified ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetResize); + } else { + ods("D3D10: Failed to verify ResizeBuffers offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->iOffsetResize); + } } } + + vtbl = (void ***) pDevice; + + void *pAddRef = (*vtbl)[1]; + offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pAddRef), d3d10->wcFileName, ARRAY_NUM_ELEMENTS(d3d10->wcFileName), "D3D10", "AddRef"); + if (offset >= 0) { + d3d10->iOffsetAddRef = offset; + ods("D3D10: Successfully found AddRef offset: %ls: %d", d3d10->wcFileName, d3d10->iOffsetAddRef); + } + + void *pRelease = (*vtbl)[2]; + offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pRelease), d3d10->wcFileName, ARRAY_NUM_ELEMENTS(d3d10->wcFileName), "D3D10", "Release"); + if (offset >= 0) { + d3d10->iOffsetRelease = offset; + ods("D3D10: Successfully found Release offset: %ls: %d", d3d10->wcFileName, d3d10->iOffsetRelease); + } } + + if (pDevice) + pDevice->Release(); + if (pSwapChain) + pSwapChain->Release(); + DestroyWindow(hwnd); } else { - ods("D3D10: No DXGI pre-Vista - skipping prepare"); + FreeLibrary(hD3D10); } } diff --git a/overlay/d3d9.cpp b/overlay/d3d9.cpp index 3c1ebbf14..d1df235a2 100644 --- a/overlay/d3d9.cpp +++ b/overlay/d3d9.cpp @@ -37,6 +37,23 @@ Direct3D9Data *d3dd = NULL; typedef IDirect3D9* (WINAPI *pDirect3DCreate9)(UINT SDKVersion) ; typedef HRESULT (WINAPI *pDirect3DCreate9Ex)(UINT SDKVersion, IDirect3D9Ex **ppD3D) ; +template<class T> +class Stash { +public: + Stash(T *variable, T newValue) + : var(variable) + , value(*var) { + *variable = newValue; + } + ~Stash() { + *var = value; + } + +private: + T *var; + T value; +}; + struct D3DTLVERTEX { float x, y, z, rhw; // Position float tu, tv; // Texture coordinates @@ -48,9 +65,12 @@ class DevState : public Pipe { IDirect3DDevice9 *dev; IDirect3DStateBlock9 *pSB; + /// Non-Win8: Initial ref count, directly after device creation. + /// Win8: Unused LONG initRefCount; LONG refCount; - // Thread-specific threadcount + /// Refcount of self (library). Used to make the overlay transparent + /// (invisible) to the outside. LONG myRefCount; DWORD dwMyThread; @@ -66,14 +86,21 @@ class DevState : public Pipe { void releaseData(); void releaseAll(); void draw(); - void postDraw(); virtual void blit(unsigned int x, unsigned int y, unsigned int w, unsigned int h); virtual void setRect(); virtual void newTexture(unsigned int width, unsigned int height); }; -static map<IDirect3DDevice9 *, DevState *> devMap; +/// Vtable offset; see d3d9.h of win-/D3D-API. +/// 3 from IUnknown + 14 in IDirect3D9, 0-based => 16 +static const int VTABLE_OFFSET_ID3D_CREATEDEVICE = 16; +/// Vtable offset; see d3d9.h of win-/D3D-API. +/// Offset: 3 from IUnknown + 14 from IDirect3D9 + 4 in IDirect3D9Ex, 0-based => 20 +static const int VTABLE_OFFSET_ID3D_CREATEDEVICE_EX = 20; + +typedef map<IDirect3DDevice9 *, DevState *> DevMapType; +static DevMapType devMap; static bool bHooked = false; DevState::DevState() { @@ -88,7 +115,7 @@ DevState::DevState() { timeT = clock(); frameCount = 0; - for (int i=0;i<4;++i) { + for (int i = 0; i < 4; ++i) { vertices[i].x = vertices[i].y = 0.0f; vertices[i].tu = vertices[i].tv = 0.0f; vertices[i].z = vertices[i].rhw = 1.0f; @@ -186,7 +213,7 @@ void DevState::newTexture(unsigned int width, unsigned int height) { dev->CreateTexture(uiWidth, uiHeight, 1, D3DUSAGE_DYNAMIC, D3DFMT_A8R8G8B8, D3DPOOL_DEFAULT, &texTexture, NULL); - for (int i=0;i<4;++i) { + for (int i = 0; i < 4; ++i) { vertices[i].x = vertices[i].y = vertices[i].z = 0.0f; vertices[i].tu = vertices[i].tv = 0.0f; vertices[i].rhw = 1.0f; @@ -247,11 +274,10 @@ void DevState::draw() { } void DevState::createCleanState() { - DWORD dwOldThread = dwMyThread; - if (dwOldThread) { + if (dwMyThread != 0) { ods("D3D9: CreateCleanState from other thread."); } - dwMyThread = GetCurrentThreadId(); + Stash<DWORD> stashThread(&dwMyThread, GetCurrentThreadId()); if (pSB) pSB->Release(); @@ -304,8 +330,6 @@ void DevState::createCleanState() { pStateBlock->Apply(); pStateBlock->Release(); - - dwMyThread = dwOldThread; } static HardHook hhCreateDevice; @@ -319,13 +343,14 @@ static HardHook hhPresentEx; static HardHook hhSwapPresent; static void doPresent(IDirect3DDevice9 *idd) { - DevState *ds = devMap[idd]; + DevMapType::iterator it = devMap.find(idd); + DevState *ds = it != devMap.end() ? it->second : NULL; if (ds && ds->pSB) { - DWORD dwOldThread = ds->dwMyThread; - if (dwOldThread) + if (ds->dwMyThread != 0) { ods("D3D9: doPresent from other thread"); - ds->dwMyThread = GetCurrentThreadId(); + } + Stash<DWORD> stashThread(&(ds->dwMyThread), GetCurrentThreadId()); IDirect3DSurface9 *pTarget = NULL; IDirect3DSurface9 *pRenderTarget = NULL; @@ -357,13 +382,12 @@ static void doPresent(IDirect3DDevice9 *idd) { pTarget->Release(); // ods("D3D9: Finished ref is %d %d", ds->myRefCount, ds->refCount); - ds->dwMyThread = dwOldThread; } } typedef HRESULT(__stdcall *SwapPresentType)(IDirect3DSwapChain9 *, CONST RECT *, CONST RECT *, HWND, CONST RGNDATA *, DWORD); -static HRESULT __stdcall mySwapPresent(IDirect3DSwapChain9 * ids, CONST RECT *pSourceRect, CONST RECT *pDestRect, HWND hDestWindowOverride, CONST RGNDATA *pDirtyRegion, DWORD dwFlags) { +static HRESULT __stdcall mySwapPresent(IDirect3DSwapChain9 *ids, CONST RECT *pSourceRect, CONST RECT *pDestRect, HWND hDestWindowOverride, CONST RGNDATA *pDirtyRegion, DWORD dwFlags) { // Present is called for each frame. Thus, we do not want to always log here. #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT ods("D3D9: SwapChain Present"); @@ -380,14 +404,14 @@ static HRESULT __stdcall mySwapPresent(IDirect3DSwapChain9 * ids, CONST RECT *pS // Call base without active hook in case of no trampoline. SwapPresentType oSwapPresent = (SwapPresentType) hhSwapPresent.call; hhSwapPresent.restore(); - HRESULT hr = oSwapPresent(ids, pSourceRect,pDestRect,hDestWindowOverride,pDirtyRegion,dwFlags); + HRESULT hr = oSwapPresent(ids, pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion, dwFlags); hhSwapPresent.inject(); return hr; } typedef HRESULT(__stdcall *PresentType)(IDirect3DDevice9 *, CONST RECT *, CONST RECT *, HWND, CONST RGNDATA *); -static HRESULT __stdcall myPresent(IDirect3DDevice9 * idd, CONST RECT* pSourceRect,CONST RECT* pDestRect,HWND hDestWindowOverride,CONST RGNDATA* pDirtyRegion) { +static HRESULT __stdcall myPresent(IDirect3DDevice9 *idd, CONST RECT *pSourceRect, CONST RECT *pDestRect, HWND hDestWindowOverride,CONST RGNDATA *pDirtyRegion) { // Present is called for each frame. Thus, we do not want to always log here. #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT ods("D3D9: Device Present"); @@ -399,14 +423,14 @@ static HRESULT __stdcall myPresent(IDirect3DDevice9 * idd, CONST RECT* pSourceRe // Call base without active hook in case of no trampoline. PresentType oPresent = (PresentType) hhPresent.call; hhPresent.restore(); - HRESULT hr = oPresent(idd,pSourceRect,pDestRect,hDestWindowOverride,pDirtyRegion); + HRESULT hr = oPresent(idd, pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion); hhPresent.inject(); return hr; } typedef HRESULT(__stdcall *PresentExType)(IDirect3DDevice9Ex *, CONST RECT *, CONST RECT *, HWND, CONST RGNDATA *, DWORD); -static HRESULT __stdcall myPresentEx(IDirect3DDevice9Ex * idd, CONST RECT* pSourceRect, CONST RECT* pDestRect, HWND hDestWindowOverride, CONST RGNDATA* pDirtyRegion, DWORD dwFlags) { +static HRESULT __stdcall myPresentEx(IDirect3DDevice9Ex *idd, CONST RECT *pSourceRect, CONST RECT *pDestRect, HWND hDestWindowOverride, CONST RGNDATA *pDirtyRegion, DWORD dwFlags) { // Present is called for each frame. Thus, we do not want to always log here. #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT ods("D3D9: Device Present Ex"); @@ -415,26 +439,26 @@ static HRESULT __stdcall myPresentEx(IDirect3DDevice9Ex * idd, CONST RECT* pSour doPresent(idd); PresentExType oPresentEx = (PresentExType) hhPresentEx.call; - hhPresentEx.restore(); HRESULT hr = oPresentEx(idd, pSourceRect, pDestRect, hDestWindowOverride, pDirtyRegion, dwFlags); hhPresentEx.inject(); + return hr; } typedef HRESULT(__stdcall *ResetType)(IDirect3DDevice9 *, D3DPRESENT_PARAMETERS *); -static HRESULT __stdcall myReset(IDirect3DDevice9 * idd, D3DPRESENT_PARAMETERS *param) { +static HRESULT __stdcall myReset(IDirect3DDevice9 *idd, D3DPRESENT_PARAMETERS *param) { ods("D3D9: Chaining Reset"); - DevState *ds = devMap[idd]; + DevMapType::iterator it = devMap.find(idd); + DevState *ds = it != devMap.end() ? it->second : NULL; if (ds) { - DWORD dwOldThread = ds->dwMyThread; - if (dwOldThread) + if (ds->dwMyThread != 0) { ods("D3D9: myReset from other thread"); - ds->dwMyThread = GetCurrentThreadId(); + } + Stash<DWORD> stashThread(&(ds->dwMyThread), GetCurrentThreadId()); ds->releaseAll(); - ds->dwMyThread = dwOldThread; } //TODO: Move logic to HardHook. @@ -451,18 +475,18 @@ static HRESULT __stdcall myReset(IDirect3DDevice9 * idd, D3DPRESENT_PARAMETERS * } typedef HRESULT(__stdcall *ResetExType)(IDirect3DDevice9Ex *, D3DPRESENT_PARAMETERS *, D3DDISPLAYMODEEX *); -static HRESULT __stdcall myResetEx(IDirect3DDevice9Ex * idd, D3DPRESENT_PARAMETERS *param, D3DDISPLAYMODEEX * param2) { +static HRESULT __stdcall myResetEx(IDirect3DDevice9Ex *idd, D3DPRESENT_PARAMETERS *param, D3DDISPLAYMODEEX *param2) { ods("D3D9: Chaining ResetEx"); - DevState *ds = devMap[idd]; + DevMapType::iterator it = devMap.find(idd); + DevState *ds = it != devMap.end() ? it->second : NULL; if (ds) { - DWORD dwOldThread = ds->dwMyThread; - if (dwOldThread) + if (ds->dwMyThread) { ods("D3D9: myResetEx from other thread"); - ds->dwMyThread = GetCurrentThreadId(); + } + Stash<DWORD> stashThread(&(ds->dwMyThread), GetCurrentThreadId()); ds->releaseAll(); - ds->dwMyThread = dwOldThread; } //TODO: Move logic to HardHook. @@ -479,22 +503,24 @@ static HRESULT __stdcall myResetEx(IDirect3DDevice9Ex * idd, D3DPRESENT_PARAMETE } typedef ULONG(__stdcall *AddRefType)(IDirect3DDevice9 *); + static ULONG __stdcall myAddRef(IDirect3DDevice9 *idd) { Mutex m; - // AddRef is called very often. Thus, we do not want to always log here. - #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT - ods("D3D9: Chaining AddRef"); - #endif - - DevState *ds = devMap[idd]; + DevMapType::iterator it = devMap.find(idd); + DevState *ds = it != devMap.end() ? it->second : NULL; if (ds) { + // AddRef is called very often. Thus, we do not want to always log here. + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + ods("D3D9: Using own Refcount implementation for call to AddRef."); + #endif + if (ds->dwMyThread == GetCurrentThreadId()) { ds->myRefCount++; } else { ds->refCount++; } - return ds->refCount + ds->initRefCount; + return ds->initRefCount + ds->refCount; } //TODO: Move logic to HardHook. @@ -515,13 +541,14 @@ static ULONG __stdcall myAddRef(IDirect3DDevice9 *idd) { static ULONG __stdcall myWin8AddRef(IDirect3DDevice9 *idd) { Mutex m; - // AddRef is called very often. Thus, we do not want to always log here. - #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT - ods("D3D9: Chaining AddRef (Win8)"); - #endif - - DevState *ds = devMap[idd]; + DevMapType::iterator it = devMap.find(idd); + DevState *ds = it != devMap.end() ? it->second : NULL; if (ds && ds->dwMyThread == GetCurrentThreadId()) { + // AddRef is called very often. Thus, we do not want to always log here. + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + ods("D3D9: Using own Refcount implementation for call to AddRef (Win8)."); + #endif + ds->myRefCount++; return ds->refCount; } @@ -545,19 +572,21 @@ static ULONG __stdcall myWin8AddRef(IDirect3DDevice9 *idd) { } typedef ULONG(__stdcall *ReleaseType)(IDirect3DDevice9 *); + static ULONG __stdcall myRelease(IDirect3DDevice9 *idd) { Mutex m; - // Release is called very often. Thus, we do not want to always log here. - #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT - ods("D3D9: Chaining Release"); - #endif - - DevState *ds = devMap[idd]; + DevMapType::iterator it = devMap.find(idd); + DevState *ds = it != devMap.end() ? it->second : NULL; if (ds) { + // Release is called very often. Thus, we do not want to always log here. + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + ods("D3D9: Using own Refcount implementation for call to Release."); + #endif + if (ds->dwMyThread == GetCurrentThreadId()) { ds->myRefCount--; - return ds->refCount + ds->initRefCount; + return ds->initRefCount + ds->refCount; } else { ds->refCount--; } @@ -566,24 +595,28 @@ static ULONG __stdcall myRelease(IDirect3DDevice9 *idd) { ds->disconnect(); } - if (ds->refCount >= 0) - return ds->refCount + ds->initRefCount; + if (ds->refCount >= 0) { + return ds->initRefCount + ds->refCount; + } ods("D3D9: Final release is following. MyRefs = %d, Tot = %d", ds->myRefCount, ds->refCount); - DWORD dwOldThread = ds->dwMyThread; - if (dwOldThread) + if (ds->dwMyThread != 0) { ods("D3D9: finalRelease from other thread"); - ds->dwMyThread = GetCurrentThreadId(); + } - ds->releaseAll(); + // Codeblock for stashing threadid + { + Stash<DWORD> stashThread(&(ds->dwMyThread), GetCurrentThreadId()); - ds->dwMyThread = dwOldThread; + ds->releaseAll(); + } - ods("D3D9: Final release. MyRefs = %d Tot = %d", ds->myRefCount, ds->refCount); + ods("D3D9: Final release of %p. MyRefs = %d Tot = %d", idd, ds->myRefCount, ds->refCount); - devMap.erase(idd); + devMap.erase(it); delete ds; + ds = NULL; } //TODO: Move logic to HardHook. @@ -604,13 +637,14 @@ static ULONG __stdcall myRelease(IDirect3DDevice9 *idd) { static ULONG __stdcall myWin8Release(IDirect3DDevice9 *idd) { Mutex m; - // Release is called very often. Thus, we do not want to always log here. - #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT - ods("D3D9: Chaining Release (Win8)"); - #endif - - DevState *ds = devMap[idd]; + DevMapType::iterator it = devMap.find(idd); + DevState *ds = it != devMap.end() ? it->second : NULL; if (ds) { + // Release is called very often. Thus, we do not want to always log here. + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + ods("D3D9: Using own Refcount implementation for call to Release."); + #endif + if (ds->dwMyThread == GetCurrentThreadId()) { ds->myRefCount--; return ds->refCount; @@ -619,15 +653,21 @@ static ULONG __stdcall myWin8Release(IDirect3DDevice9 *idd) { ds->disconnect(); ods("D3D9: Final release. MyRefs = %d, Tot = %d", ds->myRefCount, ds->refCount); - DWORD dwOldThread = ds->dwMyThread; - if (dwOldThread) + + if (ds->dwMyThread != 0) { ods("D3D9: finalRelease from other thread"); - ds->dwMyThread = GetCurrentThreadId(); - ds->releaseAll(); - ds->dwMyThread = dwOldThread; - ods("D3D9: Final release, MyRefs = %d Tot = %d", ds->myRefCount, ds->refCount); + } + + // Codeblock for stashing threadid + { + Stash<DWORD> stashThread(&(ds->dwMyThread), GetCurrentThreadId()); + + ds->releaseAll(); + } + + ods("D3D9: Final release of %p. MyRefs = %d Tot = %d", idd, ds->myRefCount, ds->refCount); - devMap.erase(idd); + devMap.erase(it); delete ds; ds = NULL; } @@ -651,8 +691,37 @@ static ULONG __stdcall myWin8Release(IDirect3DDevice9 *idd) { return res; } +static IDirect3DDevice9* findOriginalDevice(IDirect3DDevice9 *device) { + + IDirect3DSwapChain9 *pSwap = NULL; + device->GetSwapChain(0, &pSwap); + if (pSwap) { + + IDirect3DDevice9 *originalDevice = NULL; + if (SUCCEEDED(pSwap->GetDevice(&originalDevice))) { + + if (originalDevice == device) { + // Found the original device. Release responsibility is passed + // to the caller. + } else { + device = findOriginalDevice(originalDevice); + originalDevice->Release(); + } + + } else { + ods("D3D9: Failed to recurse to find original device. Could not get Device from Swapchain."); + } + + pSwap->Release(); + } else { + ods("D3D9: Failed to recurse to find original device. Could not get Swapchain."); + } + + return device; +} + typedef HRESULT(__stdcall *CreateDeviceType)(IDirect3D9 *, UINT, D3DDEVTYPE, HWND, DWORD, D3DPRESENT_PARAMETERS *, IDirect3DDevice9 **); -static HRESULT __stdcall myCreateDevice(IDirect3D9 * id3d, UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow, DWORD BehaviorFlags, D3DPRESENT_PARAMETERS *pPresentationParameters, IDirect3DDevice9 **ppReturnedDeviceInterface) { +static HRESULT __stdcall myCreateDevice(IDirect3D9 *id3d, UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow, DWORD BehaviorFlags, D3DPRESENT_PARAMETERS *pPresentationParameters, IDirect3DDevice9 **ppReturnedDeviceInterface) { Mutex m; ods("D3D9: Chaining CreateDevice"); @@ -672,27 +741,11 @@ static HRESULT __stdcall myCreateDevice(IDirect3D9 * id3d, UINT Adapter, D3DDEVT IDirect3DDevice9 *idd = *ppReturnedDeviceInterface; // Get real interface, please. - bool bfound = false; - do { - bfound = false; - IDirect3DSwapChain9 *pSwap = NULL; - idd->GetSwapChain(0, &pSwap); - if (pSwap) { - IDirect3DDevice9 *idorig = NULL; - if (SUCCEEDED(pSwap->GetDevice(&idorig))) { - if (idorig != idd) { - ods("D3D9: Prepatched device, using original. %p => %p", idorig, idd); - if (idd != *ppReturnedDeviceInterface) - idd->Release(); - idd = idorig; - bfound = true; - } else { - idorig->Release(); - } - } - pSwap->Release(); - } - } while (bfound); + IDirect3DDevice9 *originalDevice = findOriginalDevice(idd); + if (idd != originalDevice) { + ods("D3D9: Prepatched device, using original. %p => %p", idd, originalDevice); + idd = originalDevice; + } DevState *ds = new DevState; ds->dev = idd; @@ -700,9 +753,11 @@ static HRESULT __stdcall myCreateDevice(IDirect3D9 * id3d, UINT Adapter, D3DDEVT idd->AddRef(); ds->initRefCount = idd->Release(); - if (devMap[idd] != NULL) { - ods("Device exists in devMap already - canceling injection into device"); + DevMapType::iterator it = devMap.find(idd); + if (it != devMap.end()) { + ods("Device exists in devMap already - canceling injection into device. Thread prev: %d ; new: %d", it->second->dwMyThread, GetCurrentThreadId()); delete ds; + return hr; } devMap[idd] = ds; @@ -736,11 +791,12 @@ static HRESULT __stdcall myCreateDevice(IDirect3D9 * id3d, UINT Adapter, D3DDEVT } ds->createCleanState(); + return hr; } typedef HRESULT(__stdcall *CreateDeviceExType)(IDirect3D9Ex *, UINT, D3DDEVTYPE, HWND, DWORD, D3DPRESENT_PARAMETERS *, D3DDISPLAYMODEEX *, IDirect3DDevice9Ex **); -static HRESULT __stdcall myCreateDeviceEx(IDirect3D9Ex * id3d, UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow, DWORD BehaviorFlags, D3DPRESENT_PARAMETERS *pPresentationParameters, D3DDISPLAYMODEEX* pFullscreenDisplayMode, IDirect3DDevice9Ex** ppReturnedDeviceInterface) { +static HRESULT __stdcall myCreateDeviceEx(IDirect3D9Ex *id3d, UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow, DWORD BehaviorFlags, D3DPRESENT_PARAMETERS *pPresentationParameters, D3DDISPLAYMODEEX *pFullscreenDisplayMode, IDirect3DDevice9Ex **ppReturnedDeviceInterface) { Mutex m; ods("D3D9: Chaining CreateDeviceEx"); @@ -764,9 +820,11 @@ static HRESULT __stdcall myCreateDeviceEx(IDirect3D9Ex * id3d, UINT Adapter, D3D idd->AddRef(); ds->initRefCount = idd->Release(); - if (devMap[idd] != NULL) { - ods("Device exists in devMap already - canceling injection into device"); + DevMapType::iterator it = devMap.find(idd); + if (it != devMap.end()) { + ods("Device exists in devMap already - canceling injection into device. Thread prev: %d ; new: %d", it->second->dwMyThread, GetCurrentThreadId()); delete ds; + return hr; } devMap[idd] = ds; @@ -821,17 +879,20 @@ static void HookCreateRawEx(voidFunc vfCreate) { static void HookCreate(IDirect3D9 *pD3D) { ods("D3D9: Injecting CreateDevice"); - hhCreateDevice.setupInterface(pD3D, 16, reinterpret_cast<voidFunc>(myCreateDevice)); + hhCreateDevice.setupInterface(pD3D, VTABLE_OFFSET_ID3D_CREATEDEVICE, reinterpret_cast<voidFunc>(myCreateDevice)); } static void HookCreateEx(IDirect3D9Ex *pD3D) { ods("D3D9Ex: Injecting CreateDevice / CreateDeviceEx"); - hhCreateDevice.setupInterface(pD3D, 16, reinterpret_cast<voidFunc>(myCreateDevice)); - hhCreateDeviceEx.setupInterface(pD3D, 20, reinterpret_cast<voidFunc>(myCreateDeviceEx)); + // Best effort hooking for the non-ex create method. If it was set up + // previously this setup call is safely ignored by the hook. + HookCreate(pD3D); + + hhCreateDeviceEx.setupInterface(pD3D, VTABLE_OFFSET_ID3D_CREATEDEVICE_EX, reinterpret_cast<voidFunc>(myCreateDeviceEx)); } -void hookD3D9(HMODULE hD3D, bool preonly); +static void hookD3D9(HMODULE hD3D, bool preonly); // @param preonly If rawpatching the createdevice-functions fails, don't try to // patch Direct3DCreate9. @@ -851,33 +912,41 @@ void checkD3D9Hook(bool preonly) { if (! bHooked) { hookD3D9(hD3D, preonly); } + } else { + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + ods("D3D9: No D3D9.DLL found as loaded. No hooking at this point."); + #endif } bCheckHookActive = false; } -void hookD3D9(HMODULE hD3D, bool preonly) { - const int procnamesize = 2048; - char procname[procnamesize]; - GetModuleFileName(NULL, procname, procnamesize); - ods("D3D9: hookD3D9 in App %s", procname); +static void hookD3D9(HMODULE hD3D, bool preonly) { + + char procname[MODULEFILEPATH_BUFLEN]; + GetModuleFileName(NULL, procname, ARRAY_NUM_ELEMENTS(procname)); + ods("D3D9: hookD3D9 in App '%s'", procname); // Add a ref to ourselves; we do NOT want to get unloaded directly from this process. - GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, reinterpret_cast<char *>(&HookCreate), &hSelf); + HMODULE hTempSelf = NULL; + GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, reinterpret_cast<char *>(&hookD3D9), &hTempSelf); bHooked = true; // Can we use the prepatch data? - GetModuleFileName(hD3D, procname, procnamesize); - if (_stricmp(d3dd->cFileName, procname) == 0) { + wchar_t modulename[MODULEFILEPATH_BUFLEN]; + GetModuleFileNameW(hD3D, modulename, ARRAY_NUM_ELEMENTS(modulename)); + if (_wcsicmp(d3dd->wcFileName, modulename) == 0) { // The module seems to match the one we prepared d3dd for. unsigned char *raw = (unsigned char *) hD3D; HookCreateRaw((voidFunc)(raw + d3dd->iOffsetCreate)); - if (d3dd->iOffsetCreateEx) + if (d3dd->iOffsetCreateEx) { HookCreateRawEx((voidFunc)(raw + d3dd->iOffsetCreateEx)); + } + } else if (! preonly) { - ods("D3D9: Interface changed, can't rawpatch."); + ods("D3D9: Interface changed, can't rawpatch. Current: %ls ; Previously: %ls", modulename, d3dd->wcFileName); pDirect3DCreate9 d3dcreate9 = reinterpret_cast<pDirect3DCreate9>(GetProcAddress(hD3D, "Direct3DCreate9")); if (d3dcreate9) { @@ -894,8 +963,28 @@ void hookD3D9(HMODULE hD3D, bool preonly) { ods("D3D9: Library without Direct3DCreate9?"); } - //TODO: hook for Direct3DCreate9Ex - // pDirect3DCreate9Ex d3dcreate9ex = reinterpret_cast<pDirect3DCreate9Ex>(GetProcAddress(hD3D, "Direct3DCreate9Ex")); + pDirect3DCreate9Ex d3dcreate9ex = reinterpret_cast<pDirect3DCreate9Ex>(GetProcAddress(hD3D, "Direct3DCreate9Ex")); + if (d3dcreate9ex) { + ods("D3D9: Got %p for Direct3DCreate9Ex", d3dcreate9ex); + IDirect3D9Ex** id3d9ex = 0; + HRESULT hr = d3dcreate9ex(D3D_SDK_VERSION, id3d9ex); + if (SUCCEEDED(hr)) { + HookCreateEx(*id3d9ex); + (*id3d9ex)->Release(); + } else { + switch (hr) { + case D3DERR_OUTOFVIDEOMEMORY: + ods("D3D11: Direct3DCreate9Ex returned with out of memory error."); + break; + case D3DERR_NOTAVAILABLE: + ods("D3D11: Direct3DCreate9Ex is not available."); + break; + default: + ods("D3D11: Unexpected return result from Direct3DCreate9Ex"); + break; + } + } + } } else { bHooked = false; } @@ -920,24 +1009,6 @@ void freeD3D9Hook(HMODULE hModule) { } } -// Checks if the module of the fnptr equals the name/path of the one saved in @global d3dd. -bool IsFnInModule(char* refmodulepath, const char* fnptr, const std::string & textindicator) { - char modulename[2048]; - // A handle to the module. - HMODULE hRef = NULL; - - BOOL success = GetModuleHandleEx( - GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - fnptr, &hRef); - if (!success) { - ods(("D3D9: Failed to get module for " + textindicator).c_str()); - } else { - GetModuleFileName(hRef, modulename, 2048); - return _stricmp(refmodulepath, modulename) == 0; - } - return false; -} - extern "C" __declspec(dllexport) void __cdecl PrepareD3D9() { if (! d3dd) return; @@ -948,29 +1019,28 @@ extern "C" __declspec(dllexport) void __cdecl PrepareD3D9() { if (hD3D != NULL) { - GetModuleFileName(hD3D, d3dd->cFileName, 2048); + GetModuleFileNameW(hD3D, d3dd->wcFileName, ARRAY_NUM_ELEMENTS(d3dd->wcFileName)); std::string d3d9FnName("Direct3DCreate9"); pDirect3DCreate9 d3dcreate9 = reinterpret_cast<pDirect3DCreate9>(GetProcAddress(hD3D, d3d9FnName.c_str())); if (! d3dcreate9) { ods(("D3D9: Library without " + d3d9FnName).c_str()); } else { - if (!IsFnInModule(d3dd->cFileName, (const char*)d3dcreate9, "D3D9")) { + if (!IsFnInModule(reinterpret_cast<voidFunc>(d3dcreate9), d3dd->wcFileName, "D3D9", d3d9FnName)) { ods(("D3D9: " + d3d9FnName + " is not in D3D9 library").c_str()); } else { IDirect3D9 *id3d9 = d3dcreate9(D3D_SDK_VERSION); if (id3d9) { void ***vtbl = (void ***) id3d9; - // vtable offset: CreateDevice is 17th method (0 based 16th) - // in IDirect3D9. See d3d9.h of win-/D3D-API. - void *pCreate = (*vtbl)[16]; - if (!IsFnInModule(d3dd->cFileName, (const char*)pCreate, "CreateDevice")) { + void *pCreate = (*vtbl)[VTABLE_OFFSET_ID3D_CREATEDEVICE]; + + if (!IsFnInModule(reinterpret_cast<voidFunc>(pCreate), d3dd->wcFileName, "D3D9", "CreateDevice")) { ods("D3D9: CreateDevice is not in D3D9 library"); } else { - unsigned char *b = (unsigned char *) pCreate; - unsigned char *a = (unsigned char *) hD3D; - d3dd->iOffsetCreate = b-a; + unsigned char *fn = reinterpret_cast<unsigned char *>(pCreate); + unsigned char *base = reinterpret_cast<unsigned char *>(hD3D); + d3dd->iOffsetCreate = fn - base; ods("D3D9: Successfully found prepatch offset: %p %p %p: %d", hD3D, d3dcreate9, pCreate, d3dd->iOffsetCreate); } id3d9->Release(); @@ -983,26 +1053,21 @@ extern "C" __declspec(dllexport) void __cdecl PrepareD3D9() { if (! d3dcreate9ex) { ods(("D3D9: Library without " + d3d9exFnName).c_str()); } else { - if (!IsFnInModule(d3dd->cFileName, (const char*)d3dcreate9ex, "D3D9")) { + if (!IsFnInModule(reinterpret_cast<voidFunc>(d3dcreate9ex), d3dd->wcFileName, "D3D9", d3d9exFnName)) { ods(("D3D9: " + d3d9exFnName + " is not in D3D9 library").c_str()); } else { IDirect3D9Ex *id3d9 = NULL; d3dcreate9ex(D3D_SDK_VERSION, &id3d9); if (id3d9) { void ***vtbl = (void ***) id3d9; - // vtable offset: CreateDeviceEx is 20th method (0 based 19th) - // in IDirect3D9Ex as declared in d3d9.h of win-/D3D-API, - // but is actually the 21th. TODO: How come? - // CreateDeviceEx defines one less method before-hand than - // CreateDevice. Maybe that one comes in anyway? - void *pCreateEx = (*vtbl)[20]; - - if (!IsFnInModule(d3dd->cFileName, (const char*)pCreateEx, "CreateDeviceEx")) { + void *pCreateEx = (*vtbl)[VTABLE_OFFSET_ID3D_CREATEDEVICE_EX]; + + if (!IsFnInModule(reinterpret_cast<voidFunc>(pCreateEx), d3dd->wcFileName, "D3D9", "CreateDeviceEx")) { ods("D3D9: CreateDeviceEx is not in D3D9 library"); } else { - unsigned char *b = (unsigned char *) pCreateEx; - unsigned char *a = (unsigned char *) hD3D; - d3dd->iOffsetCreateEx = b-a; + unsigned char *fn = reinterpret_cast<unsigned char *>(pCreateEx); + unsigned char *base = reinterpret_cast<unsigned char *>(hD3D); + d3dd->iOffsetCreateEx = fn - base; ods("D3D9: Successfully found prepatch ex offset: %p %p %p: %d", hD3D, d3dcreate9ex, pCreateEx, d3dd->iOffsetCreateEx); } diff --git a/overlay/dxgi.cpp b/overlay/dxgi.cpp new file mode 100644 index 000000000..f959f9c88 --- /dev/null +++ b/overlay/dxgi.cpp @@ -0,0 +1,214 @@ +/* Copyright (C) 2005-2010, Thorvald Natvig <thorvald@natvig.com> + Copyright (C) 2011, Kissaki <kissaki@gmx.de> + Copyright (C) 2011, Nye Liu <nyet@nyet.org> + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + - Neither the name of the Mumble Developers nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "lib.h" +#include <dxgi.h> +#include <time.h> + +DXGIData *dxgi = NULL; + +static bool bHooked = false; +static HardHook hhPresent; +static HardHook hhResize; + +typedef HRESULT(__stdcall *CreateDXGIFactory1Type)(REFIID, void **); + +typedef HRESULT(__stdcall *PresentType)(IDXGISwapChain *, UINT, UINT); +typedef HRESULT(__stdcall *ResizeBuffersType)(IDXGISwapChain *, UINT, UINT, UINT, DXGI_FORMAT, UINT); + +#define HMODREF(mod, func) func##Type p##func = (func##Type) GetProcAddress(mod, #func) + +// From d3d10.cpp +extern HRESULT presentD3D10(IDXGISwapChain *pSwapChain); + +static HRESULT __stdcall myPresent(IDXGISwapChain *pSwapChain, UINT SyncInterval, UINT Flags) { + // Present is called for each frame. Thus, we do not want to always log here. + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + ods("DXGI: Call to Present; Drawing and then chaining the call to the original logic"); + #endif + + presentD3D10(pSwapChain); + + //TODO: Move logic to HardHook. + // Call base without active hook in case of no trampoline. + PresentType oPresent = (PresentType) hhPresent.call; + hhPresent.restore(); + HRESULT hr = oPresent(pSwapChain, SyncInterval, Flags); + hhPresent.inject(); + + return hr; +} + +// From d3d10.cpp +extern void resizeD3D10(IDXGISwapChain *pSwapChain); + +static HRESULT __stdcall myResize(IDXGISwapChain *pSwapChain, UINT BufferCount, UINT Width, UINT Height, DXGI_FORMAT NewFormat, UINT SwapChainFlags) { + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + ods("DXGI: Call to Resize. Forwarding to D3D10 custom implementation before calling original logic ..."); + #endif + + resizeD3D10(pSwapChain); + + //TODO: Move logic to HardHook. + // Call base without active hook in case of no trampoline. + ResizeBuffersType oResize = (ResizeBuffersType) hhResize.call; + hhResize.restore(); + HRESULT hr = oResize(pSwapChain, BufferCount, Width, Height, NewFormat, SwapChainFlags); + hhResize.inject(); + return hr; +} + +static void HookPresentRaw(voidFunc vfPresent) { + ods("DXGI: Injecting Present"); + hhPresent.setup(vfPresent, reinterpret_cast<voidFunc>(myPresent)); +} + +static void HookResizeRaw(voidFunc vfResize) { + ods("DXGI: Injecting ResizeBuffers Raw"); + hhResize.setup(vfResize, reinterpret_cast<voidFunc>(myResize)); +} + +void hookDXGI(HMODULE hDXGI, bool preonly); + +void checkDXGIHook(bool preonly) { + static bool bCheckHookActive = false; + if (bCheckHookActive) { + ods("DXGI: Recursion in checkDXGIHook"); + return; + } + + if (dxgi->iOffsetPresent == 0 || dxgi->iOffsetResize == 0) + return; + + bCheckHookActive = true; + + HMODULE hDXGI = GetModuleHandleW(L"DXGI.DLL"); + + if (hDXGI) { + if (! bHooked) { + hookDXGI(hDXGI, preonly); + } + #ifdef EXTENDED_OVERLAY_DEBUGOUTPUT + } else { + ods("DXGI: No DXGI.DLL found as loaded. No hooking at this point."); + #endif + } + + bCheckHookActive = false; +} + +/// @param hDXGI must be a valid module handle. +void hookDXGI(HMODULE hDXGI, bool preonly) { + wchar_t modulename[MODULEFILEPATH_BUFLEN]; + GetModuleFileNameW(NULL, modulename, ARRAY_NUM_ELEMENTS(modulename)); + ods("DXGI: hookDXGI in App '%ls'", modulename); + + // Add a ref to ourselves; we do NOT want to get unloaded directly from this process. + HMODULE hTempSelf = NULL; + GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, reinterpret_cast<char *>(&hookDXGI), &hTempSelf); + + bHooked = true; + + // Can we use the prepatch data? + GetModuleFileNameW(hDXGI, modulename, ARRAY_NUM_ELEMENTS(modulename)); + if (_wcsicmp(dxgi->wcFileName, modulename) == 0) { + // The module seems to match the one we prepared d3dd for. + + unsigned char *raw = (unsigned char *) hDXGI; + HookPresentRaw((voidFunc)(raw + dxgi->iOffsetPresent)); + HookResizeRaw((voidFunc)(raw + dxgi->iOffsetResize)); + + } else if (! preonly) { + ods("DXGI: Interface changed, can't rawpatch. Current: %ls ; Previously: %ls", modulename, dxgi->wcFileName); + } else { + bHooked = false; + } +} + +// From d3d10.cpp +extern void PrepareDXGI10(IDXGIAdapter1 *pAdapter, bool initializeDXGIData); + +/// This function is called by the Mumble client in Mumble's scope +/// mainly to extract the offsets of various functions in the IDXGISwapChain +/// and IDXGIObject interfaces that need to be hooked in target +/// applications. The data is stored in the dxgi shared memory structure. +extern "C" __declspec(dllexport) void __cdecl PrepareDXGI() { + if (! dxgi) + return; + + ods("DXGI: Preparing static data for DXGI Injection"); + + dxgi->wcFileName[0] = 0; + dxgi->iOffsetPresent = 0; + dxgi->iOffsetResize = 0; + + // Make sure this is Vista or greater as quite a number of <=WinXP users have fake DX10 libs installed + OSVERSIONINFOEXW ovi; + memset(&ovi, 0, sizeof(ovi)); + ovi.dwOSVersionInfoSize = sizeof(ovi); + GetVersionExW(reinterpret_cast<OSVERSIONINFOW *>(&ovi)); + if (ovi.dwMajorVersion < 6 || (ovi.dwMajorVersion == 6 && ovi.dwBuildNumber < 6001)) { + ods("DXGI: No DXGI pre-Vista - skipping prepare"); + return; + } + + HMODULE hDXGI = LoadLibrary("DXGI.DLL"); + + if (hDXGI != NULL) { + GetModuleFileNameW(hDXGI, dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName)); + + CreateDXGIFactory1Type pCreateDXGIFactory1 = reinterpret_cast<CreateDXGIFactory1Type>(GetProcAddress(hDXGI, "CreateDXGIFactory1")); + ods("DXGI: Got CreateDXGIFactory1 at %p", pCreateDXGIFactory1); + if (pCreateDXGIFactory1) { + IDXGIFactory1 * pFactory; + HRESULT hr = pCreateDXGIFactory1(__uuidof(IDXGIFactory1), (void**)(&pFactory)); + if (FAILED(hr)) + ods("DXGI: Call to pCreateDXGIFactory1 failed!"); + if (pFactory) { + IDXGIAdapter1 *pAdapter = NULL; + pFactory->EnumAdapters1(0, &pAdapter); + + /// Offsets have to be identified and initialized only once. + bool initializeDXGIData = !dxgi->iOffsetPresent && !dxgi->iOffsetResize; + PrepareDXGI10(pAdapter, initializeDXGIData); + + pFactory->Release(); + } else { + FreeLibrary(hDXGI); + } + } else { + FreeLibrary(hDXGI); + } + } else { + FreeLibrary(hDXGI); + } +} diff --git a/overlay/lib.cpp b/overlay/lib.cpp index 020d838e1..938ad91fa 100644 --- a/overlay/lib.cpp +++ b/overlay/lib.cpp @@ -36,7 +36,6 @@ static HANDLE hMapObject = NULL; static HANDLE hHookMutex = NULL; static HHOOK hhookWnd = 0; -HMODULE hSelf = NULL; BOOL bIsWin8 = FALSE; static BOOL bMumble = FALSE; @@ -47,7 +46,7 @@ static HardHook hhLoad; static HardHook hhLoadW; static HardHook hhFree; -static SharedData *sd; +static SharedData *sd = NULL; FakeInterface::FakeInterface(IUnknown *orig, int entries) { this->pOriginal = orig; @@ -406,6 +405,13 @@ void Pipe::checkMessage(unsigned int width, unsigned int height) { blit((*i).left, (*i).top, (*i).right - (*i).left, (*i).bottom - (*i).top); } +static void checkHooks(bool preonly) { + checkD3D9Hook(preonly); + checkDXGIHook(preonly); + checkDXGI10Hook(preonly); + checkOpenGLHook(); +} + typedef HMODULE(__stdcall *LoadLibraryAType)(const char *); static HMODULE WINAPI MyLoadLibrary(const char *lpFileName) { //TODO: Move logic to HardHook. @@ -418,9 +424,7 @@ static HMODULE WINAPI MyLoadLibrary(const char *lpFileName) { ods("Lib: Library %s loaded to %p", lpFileName, h); if (! bBlackListed) { - checkD3D9Hook(); - checkDXGIHook(); - checkOpenGLHook(); + checkHooks(); } return h; @@ -440,9 +444,7 @@ static HMODULE WINAPI MyLoadLibraryW(const wchar_t *lpFileName) { checkForWPF(); if (! bBlackListed) { - checkD3D9Hook(); - checkDXGIHook(); - checkOpenGLHook(); + checkHooks(); } return h; @@ -486,6 +488,7 @@ extern "C" __declspec(dllexport) void __cdecl InstallHooks() { DWORD dwWaitResult = WaitForSingleObject(hHookMutex, 1000L); if (dwWaitResult == WAIT_OBJECT_0) { if (! sd->bHooked) { + HMODULE hSelf = NULL; GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (char *) &InstallHooks, &hSelf); if (hSelf == NULL) { ods("Lib: Failed to find myself"); @@ -505,9 +508,10 @@ extern "C" __declspec(dllexport) unsigned int __cdecl GetOverlayMagicVersion() { return OVERLAY_MAGIC_NUMBER; } -bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char* p); +static bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char *p); +static void createSharedDataMap(); -void dllmainProcAttach(char* procname) { +static void dllmainProcAttach(char *procname) { Mutex::init(); char *p = strrchr(procname, '\\'); @@ -529,7 +533,7 @@ void dllmainProcAttach(char* procname) { memset(&ovi, 0, sizeof(ovi)); ovi.dwOSVersionInfoSize = sizeof(ovi); GetVersionEx(reinterpret_cast<OSVERSIONINFO *>(&ovi)); - bIsWin8 = (ovi.dwMajorVersion >= 7) || ((ovi.dwMajorVersion == 6) &&(ovi.dwBuildNumber >= 9200)); + bIsWin8 = (ovi.dwMajorVersion >= 7) || ((ovi.dwMajorVersion == 6) && (ovi.dwBuildNumber >= 9200)); ods("Lib: bIsWin8: %i", bIsWin8); @@ -540,30 +544,7 @@ void dllmainProcAttach(char* procname) { return; } - DWORD dwSharedSize = sizeof(SharedData) + sizeof(Direct3D9Data) + sizeof(DXGIData); - - hMapObject = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, dwSharedSize, "MumbleOverlayPrivate"); - if (hMapObject == NULL) { - ods("Lib: CreateFileMapping failed"); - return; - } - - bool bInit = (GetLastError() != ERROR_ALREADY_EXISTS); - - sd = static_cast<SharedData *>(MapViewOfFile(hMapObject, FILE_MAP_ALL_ACCESS, 0, 0, dwSharedSize)); - - if (sd == NULL) { - ods("Lib: MapViewOfFile Failed"); - return; - } - - if (bInit) - memset(sd, 0, dwSharedSize); - - unsigned char *raw = (unsigned char *) sd; - d3dd = reinterpret_cast<Direct3D9Data *>(raw + sizeof(SharedData)); - dxgi = reinterpret_cast<DXGIData *>(raw + sizeof(SharedData) + sizeof(Direct3D9Data)); - + createSharedDataMap(); if (! bMumble) { // Hook our own LoadLibrary functions so we notice when a new library (like the d3d ones) is loaded. @@ -571,15 +552,13 @@ void dllmainProcAttach(char* procname) { hhLoadW.setup(reinterpret_cast<voidFunc>(LoadLibraryW), reinterpret_cast<voidFunc>(MyLoadLibraryW)); hhFree.setup(reinterpret_cast<voidFunc>(FreeLibrary), reinterpret_cast<voidFunc>(MyFreeLibrary)); - checkD3D9Hook(true); - checkDXGIHook(true); - checkOpenGLHook(); + checkHooks(true); ods("Lib: Injected into %s", procname); } } // Is the process black(listed)? -bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char* p) { +static bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char *p) { DWORD buffsize = MAX_PATH * 20; // Initial buffer size for registry operation bool usewhitelist = false; @@ -668,11 +647,14 @@ bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char* p) { // check if there is a "nooverlay" file in the executables folder, which would disable/blacklist the overlay // Same buffersize as procname; which we copy from. - char fname[1024 + 64]; - p = fname + (p - procname); - strncpy_s(fname, sizeof(fname), procname, p - procname + 1); + char fname[PROCNAMEFILEPATH_EXTENDED_BUFFER_BUFLEN]; - strcpy_s(p+1, 64, "nooverlay"); + int pathlength = p - procname; + p = fname + pathlength; + strncpy_s(fname, sizeof(fname), procname, pathlength + 1); + + + strcpy_s(p+1, PROCNAMEFILEPATH_EXTENDED_EXTLEN, "nooverlay"); HANDLE h = CreateFile(fname, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (h != INVALID_HANDLE_VALUE) { CloseHandle(h); @@ -682,7 +664,7 @@ bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char* p) { } // check for "debugoverlay" file, which would enable overlay debugging - strcpy_s(p+1, 64, "debugoverlay"); + strcpy_s(p+1, PROCNAMEFILEPATH_EXTENDED_EXTLEN, "debugoverlay"); h = CreateFile(fname, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (h != INVALID_HANDLE_VALUE) { CloseHandle(h); @@ -699,7 +681,42 @@ bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char* p) { return false; } -void dllmainProcDetach() { +static void createSharedDataMap() { + DWORD dwSharedSize = sizeof(SharedData) + sizeof(Direct3D9Data) + sizeof(DXGIData) + sizeof(D3D10Data); + + hMapObject = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, dwSharedSize, "MumbleOverlayPrivate"); + if (hMapObject == NULL) { + ods("Lib: CreateFileMapping failed"); + return; + } + + bool bInit = (GetLastError() != ERROR_ALREADY_EXISTS); + + unsigned char *rawSharedPointer = static_cast<unsigned char *>( + MapViewOfFile(hMapObject, FILE_MAP_ALL_ACCESS, 0, 0, dwSharedSize)); + + if (rawSharedPointer == NULL) { + ods("Lib: MapViewOfFile Failed"); + return; + } + + if (bInit) + memset(rawSharedPointer, 0, dwSharedSize); + + sd = reinterpret_cast<SharedData *>(rawSharedPointer); + rawSharedPointer += sizeof(SharedData); + + d3dd = reinterpret_cast<Direct3D9Data *>(rawSharedPointer); + rawSharedPointer += sizeof(Direct3D9Data); + + dxgi = reinterpret_cast<DXGIData *>(rawSharedPointer); + rawSharedPointer += sizeof(DXGIData); + + d3d10 = reinterpret_cast<D3D10Data *>(rawSharedPointer); + rawSharedPointer += sizeof(D3D10Data); +} + +static void dllmainProcDetach() { hhLoad.restore(true); hhLoad.reset(); @@ -716,28 +733,26 @@ void dllmainProcDetach() { CloseHandle(hHookMutex); } -void dllmainThreadAttach() { +static void dllmainThreadAttach() { static bool bTriedHook = false; if (!bBlackListed && sd && ! bTriedHook) { bTriedHook = true; checkForWPF(); if (!bBlackListed) { - checkD3D9Hook(); - checkDXGIHook(); - checkOpenGLHook(); - ods("Lib: Injected to thread"); + ods("Lib: Checking for hooks, potentially injecting"); + checkHooks(); } } } extern "C" BOOL WINAPI DllMain(HINSTANCE, DWORD fdwReason, LPVOID) { - // 1024 for the procname, 64 additional for more logic in the buffer deep inside - char procname[1024+64]; - GetModuleFileNameA(NULL, procname, 1024); + + char procname[PROCNAMEFILEPATH_EXTENDED_BUFFER_BUFLEN]; + GetModuleFileNameA(NULL, procname, ARRAY_NUM_ELEMENTS(procname)); // Fix for windows XP; on length nSize does not include null-termination // @see http://msdn.microsoft.com/en-us/library/windows/desktop/ms683197%28v=vs.85%29.aspx - procname[1023] = '\0'; + procname[ARRAY_NUM_ELEMENTS(procname) - 1] = '\0'; switch (fdwReason) { case DLL_PROCESS_ATTACH: @@ -757,3 +772,46 @@ extern "C" BOOL WINAPI DllMain(HINSTANCE, DWORD fdwReason, LPVOID) { } return TRUE; } + +bool IsFnInModule(voidFunc fnptr, wchar_t *refmodulepath, const std::string &logPrefix, const std::string &fnName) { + + HMODULE hModule = NULL; + + BOOL success = GetModuleHandleEx( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast<LPCSTR>(fnptr), &hModule); + if (!success) { + ods((logPrefix + ": Failed to get module for " + fnName).c_str()); + } else { + wchar_t modulename[MODULEFILEPATH_BUFLEN]; + GetModuleFileNameW(hModule, modulename, ARRAY_NUM_ELEMENTS(modulename)); + return _wcsicmp(modulename, refmodulepath) == 0; + } + return false; +} + +int GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int refmodulepathLen, const std::string &logPrefix, const std::string &fnName) { + + HMODULE hModule = NULL; + + if (! GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (char *) fnptr, &hModule)) { + ods((logPrefix + ": Failed to get module for " + fnName).c_str()); + return -1; + } + + const bool bInit = refmodulepath[0] == '\0'; + if (bInit) { + GetModuleFileNameW(hModule, refmodulepath, refmodulepathLen); + } else { + wchar_t modulename[MODULEFILEPATH_BUFLEN]; + GetModuleFileNameW(hModule, modulename, ARRAY_NUM_ELEMENTS(modulename)); + if (_wcsicmp(modulename, refmodulepath) != 0) { + ods((logPrefix + ": " + fnName + " functions module path does not match previously found. Now: '%ls', Previously: '%ls'").c_str(), modulename, refmodulepath); + return -2; + } + } + + unsigned char *fn = reinterpret_cast<unsigned char *>(fnptr); + unsigned char *base = reinterpret_cast<unsigned char *>(hModule); + return fn - base; +} diff --git a/overlay/lib.h b/overlay/lib.h index 43321b03b..ac5a1aa01 100644 --- a/overlay/lib.h +++ b/overlay/lib.h @@ -47,26 +47,37 @@ #define lround(x) static_cast<long int>((x) + (((x) >= 0.0) ? 0.5 : -0.5)) +#define ARRAY_SIZE_BYTES(x) sizeof(x) +#define ARRAY_NUM_ELEMENTS(x) (sizeof(x)/sizeof((x)[0])) + using namespace std; void __cdecl ods(const char *format, ...); +const int MODULEFILEPATH_BUFLEN = 2048; +const int PROCNAMEFILEPATH_BUFLEN = 1024; +const int PROCNAMEFILEPATH_EXTENDED_EXTLEN = 64; +const int PROCNAMEFILEPATH_EXTENDED_BUFFER_BUFLEN = PROCNAMEFILEPATH_BUFLEN + PROCNAMEFILEPATH_EXTENDED_EXTLEN; + struct Direct3D9Data { - // Offset from module address to Create method. + /// Filepath of the module the offsets are for. + wchar_t wcFileName[MODULEFILEPATH_BUFLEN]; int iOffsetCreate; - // Offset from module address to CreateEx method. int iOffsetCreateEx; - // Filename of the module. - char cFileName[2048]; }; struct DXGIData { + /// Filepath of the module the offsets are for. + wchar_t wcFileName[MODULEFILEPATH_BUFLEN]; int iOffsetPresent; int iOffsetResize; +}; + +struct D3D10Data { + /// Filepath of the module the offsets are for. + wchar_t wcFileName[MODULEFILEPATH_BUFLEN]; int iOffsetAddRef; int iOffsetRelease; - wchar_t wcDXGIFileName[2048]; - wchar_t wcD3D10FileName[2048]; }; struct SharedData { @@ -112,21 +123,48 @@ class Pipe { virtual void setRect() = 0; virtual void newTexture(unsigned int w, unsigned int h) = 0; Pipe(); - ~Pipe(); + virtual ~Pipe(); public: void disconnect(); }; +// From lib.cpp +extern void checkHooks(bool preonly = false); +// From dxgi.cpp extern void checkDXGIHook(bool preonly = false); +// From d3d10.cpp +extern void checkDXGI10Hook(bool preonly = false); +// From d3d9.cpp extern void checkD3D9Hook(bool preonly = false); +// From opengl.cpp extern void checkOpenGLHook(); +// From d3d9.cpp extern void freeD3D9Hook(HMODULE hModule); +// From d3d9.cpp extern Direct3D9Data *d3dd; +// From dxgi.cpp extern DXGIData *dxgi; -extern HMODULE hSelf; +// From d3d10.cpp +extern D3D10Data *d3d10; +// From lib.cpp extern BOOL bIsWin8; -extern unsigned int uiAudioCount; -extern bool bVideoHooked; + +// From lib.cpp +/// Checks if the module of the function pointer fnptr equals the module filepath +/// of refmodulepath. +/// +/// @param fnptr +/// @param refmodulepath the module path to compare against +/// @param logPrefix Used for debug logging. +/// @param fnName name of the method fnptr points to. used for debug logging +/// @return true if the module filepath of the function pointer matches the reference one +extern bool IsFnInModule(voidFunc fnptr, wchar_t *refmodulepath, const std::string &logPrefix, const std::string &fnName); + +// From lib.cpp +/// Checks fnptr is in a loaded module with module path refmodulepath. +/// +/// @return Offset as int or < 0 on failure. +extern int GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int refmodulepathLen, const std::string &logPrefix, const std::string &fnName); #endif diff --git a/overlay/opengl.cpp b/overlay/opengl.cpp index 89dcc5e8d..d28a9df4e 100644 --- a/overlay/opengl.cpp +++ b/overlay/opengl.cpp @@ -340,25 +340,31 @@ static void doSwap(HDC hdc) { static BOOL __stdcall mywglSwapBuffers(HDC hdc) { ods("OpenGL: wglSwapBuffers"); doSwap(hdc); + hhwglSwapBuffers.restore(); BOOL ret=owglSwapBuffers(hdc); hhwglSwapBuffers.inject(); + return ret; } static BOOL __stdcall mySwapBuffers(HDC hdc) { ods("OpenGL: SwapBuffers"); + hhSwapBuffers.restore(); BOOL ret=oSwapBuffers(hdc); hhSwapBuffers.inject(); + return ret; } static BOOL __stdcall mywglSwapLayerBuffers(HDC hdc, UINT fuPlanes) { ods("OpenGL: SwapLayerBuffers %x",fuPlanes); + hhwglSwapLayerBuffers.restore(); BOOL ret=owglSwapLayerBuffers(hdc, fuPlanes); hhwglSwapLayerBuffers.inject(); + return ret; } @@ -369,7 +375,7 @@ static BOOL __stdcall mywglSwapLayerBuffers(HDC hdc, UINT fuPlanes) { void checkOpenGLHook() { static bool bCheckHookActive = false; if (bCheckHookActive) { - ods("Recursion in checkOpenGLHook"); + ods("OpenGL: Recursion in checkOpenGLHook"); return; } @@ -385,7 +391,8 @@ void checkOpenGLHook() { bHooked = true; // Add a ref to ourselves; we do NOT want to get unloaded directly from this process. - GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, reinterpret_cast<char *>(&checkOpenGLHook), &hSelf); + HMODULE hTempSelf = NULL; + GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, reinterpret_cast<char *>(&checkOpenGLHook), &hTempSelf); INJECT(wglSwapBuffers); // INJECT(wglSwapLayerBuffers); diff --git a/overlay/overlay.pro b/overlay/overlay.pro index 46cd8b2fe..254581c8d 100644 --- a/overlay/overlay.pro +++ b/overlay/overlay.pro @@ -7,7 +7,7 @@ CONFIG -= qt CONFIG *= dll shared debug_and_release warn_on CONFIG -= embed_manifest_dll TARGET = mumble_ol -SOURCES = lib.cpp d3d9.cpp d3d10.cpp ods.cpp opengl.cpp HardHook.cpp +SOURCES = lib.cpp d3d9.cpp dxgi.cpp d3d10.cpp ods.cpp opengl.cpp HardHook.cpp HEADERS = lib.h ods.h HardHook.h overlay_blacklist.h EFFECTS = overlay.fx DIST = overlay.h overlay.fx HardHook.h @@ -25,12 +25,12 @@ INCLUDEPATH *= "$$(DXSDK_DIR)Include" LIBS *= -l"$$(DXSDK_DIR)Lib/x86/dxguid" -luuid -lole32 -luser32 -ladvapi32 CONFIG(release, debug|release) { - DESTDIR = ../release + DESTDIR = ../release } CONFIG(debug, debug|release) { - DESTDIR = ../release - DEFINES *= DEBUG + DESTDIR = ../debug + DEFINES *= DEBUG } fxc.output = ${QMAKE_FILE_BASE}.hex |