From 7e52ec46b1618de4231b46be3750e89610e593b0 Mon Sep 17 00:00:00 2001 From: matcool <26722564+matcool@users.noreply.github.com> Date: Mon, 3 Jun 2024 01:10:51 -0300 Subject: [PATCH] create Windows64Convention for default windows x64 cc --- include/tulip/platform/PlatformConvention.hpp | 2 + .../tulip/platform/Windows64Convention.hpp | 16 ++++-- src/Main.cpp | 2 + src/convention/Windows64Convention.cpp | 51 ++++++++++++------- test/Hook.cpp | 3 +- 5 files changed, 49 insertions(+), 25 deletions(-) diff --git a/include/tulip/platform/PlatformConvention.hpp b/include/tulip/platform/PlatformConvention.hpp index 391bb2c..4670f72 100644 --- a/include/tulip/platform/PlatformConvention.hpp +++ b/include/tulip/platform/PlatformConvention.hpp @@ -9,6 +9,8 @@ namespace tulip::hook { using PlatformConvention = CdeclConvention; #elif defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64) using PlatformConvention = SystemVConvention; +#elif defined(TULIP_HOOK_WINDOWS) && defined(TULIP_HOOK_X64) + using PlatformConvention = Windows64Convention; #else using PlatformConvention = DefaultConvention; #endif diff --git a/include/tulip/platform/Windows64Convention.hpp b/include/tulip/platform/Windows64Convention.hpp index c1fa4b0..49a68c9 100644 --- a/include/tulip/platform/Windows64Convention.hpp +++ b/include/tulip/platform/Windows64Convention.hpp @@ -4,7 +4,7 @@ #if defined(TULIP_HOOK_WINDOWS) && defined(TULIP_HOOK_X64) -#include "../CallingConvention.hpp" +#include "DefaultConvention.hpp" #include #include @@ -12,14 +12,22 @@ namespace tulip::hook { class AbstractFunction; - class TULIP_HOOK_DLL ThiscallConvention : public CallingConvention { + class TULIP_HOOK_DLL Windows64Convention : public DefaultConvention { public: - ~ThiscallConvention() override; + ~Windows64Convention() override; void generateDefaultCleanup(BaseAssembler& a, AbstractFunction const& function) override; + void generateIntoDefault(BaseAssembler& a, AbstractFunction const& function) override; + + static std::shared_ptr create(); + }; + + class TULIP_HOOK_DLL ThiscallConvention : public Windows64Convention { + public: + ~ThiscallConvention() override; + void generateIntoDefault(BaseAssembler& a, AbstractFunction const& function) override; void generateIntoOriginal(BaseAssembler& a, AbstractFunction const& function) override; - void generateOriginalCleanup(BaseAssembler& a, AbstractFunction const& function) override; bool needsWrapper(AbstractFunction const& function) const override; static std::shared_ptr create(); diff --git a/src/Main.cpp b/src/Main.cpp index 1fd9ef8..8d0a124 100644 --- a/src/Main.cpp +++ b/src/Main.cpp @@ -63,6 +63,8 @@ std::shared_ptr tulip::hook::createConvention(TulipConvention default: #if defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64) return SystemVConvention::create(); +#elif defined(TULIP_HOOK_WINDOWS) && defined(TULIP_HOOK_X64) + return Windows64Convention::create(); #else return DefaultConvention::create(); #endif diff --git a/src/convention/Windows64Convention.cpp b/src/convention/Windows64Convention.cpp index f306de1..0b6349c 100644 --- a/src/convention/Windows64Convention.cpp +++ b/src/convention/Windows64Convention.cpp @@ -30,9 +30,9 @@ namespace { } } -ThiscallConvention::~ThiscallConvention() {} +Windows64Convention::~Windows64Convention() {} -void ThiscallConvention::generateDefaultCleanup(BaseAssembler& a_, AbstractFunction const& function) { +void Windows64Convention::generateDefaultCleanup(BaseAssembler& a_, AbstractFunction const& function) { size_t paddedSize = getPaddedStackParamSize(function); if (paddedSize > 0) { auto& a = static_cast(a_); @@ -41,25 +41,11 @@ void ThiscallConvention::generateDefaultCleanup(BaseAssembler& a_, AbstractFunct } } -// Member functions deal with struct return differently, since in the windows x64 convention -// a struct return is as a hidden first parameter, member functions end up considering the first parameter -// the one after the `this`, whereas static functions do not. -// -// So where a static function would behave like this: -// SomeStruct* func(SomeStruct* ret_ptr, Class* self, int a, int b); -// a member function would behave like this: -// SomeStruct* Class::method(Class* this, SomeStruct* ret_ptr, int a, int b); -// so to undo this we just swap the first two parameters (RCX and RDX). - -void ThiscallConvention::generateIntoDefault(BaseAssembler& a_, AbstractFunction const& function) { +void Windows64Convention::generateIntoDefault(BaseAssembler& a_, AbstractFunction const& function) { auto& a = static_cast(a_); using enum X64Register; RegMem64 m; - if (function.m_return.m_kind == AbstractTypeKind::Other) { - a.xchg(RCX, RDX); - } - size_t stackParamSize = getStackParamSize(function); if (stackParamSize > 0) { auto const paddedSize = (stackParamSize % 16) ? stackParamSize + 8 : stackParamSize; @@ -81,7 +67,23 @@ void ThiscallConvention::generateIntoDefault(BaseAssembler& a_, AbstractFunction } } -void ThiscallConvention::generateIntoOriginal(BaseAssembler& a_, AbstractFunction const& function) { +std::shared_ptr Windows64Convention::create() { + return std::make_shared(); +} + +// Member functions deal with struct return differently, since in the windows x64 convention +// a struct return is as a hidden first parameter, member functions end up considering the first parameter +// the one after the `this`, whereas static functions do not. +// +// So where a static function would behave like this: +// SomeStruct* func(SomeStruct* ret_ptr, Class* self, int a, int b); +// a member function would behave like this: +// SomeStruct* Class::method(Class* this, SomeStruct* ret_ptr, int a, int b); +// so to undo this we just swap the first two parameters (RCX and RDX). + +ThiscallConvention::~ThiscallConvention() {} + +void ThiscallConvention::generateIntoDefault(BaseAssembler& a_, AbstractFunction const& function) { auto& a = static_cast(a_); using enum X64Register; RegMem64 m; @@ -89,9 +91,20 @@ void ThiscallConvention::generateIntoOriginal(BaseAssembler& a_, AbstractFunctio if (function.m_return.m_kind == AbstractTypeKind::Other) { a.xchg(RCX, RDX); } + + Windows64Convention::generateIntoDefault(a, function); } -void ThiscallConvention::generateOriginalCleanup(BaseAssembler& a, AbstractFunction const& function) { +void ThiscallConvention::generateIntoOriginal(BaseAssembler& a_, AbstractFunction const& function) { + auto& a = static_cast(a_); + using enum X64Register; + RegMem64 m; + + if (function.m_return.m_kind == AbstractTypeKind::Other) { + a.xchg(RCX, RDX); + } + + Windows64Convention::generateIntoOriginal(a, function); } bool ThiscallConvention::needsWrapper(AbstractFunction const& function) const { diff --git a/test/Hook.cpp b/test/Hook.cpp index 0429997..e09f7c3 100644 --- a/test/Hook.cpp +++ b/test/Hook.cpp @@ -223,8 +223,7 @@ int checkParamsHook(int a, int b, int c, int d, int e, int f, int g, float h, in TEST(HookTest, SingleHookCheckParams) { HandlerMetadata handlerMetadata; - // cheating using thiscall because default convention is not properly implemented - handlerMetadata.m_convention = tulip::hook::createConvention(tulip::hook::TulipConvention::Thiscall); + handlerMetadata.m_convention = std::make_unique(); handlerMetadata.m_abstract = AbstractFunction::from(&checkParams); auto handleRes = createHandler(reinterpret_cast(&checkParams), handlerMetadata);