[release/6.0] Use x64 directory if running x64 process on arm64 apphost (#68583)

* Use x64 directory if running x64 process on arm64 apphost

Backport of #59890

* Fix is_emulating_x64 on Windows (#68671)

Co-authored-by: Mateo Torres-Ruiz <[email protected]>

Author: elinor-fung

src/native/corehost/hostmisc/pal.h

@@ -310,6 +310,7 @@ namespace pal
     void unload_library(dll_t library);
 
     bool is_running_in_wow64();
+    bool is_emulating_x64();
 
     bool are_paths_equal_with_normalized_casing(const string_t& path1, const string_t& path2);
 }

src/native/corehost/hostmisc/pal.unix.cpp

@@ -505,6 +505,10 @@ bool pal::get_default_installation_dir(pal::string_t* recv)
 
 #if defined(TARGET_OSX)
     recv->assign(_X("/usr/local/share/dotnet"));
+    if (pal::is_emulating_x64())
+    {
+        append_path(recv, _X("x64"));
+    }
 #else
     recv->assign(_X("/usr/share/dotnet"));
 #endif
@@ -979,6 +983,26 @@ bool pal::is_running_in_wow64()
     return false;
 }
 
+bool pal::is_emulating_x64()
+{
+    int is_translated_process = 0;
+#if defined(TARGET_OSX)
+    size_t size = sizeof(is_translated_process);
+    if (sysctlbyname("sysctl.proc_translated", &is_translated_process, &size, NULL, 0) == -1)
+    {
+        trace::info(_X("Could not determine whether the current process is running under Rosetta."));
+        if (errno != ENOENT)
+        {
+            trace::info(_X("Call to sysctlbyname failed: %s"), strerror(errno));
+        }
+
+        return false;
+    }
+#endif
+
+    return is_translated_process == 1;
+}
+
 bool pal::are_paths_equal_with_normalized_casing(const string_t& path1, const string_t& path2)
 {
 #if defined(TARGET_OSX)

src/native/corehost/hostmisc/pal.windows.cpp

@@ -292,6 +292,11 @@ bool pal::get_default_installation_dir(pal::string_t* recv)
     }
 
     append_path(recv, _X("dotnet"));
+    if (pal::is_emulating_x64())
+    {
+        // Install location for emulated x64 should be %ProgramFiles%\dotnet\x64.
+        append_path(recv, _X("x64"));
+    }
 
     return true;
 }
@@ -783,6 +788,46 @@ bool pal::is_running_in_wow64()
     return (fWow64Process != FALSE);
 }
 
+typedef BOOL (WINAPI* is_wow64_process2)(
+    HANDLE hProcess,
+    USHORT *pProcessMachine,
+    USHORT *pNativeMachine
+);
+
+bool pal::is_emulating_x64()
+{
+#if defined(TARGET_AMD64)
+    auto kernel32 = LoadLibraryExW(L"kernel32.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32);
+    if (kernel32 == nullptr)
+    {
+        // Loading kernel32.dll failed, log the error and continue.
+        trace::info(_X("Could not load 'kernel32.dll': %u"), GetLastError());
+        return false;
+    }
+
+    is_wow64_process2 is_wow64_process2_func = (is_wow64_process2)::GetProcAddress(kernel32, "IsWow64Process2");
+    if (is_wow64_process2_func == nullptr)
+    {
+        // Could not find IsWow64Process2.
+        return false;
+    }
+
+    USHORT process_machine;
+    USHORT native_machine;
+    if (!is_wow64_process2_func(GetCurrentProcess(), &process_machine, &native_machine))
+    {
+        // IsWow64Process2 failed. Log the error and continue.
+        trace::info(_X("Call to IsWow64Process2 failed: %u"), GetLastError());
+        return false;
+    }
+
+    // If we are running targeting x64 on a non-x64 machine, we are emulating
+    return native_machine != IMAGE_FILE_MACHINE_AMD64;
+#else
+    return false;
+#endif
+}
+
 bool pal::are_paths_equal_with_normalized_casing(const string_t& path1, const string_t& path2)
 {
     // On Windows, paths are case-insensitive