Auto merge of rust-lang#86419 - ricobbe:raw-dylib-stdcall, r=petrochenkov

Add support for raw-dylib with stdcall, fastcall functions

Next stage of work for rust-lang#58713: allow `extern "stdcall"` and `extern "fastcall"` with `#[link(kind = "raw-dylib")]`.

I've deliberately omitted support for vectorcall, as that doesn't currently work, and I wanted to get this out for review.  (I haven't really investigated the vectorcall failure much yet, but at first (very cursory) glance it appears that the problem is elsewhere.)

Author: bors

compiler/rustc_codegen_llvm/src/back/archive.rs

@@ -12,7 +12,7 @@ use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport};
 use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder};
 use rustc_codegen_ssa::{looks_like_rust_object_file, METADATA_FILENAME};
 use rustc_data_structures::temp_dir::MaybeTempDir;
-use rustc_middle::middle::cstore::DllImport;
+use rustc_middle::middle::cstore::{DllCallingConvention, DllImport};
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
 
@@ -208,10 +208,12 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
         // have any \0 characters
         let import_name_vector: Vec<CString> = dll_imports
             .iter()
-            .map(if self.config.sess.target.arch == "x86" {
-                |import: &DllImport| CString::new(format!("_{}", import.name.to_string())).unwrap()
-            } else {
-                |import: &DllImport| CString::new(import.name.to_string()).unwrap()
+            .map(|import: &DllImport| {
+                if self.config.sess.target.arch == "x86" {
+                    LlvmArchiveBuilder::i686_decorated_name(import)
+                } else {
+                    CString::new(import.name.to_string()).unwrap()
+                }
             })
             .collect();
 
@@ -391,6 +393,21 @@ impl<'a> LlvmArchiveBuilder<'a> {
             ret
         }
     }
+
+    fn i686_decorated_name(import: &DllImport) -> CString {
+        let name = import.name;
+        // We verified during construction that `name` does not contain any NULL characters, so the
+        // conversion to CString is guaranteed to succeed.
+        CString::new(match import.calling_convention {
+            DllCallingConvention::C => format!("_{}", name),
+            DllCallingConvention::Stdcall(arg_list_size) => format!("_{}@{}", name, arg_list_size),
+            DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size),
+            DllCallingConvention::Vectorcall(arg_list_size) => {
+                format!("{}@@{}", name, arg_list_size)
+            }
+        })
+        .unwrap()
+    }
 }
 
 fn string_to_io_error(s: String) -> io::Error {

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -3,7 +3,7 @@ use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_errors::Handler;
 use rustc_fs_util::fix_windows_verbatim_for_gcc;
 use rustc_hir::def_id::CrateNum;
-use rustc_middle::middle::cstore::DllImport;
+use rustc_middle::middle::cstore::{DllCallingConvention, DllImport};
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip};
 use rustc_session::config::{OutputFilenames, OutputType, PrintRequest};
@@ -34,8 +34,8 @@ use object::write::Object;
 use object::{Architecture, BinaryFormat, Endianness, FileFlags, SectionFlags, SectionKind};
 use tempfile::Builder as TempFileBuilder;
 
-use std::cmp::Ordering;
 use std::ffi::OsString;
+use std::iter::FromIterator;
 use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Output, Stdio};
 use std::{ascii, char, env, fmt, fs, io, mem, str};
@@ -259,7 +259,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
     }
 
     for (raw_dylib_name, raw_dylib_imports) in
-        collate_raw_dylibs(&codegen_results.crate_info.used_libraries)
+        collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)
     {
         ab.inject_dll_import_lib(&raw_dylib_name, &raw_dylib_imports, tmpdir);
     }
@@ -451,8 +451,11 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
 /// then the CodegenResults value contains one NativeLib instance for each block.  However, the
 /// linker appears to expect only a single import library for each library used, so we need to
 /// collate the symbols together by library name before generating the import libraries.
-fn collate_raw_dylibs(used_libraries: &[NativeLib]) -> Vec<(String, Vec<DllImport>)> {
-    let mut dylib_table: FxHashMap<String, FxHashSet<Symbol>> = FxHashMap::default();
+fn collate_raw_dylibs(
+    sess: &Session,
+    used_libraries: &[NativeLib],
+) -> Vec<(String, Vec<DllImport>)> {
+    let mut dylib_table: FxHashMap<String, FxHashSet<DllImport>> = FxHashMap::default();
 
     for lib in used_libraries {
         if lib.kind == NativeLibKind::RawDylib {
@@ -464,35 +467,51 @@ fn collate_raw_dylibs(used_libraries: &[NativeLib]) -> Vec<(String, Vec<DllImpor
             } else {
                 format!("{}.dll", name)
             };
-            dylib_table
-                .entry(name)
-                .or_default()
-                .extend(lib.dll_imports.iter().map(|import| import.name));
+            dylib_table.entry(name).or_default().extend(lib.dll_imports.iter().cloned());
         }
     }
 
-    // FIXME: when we add support for ordinals, fix this to propagate ordinals.  Also figure out
-    // what we should do if we have two DllImport values with the same name but different
-    // ordinals.
-    let mut result = dylib_table
+    // Rustc already signals an error if we have two imports with the same name but different
+    // calling conventions (or function signatures), so we don't have pay attention to those
+    // when ordering.
+    // FIXME: when we add support for ordinals, figure out if we need to do anything if we
+    // have two DllImport values with the same name but different ordinals.
+    let mut result: Vec<(String, Vec<DllImport>)> = dylib_table
         .into_iter()
-        .map(|(lib_name, imported_names)| {
-            let mut names = imported_names
-                .iter()
-                .map(|name| DllImport { name: *name, ordinal: None })
-                .collect::<Vec<_>>();
-            names.sort_unstable_by(|a: &DllImport, b: &DllImport| {
-                match a.name.as_str().cmp(&b.name.as_str()) {
-                    Ordering::Equal => a.ordinal.cmp(&b.ordinal),
-                    x => x,
-                }
-            });
-            (lib_name, names)
+        .map(|(lib_name, import_table)| {
+            let mut imports = Vec::from_iter(import_table.into_iter());
+            imports.sort_unstable_by_key(|x: &DllImport| x.name.as_str());
+            (lib_name, imports)
         })
         .collect::<Vec<_>>();
     result.sort_unstable_by(|a: &(String, Vec<DllImport>), b: &(String, Vec<DllImport>)| {
         a.0.cmp(&b.0)
     });
+    let result = result;
+
+    // Check for multiple imports with the same name but different calling conventions or
+    // (when relevant) argument list sizes.  Rustc only signals an error for this if the
+    // declarations are at the same scope level; if one shadows the other, we only get a lint
+    // warning.
+    for (library, imports) in &result {
+        let mut import_table: FxHashMap<Symbol, DllCallingConvention> = FxHashMap::default();
+        for import in imports {
+            if let Some(old_convention) =
+                import_table.insert(import.name, import.calling_convention)
+            {
+                if import.calling_convention != old_convention {
+                    sess.span_fatal(
+                        import.span,
+                        &format!(
+                            "multiple definitions of external function `{}` from library `{}` have different calling conventions",
+                            import.name,
+                            library,
+                    ));
+                }
+            }
+        }
+    }
+
     result
 }
 

compiler/rustc_metadata/src/native_libs.rs

@@ -3,8 +3,8 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::itemlikevisit::ItemLikeVisitor;
-use rustc_middle::middle::cstore::{DllImport, NativeLib};
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::middle::cstore::{DllCallingConvention, DllImport, NativeLib};
+use rustc_middle::ty::{List, ParamEnv, ParamEnvAnd, Ty, TyCtxt};
 use rustc_session::parse::feature_err;
 use rustc_session::utils::NativeLibKind;
 use rustc_session::Session;
@@ -199,22 +199,10 @@ impl ItemLikeVisitor<'tcx> for Collector<'tcx> {
             }
 
             if lib.kind == NativeLibKind::RawDylib {
-                match abi {
-                    Abi::C { .. } => (),
-                    Abi::Cdecl => (),
-                    _ => {
-                        if sess.target.arch == "x86" {
-                            sess.span_fatal(
-                                it.span,
-                                r#"`#[link(kind = "raw-dylib")]` only supports C and Cdecl ABIs"#,
-                            );
-                        }
-                    }
-                };
                 lib.dll_imports.extend(
                     foreign_mod_items
                         .iter()
-                        .map(|child_item| DllImport { name: child_item.ident.name, ordinal: None }),
+                        .map(|child_item| self.build_dll_import(abi, child_item)),
                 );
             }
 
@@ -396,4 +384,58 @@ impl Collector<'tcx> {
             }
         }
     }
+
+    fn i686_arg_list_size(&self, item: &hir::ForeignItemRef<'_>) -> usize {
+        let argument_types: &List<Ty<'_>> = self.tcx.erase_late_bound_regions(
+            self.tcx
+                .type_of(item.id.def_id)
+                .fn_sig(self.tcx)
+                .inputs()
+                .map_bound(|slice| self.tcx.mk_type_list(slice.iter())),
+        );
+
+        argument_types
+            .iter()
+            .map(|ty| {
+                let layout = self
+                    .tcx
+                    .layout_of(ParamEnvAnd { param_env: ParamEnv::empty(), value: ty })
+                    .expect("layout")
+                    .layout;
+                // In both stdcall and fastcall, we always round up the argument size to the
+                // nearest multiple of 4 bytes.
+                (layout.size.bytes_usize() + 3) & !3
+            })
+            .sum()
+    }
+
+    fn build_dll_import(&self, abi: Abi, item: &hir::ForeignItemRef<'_>) -> DllImport {
+        let calling_convention = if self.tcx.sess.target.arch == "x86" {
+            match abi {
+                Abi::C { .. } | Abi::Cdecl => DllCallingConvention::C,
+                Abi::Stdcall { .. } | Abi::System { .. } => {
+                    DllCallingConvention::Stdcall(self.i686_arg_list_size(item))
+                }
+                Abi::Fastcall => DllCallingConvention::Fastcall(self.i686_arg_list_size(item)),
+                // Vectorcall is intentionally not supported at this time.
+                _ => {
+                    self.tcx.sess.span_fatal(
+                        item.span,
+                        r#"ABI not supported by `#[link(kind = "raw-dylib")]` on i686"#,
+                    );
+                }
+            }
+        } else {
+            match abi {
+                Abi::C { .. } | Abi::Win64 | Abi::System { .. } => DllCallingConvention::C,
+                _ => {
+                    self.tcx.sess.span_fatal(
+                        item.span,
+                        r#"ABI not supported by `#[link(kind = "raw-dylib")]` on this architecture"#,
+                    );
+                }
+            }
+        };
+        DllImport { name: item.ident.name, ordinal: None, calling_convention, span: item.span }
+    }
 }

compiler/rustc_middle/src/middle/cstore.rs

@@ -77,10 +77,29 @@ pub struct NativeLib {
     pub dll_imports: Vec<DllImport>,
 }
 
-#[derive(Clone, Debug, Encodable, Decodable, HashStable)]
+#[derive(Clone, Debug, PartialEq, Eq, Encodable, Decodable, Hash, HashStable)]
 pub struct DllImport {
     pub name: Symbol,
     pub ordinal: Option<u16>,
+    /// Calling convention for the function.
+    ///
+    /// On x86_64, this is always `DllCallingConvention::C`; on i686, it can be any
+    /// of the values, and we use `DllCallingConvention::C` to represent `"cdecl"`.
+    pub calling_convention: DllCallingConvention,
+    /// Span of import's "extern" declaration; used for diagnostics.
+    pub span: Span,
+}
+
+/// Calling convention for a function defined in an external library.
+///
+/// The usize value, where present, indicates the size of the function's argument list
+/// in bytes.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Encodable, Decodable, Hash, HashStable)]
+pub enum DllCallingConvention {
+    C,
+    Stdcall(usize),
+    Fastcall(usize),
+    Vectorcall(usize),
 }
 
 #[derive(Clone, TyEncodable, TyDecodable, HashStable, Debug)]

src/test/run-make/raw-dylib-alt-calling-convention/Makefile

@@ -0,0 +1,18 @@
+# Test the behavior of #[link(.., kind = "raw-dylib")] with alternative calling conventions.
+
+# only-i686-pc-windows-msvc
+
+-include ../../run-make-fulldeps/tools.mk
+
+all:
+	$(call COMPILE_OBJ,"$(TMPDIR)"/extern.obj,extern.c)
+	$(CC) "$(TMPDIR)"/extern.obj -link -dll -out:"$(TMPDIR)"/extern.dll
+	$(RUSTC) --crate-type lib --crate-name raw_dylib_alt_calling_convention_test lib.rs
+	$(RUSTC) --crate-type bin driver.rs -L "$(TMPDIR)"
+	"$(TMPDIR)"/driver > "$(TMPDIR)"/output.txt
+
+ifdef RUSTC_BLESS_TEST
+	cp "$(TMPDIR)"/output.txt output.txt
+else
+	$(DIFF) output.txt "$(TMPDIR)"/output.txt
+endif

src/test/run-make/raw-dylib-alt-calling-convention/driver.rs

@@ -0,0 +1,5 @@
+extern crate raw_dylib_alt_calling_convention_test;
+
+fn main() {
+    raw_dylib_alt_calling_convention_test::library_function();
+}