Wrap all intrinsics in the intrinsics! macro

This ensures that each intrinsic ends up in a separate module, which in
turn (because rustc treats compiler_builtins specially) will result in
each intrinsic ending up in its own object file. This allows the linker
to only pick up object files for intrinsics that are missing and avoids
duplicate symbol definition errors.

src/arm.rs

@@ -3,251 +3,181 @@
 
 use core::intrinsics;
 
-// NOTE This function and the ones below are implemented using assembly because they are using a
-// custom calling convention which can't be implemented using a normal Rust function.
-// NOTE The only difference between the iOS and non-iOS versions of those functions is that the iOS
-// versions use 3 leading underscores in the names of called functions instead of 2.
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_uidivmod() {
-    core::arch::asm!(
-        "push {{lr}}",
-        "sub sp, sp, #4",
-        "mov r2, sp",
-        "bl __udivmodsi4",
-        "ldr r1, [sp]",
-        "add sp, sp, #4",
-        "pop {{pc}}",
-        options(noreturn)
-    );
-}
-
+// iOS symbols have a leading underscore.
 #[cfg(target_os = "ios")]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_uidivmod() {
-    core::arch::asm!(
-        "push {{lr}}",
-        "sub sp, sp, #4",
-        "mov r2, sp",
-        "bl ___udivmodsi4",
-        "ldr r1, [sp]",
-        "add sp, sp, #4",
-        "pop {{pc}}",
-        options(noreturn)
-    );
+macro_rules! bl {
+    ($func:literal) => {
+        concat!("bl _", $func)
+    };
 }
-
 #[cfg(not(target_os = "ios"))]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_uldivmod() {
-    core::arch::asm!(
-        "push {{r4, lr}}",
-        "sub sp, sp, #16",
-        "add r4, sp, #8",
-        "str r4, [sp]",
-        "bl __udivmoddi4",
-        "ldr r2, [sp, #8]",
-        "ldr r3, [sp, #12]",
-        "add sp, sp, #16",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
-
-#[cfg(target_os = "ios")]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_uldivmod() {
-    core::arch::asm!(
-        "push {{r4, lr}}",
-        "sub sp, sp, #16",
-        "add r4, sp, #8",
-        "str r4, [sp]",
-        "bl ___udivmoddi4",
-        "ldr r2, [sp, #8]",
-        "ldr r3, [sp, #12]",
-        "add sp, sp, #16",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
-
-#[cfg(not(target_os = "ios"))]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_idivmod() {
-    core::arch::asm!(
-        "push {{r0, r1, r4, lr}}",
-        "bl __aeabi_idiv",
-        "pop {{r1, r2}}",
-        "muls r2, r2, r0",
-        "subs r1, r1, r2",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
-
-#[cfg(target_os = "ios")]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_idivmod() {
-    core::arch::asm!(
-        "push {{r0, r1, r4, lr}}",
-        "bl ___aeabi_idiv",
-        "pop {{r1, r2}}",
-        "muls r2, r2, r0",
-        "subs r1, r1, r2",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
+macro_rules! bl {
+    ($func:literal) => {
+        concat!("bl ", $func)
+    };
+}
+
+intrinsics! {
+    // NOTE This function and the ones below are implemented using assembly because they are using a
+    // custom calling convention which can't be implemented using a normal Rust function.
+    #[naked]
+    #[cfg(not(target_env = "msvc"))]
+    pub unsafe extern "C" fn __aeabi_uidivmod() {
+        core::arch::asm!(
+            "push {{lr}}",
+            "sub sp, sp, #4",
+            "mov r2, sp",
+            bl!("__udivmodsi4"),
+            "ldr r1, [sp]",
+            "add sp, sp, #4",
+            "pop {{pc}}",
+            options(noreturn)
+        );
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_ldivmod() {
-    core::arch::asm!(
-        "push {{r4, lr}}",
-        "sub sp, sp, #16",
-        "add r4, sp, #8",
-        "str r4, [sp]",
-        "bl __divmoddi4",
-        "ldr r2, [sp, #8]",
-        "ldr r3, [sp, #12]",
-        "add sp, sp, #16",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
+    #[naked]
+    pub unsafe extern "C" fn __aeabi_uldivmod() {
+        core::arch::asm!(
+            "push {{r4, lr}}",
+            "sub sp, sp, #16",
+            "add r4, sp, #8",
+            "str r4, [sp]",
+            bl!("__udivmodsi4"),
+            "ldr r2, [sp, #8]",
+            "ldr r3, [sp, #12]",
+            "add sp, sp, #16",
+            "pop {{r4, pc}}",
+            options(noreturn)
+        );
+    }
 
-#[cfg(target_os = "ios")]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_ldivmod() {
-    core::arch::asm!(
-        "push {{r4, lr}}",
-        "sub sp, sp, #16",
-        "add r4, sp, #8",
-        "str r4, [sp]",
-        "bl ___divmoddi4",
-        "ldr r2, [sp, #8]",
-        "ldr r3, [sp, #12]",
-        "add sp, sp, #16",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
+    #[naked]
+    pub unsafe extern "C" fn __aeabi_idivmod() {
+        core::arch::asm!(
+            "push {{r0, r1, r4, lr}}",
+            bl!("__aeabi_idiv"),
+            "pop {{r1, r2}}",
+            "muls r2, r2, r0",
+            "subs r1, r1, r2",
+            "pop {{r4, pc}}",
+            options(noreturn)
+        );
+    }
 
-// The following functions use weak linkage to allow users to override
-// with custom implementation.
-// FIXME: The `*4` and `*8` variants should be defined as aliases.
+    #[naked]
+    pub unsafe extern "C" fn __aeabi_ldivmod() {
+        core::arch::asm!(
+            "push {{r4, lr}}",
+            "sub sp, sp, #16",
+            "add r4, sp, #8",
+            "str r4, [sp]",
+            bl!("__divmoddi4"),
+            "ldr r2, [sp, #8]",
+            "ldr r3, [sp, #12]",
+            "add sp, sp, #16",
+            "pop {{r4, pc}}",
+            options(noreturn)
+        );
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) {
-    ::mem::memcpy(dest, src, n);
-}
+    // The following functions use weak linkage to allow users to override
+    // with custom implementation.
+    // FIXME: The `*4` and `*8` variants should be defined as aliases.
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, mut n: usize) {
-    // We are guaranteed 4-alignment, so accessing at u32 is okay.
-    let mut dest = dest as *mut u32;
-    let mut src = src as *mut u32;
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) {
+        ::mem::memcpy(dest, src, n);
+    }
 
-    while n >= 4 {
-        *dest = *src;
-        dest = dest.offset(1);
-        src = src.offset(1);
-        n -= 4;
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, n: usize) {
+        // We are guaranteed 4-alignment, so accessing at u32 is okay.
+        let mut dest = dest as *mut u32;
+        let mut src = src as *mut u32;
+        let mut n = n;
+
+        while n >= 4 {
+            *dest = *src;
+            dest = dest.offset(1);
+            src = src.offset(1);
+            n -= 4;
+        }
+
+        __aeabi_memcpy(dest as *mut u8, src as *const u8, n);
     }
 
-    __aeabi_memcpy(dest as *mut u8, src as *const u8, n);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) {
+        __aeabi_memcpy4(dest, src, n);
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) {
-    __aeabi_memcpy4(dest, src, n);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) {
+        ::mem::memmove(dest, src, n);
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) {
-    ::mem::memmove(dest, src, n);
-}
+    #[cfg(not(any(target_os = "ios", target_env = "msvc")))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) {
+        __aeabi_memmove(dest, src, n);
+    }
 
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) {
-    __aeabi_memmove(dest, src, n);
-}
+    #[cfg(not(any(target_os = "ios", target_env = "msvc")))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) {
+        __aeabi_memmove(dest, src, n);
+    }
 
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) {
-    __aeabi_memmove(dest, src, n);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) {
+        // Note the different argument order
+        ::mem::memset(dest, c, n);
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) {
-    // Note the different argument order
-    ::mem::memset(dest, c, n);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, n: usize, c: i32) {
+        let mut dest = dest as *mut u32;
+        let mut n = n;
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, mut n: usize, c: i32) {
-    let mut dest = dest as *mut u32;
+        let byte = (c as u32) & 0xff;
+        let c = (byte << 24) | (byte << 16) | (byte << 8) | byte;
 
-    let byte = (c as u32) & 0xff;
-    let c = (byte << 24) | (byte << 16) | (byte << 8) | byte;
+        while n >= 4 {
+            *dest = c;
+            dest = dest.offset(1);
+            n -= 4;
+        }
 
-    while n >= 4 {
-        *dest = c;
-        dest = dest.offset(1);
-        n -= 4;
+        __aeabi_memset(dest as *mut u8, n, byte as i32);
     }
 
-    __aeabi_memset(dest as *mut u8, n, byte as i32);
-}
-
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) {
-    __aeabi_memset4(dest, n, c);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) {
+        __aeabi_memset4(dest, n, c);
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) {
-    __aeabi_memset(dest, n, 0);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) {
+        __aeabi_memset(dest, n, 0);
+    }
 
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) {
-    __aeabi_memset4(dest, n, 0);
-}
+    #[cfg(not(any(target_os = "ios", target_env = "msvc")))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) {
+        __aeabi_memset4(dest, n, 0);
+    }
 
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) {
-    __aeabi_memset4(dest, n, 0);
+    #[cfg(not(any(target_os = "ios", target_env = "msvc")))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) {
+        __aeabi_memset4(dest, n, 0);
+    }
 }

src/arm_linux.rs

@@ -90,17 +90,19 @@ unsafe fn atomic_cmpxchg<T>(ptr: *mut T, oldval: u32, newval: u32) -> u32 {
 
 macro_rules! atomic_rmw {
     ($name:ident, $ty:ty, $op:expr) => {
-        #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-        pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty {
-            atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty
+        intrinsics! {
+            pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty {
+                atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty
+            }
         }
     };
 }
 macro_rules! atomic_cmpxchg {
     ($name:ident, $ty:ty) => {
-        #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-        pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty {
-            atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty
+        intrinsics! {
+            pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty {
+                atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty
+            }
         }
     };
 }
@@ -205,7 +207,8 @@ atomic_cmpxchg!(__sync_val_compare_and_swap_1, u8);
 atomic_cmpxchg!(__sync_val_compare_and_swap_2, u16);
 atomic_cmpxchg!(__sync_val_compare_and_swap_4, u32);
 
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __sync_synchronize() {
-    __kuser_memory_barrier();
+intrinsics! {
+    pub unsafe extern "C" fn __sync_synchronize() {
+        __kuser_memory_barrier();
+    }
 }

src/macros.rs

@@ -76,7 +76,7 @@ macro_rules! intrinsics {
     (
         #[maybe_use_optimized_c_shim]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
@@ -84,9 +84,9 @@ macro_rules! intrinsics {
     ) => (
 
         #[cfg($name = "optimized-c")]
-        pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+        pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
             extern $abi {
-                fn $name($($argname: $ty),*) -> $ret;
+                fn $name($($argname: $ty),*) $(-> $ret)?;
             }
             unsafe {
                 $name($($argname),*)
@@ -96,7 +96,7 @@ macro_rules! intrinsics {
         #[cfg(not($name = "optimized-c"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -110,7 +110,7 @@ macro_rules! intrinsics {
     (
         #[aapcs_on_arm]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
@@ -119,7 +119,7 @@ macro_rules! intrinsics {
         #[cfg(target_arch = "arm")]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern "aapcs" fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern "aapcs" fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -127,7 +127,7 @@ macro_rules! intrinsics {
         #[cfg(not(target_arch = "arm"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -140,7 +140,7 @@ macro_rules! intrinsics {
     (
         #[unadjusted_on_win64]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
@@ -149,7 +149,7 @@ macro_rules! intrinsics {
         #[cfg(all(windows, target_pointer_width = "64"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern "unadjusted" fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -157,7 +157,7 @@ macro_rules! intrinsics {
         #[cfg(not(all(windows, target_pointer_width = "64")))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -175,7 +175,7 @@ macro_rules! intrinsics {
     (
         #[win64_128bit_abi_hack]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
@@ -183,7 +183,7 @@ macro_rules! intrinsics {
     ) => (
         #[cfg(all(windows, target_arch = "x86_64"))]
         $(#[$($attr)*])*
-        pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+        pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
             $($body)*
         }
 
@@ -193,7 +193,7 @@ macro_rules! intrinsics {
             pub extern $abi fn $name( $($argname: $ty),* )
                 -> ::macros::win64_128bit_abi_hack::U64x2
             {
-                let e: $ret = super::$name($($argname),*);
+                let e: $($ret)? = super::$name($($argname),*);
                 ::macros::win64_128bit_abi_hack::U64x2::from(e)
             }
         }
@@ -201,7 +201,7 @@ macro_rules! intrinsics {
         #[cfg(not(all(windows, target_arch = "x86_64")))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -216,21 +216,21 @@ macro_rules! intrinsics {
     (
         #[arm_aeabi_alias = $alias:ident]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
         $($rest:tt)*
     ) => (
         #[cfg(target_arch = "arm")]
-        pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+        pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
             $($body)*
         }
 
         #[cfg(target_arch = "arm")]
         pub mod $name {
             #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 super::$name($($argname),*)
             }
         }
@@ -238,7 +238,7 @@ macro_rules! intrinsics {
         #[cfg(target_arch = "arm")]
         pub mod $alias {
             #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-            pub extern "aapcs" fn $alias( $($argname: $ty),* ) -> $ret {
+            pub extern "aapcs" fn $alias( $($argname: $ty),* ) $(-> $ret)? {
                 super::$name($($argname),*)
             }
         }
@@ -246,7 +246,57 @@ macro_rules! intrinsics {
         #[cfg(not(target_arch = "arm"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+                $($body)*
+            }
+        }
+
+        intrinsics!($($rest)*);
+    );
+
+    // C mem* functions are only generated when the "mem" feature is enabled.
+    (
+        #[mem_builtin]
+        $(#[$($attr:tt)*])*
+        pub unsafe extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
+            $($body:tt)*
+        }
+
+        $($rest:tt)*
+    ) => (
+        $(#[$($attr)*])*
+        pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+            $($body)*
+        }
+
+        #[cfg(feature = "mem")]
+        pub mod $name {
+            $(#[$($attr)*])*
+            #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
+            pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+                super::$name($($argname),*)
+            }
+        }
+
+        intrinsics!($($rest)*);
+    );
+
+    // Naked functions are special: we can't generate wrappers for them since
+    // they use a custom calling convention.
+    (
+        #[naked]
+        $(#[$($attr:tt)*])*
+        pub unsafe extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
+            $($body:tt)*
+        }
+
+        $($rest:tt)*
+    ) => (
+        pub mod $name {
+            #[naked]
+            $(#[$($attr)*])*
+            #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
+            pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -268,21 +318,46 @@ macro_rules! intrinsics {
     // input we were given.
     (
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
+            $($body:tt)*
+        }
+
+        $($rest:tt)*
+    ) => (
+        $(#[$($attr)*])*
+        pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+            $($body)*
+        }
+
+        pub mod $name {
+            $(#[$($attr)*])*
+            #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+                super::$name($($argname),*)
+            }
+        }
+
+        intrinsics!($($rest)*);
+    );
+
+    // Same as the above for unsafe functions.
+    (
+        $(#[$($attr:tt)*])*
+        pub unsafe extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
         $($rest:tt)*
     ) => (
         $(#[$($attr)*])*
-        pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+        pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
             $($body)*
         }
 
         pub mod $name {
             $(#[$($attr)*])*
             #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 super::$name($($argname),*)
             }
         }

src/mem/mod.rs

@@ -19,53 +19,55 @@ use core::ops::{BitOr, Shl};
 )]
 mod impls;
 
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
-    impls::copy_forward(dest, src, n);
-    dest
-}
-
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
-    let delta = (dest as usize).wrapping_sub(src as usize);
-    if delta >= n {
-        // We can copy forwards because either dest is far enough ahead of src,
-        // or src is ahead of dest (and delta overflowed).
+intrinsics! {
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
         impls::copy_forward(dest, src, n);
-    } else {
-        impls::copy_backward(dest, src, n);
+        dest
     }
-    dest
-}
 
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn memset(s: *mut u8, c: c_int, n: usize) -> *mut u8 {
-    impls::set_bytes(s, c as u8, n);
-    s
-}
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
+        let delta = (dest as usize).wrapping_sub(src as usize);
+        if delta >= n {
+            // We can copy forwards because either dest is far enough ahead of src,
+            // or src is ahead of dest (and delta overflowed).
+            impls::copy_forward(dest, src, n);
+        } else {
+            impls::copy_backward(dest, src, n);
+        }
+        dest
+    }
 
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
-    let mut i = 0;
-    while i < n {
-        let a = *s1.add(i);
-        let b = *s2.add(i);
-        if a != b {
-            return a as i32 - b as i32;
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn memset(s: *mut u8, c: crate::mem::c_int, n: usize) -> *mut u8 {
+        impls::set_bytes(s, c as u8, n);
+        s
+    }
+
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
+        let mut i = 0;
+        while i < n {
+            let a = *s1.add(i);
+            let b = *s2.add(i);
+            if a != b {
+                return a as i32 - b as i32;
+            }
+            i += 1;
         }
-        i += 1;
+        0
     }
-    0
-}
 
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
-    memcmp(s1, s2, n)
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
+        memcmp(s1, s2, n)
+    }
 }
 
 // `bytes` must be a multiple of `mem::size_of::<T>()`
@@ -133,65 +135,65 @@ where
 
 intrinsics! {
     #[cfg(target_has_atomic_load_store = "8")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "16")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "32")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "64")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "128")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
 
     #[cfg(target_has_atomic_load_store = "8")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "16")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "32")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "64")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "128")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
 
     #[cfg(target_has_atomic_load_store = "8")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
     #[cfg(target_has_atomic_load_store = "16")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
     #[cfg(target_has_atomic_load_store = "32")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
     #[cfg(target_has_atomic_load_store = "64")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
     #[cfg(target_has_atomic_load_store = "128")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
 }

src/x86.rs

@@ -8,82 +8,78 @@ use core::intrinsics;
 // NOTE These functions are never mangled as they are not tested against compiler-rt
 // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn ___chkstk_ms() {
-    core::arch::asm!(
-        "push   %ecx",
-        "push   %eax",
-        "cmp    $0x1000,%eax",
-        "lea    12(%esp),%ecx",
-        "jb     1f",
-        "2:",
-        "sub    $0x1000,%ecx",
-        "test   %ecx,(%ecx)",
-        "sub    $0x1000,%eax",
-        "cmp    $0x1000,%eax",
-        "ja     2b",
-        "1:",
-        "sub    %eax,%ecx",
-        "test   %ecx,(%ecx)",
-        "pop    %eax",
-        "pop    %ecx",
-        "ret",
-        options(noreturn, att_syntax)
-    );
-}
+intrinsics! {
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn ___chkstk_ms() {
+        core::arch::asm!(
+            "push   %ecx",
+            "push   %eax",
+            "cmp    $0x1000,%eax",
+            "lea    12(%esp),%ecx",
+            "jb     1f",
+            "2:",
+            "sub    $0x1000,%ecx",
+            "test   %ecx,(%ecx)",
+            "sub    $0x1000,%eax",
+            "cmp    $0x1000,%eax",
+            "ja     2b",
+            "1:",
+            "sub    %eax,%ecx",
+            "test   %ecx,(%ecx)",
+            "pop    %eax",
+            "pop    %ecx",
+            "ret",
+            options(noreturn, att_syntax)
+        );
+    }
 
-// FIXME: __alloca should be an alias to __chkstk
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn __alloca() {
-    core::arch::asm!(
-        "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
-        options(noreturn, att_syntax)
-    );
-}
+    // FIXME: __alloca should be an alias to __chkstk
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn __alloca() {
+        core::arch::asm!(
+            "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
+            options(noreturn, att_syntax)
+        );
+    }
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn ___chkstk() {
-    core::arch::asm!(
-        "push   %ecx",
-        "cmp    $0x1000,%eax",
-        "lea    8(%esp),%ecx", // esp before calling this routine -> ecx
-        "jb     1f",
-        "2:",
-        "sub    $0x1000,%ecx",
-        "test   %ecx,(%ecx)",
-        "sub    $0x1000,%eax",
-        "cmp    $0x1000,%eax",
-        "ja     2b",
-        "1:",
-        "sub    %eax,%ecx",
-        "test   %ecx,(%ecx)",
-        "lea    4(%esp),%eax",  // load pointer to the return address into eax
-        "mov    %ecx,%esp",     // install the new top of stack pointer into esp
-        "mov    -4(%eax),%ecx", // restore ecx
-        "push   (%eax)",        // push return address onto the stack
-        "sub    %esp,%eax",     // restore the original value in eax
-        "ret",
-        options(noreturn, att_syntax)
-    );
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn ___chkstk() {
+        core::arch::asm!(
+            "push   %ecx",
+            "cmp    $0x1000,%eax",
+            "lea    8(%esp),%ecx", // esp before calling this routine -> ecx
+            "jb     1f",
+            "2:",
+            "sub    $0x1000,%ecx",
+            "test   %ecx,(%ecx)",
+            "sub    $0x1000,%eax",
+            "cmp    $0x1000,%eax",
+            "ja     2b",
+            "1:",
+            "sub    %eax,%ecx",
+            "test   %ecx,(%ecx)",
+            "lea    4(%esp),%eax",  // load pointer to the return address into eax
+            "mov    %ecx,%esp",     // install the new top of stack pointer into esp
+            "mov    -4(%eax),%ecx", // restore ecx
+            "push   (%eax)",        // push return address onto the stack
+            "sub    %esp,%eax",     // restore the original value in eax
+            "ret",
+            options(noreturn, att_syntax)
+        );
+    }
 }

src/x86_64.rs

@@ -8,89 +8,87 @@ use core::intrinsics;
 // NOTE These functions are never mangled as they are not tested against compiler-rt
 // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn ___chkstk_ms() {
-    core::arch::asm!(
-        "push   %rcx",
-        "push   %rax",
-        "cmp    $0x1000,%rax",
-        "lea    24(%rsp),%rcx",
-        "jb     1f",
-        "2:",
-        "sub    $0x1000,%rcx",
-        "test   %rcx,(%rcx)",
-        "sub    $0x1000,%rax",
-        "cmp    $0x1000,%rax",
-        "ja     2b",
-        "1:",
-        "sub    %rax,%rcx",
-        "test   %rcx,(%rcx)",
-        "pop    %rax",
-        "pop    %rcx",
-        "ret",
-        options(noreturn, att_syntax)
-    );
-}
+intrinsics! {
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn ___chkstk_ms() {
+        core::arch::asm!(
+            "push   %rcx",
+            "push   %rax",
+            "cmp    $0x1000,%rax",
+            "lea    24(%rsp),%rcx",
+            "jb     1f",
+            "2:",
+            "sub    $0x1000,%rcx",
+            "test   %rcx,(%rcx)",
+            "sub    $0x1000,%rax",
+            "cmp    $0x1000,%rax",
+            "ja     2b",
+            "1:",
+            "sub    %rax,%rcx",
+            "test   %rcx,(%rcx)",
+            "pop    %rax",
+            "pop    %rcx",
+            "ret",
+            options(noreturn, att_syntax)
+        );
+    }
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn __alloca() {
-    core::arch::asm!(
-        "mov    %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx
-        "jmp    ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
-        options(noreturn, att_syntax)
-    );
-}
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn __alloca() {
+        core::arch::asm!(
+            "mov    %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx
+            "jmp    ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
+            options(noreturn, att_syntax)
+        );
+    }
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn ___chkstk() {
-    core::arch::asm!(
-        "push   %rcx",
-        "cmp    $0x1000,%rax",
-        "lea    16(%rsp),%rcx", // rsp before calling this routine -> rcx
-        "jb     1f",
-        "2:",
-        "sub    $0x1000,%rcx",
-        "test   %rcx,(%rcx)",
-        "sub    $0x1000,%rax",
-        "cmp    $0x1000,%rax",
-        "ja     2b",
-        "1:",
-        "sub    %rax,%rcx",
-        "test   %rcx,(%rcx)",
-        "lea    8(%rsp),%rax",  // load pointer to the return address into rax
-        "mov    %rcx,%rsp",     // install the new top of stack pointer into rsp
-        "mov    -8(%rax),%rcx", // restore rcx
-        "push   (%rax)",        // push return address onto the stack
-        "sub    %rsp,%rax",     // restore the original value in rax
-        "ret",
-        options(noreturn, att_syntax)
-    );
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn ___chkstk() {
+        core::arch::asm!(
+            "push   %rcx",
+            "cmp    $0x1000,%rax",
+            "lea    16(%rsp),%rcx", // rsp before calling this routine -> rcx
+            "jb     1f",
+            "2:",
+            "sub    $0x1000,%rcx",
+            "test   %rcx,(%rcx)",
+            "sub    $0x1000,%rax",
+            "cmp    $0x1000,%rax",
+            "ja     2b",
+            "1:",
+            "sub    %rax,%rcx",
+            "test   %rcx,(%rcx)",
+            "lea    8(%rsp),%rax",  // load pointer to the return address into rax
+            "mov    %rcx,%rsp",     // install the new top of stack pointer into rsp
+            "mov    -8(%rax),%rcx", // restore rcx
+            "push   (%rax)",        // push return address onto the stack
+            "sub    %rsp,%rax",     // restore the original value in rax
+            "ret",
+            options(noreturn, att_syntax)
+        );
+    }
 }
 
 // HACK(https://github.com/rust-lang/rust/issues/62785): x86_64-unknown-uefi needs special LLVM
 // support unless we emit the _fltused
-#[no_mangle]
-#[used]
-#[cfg(target_os = "uefi")]
-static _fltused: i32 = 0;
+mod _fltused {
+    #[no_mangle]
+    #[used]
+    #[cfg(target_os = "uefi")]
+    static _fltused: i32 = 0;
+}