Merge pull request #452 from Amanieu/intrinsic_mod

ci/run-docker.sh

@@ -19,7 +19,7 @@ run() {
            -e CARGO_HOME=/cargo \
            -e CARGO_TARGET_DIR=/target \
            -e RUST_COMPILER_RT_ROOT \
-           -v $(dirname $(dirname `which cargo`)):/cargo \
+           -v "${HOME}/.cargo":/cargo \
            -v `pwd`/target:/target \
            -v `pwd`:/checkout:ro \
            -v `rustc --print sysroot`:/rust:ro \

src/arm.rs

@@ -3,251 +3,181 @@
 
 use core::intrinsics;
 
-// NOTE This function and the ones below are implemented using assembly because they are using a
-// custom calling convention which can't be implemented using a normal Rust function.
-// NOTE The only difference between the iOS and non-iOS versions of those functions is that the iOS
-// versions use 3 leading underscores in the names of called functions instead of 2.
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_uidivmod() {
-    core::arch::asm!(
-        "push {{lr}}",
-        "sub sp, sp, #4",
-        "mov r2, sp",
-        "bl __udivmodsi4",
-        "ldr r1, [sp]",
-        "add sp, sp, #4",
-        "pop {{pc}}",
-        options(noreturn)
-    );
-}
-
+// iOS symbols have a leading underscore.
 #[cfg(target_os = "ios")]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_uidivmod() {
-    core::arch::asm!(
-        "push {{lr}}",
-        "sub sp, sp, #4",
-        "mov r2, sp",
-        "bl ___udivmodsi4",
-        "ldr r1, [sp]",
-        "add sp, sp, #4",
-        "pop {{pc}}",
-        options(noreturn)
-    );
+macro_rules! bl {
+    ($func:literal) => {
+        concat!("bl _", $func)
+    };
 }
-
 #[cfg(not(target_os = "ios"))]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_uldivmod() {
-    core::arch::asm!(
-        "push {{r4, lr}}",
-        "sub sp, sp, #16",
-        "add r4, sp, #8",
-        "str r4, [sp]",
-        "bl __udivmoddi4",
-        "ldr r2, [sp, #8]",
-        "ldr r3, [sp, #12]",
-        "add sp, sp, #16",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
-
-#[cfg(target_os = "ios")]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_uldivmod() {
-    core::arch::asm!(
-        "push {{r4, lr}}",
-        "sub sp, sp, #16",
-        "add r4, sp, #8",
-        "str r4, [sp]",
-        "bl ___udivmoddi4",
-        "ldr r2, [sp, #8]",
-        "ldr r3, [sp, #12]",
-        "add sp, sp, #16",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
-
-#[cfg(not(target_os = "ios"))]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_idivmod() {
-    core::arch::asm!(
-        "push {{r0, r1, r4, lr}}",
-        "bl __aeabi_idiv",
-        "pop {{r1, r2}}",
-        "muls r2, r2, r0",
-        "subs r1, r1, r2",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
-
-#[cfg(target_os = "ios")]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_idivmod() {
-    core::arch::asm!(
-        "push {{r0, r1, r4, lr}}",
-        "bl ___aeabi_idiv",
-        "pop {{r1, r2}}",
-        "muls r2, r2, r0",
-        "subs r1, r1, r2",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
+macro_rules! bl {
+    ($func:literal) => {
+        concat!("bl ", $func)
+    };
+}
+
+intrinsics! {
+    // NOTE This function and the ones below are implemented using assembly because they are using a
+    // custom calling convention which can't be implemented using a normal Rust function.
+    #[naked]
+    #[cfg(not(target_env = "msvc"))]
+    pub unsafe extern "C" fn __aeabi_uidivmod() {
+        core::arch::asm!(
+            "push {{lr}}",
+            "sub sp, sp, #4",
+            "mov r2, sp",
+            bl!("__udivmodsi4"),
+            "ldr r1, [sp]",
+            "add sp, sp, #4",
+            "pop {{pc}}",
+            options(noreturn)
+        );
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_ldivmod() {
-    core::arch::asm!(
-        "push {{r4, lr}}",
-        "sub sp, sp, #16",
-        "add r4, sp, #8",
-        "str r4, [sp]",
-        "bl __divmoddi4",
-        "ldr r2, [sp, #8]",
-        "ldr r3, [sp, #12]",
-        "add sp, sp, #16",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
+    #[naked]
+    pub unsafe extern "C" fn __aeabi_uldivmod() {
+        core::arch::asm!(
+            "push {{r4, lr}}",
+            "sub sp, sp, #16",
+            "add r4, sp, #8",
+            "str r4, [sp]",
+            bl!("__udivmodsi4"),
+            "ldr r2, [sp, #8]",
+            "ldr r3, [sp, #12]",
+            "add sp, sp, #16",
+            "pop {{r4, pc}}",
+            options(noreturn)
+        );
+    }
 
-#[cfg(target_os = "ios")]
-#[naked]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __aeabi_ldivmod() {
-    core::arch::asm!(
-        "push {{r4, lr}}",
-        "sub sp, sp, #16",
-        "add r4, sp, #8",
-        "str r4, [sp]",
-        "bl ___divmoddi4",
-        "ldr r2, [sp, #8]",
-        "ldr r3, [sp, #12]",
-        "add sp, sp, #16",
-        "pop {{r4, pc}}",
-        options(noreturn)
-    );
-}
+    #[naked]
+    pub unsafe extern "C" fn __aeabi_idivmod() {
+        core::arch::asm!(
+            "push {{r0, r1, r4, lr}}",
+            bl!("__aeabi_idiv"),
+            "pop {{r1, r2}}",
+            "muls r2, r2, r0",
+            "subs r1, r1, r2",
+            "pop {{r4, pc}}",
+            options(noreturn)
+        );
+    }
 
-// The following functions use weak linkage to allow users to override
-// with custom implementation.
-// FIXME: The `*4` and `*8` variants should be defined as aliases.
+    #[naked]
+    pub unsafe extern "C" fn __aeabi_ldivmod() {
+        core::arch::asm!(
+            "push {{r4, lr}}",
+            "sub sp, sp, #16",
+            "add r4, sp, #8",
+            "str r4, [sp]",
+            bl!("__divmoddi4"),
+            "ldr r2, [sp, #8]",
+            "ldr r3, [sp, #12]",
+            "add sp, sp, #16",
+            "pop {{r4, pc}}",
+            options(noreturn)
+        );
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) {
-    ::mem::memcpy(dest, src, n);
-}
+    // The following functions use weak linkage to allow users to override
+    // with custom implementation.
+    // FIXME: The `*4` and `*8` variants should be defined as aliases.
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, mut n: usize) {
-    // We are guaranteed 4-alignment, so accessing at u32 is okay.
-    let mut dest = dest as *mut u32;
-    let mut src = src as *mut u32;
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) {
+        ::mem::memcpy(dest, src, n);
+    }
 
-    while n >= 4 {
-        *dest = *src;
-        dest = dest.offset(1);
-        src = src.offset(1);
-        n -= 4;
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, n: usize) {
+        // We are guaranteed 4-alignment, so accessing at u32 is okay.
+        let mut dest = dest as *mut u32;
+        let mut src = src as *mut u32;
+        let mut n = n;
+
+        while n >= 4 {
+            *dest = *src;
+            dest = dest.offset(1);
+            src = src.offset(1);
+            n -= 4;
+        }
+
+        __aeabi_memcpy(dest as *mut u8, src as *const u8, n);
     }
 
-    __aeabi_memcpy(dest as *mut u8, src as *const u8, n);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) {
+        __aeabi_memcpy4(dest, src, n);
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) {
-    __aeabi_memcpy4(dest, src, n);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) {
+        ::mem::memmove(dest, src, n);
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) {
-    ::mem::memmove(dest, src, n);
-}
+    #[cfg(not(any(target_os = "ios", target_env = "msvc")))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) {
+        __aeabi_memmove(dest, src, n);
+    }
 
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) {
-    __aeabi_memmove(dest, src, n);
-}
+    #[cfg(not(any(target_os = "ios", target_env = "msvc")))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) {
+        __aeabi_memmove(dest, src, n);
+    }
 
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) {
-    __aeabi_memmove(dest, src, n);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) {
+        // Note the different argument order
+        ::mem::memset(dest, c, n);
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) {
-    // Note the different argument order
-    ::mem::memset(dest, c, n);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, n: usize, c: i32) {
+        let mut dest = dest as *mut u32;
+        let mut n = n;
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, mut n: usize, c: i32) {
-    let mut dest = dest as *mut u32;
+        let byte = (c as u32) & 0xff;
+        let c = (byte << 24) | (byte << 16) | (byte << 8) | byte;
 
-    let byte = (c as u32) & 0xff;
-    let c = (byte << 24) | (byte << 16) | (byte << 8) | byte;
+        while n >= 4 {
+            *dest = c;
+            dest = dest.offset(1);
+            n -= 4;
+        }
 
-    while n >= 4 {
-        *dest = c;
-        dest = dest.offset(1);
-        n -= 4;
+        __aeabi_memset(dest as *mut u8, n, byte as i32);
     }
 
-    __aeabi_memset(dest as *mut u8, n, byte as i32);
-}
-
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) {
-    __aeabi_memset4(dest, n, c);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) {
+        __aeabi_memset4(dest, n, c);
+    }
 
-#[cfg(not(target_os = "ios"))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) {
-    __aeabi_memset(dest, n, 0);
-}
+    #[cfg(not(target_os = "ios"))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) {
+        __aeabi_memset(dest, n, 0);
+    }
 
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) {
-    __aeabi_memset4(dest, n, 0);
-}
+    #[cfg(not(any(target_os = "ios", target_env = "msvc")))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) {
+        __aeabi_memset4(dest, n, 0);
+    }
 
-#[cfg(not(any(target_os = "ios", target_env = "msvc")))]
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-#[linkage = "weak"]
-pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) {
-    __aeabi_memset4(dest, n, 0);
+    #[cfg(not(any(target_os = "ios", target_env = "msvc")))]
+    #[linkage = "weak"]
+    pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) {
+        __aeabi_memset4(dest, n, 0);
+    }
 }

src/arm_linux.rs

@@ -90,17 +90,19 @@ unsafe fn atomic_cmpxchg<T>(ptr: *mut T, oldval: u32, newval: u32) -> u32 {
 
 macro_rules! atomic_rmw {
     ($name:ident, $ty:ty, $op:expr) => {
-        #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-        pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty {
-            atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty
+        intrinsics! {
+            pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty {
+                atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty
+            }
         }
     };
 }
 macro_rules! atomic_cmpxchg {
     ($name:ident, $ty:ty) => {
-        #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-        pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty {
-            atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty
+        intrinsics! {
+            pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty {
+                atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty
+            }
         }
     };
 }
@@ -205,7 +207,8 @@ atomic_cmpxchg!(__sync_val_compare_and_swap_1, u8);
 atomic_cmpxchg!(__sync_val_compare_and_swap_2, u16);
 atomic_cmpxchg!(__sync_val_compare_and_swap_4, u32);
 
-#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-pub unsafe extern "C" fn __sync_synchronize() {
-    __kuser_memory_barrier();
+intrinsics! {
+    pub unsafe extern "C" fn __sync_synchronize() {
+        __kuser_memory_barrier();
+    }
 }

src/macros.rs

@@ -76,7 +76,7 @@ macro_rules! intrinsics {
     (
         #[maybe_use_optimized_c_shim]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
@@ -84,9 +84,9 @@ macro_rules! intrinsics {
     ) => (
 
         #[cfg($name = "optimized-c")]
-        pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+        pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
             extern $abi {
-                fn $name($($argname: $ty),*) -> $ret;
+                fn $name($($argname: $ty),*) $(-> $ret)?;
             }
             unsafe {
                 $name($($argname),*)
@@ -96,7 +96,7 @@ macro_rules! intrinsics {
         #[cfg(not($name = "optimized-c"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -110,7 +110,7 @@ macro_rules! intrinsics {
     (
         #[aapcs_on_arm]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
@@ -119,7 +119,7 @@ macro_rules! intrinsics {
         #[cfg(target_arch = "arm")]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern "aapcs" fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern "aapcs" fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -127,7 +127,7 @@ macro_rules! intrinsics {
         #[cfg(not(target_arch = "arm"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -140,7 +140,7 @@ macro_rules! intrinsics {
     (
         #[unadjusted_on_win64]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
@@ -149,7 +149,7 @@ macro_rules! intrinsics {
         #[cfg(all(windows, target_pointer_width = "64"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern "unadjusted" fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -157,7 +157,7 @@ macro_rules! intrinsics {
         #[cfg(not(all(windows, target_pointer_width = "64")))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -175,7 +175,7 @@ macro_rules! intrinsics {
     (
         #[win64_128bit_abi_hack]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
@@ -183,7 +183,7 @@ macro_rules! intrinsics {
     ) => (
         #[cfg(all(windows, target_arch = "x86_64"))]
         $(#[$($attr)*])*
-        pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+        pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
             $($body)*
         }
 
@@ -193,7 +193,7 @@ macro_rules! intrinsics {
             pub extern $abi fn $name( $($argname: $ty),* )
                 -> ::macros::win64_128bit_abi_hack::U64x2
             {
-                let e: $ret = super::$name($($argname),*);
+                let e: $($ret)? = super::$name($($argname),*);
                 ::macros::win64_128bit_abi_hack::U64x2::from(e)
             }
         }
@@ -201,7 +201,7 @@ macro_rules! intrinsics {
         #[cfg(not(all(windows, target_arch = "x86_64")))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -216,21 +216,21 @@ macro_rules! intrinsics {
     (
         #[arm_aeabi_alias = $alias:ident]
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
         $($rest:tt)*
     ) => (
         #[cfg(target_arch = "arm")]
-        pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+        pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
             $($body)*
         }
 
         #[cfg(target_arch = "arm")]
         pub mod $name {
             #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 super::$name($($argname),*)
             }
         }
@@ -238,7 +238,7 @@ macro_rules! intrinsics {
         #[cfg(target_arch = "arm")]
         pub mod $alias {
             #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-            pub extern "aapcs" fn $alias( $($argname: $ty),* ) -> $ret {
+            pub extern "aapcs" fn $alias( $($argname: $ty),* ) $(-> $ret)? {
                 super::$name($($argname),*)
             }
         }
@@ -246,7 +246,57 @@ macro_rules! intrinsics {
         #[cfg(not(target_arch = "arm"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+                $($body)*
+            }
+        }
+
+        intrinsics!($($rest)*);
+    );
+
+    // C mem* functions are only generated when the "mem" feature is enabled.
+    (
+        #[mem_builtin]
+        $(#[$($attr:tt)*])*
+        pub unsafe extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
+            $($body:tt)*
+        }
+
+        $($rest:tt)*
+    ) => (
+        $(#[$($attr)*])*
+        pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+            $($body)*
+        }
+
+        #[cfg(feature = "mem")]
+        pub mod $name {
+            $(#[$($attr)*])*
+            #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
+            pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+                super::$name($($argname),*)
+            }
+        }
+
+        intrinsics!($($rest)*);
+    );
+
+    // Naked functions are special: we can't generate wrappers for them since
+    // they use a custom calling convention.
+    (
+        #[naked]
+        $(#[$($attr:tt)*])*
+        pub unsafe extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
+            $($body:tt)*
+        }
+
+        $($rest:tt)*
+    ) => (
+        pub mod $name {
+            #[naked]
+            $(#[$($attr)*])*
+            #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
+            pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }
@@ -268,21 +318,46 @@ macro_rules! intrinsics {
     // input we were given.
     (
         $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) -> $ret:ty {
+        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
+            $($body:tt)*
+        }
+
+        $($rest:tt)*
+    ) => (
+        $(#[$($attr)*])*
+        pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+            $($body)*
+        }
+
+        pub mod $name {
+            $(#[$($attr)*])*
+            #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
+            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
+                super::$name($($argname),*)
+            }
+        }
+
+        intrinsics!($($rest)*);
+    );
+
+    // Same as the above for unsafe functions.
+    (
+        $(#[$($attr:tt)*])*
+        pub unsafe extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
         }
 
         $($rest:tt)*
     ) => (
         $(#[$($attr)*])*
-        pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+        pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
             $($body)*
         }
 
         pub mod $name {
             $(#[$($attr)*])*
             #[cfg_attr(not(feature = "mangled-names"), no_mangle)]
-            pub extern $abi fn $name( $($argname: $ty),* ) -> $ret {
+            pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
                 super::$name($($argname),*)
             }
         }

src/mem/mod.rs

@@ -19,53 +19,55 @@ use core::ops::{BitOr, Shl};
 )]
 mod impls;
 
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
-    impls::copy_forward(dest, src, n);
-    dest
-}
-
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
-    let delta = (dest as usize).wrapping_sub(src as usize);
-    if delta >= n {
-        // We can copy forwards because either dest is far enough ahead of src,
-        // or src is ahead of dest (and delta overflowed).
+intrinsics! {
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
         impls::copy_forward(dest, src, n);
-    } else {
-        impls::copy_backward(dest, src, n);
+        dest
     }
-    dest
-}
 
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn memset(s: *mut u8, c: c_int, n: usize) -> *mut u8 {
-    impls::set_bytes(s, c as u8, n);
-    s
-}
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 {
+        let delta = (dest as usize).wrapping_sub(src as usize);
+        if delta >= n {
+            // We can copy forwards because either dest is far enough ahead of src,
+            // or src is ahead of dest (and delta overflowed).
+            impls::copy_forward(dest, src, n);
+        } else {
+            impls::copy_backward(dest, src, n);
+        }
+        dest
+    }
 
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
-    let mut i = 0;
-    while i < n {
-        let a = *s1.add(i);
-        let b = *s2.add(i);
-        if a != b {
-            return a as i32 - b as i32;
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn memset(s: *mut u8, c: crate::mem::c_int, n: usize) -> *mut u8 {
+        impls::set_bytes(s, c as u8, n);
+        s
+    }
+
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
+        let mut i = 0;
+        while i < n {
+            let a = *s1.add(i);
+            let b = *s2.add(i);
+            if a != b {
+                return a as i32 - b as i32;
+            }
+            i += 1;
         }
-        i += 1;
+        0
     }
-    0
-}
 
-#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)]
-#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
-pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
-    memcmp(s1, s2, n)
+    #[mem_builtin]
+    #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")]
+    pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
+        memcmp(s1, s2, n)
+    }
 }
 
 // `bytes` must be a multiple of `mem::size_of::<T>()`
@@ -133,65 +135,65 @@ where
 
 intrinsics! {
     #[cfg(target_has_atomic_load_store = "8")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "16")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "32")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "64")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "128")]
-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
         memcpy_element_unordered_atomic(dest, src, bytes);
     }
 
     #[cfg(target_has_atomic_load_store = "8")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "16")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "32")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "64")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
     #[cfg(target_has_atomic_load_store = "128")]
-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
         memmove_element_unordered_atomic(dest, src, bytes);
     }
 
     #[cfg(target_has_atomic_load_store = "8")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
     #[cfg(target_has_atomic_load_store = "16")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
     #[cfg(target_has_atomic_load_store = "32")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
     #[cfg(target_has_atomic_load_store = "64")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
     #[cfg(target_has_atomic_load_store = "128")]
-    pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
+    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
         memset_element_unordered_atomic(s, c, bytes);
     }
 }

src/x86.rs

@@ -8,82 +8,78 @@ use core::intrinsics;
 // NOTE These functions are never mangled as they are not tested against compiler-rt
 // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn ___chkstk_ms() {
-    core::arch::asm!(
-        "push   %ecx",
-        "push   %eax",
-        "cmp    $0x1000,%eax",
-        "lea    12(%esp),%ecx",
-        "jb     1f",
-        "2:",
-        "sub    $0x1000,%ecx",
-        "test   %ecx,(%ecx)",
-        "sub    $0x1000,%eax",
-        "cmp    $0x1000,%eax",
-        "ja     2b",
-        "1:",
-        "sub    %eax,%ecx",
-        "test   %ecx,(%ecx)",
-        "pop    %eax",
-        "pop    %ecx",
-        "ret",
-        options(noreturn, att_syntax)
-    );
-}
+intrinsics! {
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn ___chkstk_ms() {
+        core::arch::asm!(
+            "push   %ecx",
+            "push   %eax",
+            "cmp    $0x1000,%eax",
+            "lea    12(%esp),%ecx",
+            "jb     1f",
+            "2:",
+            "sub    $0x1000,%ecx",
+            "test   %ecx,(%ecx)",
+            "sub    $0x1000,%eax",
+            "cmp    $0x1000,%eax",
+            "ja     2b",
+            "1:",
+            "sub    %eax,%ecx",
+            "test   %ecx,(%ecx)",
+            "pop    %eax",
+            "pop    %ecx",
+            "ret",
+            options(noreturn, att_syntax)
+        );
+    }
 
-// FIXME: __alloca should be an alias to __chkstk
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn __alloca() {
-    core::arch::asm!(
-        "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
-        options(noreturn, att_syntax)
-    );
-}
+    // FIXME: __alloca should be an alias to __chkstk
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn __alloca() {
+        core::arch::asm!(
+            "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
+            options(noreturn, att_syntax)
+        );
+    }
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn ___chkstk() {
-    core::arch::asm!(
-        "push   %ecx",
-        "cmp    $0x1000,%eax",
-        "lea    8(%esp),%ecx", // esp before calling this routine -> ecx
-        "jb     1f",
-        "2:",
-        "sub    $0x1000,%ecx",
-        "test   %ecx,(%ecx)",
-        "sub    $0x1000,%eax",
-        "cmp    $0x1000,%eax",
-        "ja     2b",
-        "1:",
-        "sub    %eax,%ecx",
-        "test   %ecx,(%ecx)",
-        "lea    4(%esp),%eax",  // load pointer to the return address into eax
-        "mov    %ecx,%esp",     // install the new top of stack pointer into esp
-        "mov    -4(%eax),%ecx", // restore ecx
-        "push   (%eax)",        // push return address onto the stack
-        "sub    %esp,%eax",     // restore the original value in eax
-        "ret",
-        options(noreturn, att_syntax)
-    );
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn ___chkstk() {
+        core::arch::asm!(
+            "push   %ecx",
+            "cmp    $0x1000,%eax",
+            "lea    8(%esp),%ecx", // esp before calling this routine -> ecx
+            "jb     1f",
+            "2:",
+            "sub    $0x1000,%ecx",
+            "test   %ecx,(%ecx)",
+            "sub    $0x1000,%eax",
+            "cmp    $0x1000,%eax",
+            "ja     2b",
+            "1:",
+            "sub    %eax,%ecx",
+            "test   %ecx,(%ecx)",
+            "lea    4(%esp),%eax",  // load pointer to the return address into eax
+            "mov    %ecx,%esp",     // install the new top of stack pointer into esp
+            "mov    -4(%eax),%ecx", // restore ecx
+            "push   (%eax)",        // push return address onto the stack
+            "sub    %esp,%eax",     // restore the original value in eax
+            "ret",
+            options(noreturn, att_syntax)
+        );
+    }
 }

src/x86_64.rs

@@ -8,89 +8,87 @@ use core::intrinsics;
 // NOTE These functions are never mangled as they are not tested against compiler-rt
 // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn ___chkstk_ms() {
-    core::arch::asm!(
-        "push   %rcx",
-        "push   %rax",
-        "cmp    $0x1000,%rax",
-        "lea    24(%rsp),%rcx",
-        "jb     1f",
-        "2:",
-        "sub    $0x1000,%rcx",
-        "test   %rcx,(%rcx)",
-        "sub    $0x1000,%rax",
-        "cmp    $0x1000,%rax",
-        "ja     2b",
-        "1:",
-        "sub    %rax,%rcx",
-        "test   %rcx,(%rcx)",
-        "pop    %rax",
-        "pop    %rcx",
-        "ret",
-        options(noreturn, att_syntax)
-    );
-}
+intrinsics! {
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn ___chkstk_ms() {
+        core::arch::asm!(
+            "push   %rcx",
+            "push   %rax",
+            "cmp    $0x1000,%rax",
+            "lea    24(%rsp),%rcx",
+            "jb     1f",
+            "2:",
+            "sub    $0x1000,%rcx",
+            "test   %rcx,(%rcx)",
+            "sub    $0x1000,%rax",
+            "cmp    $0x1000,%rax",
+            "ja     2b",
+            "1:",
+            "sub    %rax,%rcx",
+            "test   %rcx,(%rcx)",
+            "pop    %rax",
+            "pop    %rcx",
+            "ret",
+            options(noreturn, att_syntax)
+        );
+    }
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn __alloca() {
-    core::arch::asm!(
-        "mov    %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx
-        "jmp    ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
-        options(noreturn, att_syntax)
-    );
-}
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn __alloca() {
+        core::arch::asm!(
+            "mov    %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx
+            "jmp    ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable"
+            options(noreturn, att_syntax)
+        );
+    }
 
-#[cfg(all(
-    windows,
-    target_env = "gnu",
-    not(feature = "no-asm"),
-    not(feature = "mangled-names")
-))]
-#[naked]
-#[no_mangle]
-pub unsafe extern "C" fn ___chkstk() {
-    core::arch::asm!(
-        "push   %rcx",
-        "cmp    $0x1000,%rax",
-        "lea    16(%rsp),%rcx", // rsp before calling this routine -> rcx
-        "jb     1f",
-        "2:",
-        "sub    $0x1000,%rcx",
-        "test   %rcx,(%rcx)",
-        "sub    $0x1000,%rax",
-        "cmp    $0x1000,%rax",
-        "ja     2b",
-        "1:",
-        "sub    %rax,%rcx",
-        "test   %rcx,(%rcx)",
-        "lea    8(%rsp),%rax",  // load pointer to the return address into rax
-        "mov    %rcx,%rsp",     // install the new top of stack pointer into rsp
-        "mov    -8(%rax),%rcx", // restore rcx
-        "push   (%rax)",        // push return address onto the stack
-        "sub    %rsp,%rax",     // restore the original value in rax
-        "ret",
-        options(noreturn, att_syntax)
-    );
+    #[naked]
+    #[cfg(all(
+        windows,
+        target_env = "gnu",
+        not(feature = "no-asm")
+    ))]
+    pub unsafe extern "C" fn ___chkstk() {
+        core::arch::asm!(
+            "push   %rcx",
+            "cmp    $0x1000,%rax",
+            "lea    16(%rsp),%rcx", // rsp before calling this routine -> rcx
+            "jb     1f",
+            "2:",
+            "sub    $0x1000,%rcx",
+            "test   %rcx,(%rcx)",
+            "sub    $0x1000,%rax",
+            "cmp    $0x1000,%rax",
+            "ja     2b",
+            "1:",
+            "sub    %rax,%rcx",
+            "test   %rcx,(%rcx)",
+            "lea    8(%rsp),%rax",  // load pointer to the return address into rax
+            "mov    %rcx,%rsp",     // install the new top of stack pointer into rsp
+            "mov    -8(%rax),%rcx", // restore rcx
+            "push   (%rax)",        // push return address onto the stack
+            "sub    %rsp,%rax",     // restore the original value in rax
+            "ret",
+            options(noreturn, att_syntax)
+        );
+    }
 }
 
 // HACK(https://github.com/rust-lang/rust/issues/62785): x86_64-unknown-uefi needs special LLVM
 // support unless we emit the _fltused
-#[no_mangle]
-#[used]
-#[cfg(target_os = "uefi")]
-static _fltused: i32 = 0;
+mod _fltused {
+    #[no_mangle]
+    #[used]
+    #[cfg(target_os = "uefi")]
+    static _fltused: i32 = 0;
+}