ld_so: significant refactor to prepare for pthread_create support of TLS

Cargo.lock

@@ -141,6 +141,14 @@ dependencies = [
  "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.6"
@@ -263,6 +271,7 @@ dependencies = [
  "core_io 0.1.20181107",
  "goblin 0.0.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memoffset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "posix-regex 0.1.0",
  "ralloc 1.0.0",
  "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -544,6 +553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)" = "bedcc7a809076656486ffe045abeeac163da1b558e963a31e29fbfbeba916917"
 "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
 "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6"
+"checksum memoffset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7efacc914ca612fc1022f27b7dc51585e1a9f94c08fd5d322cfd741399260ce0"
 "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1"
 "checksum plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
 "checksum proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cd07deb3c6d1d9ff827999c7f9b04cdfd66b1b17ae508e14fe47b620f2282ae0"

Cargo.toml

@@ -18,6 +18,7 @@ cc = "1.0.25"
 cbitset = "0.1.0"
 core_io = { path = "core_io", features = ["collections"] }
 lazy_static = { version = "1.2.0", features = ["nightly", "spin_no_std"] }
+memoffset = "0.3.0"
 posix-regex = { path = "posix-regex", features = ["no_std"] }
 rand = { version = "0.5.5", default-features = false }
 va_list = { path = "va_list", features = ["no_std"] }

src/ld_so/linker.rs

@@ -12,7 +12,8 @@ use header::{fcntl, sys_mman, unistd};
 use io::Read;
 use platform::types::c_void;
 
-const PAGE_SIZE: usize = 4096;
+use super::PAGE_SIZE;
+use super::tcb::{Tcb, Master};
 
 #[cfg(target_os = "redox")]
 const PATH_SEP: char = ';';
@@ -20,61 +21,6 @@ const PATH_SEP: char = ';';
 #[cfg(target_os = "linux")]
 const PATH_SEP: char = ':';
 
-// On Linux, a new TCB is required
-#[cfg(target_os = "linux")]
-unsafe fn allocate_tls(size: usize) -> Result<&'static mut [u8]> {
-    let ptr = sys_mman::mmap(
-        ptr::null_mut(),
-        size + PAGE_SIZE /* TLS and TCB */,
-        sys_mman::PROT_READ | sys_mman::PROT_WRITE,
-        sys_mman::MAP_ANONYMOUS | sys_mman::MAP_PRIVATE,
-        -1,
-        0
-    );
-    if ptr as usize == !0 /* MAP_FAILED */ {
-        return Err(Error::Malformed(
-            format!("failed to map tls")
-        ));
-    }
-
-    let mut tls = slice::from_raw_parts_mut(ptr as *mut u8, size);
-    let mut tcb = slice::from_raw_parts_mut((ptr as *mut u8).add(size), PAGE_SIZE);
-    *(tcb.as_mut_ptr() as *mut *mut u8) = tls.as_mut_ptr().add(size);
-
-    #[cfg(target_arch = "x86_64")]
-    {
-        const ARCH_SET_FS: usize = 0x1002;
-        syscall!(ARCH_PRCTL, ARCH_SET_FS, tcb.as_mut_ptr());
-    }
-
-    Ok(tls)
-}
-
-// On Redox, reuse the current TCB
-// TODO: Consider adopting Linux behavior
-#[cfg(target_os = "redox")]
-unsafe fn allocate_tls(size: usize) -> Result<&'static mut [u8]> {
-    let ptr = sys_mman::mmap(
-        ptr::null_mut(),
-        size,
-        sys_mman::PROT_READ | sys_mman::PROT_WRITE,
-        sys_mman::MAP_ANONYMOUS | sys_mman::MAP_PRIVATE,
-        -1,
-        0
-    );
-    if ptr as usize == !0 /* MAP_FAILED */ {
-        return Err(Error::Malformed(
-            format!("failed to map tls")
-        ));
-    }
-
-    let mut tls = slice::from_raw_parts_mut(ptr as *mut u8, size);
-    let mut tcb = slice::from_raw_parts_mut(0xB000_0000 as *mut u8, PAGE_SIZE);
-    *(tcb.as_mut_ptr() as *mut *mut u8) = tls.as_mut_ptr().add(size);
-
-    Ok(tls)
-}
-
 pub struct Linker {
     library_path: String,
     objects: BTreeMap<String, Box<[u8]>>
@@ -263,11 +209,12 @@ impl Linker {
         }
 
         // Allocate TLS
-        let mut tls = unsafe { allocate_tls(tls_size)? };
-        println!("tls {:p}, {:#x}", tls.as_mut_ptr(), tls.len());
+        let mut tcb = unsafe { Tcb::new(tls_size)? };
+        println!("tcb {:x?}", tcb);
 
         // Copy data
         let mut tls_offset = tls_primary;
+        let mut tcb_masters = Vec::new();
         let mut tls_index = 0;
         let mut tls_ranges = BTreeMap::new();
         for (elf_name, elf) in elfs.iter() {
@@ -322,46 +269,42 @@ impl Linker {
                             ph.p_memsz
                         } as usize;
 
-                        let obj_data = {
-                            let range = ph.file_range();
-                            match object.get(range.clone()) {
-                                Some(some) => some,
-                                None => return Err(Error::Malformed(
-                                    format!("failed to read {:?}", range)
-                                )),
-                            }
-                        };
-
-                        let tls_data = {
-                            let (index, start) = if *elf_name == primary {
-                                (0, tls.len() - valign)
-                            } else {
-                                let start = tls.len() - (tls_offset + valign);
-                                tls_offset += vsize;
-                                tls_index += 1;
-                                (tls_index, start)
-                            };
-                            let range = start..start + obj_data.len();
-                            match tls.get_mut(range.clone()) {
-                                Some(some) => {
-                                    tls_ranges.insert(elf_name, (index, range));
-                                    some
-                                },
-                                None => return Err(Error::Malformed(
-                                    format!("failed to write tls {:?}", range)
-                                )),
-                            }
+                        let mut tcb_master = Master {
+                            ptr: unsafe { mmap.as_ptr().add(ph.p_vaddr as usize) },
+                            len: ph.p_filesz as usize,
+                            offset: tls_size - valign,
                         };
 
-                        println!("  copy tls {:#x}, {:#x}: {:#x}, {:#x}", vaddr, vsize, voff, obj_data.len());
-
-                        tls_data.copy_from_slice(obj_data);
+                        println!(
+                            "  tls master {:p}, {:#x}: {:#x}, {:#x}",
+                            tcb_master.ptr,
+                            tcb_master.len,
+                            tcb_master.offset,
+                            valign,
+                        );
+
+                        if *elf_name == primary {
+                            tls_ranges.insert(elf_name, (0, tcb_master.range()));
+                            tcb_masters.insert(0, tcb_master);
+                        } else {
+                            tcb_master.offset -= tls_offset;
+                            tls_offset += vsize;
+                            tls_index += 1;
+                            tls_ranges.insert(elf_name, (tls_index, tcb_master.range()));
+                            tcb_masters.push(tcb_master);
+                        }
                     },
                     _ => ()
                 }
             }
         }
 
+        // Set master images for TLS and copy TLS data
+        unsafe {
+            tcb.set_masters(tcb_masters.into_boxed_slice());
+            tcb.copy_masters()?;
+        }
+
         // Perform relocations, and protect pages
         for (elf_name, elf) in elfs.iter() {
             let mmap = match mmaps.get_mut(elf_name) {
@@ -402,10 +345,10 @@ impl Linker {
                     0
                 };
 
-                let t = if let Some((tls_index, tls_range)) = tls_ranges.get(elf_name) {
-                    tls_range.start
+                let (tm, t) = if let Some((tls_index, tls_range)) = tls_ranges.get(elf_name) {
+                    (*tls_index, tls_range.start)
                 } else {
-                    0
+                    (0, 0)
                 };
 
                 let ptr = unsafe {
@@ -480,6 +423,11 @@ impl Linker {
             }
         }
 
+        // Activate TLS
+        unsafe {
+            tcb.activate();
+        }
+
         // Perform indirect relocations (necessary evil), gather entry point
         let mut entry_opt = None;
         for (elf_name, elf) in elfs.iter() {
@@ -490,8 +438,7 @@ impl Linker {
 
             println!("entry {}", elf_name);
 
-            let is_primary = *elf_name == primary;
-            if is_primary {
+            if *elf_name == primary {
                 entry_opt = Some(mmap.as_mut_ptr() as usize + elf.header.e_entry as usize);
             }
 

src/ld_so/mod.rs

@@ -1,2 +1,5 @@
+pub const PAGE_SIZE: usize = 4096;
+
 pub mod linker;
 pub mod start;
+pub mod tcb;

src/ld_so/tcb.rs

@@ -0,0 +1,210 @@
+use alloc::boxed::Box;
+use core::{mem, ptr, slice};
+use core::ops::Range;
+use goblin::error::{Error, Result};
+
+use header::sys_mman;
+
+use super::PAGE_SIZE;
+
+#[repr(C)]
+pub struct Master {
+    /// Pointer to initial data
+    pub ptr: *const u8,
+    /// Length of initial data in bytes
+    pub len: usize,
+    /// Offset in TLS to copy initial data to
+    pub offset: usize,
+}
+
+impl Master {
+    /// The initial data for this TLS region
+    pub unsafe fn data(&self) -> &'static [u8] {
+        slice::from_raw_parts(
+            self.ptr,
+            self.len
+        )
+    }
+
+    /// The region of TLS that the master will initialize
+    pub fn range(&self) -> Range<usize> {
+        self.offset..self.offset + self.len
+    }
+}
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct Tcb {
+    /// Pointer to the end of static TLS. Must be the first member
+    tls_end: *mut u8,
+    /// Size of the memory allocated for the static TLS in bytes (multiple of PAGE_SIZE)
+    tls_len: usize,
+    /// Pointer to this structure
+    tcb_ptr: *mut Tcb,
+    /// Size of the memory allocated for this structure in bytes (should be PAGE_SIZE)
+    tcb_len: usize,
+    /// Pointer to a list of initial TLS data
+    masters_ptr: *mut Master,
+    /// Size of the masters list in bytes (multiple of mem::size_of::<Master>())
+    masters_len: usize,
+}
+
+impl Tcb {
+    /// Create a new TCB
+    pub unsafe fn new(size: usize) -> Result<&'static mut Self> {
+        let (tls, tcb_page) = Self::os_new(size)?;
+
+        let tcb_ptr = tcb_page.as_mut_ptr() as *mut Self;
+        ptr::write(tcb_ptr, Tcb {
+            tls_end: tls.as_mut_ptr().add(tls.len()),
+            tls_len: tls.len(),
+            tcb_ptr: tcb_ptr,
+            tcb_len: tcb_page.len(),
+            masters_ptr: ptr::null_mut(),
+            masters_len: 0,
+        });
+
+        Ok(&mut *tcb_ptr)
+    }
+
+    /// Get the current TCB
+    pub unsafe fn current() -> Option<&'static mut Self> {
+        let tcb_ptr = Self::arch_read(offset_of!(Self, tcb_ptr)) as *mut Self;
+        let tcb_len = Self::arch_read(offset_of!(Self, tcb_len));
+        if tcb_ptr.is_null() || tcb_len < mem::size_of::<Self>() {
+            None
+        } else {
+            Some(&mut *tcb_ptr)
+        }
+    }
+
+    /// A slice for all of the TLS data
+    pub unsafe fn tls(&self) -> Option<&'static mut [u8]> {
+        if self.tls_end.is_null() || self.tls_len == 0 {
+            None
+        } else {
+            Some(slice::from_raw_parts_mut(
+                self.tls_end.offset(-(self.tls_len as isize)),
+                self.tls_len
+            ))
+        }
+    }
+
+    /// The initial images for TLS
+    pub unsafe fn masters(&self) -> Option<&'static mut [Master]> {
+        if self.masters_ptr.is_null() || self.masters_len == 0 {
+            None
+        } else {
+            Some(slice::from_raw_parts_mut(
+                self.masters_ptr,
+                self.masters_len / mem::size_of::<Master>()
+            ))
+        }
+    }
+
+    /// Copy data from masters
+    pub unsafe fn copy_masters(&self) -> Result<()> {
+        //TODO: Complain if masters or tls exist without the other
+        if let Some(tls) = self.tls() {
+            if let Some(masters) = self.masters() {
+                for (i, master) in masters.iter().enumerate() {
+                    let range = master.range();
+                    let data = master.data();
+                    if let Some(tls_data) = tls.get_mut(range) {
+                        println!(
+                            "tls master {}: {:p}, {:#x}: {:p}, {:#x}",
+                            i,
+                            data.as_ptr(), data.len(),
+                            tls_data.as_mut_ptr(), tls_data.len()
+                        );
+
+                        tls_data.copy_from_slice(data);
+                    } else {
+                        return Err(Error::Malformed(
+                            format!("failed to copy tls master {}", i)
+                        ));
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// The initial images for TLS
+    pub unsafe fn set_masters(&mut self, mut masters: Box<[Master]>) {
+        self.masters_ptr = masters.as_mut_ptr();
+        self.masters_len = masters.len() * mem::size_of::<Master>();
+        mem::forget(masters);
+    }
+
+    /// Activate TLS
+    pub unsafe fn activate(&mut self) {
+        Self::os_arch_activate(self.tcb_ptr as usize);
+    }
+
+    /// Mapping with correct flags for TCB and TLS
+    unsafe fn map(size: usize) -> Result<&'static mut [u8]> {
+        let ptr = sys_mman::mmap(
+            ptr::null_mut(),
+            size,
+            sys_mman::PROT_READ | sys_mman::PROT_WRITE,
+            sys_mman::MAP_ANONYMOUS | sys_mman::MAP_PRIVATE,
+            -1,
+            0
+        );
+        if ptr as usize == !0 /* MAP_FAILED */ {
+            return Err(Error::Malformed(
+                format!("failed to map tls")
+            ));
+        }
+        Ok(slice::from_raw_parts_mut(ptr as *mut u8, size))
+    }
+
+    /// OS specific code to create a new TLS and TCB - Linux
+    #[cfg(target_os = "linux")]
+    unsafe fn os_new(size: usize) -> Result<(&'static mut [u8], &'static mut [u8])> {
+        let tls_tcb = Self::map(size + PAGE_SIZE)?;
+        Ok(tls_tcb.split_at_mut(size))
+    }
+
+    /// OS specific code to create a new TLS and TCB - Redox
+    #[cfg(target_os = "redox")]
+    unsafe fn os_new(size: usize) -> Result<(&'static mut [u8], &'static mut [u8])> {
+        let tls = Self::map(size)?;
+        Ok(
+            tls,
+            //TODO: Consider allocating TCB as part of TLS
+            slice::from_raw_parts_mut(0xB000_0000 as *mut u8, PAGE_SIZE)
+        )
+    }
+
+    /// Architecture specific code to read a usize from the TCB - x86_64
+    #[inline(always)]
+    #[cfg(target_arch = "x86_64")]
+    unsafe fn arch_read(offset: usize) -> usize {
+        let value;
+        asm!("
+            mov rax, [fs:rdi]
+            "
+            : "={rax}"(value)
+            : "{rax}"(offset)
+            :
+            : "intel"
+        );
+        value
+    }
+
+    /// OS and architecture specific code to activate TLS - Linux x86_64
+    #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
+    unsafe fn os_arch_activate(tp: usize) {
+        const ARCH_SET_FS: usize = 0x1002;
+        syscall!(ARCH_PRCTL, ARCH_SET_FS, tp);
+    }
+
+    /// OS and architecture specific code to activate TLS - Linux x86_64
+    #[cfg(all(target_os = "redox", target_arch = "x86_64"))]
+    unsafe fn os_arch_activate(tp: usize) {
+        //TODO: Consider setting FS offset to TCB pointer
+    }
+}

src/lib.rs

@@ -29,6 +29,8 @@ extern crate core_io;
 extern crate goblin;
 #[macro_use]
 extern crate lazy_static;
+#[macro_use]
+extern crate memoffset;
 extern crate posix_regex;
 extern crate rand;
 extern crate va_list;