jit.rs 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937
  1. // SPDX-License-Identifier: (Apache-2.0 OR MIT)
  2. // Derived from uBPF <https://github.com/iovisor/ubpf>
  3. // Copyright 2015 Big Switch Networks, Inc
  4. // (uBPF: JIT algorithm, originally in C)
  5. // Copyright 2016 6WIND S.A. <quentin.monnet@6wind.com>
  6. // (Translation to Rust, MetaBuff addition)
  7. use std;
  8. use std::mem;
  9. use std::collections::HashMap;
  10. use std::fmt::Formatter;
  11. use std::fmt::Error as FormatterError;
  12. use std::io::{Error, ErrorKind};
  13. use std::ops::{Index, IndexMut};
  14. use ebpf;
  15. use JitProgram;
  16. extern crate libc;
  17. const PAGE_SIZE: usize = 4096;
  18. // Special values for target_pc in struct Jump
  19. const TARGET_OFFSET: isize = ebpf::PROG_MAX_INSNS as isize;
  20. const TARGET_PC_EXIT: isize = TARGET_OFFSET + 1;
  21. const TARGET_PC_DIV_BY_ZERO: isize = TARGET_OFFSET + 2;
  22. #[derive(Copy, Clone)]
  23. enum OperandSize {
  24. S8 = 8,
  25. S16 = 16,
  26. S32 = 32,
  27. S64 = 64,
  28. }
  29. // Registers
  30. const RAX: u8 = 0;
  31. const RCX: u8 = 1;
  32. const RDX: u8 = 2;
  33. const RBX: u8 = 3;
  34. const RSP: u8 = 4;
  35. const RBP: u8 = 5;
  36. const RSI: u8 = 6;
  37. const RDI: u8 = 7;
  38. const R8: u8 = 8;
  39. const R9: u8 = 9;
  40. const R10: u8 = 10;
  41. const R11: u8 = 11;
  42. //const R12: u8 = 12;
  43. const R13: u8 = 13;
  44. const R14: u8 = 14;
  45. const R15: u8 = 15;
  46. const REGISTER_MAP_SIZE: usize = 11;
  47. const REGISTER_MAP: [u8;REGISTER_MAP_SIZE] = [
  48. RAX, // 0 return value
  49. RDI, // 1 arg 1
  50. RSI, // 2 arg 2
  51. RDX, // 3 arg 3
  52. R9, // 4 arg 4
  53. R8, // 5 arg 5
  54. RBX, // 6 callee-saved
  55. R13, // 7 callee-saved
  56. R14, // 8 callee-saved
  57. R15, // 9 callee-saved
  58. RBP, // 10 stack pointer
  59. // R10 and R11 are used to compute store a constant pointer to mem and to compute offset for
  60. // LD_ABS_* and LD_IND_* operations, so they are not mapped to any eBPF register.
  61. ];
  62. // Return the x86 register for the given eBPF register
  63. fn map_register(r: u8) -> u8 {
  64. assert!(r < REGISTER_MAP_SIZE as u8);
  65. REGISTER_MAP[(r % REGISTER_MAP_SIZE as u8) as usize]
  66. }
  67. macro_rules! emit_bytes {
  68. ( $jit:ident, $data:tt, $t:ty ) => {{
  69. let size = mem::size_of::<$t>() as usize;
  70. assert!($jit.offset + size <= $jit.contents.len());
  71. unsafe {
  72. let mut ptr = $jit.contents.as_ptr().add($jit.offset) as *mut $t;
  73. *ptr = $data;
  74. }
  75. $jit.offset += size;
  76. }}
  77. }
  78. #[inline]
  79. fn emit1(jit: &mut JitMemory, data: u8) {
  80. emit_bytes!(jit, data, u8);
  81. }
  82. #[inline]
  83. fn emit2(jit: &mut JitMemory, data: u16) {
  84. emit_bytes!(jit, data, u16);
  85. }
  86. #[inline]
  87. fn emit4(jit: &mut JitMemory, data: u32) {
  88. emit_bytes!(jit, data, u32);
  89. }
  90. #[inline]
  91. fn emit8(jit: &mut JitMemory, data: u64) {
  92. emit_bytes!(jit, data, u64);
  93. }
  94. #[inline]
  95. fn emit_jump_offset(jit: &mut JitMemory, target_pc: isize) {
  96. let jump = Jump { offset_loc: jit.offset, target_pc: target_pc };
  97. jit.jumps.push(jump);
  98. emit4(jit, 0);
  99. }
  100. #[inline]
  101. fn emit_modrm(jit: &mut JitMemory, modrm: u8, r: u8, m: u8) {
  102. assert_eq!((modrm | 0xc0), 0xc0);
  103. emit1(jit, (modrm & 0xc0) | ((r & 0b111) << 3) | (m & 0b111));
  104. }
  105. #[inline]
  106. fn emit_modrm_reg2reg(jit: &mut JitMemory, r: u8, m: u8) {
  107. emit_modrm(jit, 0xc0, r, m);
  108. }
  109. #[inline]
  110. fn emit_modrm_and_displacement(jit: &mut JitMemory, r: u8, m: u8, d: i32) {
  111. if d == 0 && (m & 0b111) != RBP {
  112. emit_modrm(jit, 0x00, r, m);
  113. } else if (-128..=127).contains(&d) {
  114. emit_modrm(jit, 0x40, r, m);
  115. emit1(jit, d as u8);
  116. } else {
  117. emit_modrm(jit, 0x80, r, m);
  118. emit4(jit, d as u32);
  119. }
  120. }
  121. #[inline]
  122. fn emit_rex(jit: &mut JitMemory, w: u8, r: u8, x: u8, b: u8) {
  123. assert_eq!((w | 1), 1);
  124. assert_eq!((r | 1), 1);
  125. assert_eq!((x | 1), 1);
  126. assert_eq!((b | 1), 1);
  127. emit1(jit, 0x40 | (w << 3) | (r << 2) | (x << 1) | b);
  128. }
  129. // Emits a REX prefix with the top bit of src and dst.
  130. // Skipped if no bits would be set.
  131. #[inline]
  132. fn emit_basic_rex(jit: &mut JitMemory, w: u8, src: u8, dst: u8) {
  133. if w != 0 || (src & 0b1000) != 0 || (dst & 0b1000) != 0 {
  134. let is_masked = | val, mask | { match val & mask {
  135. 0 => 0,
  136. _ => 1
  137. }};
  138. emit_rex(jit, w, is_masked(src, 8), 0, is_masked(dst, 8));
  139. }
  140. }
  141. #[inline]
  142. fn emit_push(jit: &mut JitMemory, r: u8) {
  143. emit_basic_rex(jit, 0, 0, r);
  144. emit1(jit, 0x50 | (r & 0b111));
  145. }
  146. #[inline]
  147. fn emit_pop(jit: &mut JitMemory, r: u8) {
  148. emit_basic_rex(jit, 0, 0, r);
  149. emit1(jit, 0x58 | (r & 0b111));
  150. }
  151. // REX prefix and ModRM byte
  152. // We use the MR encoding when there is a choice
  153. // 'src' is often used as an opcode extension
  154. #[inline]
  155. fn emit_alu32(jit: &mut JitMemory, op: u8, src: u8, dst: u8) {
  156. emit_basic_rex(jit, 0, src, dst);
  157. emit1(jit, op);
  158. emit_modrm_reg2reg(jit, src, dst);
  159. }
  160. // REX prefix, ModRM byte, and 32-bit immediate
  161. #[inline]
  162. fn emit_alu32_imm32(jit: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i32) {
  163. emit_alu32(jit, op, src, dst);
  164. emit4(jit, imm as u32);
  165. }
  166. // REX prefix, ModRM byte, and 8-bit immediate
  167. #[inline]
  168. fn emit_alu32_imm8(jit: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i8) {
  169. emit_alu32(jit, op, src, dst);
  170. emit1(jit, imm as u8);
  171. }
  172. // REX.W prefix and ModRM byte
  173. // We use the MR encoding when there is a choice
  174. // 'src' is often used as an opcode extension
  175. #[inline]
  176. fn emit_alu64(jit: &mut JitMemory, op: u8, src: u8, dst: u8) {
  177. emit_basic_rex(jit, 1, src, dst);
  178. emit1(jit, op);
  179. emit_modrm_reg2reg(jit, src, dst);
  180. }
  181. // REX.W prefix, ModRM byte, and 32-bit immediate
  182. #[inline]
  183. fn emit_alu64_imm32(jit: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i32) {
  184. emit_alu64(jit, op, src, dst);
  185. emit4(jit, imm as u32);
  186. }
  187. // REX.W prefix, ModRM byte, and 8-bit immediate
  188. #[inline]
  189. fn emit_alu64_imm8(jit: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i8) {
  190. emit_alu64(jit, op, src, dst);
  191. emit1(jit, imm as u8);
  192. }
  193. // Register to register mov
  194. #[inline]
  195. fn emit_mov(jit: &mut JitMemory, src: u8, dst: u8) {
  196. emit_alu64(jit, 0x89, src, dst);
  197. }
  198. #[inline]
  199. fn emit_cmp_imm32(jit: &mut JitMemory, dst: u8, imm: i32) {
  200. emit_alu64_imm32(jit, 0x81, 7, dst, imm);
  201. }
  202. #[inline]
  203. fn emit_cmp(jit: &mut JitMemory, src: u8, dst: u8) {
  204. emit_alu64(jit, 0x39, src, dst);
  205. }
  206. #[inline]
  207. fn emit_jcc(jit: &mut JitMemory, code: u8, target_pc: isize) {
  208. emit1(jit, 0x0f);
  209. emit1(jit, code);
  210. emit_jump_offset(jit, target_pc);
  211. }
  212. #[inline]
  213. fn emit_jmp(jit: &mut JitMemory, target_pc: isize) {
  214. emit1(jit, 0xe9);
  215. emit_jump_offset(jit, target_pc);
  216. }
  217. #[inline]
  218. fn set_anchor(jit: &mut JitMemory, target: isize) {
  219. jit.special_targets.insert(target, jit.offset);
  220. }
  221. #[inline]
  222. fn emit_direct_jcc(jit: &mut JitMemory, code: u8, offset: u32) {
  223. emit1(jit, 0x0f);
  224. emit1(jit, code);
  225. emit_bytes!(jit, offset, u32);
  226. }
  227. // Load [src + offset] into dst
  228. #[inline]
  229. fn emit_load(jit: &mut JitMemory, size: OperandSize, src: u8, dst: u8, offset: i32) {
  230. let data = match size {
  231. OperandSize::S64 => 1,
  232. _ => 0
  233. };
  234. emit_basic_rex(jit, data, dst, src);
  235. match size {
  236. OperandSize::S8 => {
  237. // movzx
  238. emit1(jit, 0x0f);
  239. emit1(jit, 0xb6);
  240. },
  241. OperandSize::S16 => {
  242. // movzx
  243. emit1(jit, 0x0f);
  244. emit1(jit, 0xb7);
  245. },
  246. OperandSize::S32 | OperandSize::S64 => {
  247. // mov
  248. emit1(jit, 0x8b);
  249. }
  250. }
  251. emit_modrm_and_displacement(jit, dst, src, offset);
  252. }
  253. // Load sign-extended immediate into register
  254. #[inline]
  255. fn emit_load_imm(jit: &mut JitMemory, dst: u8, imm: i64) {
  256. if imm >= std::i32::MIN as i64 && imm <= std::i32::MAX as i64 {
  257. emit_alu64_imm32(jit, 0xc7, 0, dst, imm as i32);
  258. } else {
  259. // movabs $imm,dst
  260. emit_basic_rex(jit, 1, 0, dst);
  261. emit1(jit, 0xb8 | (dst & 0b111));
  262. emit8(jit, imm as u64);
  263. }
  264. }
  265. // Store register src to [dst + offset]
  266. #[inline]
  267. fn emit_store(jit: &mut JitMemory, size: OperandSize, src: u8, dst: u8, offset: i32) {
  268. match size {
  269. OperandSize::S16 => emit1(jit, 0x66), // 16-bit override
  270. _ => {},
  271. };
  272. let (is_s8, is_u64, rexw) = match size {
  273. OperandSize::S8 => (true, false, 0),
  274. OperandSize::S64 => (false, true, 1),
  275. _ => (false, false, 0),
  276. };
  277. if is_u64 || (src & 0b1000) != 0 || (dst & 0b1000) != 0 || is_s8 {
  278. let is_masked = | val, mask | {
  279. match val & mask {
  280. 0 => 0,
  281. _ => 1
  282. }
  283. };
  284. emit_rex(jit, rexw, is_masked(src, 8), 0, is_masked(dst, 8));
  285. }
  286. match size {
  287. OperandSize::S8 => emit1(jit, 0x88),
  288. _ => emit1(jit, 0x89),
  289. };
  290. emit_modrm_and_displacement(jit, src, dst, offset);
  291. }
  292. // Store immediate to [dst + offset]
  293. #[inline]
  294. fn emit_store_imm32(jit: &mut JitMemory, size: OperandSize, dst: u8, offset: i32, imm: i32) {
  295. match size {
  296. OperandSize::S16 => emit1(jit, 0x66), // 16-bit override
  297. _ => {},
  298. };
  299. match size {
  300. OperandSize::S64 => emit_basic_rex(jit, 1, 0, dst),
  301. _ => emit_basic_rex(jit, 0, 0, dst),
  302. };
  303. match size {
  304. OperandSize::S8 => emit1(jit, 0xc6),
  305. _ => emit1(jit, 0xc7),
  306. };
  307. emit_modrm_and_displacement(jit, 0, dst, offset);
  308. match size {
  309. OperandSize::S8 => emit1(jit, imm as u8),
  310. OperandSize::S16 => emit2(jit, imm as u16),
  311. _ => emit4(jit, imm as u32),
  312. };
  313. }
  314. #[inline]
  315. fn emit_call(jit: &mut JitMemory, target: usize) {
  316. // TODO use direct call when possible
  317. emit_load_imm(jit, RAX, target as i64);
  318. // callq *%rax
  319. emit1(jit, 0xff);
  320. emit1(jit, 0xd0);
  321. }
  322. fn muldivmod(jit: &mut JitMemory, pc: u16, opc: u8, src: u8, dst: u8, imm: i32) {
  323. let mul = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::MUL32_IMM & ebpf::BPF_ALU_OP_MASK);
  324. let div = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::DIV32_IMM & ebpf::BPF_ALU_OP_MASK);
  325. let modrm = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::MOD32_IMM & ebpf::BPF_ALU_OP_MASK);
  326. let is64 = (opc & ebpf::BPF_CLS_MASK) == ebpf::BPF_ALU64;
  327. let is_reg = (opc & ebpf::BPF_X) == ebpf::BPF_X;
  328. if div && !is_reg && imm == 0 {
  329. // Division by zero returns 0
  330. // Set register to 0: xor with itself
  331. emit_alu32(jit, 0x31, dst, dst);
  332. return;
  333. }
  334. if modrm && !is_reg && imm == 0 {
  335. // Modulo remainder of division by zero keeps destination register unchanged
  336. return;
  337. }
  338. if (div || modrm) && is_reg {
  339. emit_load_imm(jit, RCX, pc as i64);
  340. // test src,src
  341. if is64 {
  342. emit_alu64(jit, 0x85, src, src);
  343. } else {
  344. emit_alu32(jit, 0x85, src, src);
  345. }
  346. if div {
  347. // No division by 0: skip next instructions
  348. emit_direct_jcc(jit, 0x85, 7);
  349. // Division by 0: set dst to 0 then go to next instruction
  350. // Set register to 0: xor with itself
  351. emit_alu32(jit, 0x31, dst, dst);
  352. emit_jmp(jit, (pc + 1) as isize);
  353. }
  354. if modrm {
  355. // Modulo by zero: keep destination register unchanged
  356. emit_jcc(jit, 0x84, (pc + 1) as isize);
  357. }
  358. }
  359. if dst != RAX {
  360. emit_push(jit, RAX);
  361. }
  362. if dst != RDX {
  363. emit_push(jit, RDX);
  364. }
  365. if imm != 0 {
  366. emit_load_imm(jit, RCX, imm as i64);
  367. } else {
  368. emit_mov(jit, src, RCX);
  369. }
  370. emit_mov(jit, dst, RAX);
  371. if div || modrm {
  372. // Set register to 0: xor %edx,%edx
  373. emit_alu32(jit, 0x31, RDX, RDX);
  374. }
  375. if is64 {
  376. emit_rex(jit, 1, 0, 0, 0);
  377. }
  378. // mul %ecx or div %ecx
  379. emit_alu32(jit, 0xf7, if mul { 4 } else { 6 }, RCX);
  380. if dst != RDX {
  381. if modrm {
  382. emit_mov(jit, RDX, dst);
  383. }
  384. emit_pop(jit, RDX);
  385. }
  386. if dst != RAX {
  387. if div || mul {
  388. emit_mov(jit, RAX, dst);
  389. }
  390. emit_pop(jit, RAX);
  391. }
  392. }
  393. #[derive(Debug)]
  394. struct Jump {
  395. offset_loc: usize,
  396. target_pc: isize,
  397. }
  398. struct JitMemory<'a> {
  399. contents: &'a mut [u8],
  400. offset: usize,
  401. pc_locs: Vec<usize>,
  402. special_targets: HashMap<isize, usize>,
  403. jumps: Vec<Jump>,
  404. }
  405. impl<'a> JitMemory<'a> {
  406. fn new(num_pages: usize) -> JitMemory<'a> {
  407. let contents: &mut[u8];
  408. let mut raw: mem::MaybeUninit<*mut libc::c_void> = mem::MaybeUninit::uninit();
  409. unsafe {
  410. let size = num_pages * PAGE_SIZE;
  411. libc::posix_memalign(raw.as_mut_ptr(), PAGE_SIZE, size);
  412. libc::mprotect(*raw.as_mut_ptr(), size, libc::PROT_EXEC | libc::PROT_READ | libc::PROT_WRITE);
  413. std::ptr::write_bytes(*raw.as_mut_ptr(), 0xc3, size); // for now, prepopulate with 'RET' calls
  414. contents = std::slice::from_raw_parts_mut(*raw.as_mut_ptr() as *mut u8, num_pages * PAGE_SIZE);
  415. raw.assume_init();
  416. }
  417. JitMemory {
  418. contents: contents,
  419. offset: 0,
  420. pc_locs: vec![],
  421. jumps: vec![],
  422. special_targets: HashMap::new(),
  423. }
  424. }
  425. fn jit_compile(&mut self, prog: &[u8], use_mbuff: bool, update_data_ptr: bool,
  426. helpers: &HashMap<u32, ebpf::Helper>) -> Result<(), Error> {
  427. emit_push(self, RBP);
  428. emit_push(self, RBX);
  429. emit_push(self, R13);
  430. emit_push(self, R14);
  431. emit_push(self, R15);
  432. // RDI: mbuff
  433. // RSI: mbuff_len
  434. // RDX: mem
  435. // RCX: mem_len
  436. // R8: mem_offset
  437. // R9: mem_end_offset
  438. // Save mem pointer for use with LD_ABS_* and LD_IND_* instructions
  439. emit_mov(self, RDX, R10);
  440. match (use_mbuff, update_data_ptr) {
  441. (false, _) => {
  442. // We do not use any mbuff. Move mem pointer into register 1.
  443. if map_register(1) != RDX {
  444. emit_mov(self, RDX, map_register(1));
  445. }
  446. },
  447. (true, false) => {
  448. // We use a mbuff already pointing to mem and mem_end: move it to register 1.
  449. if map_register(1) != RDI {
  450. emit_mov(self, RDI, map_register(1));
  451. }
  452. },
  453. (true, true) => {
  454. // We have a fixed (simulated) mbuff: update mem and mem_end offset values in it.
  455. // Store mem at mbuff + mem_offset. Trash R8.
  456. emit_alu64(self, 0x01, RDI, R8); // add mbuff to mem_offset in R8
  457. emit_store(self, OperandSize::S64, RDX, R8, 0); // set mem at mbuff + mem_offset
  458. // Store mem_end at mbuff + mem_end_offset. Trash R9.
  459. emit_load(self, OperandSize::S64, RDX, R8, 0); // load mem into R8
  460. emit_alu64(self, 0x01, RCX, R8); // add mem_len to mem (= mem_end)
  461. emit_alu64(self, 0x01, RDI, R9); // add mbuff to mem_end_offset
  462. emit_store(self, OperandSize::S64, R8, R9, 0); // store mem_end
  463. // Move rdi into register 1
  464. if map_register(1) != RDI {
  465. emit_mov(self, RDI, map_register(1));
  466. }
  467. }
  468. }
  469. // Copy stack pointer to R10
  470. emit_mov(self, RSP, map_register(10));
  471. // Allocate stack space
  472. emit_alu64_imm32(self, 0x81, 5, RSP, ebpf::STACK_SIZE as i32);
  473. self.pc_locs = vec![0; prog.len() / ebpf::INSN_SIZE + 1];
  474. let mut insn_ptr:usize = 0;
  475. while insn_ptr * ebpf::INSN_SIZE < prog.len() {
  476. let insn = ebpf::get_insn(prog, insn_ptr);
  477. self.pc_locs[insn_ptr] = self.offset;
  478. let dst = map_register(insn.dst);
  479. let src = map_register(insn.src);
  480. let target_pc = insn_ptr as isize + insn.off as isize + 1;
  481. match insn.opc {
  482. // BPF_LD class
  483. // R10 is a constant pointer to mem.
  484. ebpf::LD_ABS_B =>
  485. emit_load(self, OperandSize::S8, R10, RAX, insn.imm),
  486. ebpf::LD_ABS_H =>
  487. emit_load(self, OperandSize::S16, R10, RAX, insn.imm),
  488. ebpf::LD_ABS_W =>
  489. emit_load(self, OperandSize::S32, R10, RAX, insn.imm),
  490. ebpf::LD_ABS_DW =>
  491. emit_load(self, OperandSize::S64, R10, RAX, insn.imm),
  492. ebpf::LD_IND_B => {
  493. emit_mov(self, R10, R11); // load mem into R11
  494. emit_alu64(self, 0x01, src, R11); // add src to R11
  495. emit_load(self, OperandSize::S8, R11, RAX, insn.imm); // ld R0, mem[src+imm]
  496. },
  497. ebpf::LD_IND_H => {
  498. emit_mov(self, R10, R11); // load mem into R11
  499. emit_alu64(self, 0x01, src, R11); // add src to R11
  500. emit_load(self, OperandSize::S16, R11, RAX, insn.imm); // ld R0, mem[src+imm]
  501. },
  502. ebpf::LD_IND_W => {
  503. emit_mov(self, R10, R11); // load mem into R11
  504. emit_alu64(self, 0x01, src, R11); // add src to R11
  505. emit_load(self, OperandSize::S32, R11, RAX, insn.imm); // ld R0, mem[src+imm]
  506. },
  507. ebpf::LD_IND_DW => {
  508. emit_mov(self, R10, R11); // load mem into R11
  509. emit_alu64(self, 0x01, src, R11); // add src to R11
  510. emit_load(self, OperandSize::S64, R11, RAX, insn.imm); // ld R0, mem[src+imm]
  511. },
  512. ebpf::LD_DW_IMM => {
  513. insn_ptr += 1;
  514. let second_part = ebpf::get_insn(prog, insn_ptr).imm as u64;
  515. let imm = (insn.imm as u32) as u64 | second_part.wrapping_shl(32);
  516. emit_load_imm(self, dst, imm as i64);
  517. },
  518. // BPF_LDX class
  519. ebpf::LD_B_REG =>
  520. emit_load(self, OperandSize::S8, src, dst, insn.off as i32),
  521. ebpf::LD_H_REG =>
  522. emit_load(self, OperandSize::S16, src, dst, insn.off as i32),
  523. ebpf::LD_W_REG =>
  524. emit_load(self, OperandSize::S32, src, dst, insn.off as i32),
  525. ebpf::LD_DW_REG =>
  526. emit_load(self, OperandSize::S64, src, dst, insn.off as i32),
  527. // BPF_ST class
  528. ebpf::ST_B_IMM =>
  529. emit_store_imm32(self, OperandSize::S8, dst, insn.off as i32, insn.imm),
  530. ebpf::ST_H_IMM =>
  531. emit_store_imm32(self, OperandSize::S16, dst, insn.off as i32, insn.imm),
  532. ebpf::ST_W_IMM =>
  533. emit_store_imm32(self, OperandSize::S32, dst, insn.off as i32, insn.imm),
  534. ebpf::ST_DW_IMM =>
  535. emit_store_imm32(self, OperandSize::S64, dst, insn.off as i32, insn.imm),
  536. // BPF_STX class
  537. ebpf::ST_B_REG =>
  538. emit_store(self, OperandSize::S8, src, dst, insn.off as i32),
  539. ebpf::ST_H_REG =>
  540. emit_store(self, OperandSize::S16, src, dst, insn.off as i32),
  541. ebpf::ST_W_REG =>
  542. emit_store(self, OperandSize::S32, src, dst, insn.off as i32),
  543. ebpf::ST_DW_REG =>
  544. emit_store(self, OperandSize::S64, src, dst, insn.off as i32),
  545. ebpf::ST_W_XADD => unimplemented!(),
  546. ebpf::ST_DW_XADD => unimplemented!(),
  547. // BPF_ALU class
  548. ebpf::ADD32_IMM => emit_alu32_imm32(self, 0x81, 0, dst, insn.imm),
  549. ebpf::ADD32_REG => emit_alu32(self, 0x01, src, dst),
  550. ebpf::SUB32_IMM => emit_alu32_imm32(self, 0x81, 5, dst, insn.imm),
  551. ebpf::SUB32_REG => emit_alu32(self, 0x29, src, dst),
  552. ebpf::MUL32_IMM | ebpf::MUL32_REG |
  553. ebpf::DIV32_IMM | ebpf::DIV32_REG |
  554. ebpf::MOD32_IMM | ebpf::MOD32_REG =>
  555. muldivmod(self, insn_ptr as u16, insn.opc, src, dst, insn.imm),
  556. ebpf::OR32_IMM => emit_alu32_imm32(self, 0x81, 1, dst, insn.imm),
  557. ebpf::OR32_REG => emit_alu32(self, 0x09, src, dst),
  558. ebpf::AND32_IMM => emit_alu32_imm32(self, 0x81, 4, dst, insn.imm),
  559. ebpf::AND32_REG => emit_alu32(self, 0x21, src, dst),
  560. ebpf::LSH32_IMM => emit_alu32_imm8(self, 0xc1, 4, dst, insn.imm as i8),
  561. ebpf::LSH32_REG => {
  562. emit_mov(self, src, RCX);
  563. emit_alu32(self, 0xd3, 4, dst);
  564. },
  565. ebpf::RSH32_IMM => emit_alu32_imm8(self, 0xc1, 5, dst, insn.imm as i8),
  566. ebpf::RSH32_REG => {
  567. emit_mov(self, src, RCX);
  568. emit_alu32(self, 0xd3, 5, dst);
  569. },
  570. ebpf::NEG32 => emit_alu32(self, 0xf7, 3, dst),
  571. ebpf::XOR32_IMM => emit_alu32_imm32(self, 0x81, 6, dst, insn.imm),
  572. ebpf::XOR32_REG => emit_alu32(self, 0x31, src, dst),
  573. ebpf::MOV32_IMM => emit_alu32_imm32(self, 0xc7, 0, dst, insn.imm),
  574. ebpf::MOV32_REG => emit_mov(self, src, dst),
  575. ebpf::ARSH32_IMM => emit_alu32_imm8(self, 0xc1, 7, dst, insn.imm as i8),
  576. ebpf::ARSH32_REG => {
  577. emit_mov(self, src, RCX);
  578. emit_alu32(self, 0xd3, 7, dst);
  579. },
  580. ebpf::LE => {}, // No-op
  581. ebpf::BE => {
  582. match insn.imm {
  583. 16 => {
  584. // rol
  585. emit1(self, 0x66); // 16-bit override
  586. emit_alu32_imm8(self, 0xc1, 0, dst, 8);
  587. // and
  588. emit_alu32_imm32(self, 0x81, 4, dst, 0xffff);
  589. }
  590. 32 | 64 => {
  591. // bswap
  592. let bit = match insn.imm { 64 => 1, _ => 0 };
  593. emit_basic_rex(self, bit, 0, dst);
  594. emit1(self, 0x0f);
  595. emit1(self, 0xc8 | (dst & 0b111));
  596. }
  597. _ => unreachable!() // Should have been caught by verifier
  598. }
  599. },
  600. // BPF_ALU64 class
  601. ebpf::ADD64_IMM => emit_alu64_imm32(self, 0x81, 0, dst, insn.imm),
  602. ebpf::ADD64_REG => emit_alu64(self, 0x01, src, dst),
  603. ebpf::SUB64_IMM => emit_alu64_imm32(self, 0x81, 5, dst, insn.imm),
  604. ebpf::SUB64_REG => emit_alu64(self, 0x29, src, dst),
  605. ebpf::MUL64_IMM | ebpf::MUL64_REG |
  606. ebpf::DIV64_IMM | ebpf::DIV64_REG |
  607. ebpf::MOD64_IMM | ebpf::MOD64_REG =>
  608. muldivmod(self, insn_ptr as u16, insn.opc, src, dst, insn.imm),
  609. ebpf::OR64_IMM => emit_alu64_imm32(self, 0x81, 1, dst, insn.imm),
  610. ebpf::OR64_REG => emit_alu64(self, 0x09, src, dst),
  611. ebpf::AND64_IMM => emit_alu64_imm32(self, 0x81, 4, dst, insn.imm),
  612. ebpf::AND64_REG => emit_alu64(self, 0x21, src, dst),
  613. ebpf::LSH64_IMM => emit_alu64_imm8(self, 0xc1, 4, dst, insn.imm as i8),
  614. ebpf::LSH64_REG => {
  615. emit_mov(self, src, RCX);
  616. emit_alu64(self, 0xd3, 4, dst);
  617. },
  618. ebpf::RSH64_IMM => emit_alu64_imm8(self, 0xc1, 5, dst, insn.imm as i8),
  619. ebpf::RSH64_REG => {
  620. emit_mov(self, src, RCX);
  621. emit_alu64(self, 0xd3, 5, dst);
  622. },
  623. ebpf::NEG64 => emit_alu64(self, 0xf7, 3, dst),
  624. ebpf::XOR64_IMM => emit_alu64_imm32(self, 0x81, 6, dst, insn.imm),
  625. ebpf::XOR64_REG => emit_alu64(self, 0x31, src, dst),
  626. ebpf::MOV64_IMM => emit_load_imm(self, dst, insn.imm as i64),
  627. ebpf::MOV64_REG => emit_mov(self, src, dst),
  628. ebpf::ARSH64_IMM => emit_alu64_imm8(self, 0xc1, 7, dst, insn.imm as i8),
  629. ebpf::ARSH64_REG => {
  630. emit_mov(self, src, RCX);
  631. emit_alu64(self, 0xd3, 7, dst);
  632. },
  633. // BPF_JMP class
  634. ebpf::JA => emit_jmp(self, target_pc),
  635. ebpf::JEQ_IMM => {
  636. emit_cmp_imm32(self, dst, insn.imm);
  637. emit_jcc(self, 0x84, target_pc);
  638. },
  639. ebpf::JEQ_REG => {
  640. emit_cmp(self, src, dst);
  641. emit_jcc(self, 0x84, target_pc);
  642. },
  643. ebpf::JGT_IMM => {
  644. emit_cmp_imm32(self, dst, insn.imm);
  645. emit_jcc(self, 0x87, target_pc);
  646. },
  647. ebpf::JGT_REG => {
  648. emit_cmp(self, src, dst);
  649. emit_jcc(self, 0x87, target_pc);
  650. },
  651. ebpf::JGE_IMM => {
  652. emit_cmp_imm32(self, dst, insn.imm);
  653. emit_jcc(self, 0x83, target_pc);
  654. },
  655. ebpf::JGE_REG => {
  656. emit_cmp(self, src, dst);
  657. emit_jcc(self, 0x83, target_pc);
  658. },
  659. ebpf::JLT_IMM => {
  660. emit_cmp_imm32(self, dst, insn.imm);
  661. emit_jcc(self, 0x82, target_pc);
  662. },
  663. ebpf::JLT_REG => {
  664. emit_cmp(self, src, dst);
  665. emit_jcc(self, 0x82, target_pc);
  666. },
  667. ebpf::JLE_IMM => {
  668. emit_cmp_imm32(self, dst, insn.imm);
  669. emit_jcc(self, 0x86, target_pc);
  670. },
  671. ebpf::JLE_REG => {
  672. emit_cmp(self, src, dst);
  673. emit_jcc(self, 0x86, target_pc);
  674. },
  675. ebpf::JSET_IMM => {
  676. emit_alu64_imm32(self, 0xf7, 0, dst, insn.imm);
  677. emit_jcc(self, 0x85, target_pc);
  678. },
  679. ebpf::JSET_REG => {
  680. emit_alu64(self, 0x85, src, dst);
  681. emit_jcc(self, 0x85, target_pc);
  682. },
  683. ebpf::JNE_IMM => {
  684. emit_cmp_imm32(self, dst, insn.imm);
  685. emit_jcc(self, 0x85, target_pc);
  686. },
  687. ebpf::JNE_REG => {
  688. emit_cmp(self, src, dst);
  689. emit_jcc(self, 0x85, target_pc);
  690. },
  691. ebpf::JSGT_IMM => {
  692. emit_cmp_imm32(self, dst, insn.imm);
  693. emit_jcc(self, 0x8f, target_pc);
  694. },
  695. ebpf::JSGT_REG => {
  696. emit_cmp(self, src, dst);
  697. emit_jcc(self, 0x8f, target_pc);
  698. },
  699. ebpf::JSGE_IMM => {
  700. emit_cmp_imm32(self, dst, insn.imm);
  701. emit_jcc(self, 0x8d, target_pc);
  702. },
  703. ebpf::JSGE_REG => {
  704. emit_cmp(self, src, dst);
  705. emit_jcc(self, 0x8d, target_pc);
  706. },
  707. ebpf::JSLT_IMM => {
  708. emit_cmp_imm32(self, dst, insn.imm);
  709. emit_jcc(self, 0x8c, target_pc);
  710. },
  711. ebpf::JSLT_REG => {
  712. emit_cmp(self, src, dst);
  713. emit_jcc(self, 0x8c, target_pc);
  714. },
  715. ebpf::JSLE_IMM => {
  716. emit_cmp_imm32(self, dst, insn.imm);
  717. emit_jcc(self, 0x8e, target_pc);
  718. },
  719. ebpf::JSLE_REG => {
  720. emit_cmp(self, src, dst);
  721. emit_jcc(self, 0x8e, target_pc);
  722. },
  723. ebpf::CALL => {
  724. // For JIT, helpers in use MUST be registered at compile time. They can be
  725. // updated later, but not created after compiling (we need the address of the
  726. // helper function in the JIT-compiled program).
  727. if let Some(helper) = helpers.get(&(insn.imm as u32)) {
  728. // We reserve RCX for shifts
  729. emit_mov(self, R9, RCX);
  730. emit_call(self, *helper as usize);
  731. } else {
  732. Err(Error::new(ErrorKind::Other,
  733. format!("[JIT] Error: unknown helper function (id: {:#x})",
  734. insn.imm as u32)))?;
  735. };
  736. },
  737. ebpf::TAIL_CALL => { unimplemented!() },
  738. ebpf::EXIT => {
  739. if insn_ptr != prog.len() / ebpf::INSN_SIZE - 1 {
  740. emit_jmp(self, TARGET_PC_EXIT);
  741. };
  742. },
  743. _ => {
  744. Err(Error::new(ErrorKind::Other,
  745. format!("[JIT] Error: unknown eBPF opcode {:#2x} (insn #{insn_ptr:?})",
  746. insn.opc)))?;
  747. },
  748. }
  749. insn_ptr += 1;
  750. }
  751. // Epilogue
  752. set_anchor(self, TARGET_PC_EXIT);
  753. // Move register 0 into rax
  754. if map_register(0) != RAX {
  755. emit_mov(self, map_register(0), RAX);
  756. }
  757. // Deallocate stack space
  758. emit_alu64_imm32(self, 0x81, 0, RSP, ebpf::STACK_SIZE as i32);
  759. emit_pop(self, R15);
  760. emit_pop(self, R14);
  761. emit_pop(self, R13);
  762. emit_pop(self, RBX);
  763. emit_pop(self, RBP);
  764. emit1(self, 0xc3); // ret
  765. // Division by zero handler
  766. set_anchor(self, TARGET_PC_DIV_BY_ZERO);
  767. fn log(pc: u64) -> i64 {
  768. // Write error message on stderr.
  769. // We would like to panic!() instead (but does not work here), or maybe return an
  770. // error, that is, if we also turn all other panics into errors someday.
  771. // Note: needs `use std::io::Write;`
  772. // let res = writeln!(&mut std::io::stderr(),
  773. // "[JIT] Error: division by zero (insn {:?})\n", pc);
  774. // match res {
  775. // Ok(_) => 0,
  776. // Err(_) => -1
  777. // }
  778. pc as i64 // Just to prevent warnings
  779. }
  780. emit_mov(self, RCX, RDI); // muldivmod stored pc in RCX
  781. emit_call(self, log as usize);
  782. emit_load_imm(self, map_register(0), -1);
  783. emit_jmp(self, TARGET_PC_EXIT);
  784. Ok(())
  785. }
  786. fn resolve_jumps(&mut self) -> Result<(), Error>
  787. {
  788. for jump in &self.jumps {
  789. let target_loc = match self.special_targets.get(&jump.target_pc) {
  790. Some(target) => *target,
  791. None => self.pc_locs[jump.target_pc as usize]
  792. };
  793. // Assumes jump offset is at end of instruction
  794. unsafe {
  795. let offset_loc = jump.offset_loc as i32 + std::mem::size_of::<i32>() as i32;
  796. let rel = &(target_loc as i32 - offset_loc) as *const i32;
  797. let offset_ptr = self.contents.as_ptr().add(jump.offset_loc);
  798. libc::memcpy(offset_ptr as *mut libc::c_void, rel as *const libc::c_void,
  799. std::mem::size_of::<i32>());
  800. }
  801. }
  802. Ok(())
  803. }
  804. } // struct JitMemory
  805. impl<'a> Index<usize> for JitMemory<'a> {
  806. type Output = u8;
  807. fn index(&self, _index: usize) -> &u8 {
  808. &self.contents[_index]
  809. }
  810. }
  811. impl<'a> IndexMut<usize> for JitMemory<'a> {
  812. fn index_mut(&mut self, _index: usize) -> &mut u8 {
  813. &mut self.contents[_index]
  814. }
  815. }
  816. impl<'a> std::fmt::Debug for JitMemory<'a> {
  817. fn fmt(&self, fmt: &mut Formatter) -> Result<(), FormatterError> {
  818. fmt.write_str("JIT contents: [")?;
  819. for i in self.contents as &[u8] {
  820. fmt.write_fmt(format_args!(" {i:#04x},"))?;
  821. };
  822. fmt.write_str(" ] | ")?;
  823. fmt.debug_struct("JIT state")
  824. .field("offset", &self.offset)
  825. .field("pc_locs", &self.pc_locs)
  826. .field("special_targets", &self.special_targets)
  827. .field("jumps", &self.jumps)
  828. .finish()
  829. }
  830. }
  831. // In the end, this is the only thing we export
  832. pub fn compile(prog: &[u8],
  833. helpers: &HashMap<u32, ebpf::Helper>,
  834. use_mbuff: bool, update_data_ptr: bool)
  835. -> Result<JitProgram, Error> {
  836. // TODO: check how long the page must be to be sure to support an eBPF program of maximum
  837. // possible length
  838. let mut jit = JitMemory::new(1);
  839. jit.jit_compile(prog, use_mbuff, update_data_ptr, helpers)?;
  840. jit.resolve_jumps()?;
  841. Ok(unsafe { mem::transmute(jit.contents.as_ptr()) })
  842. }