src/jit.rs: Add support for 32-bit jump instructions in x86-64 JIT

Add support and tests for JIT-compiling programs using 32-bit jump
instructions.

The tests are exactly the same as for the interpreter.

Signed-off-by: Quentin Monnet <quentin@isovalent.com>

src/jit.rs

@@ -234,6 +234,16 @@ fn emit_cmp(jit: &mut JitMemory, src: u8, dst: u8) {
     emit_alu64(jit, 0x39, src, dst);
 }
 
+#[inline]
+fn emit_cmp32_imm32(jit: &mut JitMemory, dst: u8, imm: i32) {
+    emit_alu32_imm32(jit, 0x81, 7, dst, imm);
+}
+
+#[inline]
+fn emit_cmp32(jit: &mut JitMemory, src: u8, dst: u8) {
+    emit_alu32(jit, 0x39, src, dst);
+}
+
 #[inline]
 fn emit_jcc(jit: &mut JitMemory, code: u8, target_pc: isize) {
     emit1(jit, 0x0f);
@@ -795,6 +805,97 @@ impl<'a> JitMemory<'a> {
                     emit_cmp(self, src, dst);
                     emit_jcc(self, 0x8e, target_pc);
                 },
+
+                // BPF_JMP32 class
+                ebpf::JEQ_IMM32  => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x84, target_pc);
+                },
+                ebpf::JEQ_REG32  => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x84, target_pc);
+                },
+                ebpf::JGT_IMM32  => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x87, target_pc);
+                },
+                ebpf::JGT_REG32  => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x87, target_pc);
+                },
+                ebpf::JGE_IMM32  => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x83, target_pc);
+                },
+                ebpf::JGE_REG32  => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x83, target_pc);
+                },
+                ebpf::JLT_IMM32  => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x82, target_pc);
+                },
+                ebpf::JLT_REG32  => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x82, target_pc);
+                },
+                ebpf::JLE_IMM32  => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x86, target_pc);
+                },
+                ebpf::JLE_REG32  => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x86, target_pc);
+                },
+                ebpf::JSET_IMM32 => {
+                    emit_alu32_imm32(self, 0xf7, 0, dst, insn.imm);
+                    emit_jcc(self, 0x85, target_pc);
+                },
+                ebpf::JSET_REG32 => {
+                    emit_alu32(self, 0x85, src, dst);
+                    emit_jcc(self, 0x85, target_pc);
+                },
+                ebpf::JNE_IMM32  => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x85, target_pc);
+                },
+                ebpf::JNE_REG32  => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x85, target_pc);
+                },
+                ebpf::JSGT_IMM32 => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x8f, target_pc);
+                },
+                ebpf::JSGT_REG32 => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x8f, target_pc);
+                },
+                ebpf::JSGE_IMM32 => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x8d, target_pc);
+                },
+                ebpf::JSGE_REG32 => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x8d, target_pc);
+                },
+                ebpf::JSLT_IMM32 => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x8c, target_pc);
+                },
+                ebpf::JSLT_REG32 => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x8c, target_pc);
+                },
+                ebpf::JSLE_IMM32 => {
+                    emit_cmp32_imm32(self, dst, insn.imm);
+                    emit_jcc(self, 0x8e, target_pc);
+                },
+                ebpf::JSLE_REG32 => {
+                    emit_cmp32(self, src, dst);
+                    emit_jcc(self, 0x8e, target_pc);
+                },
+
                 ebpf::CALL       => {
                     // For JIT, helpers in use MUST be registered at compile time. They can be
                     // updated later, but not created after compiling (we need the address of the

tests/ubpf_jit_x86_64.rs

@@ -910,6 +910,451 @@ fn test_jit_jslt_reg() {
     unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
 }
 
+#[test]
+fn test_jit_jeq32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0x0
+        mov32 r1, 0xa
+        jeq32 r1, 0xb, +5
+        mov32 r0, 1
+        mov r1, 0xb
+        or r1, r9
+        jeq32 r1, 0xb, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jeq32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 0xa
+        mov32 r2, 0xb
+        jeq32 r1, r2, +5
+        mov32 r0, 1
+        mov32 r1, 0xb
+        or r1, r9
+        jeq32 r1, r2, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jge32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 0xa
+        jge32 r1, 0xb, +5
+        mov32 r0, 1
+        or r1, r9
+        mov32 r1, 0xc
+        jge32 r1, 0xb, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jge32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 0xa
+        mov32 r2, 0xb
+        jge32 r1, r2, +5
+        mov32 r0, 1
+        or r1, r9
+        mov32 r1, 0xc
+        jge32 r1, r2, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jgt32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 5
+        or r1, r9
+        jgt32 r1, 6, +4
+        jgt32 r1, 5, +3
+        jgt32 r1, 4, +1
+        exit
+        mov32 r0, 1
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jgt32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov r0, 0
+        mov r1, 5
+        mov32 r1, 5
+        or r1, r9
+        mov r2, 6
+        mov r3, 4
+        jgt32 r1, r2, +4
+        jgt32 r1, r1, +3
+        jgt32 r1, r3, +1
+        exit
+        mov r0, 1
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jle32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 5
+        or r1, r9
+        jle32 r1, 4, +5
+        jle32 r1, 6, +1
+        exit
+        jle32 r1, 5, +1
+        exit
+        mov32 r0, 1
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jle32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov r0, 0
+        mov r1, 5
+        mov r2, 4
+        mov r3, 6
+        or r1, r9
+        jle32 r1, r2, +5
+        jle32 r1, r1, +1
+        exit
+        jle32 r1, r3, +1
+        exit
+        mov r0, 1
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jlt32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 5
+        or r1, r9
+        jlt32 r1, 4, +4
+        jlt32 r1, 5, +3
+        jlt32 r1, 6, +1
+        exit
+        mov32 r0, 1
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jlt32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov r0, 0
+        mov r1, 5
+        mov r2, 4
+        mov r3, 6
+        or r1, r9
+        jlt32 r1, r2, +4
+        jlt32 r1, r1, +3
+        jlt32 r1, r3, +1
+        exit
+        mov r0, 1
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jne32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 0xb
+        or r1, r9
+        jne32 r1, 0xb, +4
+        mov32 r0, 1
+        mov32 r1, 0xa
+        or r1, r9
+        jne32 r1, 0xb, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jne32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 0xb
+        or r1, r9
+        mov32 r2, 0xb
+        jne32 r1, r2, +4
+        mov32 r0, 1
+        mov32 r1, 0xa
+        or r1, r9
+        jne32 r1, r2, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jset32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 0x7
+        or r1, r9
+        jset32 r1, 0x8, +4
+        mov32 r0, 1
+        mov32 r1, 0x9
+        jset32 r1, 0x8, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jset32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, 0x7
+        or r1, r9
+        mov32 r2, 0x8
+        jset32 r1, r2, +4
+        mov32 r0, 1
+        mov32 r1, 0x9
+        jset32 r1, r2, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jsge32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, -2
+        or r1, r9
+        jsge32 r1, -1, +5
+        jsge32 r1, 0, +4
+        mov32 r0, 1
+        mov r1, -1
+        jsge32 r1, -1, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jsge32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, -2
+        or r1, r9
+        mov r2, -1
+        mov32 r3, 0
+        jsge32 r1, r2, +5
+        jsge32 r1, r3, +4
+        mov32 r0, 1
+        mov r1, r2
+        jsge32 r1, r2, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jsgt32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, -2
+        or r1, r9
+        jsgt32 r1, -1, +4
+        mov32 r0, 1
+        mov32 r1, 0
+        jsgt32 r1, -1, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jsgt32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, -2
+        or r1, r9
+        mov r2, -1
+        jsgt32 r1, r2, +4
+        mov32 r0, 1
+        mov32 r1, 0
+        jsgt32 r1, r2, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jsle32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, -2
+        or r1, r9
+        jsle32 r1, -3, +5
+        jsle32 r1, -1, +1
+        exit
+        mov32 r0, 1
+        jsle32 r1, -2, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jsle32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, -2
+        or r1, r9
+        mov r2, -3
+        mov32 r3, 0
+        jsle32 r1, r2, +6
+        jsle32 r1, r3, +1
+        exit
+        mov32 r0, 1
+        mov r1, r2
+        jsle32 r1, r2, +1
+        mov32 r0, 2
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jslt32_imm() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, -2
+        or r1, r9
+        jslt32 r1, -3, +4
+        jslt32 r1, -2, +3
+        jslt32 r1, -1, +1
+        exit
+        mov32 r0, 1
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
+#[test]
+fn test_jit_jslt32_reg() {
+    let prog = assemble("
+        mov r9, 1
+        lsh r9, 32
+        mov32 r0, 0
+        mov32 r1, -2
+        or r1, r9
+        mov r2, -3
+        mov r3, -1
+        jslt32 r1, r1, +4
+        jslt32 r1, r2, +3
+        jslt32 r1, r3, +1
+        exit
+        mov32 r0, 1
+        exit").unwrap();
+    let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap();
+    vm.jit_compile().unwrap();
+    unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x1); }
+}
+
 #[test]
 fn test_jit_lddw() {
     let prog = assemble("