4
0

relocation.rs 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200
  1. use core::{mem, ptr, str::FromStr};
  2. use alloc::{
  3. borrow::ToOwned,
  4. format,
  5. string::{String, ToString},
  6. vec,
  7. vec::Vec,
  8. };
  9. use crate::{
  10. btf::{
  11. fields_are_compatible, types_are_compatible, Array, Btf, BtfError, BtfMember, BtfType,
  12. IntEncoding, Struct, Union, MAX_SPEC_LEN,
  13. },
  14. generated::{
  15. bpf_core_relo, bpf_core_relo_kind::*, bpf_insn, BPF_ALU, BPF_ALU64, BPF_B, BPF_CALL,
  16. BPF_DW, BPF_H, BPF_JMP, BPF_K, BPF_LD, BPF_LDX, BPF_ST, BPF_STX, BPF_W, BTF_INT_SIGNED,
  17. },
  18. util::HashMap,
  19. Object, Program, ProgramSection,
  20. };
  21. #[cfg(not(feature = "std"))]
  22. use crate::std;
  23. /// The error type returned by [`Object::relocate_btf`].
  24. #[derive(thiserror::Error, Debug)]
  25. #[error("error relocating `{section}`")]
  26. pub struct BtfRelocationError {
  27. /// The function name
  28. pub section: String,
  29. #[source]
  30. /// The original error
  31. error: RelocationError,
  32. }
  33. /// Relocation failures
  34. #[derive(thiserror::Error, Debug)]
  35. enum RelocationError {
  36. #[cfg(feature = "std")]
  37. /// I/O error
  38. #[error(transparent)]
  39. IOError(#[from] std::io::Error),
  40. /// Program not found
  41. #[error("program not found")]
  42. ProgramNotFound,
  43. /// Invalid relocation access string
  44. #[error("invalid relocation access string {access_str}")]
  45. InvalidAccessString {
  46. /// The access string
  47. access_str: String,
  48. },
  49. /// Invalid instruction index referenced by relocation
  50. #[error("invalid instruction index #{index} referenced by relocation #{relocation_number}, the program contains {num_instructions} instructions")]
  51. InvalidInstructionIndex {
  52. /// The invalid instruction index
  53. index: usize,
  54. /// Number of instructions in the program
  55. num_instructions: usize,
  56. /// The relocation number
  57. relocation_number: usize,
  58. },
  59. /// Multiple candidate target types found with different memory layouts
  60. #[error("error relocating {type_name}, multiple candidate target types found with different memory layouts: {candidates:?}")]
  61. ConflictingCandidates {
  62. /// The type name
  63. type_name: String,
  64. /// The candidates
  65. candidates: Vec<String>,
  66. },
  67. /// Maximum nesting level reached evaluating candidate type
  68. #[error("maximum nesting level reached evaluating candidate type `{}`", err_type_name(.type_name))]
  69. MaximumNestingLevelReached {
  70. /// The type name
  71. type_name: Option<String>,
  72. },
  73. /// Invalid access string
  74. #[error("invalid access string `{spec}` for type `{}`: {error}", err_type_name(.type_name))]
  75. InvalidAccessIndex {
  76. /// The type name
  77. type_name: Option<String>,
  78. /// The access string
  79. spec: String,
  80. /// The index
  81. index: usize,
  82. /// The max index
  83. max_index: usize,
  84. /// The error message
  85. error: String,
  86. },
  87. /// Relocation not valid for type
  88. #[error(
  89. "relocation #{relocation_number} of kind `{relocation_kind}` not valid for type `{type_kind}`: {error}"
  90. )]
  91. InvalidRelocationKindForType {
  92. /// The relocation number
  93. relocation_number: usize,
  94. /// The relocation kind
  95. relocation_kind: String,
  96. /// The type kind
  97. type_kind: String,
  98. /// The error message
  99. error: String,
  100. },
  101. /// Invalid instruction referenced by relocation
  102. #[error(
  103. "instruction #{index} referenced by relocation #{relocation_number} is invalid: {error}"
  104. )]
  105. InvalidInstruction {
  106. /// The relocation number
  107. relocation_number: usize,
  108. /// The instruction index
  109. index: usize,
  110. /// The error message
  111. error: String,
  112. },
  113. #[error("applying relocation `{kind:?}` missing target BTF info for type `{type_id}` at instruction #{ins_index}")]
  114. MissingTargetDefinition {
  115. kind: RelocationKind,
  116. type_id: u32,
  117. ins_index: usize,
  118. },
  119. /// BTF error
  120. #[error("invalid BTF")]
  121. BtfError(#[from] BtfError),
  122. }
  123. fn err_type_name(name: &Option<String>) -> String {
  124. name.clone().unwrap_or_else(|| "[unknown name]".to_string())
  125. }
  126. #[derive(Copy, Clone, Debug)]
  127. #[repr(u32)]
  128. enum RelocationKind {
  129. FieldByteOffset = BPF_CORE_FIELD_BYTE_OFFSET,
  130. FieldByteSize = BPF_CORE_FIELD_BYTE_SIZE,
  131. FieldExists = BPF_CORE_FIELD_EXISTS,
  132. FieldSigned = BPF_CORE_FIELD_SIGNED,
  133. FieldLShift64 = BPF_CORE_FIELD_LSHIFT_U64,
  134. FieldRShift64 = BPF_CORE_FIELD_RSHIFT_U64,
  135. TypeIdLocal = BPF_CORE_TYPE_ID_LOCAL,
  136. TypeIdTarget = BPF_CORE_TYPE_ID_TARGET,
  137. TypeExists = BPF_CORE_TYPE_EXISTS,
  138. TypeSize = BPF_CORE_TYPE_SIZE,
  139. EnumVariantExists = BPF_CORE_ENUMVAL_EXISTS,
  140. EnumVariantValue = BPF_CORE_ENUMVAL_VALUE,
  141. }
  142. impl TryFrom<u32> for RelocationKind {
  143. type Error = BtfError;
  144. fn try_from(v: u32) -> Result<Self, Self::Error> {
  145. use RelocationKind::*;
  146. Ok(match v {
  147. BPF_CORE_FIELD_BYTE_OFFSET => FieldByteOffset,
  148. BPF_CORE_FIELD_BYTE_SIZE => FieldByteSize,
  149. BPF_CORE_FIELD_EXISTS => FieldExists,
  150. BPF_CORE_FIELD_SIGNED => FieldSigned,
  151. BPF_CORE_FIELD_LSHIFT_U64 => FieldLShift64,
  152. BPF_CORE_FIELD_RSHIFT_U64 => FieldRShift64,
  153. BPF_CORE_TYPE_ID_LOCAL => TypeIdLocal,
  154. BPF_CORE_TYPE_ID_TARGET => TypeIdTarget,
  155. BPF_CORE_TYPE_EXISTS => TypeExists,
  156. BPF_CORE_TYPE_SIZE => TypeSize,
  157. BPF_CORE_ENUMVAL_EXISTS => EnumVariantExists,
  158. BPF_CORE_ENUMVAL_VALUE => EnumVariantValue,
  159. kind => return Err(BtfError::InvalidRelocationKind { kind }),
  160. })
  161. }
  162. }
  163. #[derive(Debug, Copy, Clone)]
  164. pub(crate) struct Relocation {
  165. kind: RelocationKind,
  166. ins_offset: usize,
  167. type_id: u32,
  168. access_str_offset: u32,
  169. number: usize,
  170. }
  171. impl Relocation {
  172. #[allow(unused_unsafe)]
  173. pub(crate) unsafe fn parse(data: &[u8], number: usize) -> Result<Relocation, BtfError> {
  174. if mem::size_of::<bpf_core_relo>() > data.len() {
  175. return Err(BtfError::InvalidRelocationInfo);
  176. }
  177. let rel = unsafe { ptr::read_unaligned::<bpf_core_relo>(data.as_ptr() as *const _) };
  178. Ok(Relocation {
  179. kind: rel.kind.try_into()?,
  180. ins_offset: rel.insn_off as usize,
  181. type_id: rel.type_id,
  182. access_str_offset: rel.access_str_off,
  183. number,
  184. })
  185. }
  186. }
  187. impl Object {
  188. /// Relocates programs inside this object file with loaded BTF info.
  189. pub fn relocate_btf(&mut self, target_btf: &Btf) -> Result<(), BtfRelocationError> {
  190. let (local_btf, btf_ext) = match (&self.btf, &self.btf_ext) {
  191. (Some(btf), Some(btf_ext)) => (btf, btf_ext),
  192. _ => return Ok(()),
  193. };
  194. let mut candidates_cache = HashMap::<u32, Vec<Candidate>>::new();
  195. for (sec_name_off, relos) in btf_ext.relocations() {
  196. let section_name =
  197. local_btf
  198. .string_at(*sec_name_off)
  199. .map_err(|e| BtfRelocationError {
  200. section: format!("section@{sec_name_off}"),
  201. error: RelocationError::BtfError(e),
  202. })?;
  203. let program_section = match ProgramSection::from_str(&section_name) {
  204. Ok(program) => program,
  205. Err(_) => continue,
  206. };
  207. let section_name = program_section.name();
  208. let program = self
  209. .programs
  210. .get_mut(section_name)
  211. .ok_or(BtfRelocationError {
  212. section: section_name.to_owned(),
  213. error: RelocationError::ProgramNotFound,
  214. })?;
  215. match relocate_btf_program(program, relos, local_btf, target_btf, &mut candidates_cache)
  216. {
  217. Ok(_) => {}
  218. Err(error) => {
  219. return Err(BtfRelocationError {
  220. section: section_name.to_owned(),
  221. error,
  222. })
  223. }
  224. }
  225. }
  226. Ok(())
  227. }
  228. }
  229. fn relocate_btf_program<'target>(
  230. program: &mut Program,
  231. relos: &[Relocation],
  232. local_btf: &Btf,
  233. target_btf: &'target Btf,
  234. candidates_cache: &mut HashMap<u32, Vec<Candidate<'target>>>,
  235. ) -> Result<(), RelocationError> {
  236. for rel in relos {
  237. let instructions = &mut program.function.instructions;
  238. let ins_index = rel.ins_offset / mem::size_of::<bpf_insn>();
  239. if ins_index >= instructions.len() {
  240. return Err(RelocationError::InvalidInstructionIndex {
  241. index: ins_index,
  242. num_instructions: instructions.len(),
  243. relocation_number: rel.number,
  244. });
  245. }
  246. let local_ty = local_btf.type_by_id(rel.type_id)?;
  247. let local_name = &*local_btf.type_name(local_ty)?;
  248. let access_str = &*local_btf.string_at(rel.access_str_offset)?;
  249. let local_spec = AccessSpec::new(local_btf, rel.type_id, access_str, *rel)?;
  250. let matches = match rel.kind {
  251. RelocationKind::TypeIdLocal => Vec::new(), // we don't need to look at target types to relocate this value
  252. _ => {
  253. let candidates = match candidates_cache.get(&rel.type_id) {
  254. Some(cands) => cands,
  255. None => {
  256. candidates_cache.insert(
  257. rel.type_id,
  258. find_candidates(local_ty, local_name, target_btf)?,
  259. );
  260. candidates_cache.get(&rel.type_id).unwrap()
  261. }
  262. };
  263. let mut matches = Vec::new();
  264. for candidate in candidates {
  265. if let Some(candidate_spec) = match_candidate(&local_spec, candidate)? {
  266. let comp_rel =
  267. ComputedRelocation::new(rel, &local_spec, Some(&candidate_spec))?;
  268. matches.push((candidate.name.clone(), candidate_spec, comp_rel));
  269. }
  270. }
  271. matches
  272. }
  273. };
  274. let comp_rel = if !matches.is_empty() {
  275. let mut matches = matches.into_iter();
  276. let (_, target_spec, target_comp_rel) = matches.next().unwrap();
  277. // if there's more than one candidate, make sure that they all resolve to the
  278. // same value, else the relocation is ambiguous and can't be applied
  279. let conflicts = matches
  280. .filter_map(|(cand_name, cand_spec, cand_comp_rel)| {
  281. if cand_spec.bit_offset != target_spec.bit_offset {
  282. return Some(cand_name);
  283. } else if let (Some(cand_comp_rel_target), Some(target_comp_rel_target)) = (
  284. cand_comp_rel.target.as_ref(),
  285. target_comp_rel.target.as_ref(),
  286. ) {
  287. if cand_comp_rel_target.value != target_comp_rel_target.value {
  288. return Some(cand_name);
  289. }
  290. }
  291. None
  292. })
  293. .collect::<Vec<_>>();
  294. if !conflicts.is_empty() {
  295. return Err(RelocationError::ConflictingCandidates {
  296. type_name: local_name.to_string(),
  297. candidates: conflicts,
  298. });
  299. }
  300. target_comp_rel
  301. } else {
  302. // there are no candidate matches and therefore no target_spec. This might mean
  303. // that matching failed, or that the relocation can be applied looking at local
  304. // types only (eg with EnumVariantExists, FieldExists etc)
  305. ComputedRelocation::new(rel, &local_spec, None)?
  306. };
  307. comp_rel.apply(program, rel, local_btf, target_btf)?;
  308. }
  309. Ok(())
  310. }
  311. fn flavorless_name(name: &str) -> &str {
  312. name.split_once("___").map_or(name, |x| x.0)
  313. }
  314. fn find_candidates<'target>(
  315. local_ty: &BtfType,
  316. local_name: &str,
  317. target_btf: &'target Btf,
  318. ) -> Result<Vec<Candidate<'target>>, BtfError> {
  319. let mut candidates = Vec::new();
  320. let local_name = flavorless_name(local_name);
  321. for (type_id, ty) in target_btf.types().enumerate() {
  322. if local_ty.kind() != ty.kind() {
  323. continue;
  324. }
  325. let name = &*target_btf.type_name(ty)?;
  326. if local_name != flavorless_name(name) {
  327. continue;
  328. }
  329. candidates.push(Candidate {
  330. name: name.to_owned(),
  331. btf: target_btf,
  332. _ty: ty,
  333. type_id: type_id as u32,
  334. });
  335. }
  336. Ok(candidates)
  337. }
  338. fn match_candidate<'target>(
  339. local_spec: &AccessSpec,
  340. candidate: &'target Candidate,
  341. ) -> Result<Option<AccessSpec<'target>>, RelocationError> {
  342. let mut target_spec = AccessSpec {
  343. btf: candidate.btf,
  344. root_type_id: candidate.type_id,
  345. relocation: local_spec.relocation,
  346. parts: Vec::new(),
  347. accessors: Vec::new(),
  348. bit_offset: 0,
  349. };
  350. match local_spec.relocation.kind {
  351. RelocationKind::TypeIdLocal
  352. | RelocationKind::TypeIdTarget
  353. | RelocationKind::TypeExists
  354. | RelocationKind::TypeSize => {
  355. if types_are_compatible(
  356. local_spec.btf,
  357. local_spec.root_type_id,
  358. candidate.btf,
  359. candidate.type_id,
  360. )? {
  361. return Ok(Some(target_spec));
  362. } else {
  363. return Ok(None);
  364. }
  365. }
  366. RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => {
  367. let target_id = candidate.btf.resolve_type(candidate.type_id)?;
  368. let target_ty = candidate.btf.type_by_id(target_id)?;
  369. // the first accessor is guaranteed to have a name by construction
  370. let local_variant_name = local_spec.accessors[0].name.as_ref().unwrap();
  371. let match_enum =
  372. |name_offset, index, target_spec: &mut AccessSpec| -> Result<_, BtfError> {
  373. let target_variant_name = candidate.btf.string_at(name_offset)?;
  374. if flavorless_name(local_variant_name) == flavorless_name(&target_variant_name)
  375. {
  376. target_spec.parts.push(index);
  377. target_spec.accessors.push(Accessor {
  378. index,
  379. type_id: target_id,
  380. name: None,
  381. });
  382. Ok(Some(()))
  383. } else {
  384. Ok(None)
  385. }
  386. };
  387. match target_ty {
  388. BtfType::Enum(en) => {
  389. for (index, member) in en.variants.iter().enumerate() {
  390. if let Ok(Some(_)) = match_enum(member.name_offset, index, &mut target_spec)
  391. {
  392. return Ok(Some(target_spec));
  393. }
  394. }
  395. }
  396. BtfType::Enum64(en) => {
  397. for (index, member) in en.variants.iter().enumerate() {
  398. if let Ok(Some(_)) = match_enum(member.name_offset, index, &mut target_spec)
  399. {
  400. return Ok(Some(target_spec));
  401. }
  402. }
  403. }
  404. _ => return Ok(None),
  405. }
  406. }
  407. RelocationKind::FieldByteOffset
  408. | RelocationKind::FieldByteSize
  409. | RelocationKind::FieldExists
  410. | RelocationKind::FieldSigned
  411. | RelocationKind::FieldLShift64
  412. | RelocationKind::FieldRShift64 => {
  413. let mut target_id = candidate.type_id;
  414. for (i, accessor) in local_spec.accessors.iter().enumerate() {
  415. target_id = candidate.btf.resolve_type(target_id)?;
  416. if accessor.name.is_some() {
  417. if let Some(next_id) = match_member(
  418. local_spec.btf,
  419. local_spec,
  420. accessor,
  421. candidate.btf,
  422. target_id,
  423. &mut target_spec,
  424. )? {
  425. target_id = next_id;
  426. } else {
  427. return Ok(None);
  428. }
  429. } else {
  430. // i = 0 is the base struct. for i > 0, we need to potentially do bounds checking
  431. if i > 0 {
  432. let target_ty = candidate.btf.type_by_id(target_id)?;
  433. let array = match target_ty {
  434. BtfType::Array(Array { array, .. }) => array,
  435. _ => return Ok(None),
  436. };
  437. let var_len = array.len == 0 && {
  438. // an array is potentially variable length if it's the last field
  439. // of the parent struct and has 0 elements
  440. let parent = target_spec.accessors.last().unwrap();
  441. let parent_ty = candidate.btf.type_by_id(parent.type_id)?;
  442. match parent_ty {
  443. BtfType::Struct(s) => parent.index == s.members.len() - 1,
  444. _ => false,
  445. }
  446. };
  447. if !var_len && accessor.index >= array.len as usize {
  448. return Ok(None);
  449. }
  450. target_id = candidate.btf.resolve_type(array.element_type)?;
  451. }
  452. if target_spec.parts.len() == MAX_SPEC_LEN {
  453. return Err(RelocationError::MaximumNestingLevelReached {
  454. type_name: Some(candidate.name.clone()),
  455. });
  456. }
  457. target_spec.parts.push(accessor.index);
  458. target_spec.accessors.push(Accessor {
  459. index: accessor.index,
  460. type_id: target_id,
  461. name: None,
  462. });
  463. target_spec.bit_offset +=
  464. accessor.index * candidate.btf.type_size(target_id)? * 8;
  465. }
  466. }
  467. }
  468. };
  469. Ok(Some(target_spec))
  470. }
  471. fn match_member<'target>(
  472. local_btf: &Btf,
  473. local_spec: &AccessSpec<'_>,
  474. local_accessor: &Accessor,
  475. target_btf: &'target Btf,
  476. target_id: u32,
  477. target_spec: &mut AccessSpec<'target>,
  478. ) -> Result<Option<u32>, RelocationError> {
  479. let local_ty = local_btf.type_by_id(local_accessor.type_id)?;
  480. let local_member = match local_ty {
  481. // this won't panic, bounds are checked when local_spec is built in AccessSpec::new
  482. BtfType::Struct(s) => s.members.get(local_accessor.index).unwrap(),
  483. BtfType::Union(u) => u.members.get(local_accessor.index).unwrap(),
  484. _ => panic!("bug! this should only be called for structs and unions"),
  485. };
  486. let local_name = &*local_btf.string_at(local_member.name_offset)?;
  487. let target_id = target_btf.resolve_type(target_id)?;
  488. let target_ty = target_btf.type_by_id(target_id)?;
  489. let target_members: Vec<&BtfMember> = match target_ty.members() {
  490. Some(members) => members.collect(),
  491. // not a fields type, no match
  492. None => return Ok(None),
  493. };
  494. for (index, target_member) in target_members.iter().enumerate() {
  495. if target_spec.parts.len() == MAX_SPEC_LEN {
  496. let root_ty = target_spec.btf.type_by_id(target_spec.root_type_id)?;
  497. return Err(RelocationError::MaximumNestingLevelReached {
  498. type_name: target_spec.btf.err_type_name(root_ty),
  499. });
  500. }
  501. // this will not panic as we've already established these are fields types
  502. let bit_offset = target_ty.member_bit_offset(target_member).unwrap();
  503. let target_name = &*target_btf.string_at(target_member.name_offset)?;
  504. if target_name.is_empty() {
  505. let ret = match_member(
  506. local_btf,
  507. local_spec,
  508. local_accessor,
  509. target_btf,
  510. target_member.btf_type,
  511. target_spec,
  512. )?;
  513. if ret.is_some() {
  514. target_spec.bit_offset += bit_offset;
  515. target_spec.parts.push(index);
  516. return Ok(ret);
  517. }
  518. } else if local_name == target_name {
  519. if fields_are_compatible(
  520. local_spec.btf,
  521. local_member.btf_type,
  522. target_btf,
  523. target_member.btf_type,
  524. )? {
  525. target_spec.bit_offset += bit_offset;
  526. target_spec.parts.push(index);
  527. target_spec.accessors.push(Accessor {
  528. type_id: target_id,
  529. index,
  530. name: Some(target_name.to_owned()),
  531. });
  532. return Ok(Some(target_member.btf_type));
  533. } else {
  534. return Ok(None);
  535. }
  536. }
  537. }
  538. Ok(None)
  539. }
  540. #[derive(Debug)]
  541. struct AccessSpec<'a> {
  542. btf: &'a Btf,
  543. root_type_id: u32,
  544. parts: Vec<usize>,
  545. accessors: Vec<Accessor>,
  546. relocation: Relocation,
  547. bit_offset: usize,
  548. }
  549. impl<'a> AccessSpec<'a> {
  550. fn new(
  551. btf: &'a Btf,
  552. root_type_id: u32,
  553. spec: &str,
  554. relocation: Relocation,
  555. ) -> Result<AccessSpec<'a>, RelocationError> {
  556. let parts = spec
  557. .split(':')
  558. .map(|s| s.parse::<usize>())
  559. .collect::<Result<Vec<_>, _>>()
  560. .map_err(|_| RelocationError::InvalidAccessString {
  561. access_str: spec.to_string(),
  562. })?;
  563. let mut type_id = btf.resolve_type(root_type_id)?;
  564. let ty = btf.type_by_id(type_id)?;
  565. let spec = match relocation.kind {
  566. RelocationKind::TypeIdLocal
  567. | RelocationKind::TypeIdTarget
  568. | RelocationKind::TypeExists
  569. | RelocationKind::TypeSize => {
  570. if parts != [0] {
  571. return Err(RelocationError::InvalidAccessString {
  572. access_str: spec.to_string(),
  573. });
  574. }
  575. AccessSpec {
  576. btf,
  577. root_type_id,
  578. relocation,
  579. parts,
  580. accessors: Vec::new(),
  581. bit_offset: 0,
  582. }
  583. }
  584. RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => match ty {
  585. BtfType::Enum(_) | BtfType::Enum64(_) => {
  586. if parts.len() != 1 {
  587. return Err(RelocationError::InvalidAccessString {
  588. access_str: spec.to_string(),
  589. });
  590. }
  591. let index = parts[0];
  592. let (n_variants, name_offset) = match ty {
  593. BtfType::Enum(en) => (
  594. en.variants.len(),
  595. en.variants.get(index).map(|v| v.name_offset),
  596. ),
  597. BtfType::Enum64(en) => (
  598. en.variants.len(),
  599. en.variants.get(index).map(|v| v.name_offset),
  600. ),
  601. _ => unreachable!(),
  602. };
  603. if name_offset.is_none() {
  604. return Err(RelocationError::InvalidAccessIndex {
  605. type_name: btf.err_type_name(ty),
  606. spec: spec.to_string(),
  607. index,
  608. max_index: n_variants,
  609. error: "tried to access nonexistant enum variant".to_string(),
  610. });
  611. }
  612. let accessors = vec![Accessor {
  613. type_id,
  614. index,
  615. name: Some(btf.string_at(name_offset.unwrap())?.to_string()),
  616. }];
  617. AccessSpec {
  618. btf,
  619. root_type_id,
  620. relocation,
  621. parts,
  622. accessors,
  623. bit_offset: 0,
  624. }
  625. }
  626. _ => {
  627. return Err(RelocationError::InvalidRelocationKindForType {
  628. relocation_number: relocation.number,
  629. relocation_kind: format!("{:?}", relocation.kind),
  630. type_kind: format!("{:?}", ty.kind()),
  631. error: "enum relocation on non-enum type".to_string(),
  632. })
  633. }
  634. },
  635. RelocationKind::FieldByteOffset
  636. | RelocationKind::FieldByteSize
  637. | RelocationKind::FieldExists
  638. | RelocationKind::FieldSigned
  639. | RelocationKind::FieldLShift64
  640. | RelocationKind::FieldRShift64 => {
  641. let mut accessors = vec![Accessor {
  642. type_id,
  643. index: parts[0],
  644. name: None,
  645. }];
  646. let mut bit_offset = accessors[0].index * btf.type_size(type_id)?;
  647. for index in parts.iter().skip(1).cloned() {
  648. type_id = btf.resolve_type(type_id)?;
  649. let ty = btf.type_by_id(type_id)?;
  650. match ty {
  651. BtfType::Struct(Struct { members, .. })
  652. | BtfType::Union(Union { members, .. }) => {
  653. if index >= members.len() {
  654. return Err(RelocationError::InvalidAccessIndex {
  655. type_name: btf.err_type_name(ty),
  656. spec: spec.to_string(),
  657. index,
  658. max_index: members.len(),
  659. error: "out of bounds struct or union access".to_string(),
  660. });
  661. }
  662. let member = &members[index];
  663. bit_offset += ty.member_bit_offset(member).unwrap();
  664. if member.name_offset != 0 {
  665. accessors.push(Accessor {
  666. type_id,
  667. index,
  668. name: Some(btf.string_at(member.name_offset)?.to_string()),
  669. });
  670. }
  671. type_id = member.btf_type;
  672. }
  673. BtfType::Array(Array { array, .. }) => {
  674. type_id = btf.resolve_type(array.element_type)?;
  675. let var_len = array.len == 0 && {
  676. // an array is potentially variable length if it's the last field
  677. // of the parent struct and has 0 elements
  678. let parent = accessors.last().unwrap();
  679. let parent_ty = btf.type_by_id(parent.type_id)?;
  680. match parent_ty {
  681. BtfType::Struct(s) => index == s.members.len() - 1,
  682. _ => false,
  683. }
  684. };
  685. if !var_len && index >= array.len as usize {
  686. return Err(RelocationError::InvalidAccessIndex {
  687. type_name: btf.err_type_name(ty),
  688. spec: spec.to_string(),
  689. index,
  690. max_index: array.len as usize,
  691. error: "array index out of bounds".to_string(),
  692. });
  693. }
  694. accessors.push(Accessor {
  695. type_id,
  696. index,
  697. name: None,
  698. });
  699. let size = btf.type_size(type_id)?;
  700. bit_offset += index * size * 8;
  701. }
  702. rel_kind => {
  703. return Err(RelocationError::InvalidRelocationKindForType {
  704. relocation_number: relocation.number,
  705. relocation_kind: format!("{rel_kind:?}"),
  706. type_kind: format!("{:?}", ty.kind()),
  707. error: "field relocation on a type that doesn't have fields"
  708. .to_string(),
  709. });
  710. }
  711. };
  712. }
  713. AccessSpec {
  714. btf,
  715. root_type_id,
  716. parts,
  717. accessors,
  718. relocation,
  719. bit_offset,
  720. }
  721. }
  722. };
  723. Ok(spec)
  724. }
  725. }
  726. #[derive(Debug)]
  727. struct Accessor {
  728. type_id: u32,
  729. index: usize,
  730. name: Option<String>,
  731. }
  732. #[derive(Debug)]
  733. struct Candidate<'a> {
  734. name: String,
  735. btf: &'a Btf,
  736. _ty: &'a BtfType,
  737. type_id: u32,
  738. }
  739. #[derive(Debug)]
  740. struct ComputedRelocation {
  741. local: ComputedRelocationValue,
  742. target: Option<ComputedRelocationValue>,
  743. }
  744. #[derive(Debug)]
  745. struct ComputedRelocationValue {
  746. value: u64,
  747. size: u32,
  748. type_id: Option<u32>,
  749. }
  750. fn poison_insn(ins: &mut bpf_insn) {
  751. ins.code = (BPF_JMP | BPF_CALL) as u8;
  752. ins.set_dst_reg(0);
  753. ins.set_src_reg(0);
  754. ins.off = 0;
  755. ins.imm = 0xBAD2310;
  756. }
  757. impl ComputedRelocation {
  758. fn new(
  759. rel: &Relocation,
  760. local_spec: &AccessSpec,
  761. target_spec: Option<&AccessSpec>,
  762. ) -> Result<ComputedRelocation, RelocationError> {
  763. use RelocationKind::*;
  764. let ret = match rel.kind {
  765. FieldByteOffset | FieldByteSize | FieldExists | FieldSigned | FieldLShift64
  766. | FieldRShift64 => ComputedRelocation {
  767. local: Self::compute_field_relocation(rel, Some(local_spec))?,
  768. target: Self::compute_field_relocation(rel, target_spec).ok(),
  769. },
  770. TypeIdLocal | TypeIdTarget | TypeExists | TypeSize => ComputedRelocation {
  771. local: Self::compute_type_relocation(rel, local_spec, target_spec)?,
  772. target: Self::compute_type_relocation(rel, local_spec, target_spec).ok(),
  773. },
  774. EnumVariantExists | EnumVariantValue => ComputedRelocation {
  775. local: Self::compute_enum_relocation(rel, Some(local_spec))?,
  776. target: Self::compute_enum_relocation(rel, target_spec).ok(),
  777. },
  778. };
  779. Ok(ret)
  780. }
  781. fn apply(
  782. &self,
  783. program: &mut Program,
  784. rel: &Relocation,
  785. local_btf: &Btf,
  786. target_btf: &Btf,
  787. ) -> Result<(), RelocationError> {
  788. let instructions = &mut program.function.instructions;
  789. let num_instructions = instructions.len();
  790. let ins_index = rel.ins_offset / mem::size_of::<bpf_insn>();
  791. let ins =
  792. instructions
  793. .get_mut(ins_index)
  794. .ok_or(RelocationError::InvalidInstructionIndex {
  795. index: rel.ins_offset,
  796. num_instructions,
  797. relocation_number: rel.number,
  798. })?;
  799. let target = if let Some(target) = self.target.as_ref() {
  800. target
  801. } else {
  802. let is_ld_imm64 = ins.code == (BPF_LD | BPF_DW) as u8;
  803. poison_insn(ins);
  804. if is_ld_imm64 {
  805. let next_ins = instructions.get_mut(ins_index + 1).ok_or(
  806. RelocationError::InvalidInstructionIndex {
  807. index: (ins_index + 1) * mem::size_of::<bpf_insn>(),
  808. num_instructions,
  809. relocation_number: rel.number,
  810. },
  811. )?;
  812. poison_insn(next_ins);
  813. }
  814. return Ok(());
  815. };
  816. let class = (ins.code & 0x07) as u32;
  817. let target_value = target.value;
  818. match class {
  819. BPF_ALU | BPF_ALU64 => {
  820. let src_reg = ins.src_reg();
  821. if src_reg != BPF_K as u8 {
  822. return Err(RelocationError::InvalidInstruction {
  823. relocation_number: rel.number,
  824. index: ins_index,
  825. error: format!("invalid src_reg={src_reg:x} expected {BPF_K:x}"),
  826. });
  827. }
  828. ins.imm = target_value as i32;
  829. }
  830. BPF_LDX | BPF_ST | BPF_STX => {
  831. if target_value > i16::MAX as u64 {
  832. return Err(RelocationError::InvalidInstruction {
  833. relocation_number: rel.number,
  834. index: ins_index,
  835. error: format!("value `{target_value}` overflows 16 bits offset field"),
  836. });
  837. }
  838. ins.off = target_value as i16;
  839. if self.local.size != target.size {
  840. let local_ty = local_btf.type_by_id(self.local.type_id.unwrap())?;
  841. let target_ty = target_btf.type_by_id(target.type_id.unwrap())?;
  842. let unsigned = |info: u32| ((info >> 24) & 0x0F) & BTF_INT_SIGNED == 0;
  843. use BtfType::*;
  844. match (local_ty, target_ty) {
  845. (Ptr(_), Ptr(_)) => {}
  846. (Int(local), Int(target))
  847. if unsigned(local.data) && unsigned(target.data) => {}
  848. _ => {
  849. return Err(RelocationError::InvalidInstruction {
  850. relocation_number: rel.number,
  851. index: ins_index,
  852. error: format!(
  853. "original type {} has size {} but target type {} has size {}",
  854. err_type_name(&local_btf.err_type_name(local_ty)),
  855. self.local.size,
  856. err_type_name(&target_btf.err_type_name(target_ty)),
  857. target.size,
  858. ),
  859. })
  860. }
  861. }
  862. let size = match target.size {
  863. 8 => BPF_DW,
  864. 4 => BPF_W,
  865. 2 => BPF_H,
  866. 1 => BPF_B,
  867. size => {
  868. return Err(RelocationError::InvalidInstruction {
  869. relocation_number: rel.number,
  870. index: ins_index,
  871. error: format!("invalid target size {size}"),
  872. })
  873. }
  874. } as u8;
  875. ins.code = ins.code & 0xE0 | size | ins.code & 0x07;
  876. }
  877. }
  878. BPF_LD => {
  879. ins.imm = target_value as i32;
  880. let next_ins = instructions.get_mut(ins_index + 1).ok_or(
  881. RelocationError::InvalidInstructionIndex {
  882. index: ins_index + 1,
  883. num_instructions,
  884. relocation_number: rel.number,
  885. },
  886. )?;
  887. next_ins.imm = (target_value >> 32) as i32;
  888. }
  889. class => {
  890. return Err(RelocationError::InvalidInstruction {
  891. relocation_number: rel.number,
  892. index: ins_index,
  893. error: format!("invalid instruction class {class:x}"),
  894. })
  895. }
  896. };
  897. Ok(())
  898. }
  899. fn compute_enum_relocation(
  900. rel: &Relocation,
  901. spec: Option<&AccessSpec>,
  902. ) -> Result<ComputedRelocationValue, RelocationError> {
  903. use RelocationKind::*;
  904. let value = match (rel.kind, spec) {
  905. (EnumVariantExists, spec) => spec.is_some() as u64,
  906. (EnumVariantValue, Some(spec)) => {
  907. let accessor = &spec.accessors[0];
  908. match spec.btf.type_by_id(accessor.type_id)? {
  909. BtfType::Enum(en) => {
  910. let value = en.variants[accessor.index].value;
  911. if en.is_signed() {
  912. value as i32 as u64
  913. } else {
  914. value as u64
  915. }
  916. }
  917. BtfType::Enum64(en) => {
  918. let variant = &en.variants[accessor.index];
  919. (variant.value_high as u64) << 32 | variant.value_low as u64
  920. }
  921. // candidate selection ensures that rel_kind == local_kind == target_kind
  922. _ => unreachable!(),
  923. }
  924. }
  925. _ => {
  926. return Err(RelocationError::MissingTargetDefinition {
  927. kind: rel.kind,
  928. type_id: rel.type_id,
  929. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  930. })?;
  931. }
  932. };
  933. Ok(ComputedRelocationValue {
  934. value,
  935. size: 0,
  936. type_id: None,
  937. })
  938. }
  939. fn compute_field_relocation(
  940. rel: &Relocation,
  941. spec: Option<&AccessSpec>,
  942. ) -> Result<ComputedRelocationValue, RelocationError> {
  943. use RelocationKind::*;
  944. if let FieldExists = rel.kind {
  945. // this is the bpf_preserve_field_info(member_access, FIELD_EXISTENCE) case. If we
  946. // managed to build a spec, it means the field exists.
  947. return Ok(ComputedRelocationValue {
  948. value: spec.is_some() as u64,
  949. size: 0,
  950. type_id: None,
  951. });
  952. }
  953. let spec = match spec {
  954. Some(spec) => spec,
  955. None => {
  956. return Err(RelocationError::MissingTargetDefinition {
  957. kind: rel.kind,
  958. type_id: rel.type_id,
  959. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  960. })?;
  961. }
  962. };
  963. let accessor = spec.accessors.last().unwrap();
  964. if accessor.name.is_none() {
  965. // the last accessor is unnamed, meaning that this is an array access
  966. return match rel.kind {
  967. FieldByteOffset => Ok(ComputedRelocationValue {
  968. value: (spec.bit_offset / 8) as u64,
  969. size: spec.btf.type_size(accessor.type_id)? as u32,
  970. type_id: Some(accessor.type_id),
  971. }),
  972. FieldByteSize => Ok(ComputedRelocationValue {
  973. value: spec.btf.type_size(accessor.type_id)? as u64,
  974. size: 0,
  975. type_id: Some(accessor.type_id),
  976. }),
  977. rel_kind => {
  978. let ty = spec.btf.type_by_id(accessor.type_id)?;
  979. return Err(RelocationError::InvalidRelocationKindForType {
  980. relocation_number: rel.number,
  981. relocation_kind: format!("{rel_kind:?}"),
  982. type_kind: format!("{:?}", ty.kind()),
  983. error: "invalid relocation kind for array type".to_string(),
  984. });
  985. }
  986. };
  987. }
  988. let ty = spec.btf.type_by_id(accessor.type_id)?;
  989. let (ll_ty, member) = match ty {
  990. BtfType::Struct(t) => (ty, t.members.get(accessor.index).unwrap()),
  991. BtfType::Union(t) => (ty, t.members.get(accessor.index).unwrap()),
  992. _ => {
  993. return Err(RelocationError::InvalidRelocationKindForType {
  994. relocation_number: rel.number,
  995. relocation_kind: format!("{:?}", rel.kind),
  996. type_kind: format!("{:?}", ty.kind()),
  997. error: "field relocation on a type that doesn't have fields".to_string(),
  998. });
  999. }
  1000. };
  1001. let bit_off = spec.bit_offset as u32;
  1002. let member_type_id = spec.btf.resolve_type(member.btf_type)?;
  1003. let member_ty = spec.btf.type_by_id(member_type_id)?;
  1004. let mut byte_size;
  1005. let mut byte_off;
  1006. let mut bit_size = ll_ty.member_bit_field_size(member).unwrap() as u32;
  1007. let is_bitfield = bit_size > 0;
  1008. if is_bitfield {
  1009. // find out the smallest int size to load the bitfield
  1010. byte_size = member_ty.size().unwrap();
  1011. byte_off = bit_off / 8 / byte_size * byte_size;
  1012. while bit_off + bit_size - byte_off * 8 > byte_size * 8 {
  1013. if byte_size >= 8 {
  1014. // the bitfield is larger than 8 bytes!?
  1015. return Err(BtfError::InvalidTypeInfo.into());
  1016. }
  1017. byte_size *= 2;
  1018. byte_off = bit_off / 8 / byte_size * byte_size;
  1019. }
  1020. } else {
  1021. byte_size = spec.btf.type_size(member_type_id)? as u32;
  1022. bit_size = byte_size * 8;
  1023. byte_off = spec.bit_offset as u32 / 8;
  1024. }
  1025. let mut value = ComputedRelocationValue {
  1026. value: 0,
  1027. size: 0,
  1028. type_id: None,
  1029. };
  1030. #[allow(clippy::wildcard_in_or_patterns)]
  1031. match rel.kind {
  1032. FieldByteOffset => {
  1033. value.value = byte_off as u64;
  1034. if !is_bitfield {
  1035. value.size = byte_size;
  1036. value.type_id = Some(member_type_id);
  1037. }
  1038. }
  1039. FieldByteSize => {
  1040. value.value = byte_size as u64;
  1041. }
  1042. FieldSigned => match member_ty {
  1043. BtfType::Enum(en) => value.value = en.is_signed() as u64,
  1044. BtfType::Enum64(en) => value.value = en.is_signed() as u64,
  1045. BtfType::Int(i) => value.value = i.encoding() as u64 & IntEncoding::Signed as u64,
  1046. _ => (),
  1047. },
  1048. #[cfg(target_endian = "little")]
  1049. FieldLShift64 => {
  1050. value.value = 64 - (bit_off + bit_size - byte_off * 8) as u64;
  1051. }
  1052. #[cfg(target_endian = "big")]
  1053. FieldLShift64 => {
  1054. value.value = (8 - byte_size) * 8 + (bit_off - byte_off * 8);
  1055. }
  1056. FieldRShift64 => {
  1057. value.value = 64 - bit_size as u64;
  1058. }
  1059. FieldExists // this is handled at the start of the function
  1060. | _ => panic!("bug! this should not be reached"),
  1061. }
  1062. Ok(value)
  1063. }
  1064. fn compute_type_relocation(
  1065. rel: &Relocation,
  1066. local_spec: &AccessSpec,
  1067. target_spec: Option<&AccessSpec>,
  1068. ) -> Result<ComputedRelocationValue, RelocationError> {
  1069. use RelocationKind::*;
  1070. let value = match (rel.kind, target_spec) {
  1071. (TypeIdLocal, _) => local_spec.root_type_id as u64,
  1072. (TypeIdTarget, Some(target_spec)) => target_spec.root_type_id as u64,
  1073. (TypeExists, target_spec) => target_spec.is_some() as u64,
  1074. (TypeSize, Some(target_spec)) => {
  1075. target_spec.btf.type_size(target_spec.root_type_id)? as u64
  1076. }
  1077. _ => {
  1078. return Err(RelocationError::MissingTargetDefinition {
  1079. kind: rel.kind,
  1080. type_id: rel.type_id,
  1081. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  1082. })?;
  1083. }
  1084. };
  1085. Ok(ComputedRelocationValue {
  1086. value,
  1087. size: 0,
  1088. type_id: None,
  1089. })
  1090. }
  1091. }