relocation.rs 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125
  1. use core::{mem, ptr, str::FromStr};
  2. use alloc::{
  3. borrow::ToOwned,
  4. format,
  5. string::{String, ToString},
  6. vec,
  7. vec::Vec,
  8. };
  9. use crate::{
  10. btf::{
  11. fields_are_compatible, types_are_compatible, Array, Btf, BtfError, BtfMember, BtfType,
  12. IntEncoding, Struct, Union, MAX_SPEC_LEN,
  13. },
  14. generated::{
  15. bpf_core_relo, bpf_core_relo_kind::*, bpf_insn, BPF_ALU, BPF_ALU64, BPF_B, BPF_DW, BPF_H,
  16. BPF_K, BPF_LD, BPF_LDX, BPF_ST, BPF_STX, BPF_W, BTF_INT_SIGNED,
  17. },
  18. thiserror::{self, Error},
  19. util::HashMap,
  20. Object, Program, ProgramSection,
  21. };
  22. /// The error type returned by [`Object::relocate_btf`].
  23. #[derive(Error, Debug)]
  24. #[error("error relocating `{section}`")]
  25. pub struct BtfRelocationError {
  26. /// The function name
  27. pub section: String,
  28. #[source]
  29. /// The original error
  30. error: RelocationError,
  31. }
  32. /// Relocation failures
  33. #[derive(Error, Debug)]
  34. enum RelocationError {
  35. #[cfg(not(feature = "no_std"))]
  36. /// I/O error
  37. #[error(transparent)]
  38. IOError(#[from] std::io::Error),
  39. /// Program not found
  40. #[error("program not found")]
  41. ProgramNotFound,
  42. /// Invalid relocation access string
  43. #[error("invalid relocation access string {access_str}")]
  44. InvalidAccessString {
  45. /// The access string
  46. access_str: String,
  47. },
  48. /// Invalid instruction index referenced by relocation
  49. #[error("invalid instruction index #{index} referenced by relocation #{relocation_number}, the program contains {num_instructions} instructions")]
  50. InvalidInstructionIndex {
  51. /// The invalid instruction index
  52. index: usize,
  53. /// Number of instructions in the program
  54. num_instructions: usize,
  55. /// The relocation number
  56. relocation_number: usize,
  57. },
  58. /// Multiple candidate target types found with different memory layouts
  59. #[error("error relocating {type_name}, multiple candidate target types found with different memory layouts: {candidates:?}")]
  60. ConflictingCandidates {
  61. /// The type name
  62. type_name: String,
  63. /// The candidates
  64. candidates: Vec<String>,
  65. },
  66. /// Maximum nesting level reached evaluating candidate type
  67. #[error("maximum nesting level reached evaluating candidate type `{}`", err_type_name(.type_name))]
  68. MaximumNestingLevelReached {
  69. /// The type name
  70. type_name: Option<String>,
  71. },
  72. /// Invalid access string
  73. #[error("invalid access string `{spec}` for type `{}`: {error}", err_type_name(.type_name))]
  74. InvalidAccessIndex {
  75. /// The type name
  76. type_name: Option<String>,
  77. /// The access string
  78. spec: String,
  79. /// The index
  80. index: usize,
  81. /// The max index
  82. max_index: usize,
  83. /// The error message
  84. error: String,
  85. },
  86. /// Relocation not valid for type
  87. #[error(
  88. "relocation #{relocation_number} of kind `{relocation_kind}` not valid for type `{type_kind}`: {error}"
  89. )]
  90. InvalidRelocationKindForType {
  91. /// The relocation number
  92. relocation_number: usize,
  93. /// The relocation kind
  94. relocation_kind: String,
  95. /// The type kind
  96. type_kind: String,
  97. /// The error message
  98. error: String,
  99. },
  100. /// Invalid instruction referenced by relocation
  101. #[error(
  102. "instruction #{index} referenced by relocation #{relocation_number} is invalid: {error}"
  103. )]
  104. InvalidInstruction {
  105. /// The relocation number
  106. relocation_number: usize,
  107. /// The instruction index
  108. index: usize,
  109. /// The error message
  110. error: String,
  111. },
  112. #[error("applying relocation `{kind:?}` missing target BTF info for type `{type_id}` at instruction #{ins_index}")]
  113. MissingTargetDefinition {
  114. kind: RelocationKind,
  115. type_id: u32,
  116. ins_index: usize,
  117. },
  118. /// BTF error
  119. #[error("invalid BTF")]
  120. BtfError(#[from] BtfError),
  121. }
  122. fn err_type_name(name: &Option<String>) -> String {
  123. name.clone().unwrap_or_else(|| "[unknown name]".to_string())
  124. }
  125. #[derive(Copy, Clone, Debug)]
  126. #[repr(u32)]
  127. enum RelocationKind {
  128. FieldByteOffset = BPF_CORE_FIELD_BYTE_OFFSET,
  129. FieldByteSize = BPF_CORE_FIELD_BYTE_SIZE,
  130. FieldExists = BPF_CORE_FIELD_EXISTS,
  131. FieldSigned = BPF_CORE_FIELD_SIGNED,
  132. FieldLShift64 = BPF_CORE_FIELD_LSHIFT_U64,
  133. FieldRShift64 = BPF_CORE_FIELD_RSHIFT_U64,
  134. TypeIdLocal = BPF_CORE_TYPE_ID_LOCAL,
  135. TypeIdTarget = BPF_CORE_TYPE_ID_TARGET,
  136. TypeExists = BPF_CORE_TYPE_EXISTS,
  137. TypeSize = BPF_CORE_TYPE_SIZE,
  138. EnumVariantExists = BPF_CORE_ENUMVAL_EXISTS,
  139. EnumVariantValue = BPF_CORE_ENUMVAL_VALUE,
  140. }
  141. impl TryFrom<u32> for RelocationKind {
  142. type Error = BtfError;
  143. fn try_from(v: u32) -> Result<Self, Self::Error> {
  144. use RelocationKind::*;
  145. Ok(match v {
  146. BPF_CORE_FIELD_BYTE_OFFSET => FieldByteOffset,
  147. BPF_CORE_FIELD_BYTE_SIZE => FieldByteSize,
  148. BPF_CORE_FIELD_EXISTS => FieldExists,
  149. BPF_CORE_FIELD_SIGNED => FieldSigned,
  150. BPF_CORE_FIELD_LSHIFT_U64 => FieldLShift64,
  151. BPF_CORE_FIELD_RSHIFT_U64 => FieldRShift64,
  152. BPF_CORE_TYPE_ID_LOCAL => TypeIdLocal,
  153. BPF_CORE_TYPE_ID_TARGET => TypeIdTarget,
  154. BPF_CORE_TYPE_EXISTS => TypeExists,
  155. BPF_CORE_TYPE_SIZE => TypeSize,
  156. BPF_CORE_ENUMVAL_EXISTS => EnumVariantExists,
  157. BPF_CORE_ENUMVAL_VALUE => EnumVariantValue,
  158. kind => return Err(BtfError::InvalidRelocationKind { kind }),
  159. })
  160. }
  161. }
  162. #[derive(Debug, Copy, Clone)]
  163. pub(crate) struct Relocation {
  164. kind: RelocationKind,
  165. ins_offset: usize,
  166. type_id: u32,
  167. access_str_offset: u32,
  168. number: usize,
  169. }
  170. impl Relocation {
  171. #[allow(unused_unsafe)]
  172. pub(crate) unsafe fn parse(data: &[u8], number: usize) -> Result<Relocation, BtfError> {
  173. if mem::size_of::<bpf_core_relo>() > data.len() {
  174. return Err(BtfError::InvalidRelocationInfo);
  175. }
  176. let rel = unsafe { ptr::read_unaligned::<bpf_core_relo>(data.as_ptr() as *const _) };
  177. Ok(Relocation {
  178. kind: rel.kind.try_into()?,
  179. ins_offset: rel.insn_off as usize,
  180. type_id: rel.type_id,
  181. access_str_offset: rel.access_str_off,
  182. number,
  183. })
  184. }
  185. }
  186. impl Object {
  187. /// Relocates programs inside this object file with loaded BTF info.
  188. pub fn relocate_btf(&mut self, target_btf: &Btf) -> Result<(), BtfRelocationError> {
  189. let (local_btf, btf_ext) = match (&self.btf, &self.btf_ext) {
  190. (Some(btf), Some(btf_ext)) => (btf, btf_ext),
  191. _ => return Ok(()),
  192. };
  193. let mut candidates_cache = HashMap::<u32, Vec<Candidate>>::new();
  194. for (sec_name_off, relos) in btf_ext.relocations() {
  195. let section_name =
  196. local_btf
  197. .string_at(*sec_name_off)
  198. .map_err(|e| BtfRelocationError {
  199. section: format!("section@{sec_name_off}"),
  200. error: RelocationError::BtfError(e),
  201. })?;
  202. let program_section = match ProgramSection::from_str(&section_name) {
  203. Ok(program) => program,
  204. Err(_) => continue,
  205. };
  206. let section_name = program_section.name();
  207. let program = self
  208. .programs
  209. .get_mut(section_name)
  210. .ok_or(BtfRelocationError {
  211. section: section_name.to_owned(),
  212. error: RelocationError::ProgramNotFound,
  213. })?;
  214. match relocate_btf_program(program, relos, local_btf, target_btf, &mut candidates_cache)
  215. {
  216. Ok(_) => {}
  217. Err(error) => {
  218. return Err(BtfRelocationError {
  219. section: section_name.to_owned(),
  220. error,
  221. })
  222. }
  223. }
  224. }
  225. Ok(())
  226. }
  227. }
  228. fn relocate_btf_program<'target>(
  229. program: &mut Program,
  230. relos: &[Relocation],
  231. local_btf: &Btf,
  232. target_btf: &'target Btf,
  233. candidates_cache: &mut HashMap<u32, Vec<Candidate<'target>>>,
  234. ) -> Result<(), RelocationError> {
  235. for rel in relos {
  236. let instructions = &mut program.function.instructions;
  237. let ins_index = rel.ins_offset / mem::size_of::<bpf_insn>();
  238. if ins_index >= instructions.len() {
  239. return Err(RelocationError::InvalidInstructionIndex {
  240. index: ins_index,
  241. num_instructions: instructions.len(),
  242. relocation_number: rel.number,
  243. });
  244. }
  245. let local_ty = local_btf.type_by_id(rel.type_id)?;
  246. let local_name = &*local_btf.type_name(local_ty)?;
  247. let access_str = &*local_btf.string_at(rel.access_str_offset)?;
  248. let local_spec = AccessSpec::new(local_btf, rel.type_id, access_str, *rel)?;
  249. let matches = match rel.kind {
  250. RelocationKind::TypeIdLocal => Vec::new(), // we don't need to look at target types to relocate this value
  251. _ => {
  252. let candidates = match candidates_cache.get(&rel.type_id) {
  253. Some(cands) => cands,
  254. None => {
  255. candidates_cache.insert(
  256. rel.type_id,
  257. find_candidates(local_ty, local_name, target_btf)?,
  258. );
  259. candidates_cache.get(&rel.type_id).unwrap()
  260. }
  261. };
  262. let mut matches = Vec::new();
  263. for candidate in candidates {
  264. if let Some(candidate_spec) = match_candidate(&local_spec, candidate)? {
  265. let comp_rel =
  266. ComputedRelocation::new(rel, &local_spec, Some(&candidate_spec))?;
  267. matches.push((candidate.name.clone(), candidate_spec, comp_rel));
  268. }
  269. }
  270. matches
  271. }
  272. };
  273. let comp_rel = if !matches.is_empty() {
  274. let mut matches = matches.into_iter();
  275. let (_, target_spec, target_comp_rel) = matches.next().unwrap();
  276. // if there's more than one candidate, make sure that they all resolve to the
  277. // same value, else the relocation is ambiguous and can't be applied
  278. let conflicts = matches
  279. .filter_map(|(cand_name, cand_spec, cand_comp_rel)| {
  280. if cand_spec.bit_offset != target_spec.bit_offset
  281. || cand_comp_rel.target.value != target_comp_rel.target.value
  282. {
  283. Some(cand_name)
  284. } else {
  285. None
  286. }
  287. })
  288. .collect::<Vec<_>>();
  289. if !conflicts.is_empty() {
  290. return Err(RelocationError::ConflictingCandidates {
  291. type_name: local_name.to_string(),
  292. candidates: conflicts,
  293. });
  294. }
  295. target_comp_rel
  296. } else {
  297. // there are no candidate matches and therefore no target_spec. This might mean
  298. // that matching failed, or that the relocation can be applied looking at local
  299. // types only (eg with EnumVariantExists, FieldExists etc)
  300. ComputedRelocation::new(rel, &local_spec, None)?
  301. };
  302. comp_rel.apply(program, rel, local_btf, target_btf)?;
  303. }
  304. Ok(())
  305. }
  306. fn flavorless_name(name: &str) -> &str {
  307. name.split_once("___").map_or(name, |x| x.0)
  308. }
  309. fn find_candidates<'target>(
  310. local_ty: &BtfType,
  311. local_name: &str,
  312. target_btf: &'target Btf,
  313. ) -> Result<Vec<Candidate<'target>>, BtfError> {
  314. let mut candidates = Vec::new();
  315. let local_name = flavorless_name(local_name);
  316. for (type_id, ty) in target_btf.types().enumerate() {
  317. if local_ty.kind() != ty.kind() {
  318. continue;
  319. }
  320. let name = &*target_btf.type_name(ty)?;
  321. if local_name != flavorless_name(name) {
  322. continue;
  323. }
  324. candidates.push(Candidate {
  325. name: name.to_owned(),
  326. btf: target_btf,
  327. _ty: ty,
  328. type_id: type_id as u32,
  329. });
  330. }
  331. Ok(candidates)
  332. }
  333. fn match_candidate<'target>(
  334. local_spec: &AccessSpec,
  335. candidate: &'target Candidate,
  336. ) -> Result<Option<AccessSpec<'target>>, RelocationError> {
  337. let mut target_spec = AccessSpec {
  338. btf: candidate.btf,
  339. root_type_id: candidate.type_id,
  340. relocation: local_spec.relocation,
  341. parts: Vec::new(),
  342. accessors: Vec::new(),
  343. bit_offset: 0,
  344. };
  345. match local_spec.relocation.kind {
  346. RelocationKind::TypeIdLocal
  347. | RelocationKind::TypeIdTarget
  348. | RelocationKind::TypeExists
  349. | RelocationKind::TypeSize => {
  350. if types_are_compatible(
  351. local_spec.btf,
  352. local_spec.root_type_id,
  353. candidate.btf,
  354. candidate.type_id,
  355. )? {
  356. return Ok(Some(target_spec));
  357. } else {
  358. return Ok(None);
  359. }
  360. }
  361. RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => {
  362. let target_id = candidate.btf.resolve_type(candidate.type_id)?;
  363. let target_ty = candidate.btf.type_by_id(target_id)?;
  364. // the first accessor is guaranteed to have a name by construction
  365. let local_variant_name = local_spec.accessors[0].name.as_ref().unwrap();
  366. match target_ty {
  367. BtfType::Enum(en) => {
  368. for (index, member) in en.variants.iter().enumerate() {
  369. let target_variant_name = candidate.btf.string_at(member.name_offset)?;
  370. if flavorless_name(local_variant_name)
  371. == flavorless_name(&target_variant_name)
  372. {
  373. target_spec.parts.push(index);
  374. target_spec.accessors.push(Accessor {
  375. index,
  376. type_id: target_id,
  377. name: None,
  378. });
  379. return Ok(Some(target_spec));
  380. }
  381. }
  382. }
  383. _ => return Ok(None),
  384. }
  385. }
  386. RelocationKind::FieldByteOffset
  387. | RelocationKind::FieldByteSize
  388. | RelocationKind::FieldExists
  389. | RelocationKind::FieldSigned
  390. | RelocationKind::FieldLShift64
  391. | RelocationKind::FieldRShift64 => {
  392. let mut target_id = candidate.type_id;
  393. for (i, accessor) in local_spec.accessors.iter().enumerate() {
  394. target_id = candidate.btf.resolve_type(target_id)?;
  395. if accessor.name.is_some() {
  396. if let Some(next_id) = match_member(
  397. local_spec.btf,
  398. local_spec,
  399. accessor,
  400. candidate.btf,
  401. target_id,
  402. &mut target_spec,
  403. )? {
  404. target_id = next_id;
  405. } else {
  406. return Ok(None);
  407. }
  408. } else {
  409. // i = 0 is the base struct. for i > 0, we need to potentially do bounds checking
  410. if i > 0 {
  411. let target_ty = candidate.btf.type_by_id(target_id)?;
  412. let array = match target_ty {
  413. BtfType::Array(Array { array, .. }) => array,
  414. _ => return Ok(None),
  415. };
  416. let var_len = array.len == 0 && {
  417. // an array is potentially variable length if it's the last field
  418. // of the parent struct and has 0 elements
  419. let parent = target_spec.accessors.last().unwrap();
  420. let parent_ty = candidate.btf.type_by_id(parent.type_id)?;
  421. match parent_ty {
  422. BtfType::Struct(s) => parent.index == s.members.len() - 1,
  423. _ => false,
  424. }
  425. };
  426. if !var_len && accessor.index >= array.len as usize {
  427. return Ok(None);
  428. }
  429. target_id = candidate.btf.resolve_type(array.element_type)?;
  430. }
  431. if target_spec.parts.len() == MAX_SPEC_LEN {
  432. return Err(RelocationError::MaximumNestingLevelReached {
  433. type_name: Some(candidate.name.clone()),
  434. });
  435. }
  436. target_spec.parts.push(accessor.index);
  437. target_spec.accessors.push(Accessor {
  438. index: accessor.index,
  439. type_id: target_id,
  440. name: None,
  441. });
  442. target_spec.bit_offset +=
  443. accessor.index * candidate.btf.type_size(target_id)? * 8;
  444. }
  445. }
  446. }
  447. };
  448. Ok(Some(target_spec))
  449. }
  450. fn match_member<'target>(
  451. local_btf: &Btf,
  452. local_spec: &AccessSpec<'_>,
  453. local_accessor: &Accessor,
  454. target_btf: &'target Btf,
  455. target_id: u32,
  456. target_spec: &mut AccessSpec<'target>,
  457. ) -> Result<Option<u32>, RelocationError> {
  458. let local_ty = local_btf.type_by_id(local_accessor.type_id)?;
  459. let local_member = match local_ty {
  460. // this won't panic, bounds are checked when local_spec is built in AccessSpec::new
  461. BtfType::Struct(s) => s.members.get(local_accessor.index).unwrap(),
  462. BtfType::Union(u) => u.members.get(local_accessor.index).unwrap(),
  463. _ => panic!("bug! this should only be called for structs and unions"),
  464. };
  465. let local_name = &*local_btf.string_at(local_member.name_offset)?;
  466. let target_id = target_btf.resolve_type(target_id)?;
  467. let target_ty = target_btf.type_by_id(target_id)?;
  468. let target_members: Vec<&BtfMember> = match target_ty.members() {
  469. Some(members) => members.collect(),
  470. // not a fields type, no match
  471. None => return Ok(None),
  472. };
  473. for (index, target_member) in target_members.iter().enumerate() {
  474. if target_spec.parts.len() == MAX_SPEC_LEN {
  475. let root_ty = target_spec.btf.type_by_id(target_spec.root_type_id)?;
  476. return Err(RelocationError::MaximumNestingLevelReached {
  477. type_name: target_spec.btf.err_type_name(root_ty),
  478. });
  479. }
  480. // this will not panic as we've already established these are fields types
  481. let bit_offset = target_ty.member_bit_offset(target_member).unwrap();
  482. let target_name = &*target_btf.string_at(target_member.name_offset)?;
  483. if target_name.is_empty() {
  484. let ret = match_member(
  485. local_btf,
  486. local_spec,
  487. local_accessor,
  488. target_btf,
  489. target_member.btf_type,
  490. target_spec,
  491. )?;
  492. if ret.is_some() {
  493. target_spec.bit_offset += bit_offset;
  494. target_spec.parts.push(index);
  495. return Ok(ret);
  496. }
  497. } else if local_name == target_name {
  498. if fields_are_compatible(
  499. local_spec.btf,
  500. local_member.btf_type,
  501. target_btf,
  502. target_member.btf_type,
  503. )? {
  504. target_spec.bit_offset += bit_offset;
  505. target_spec.parts.push(index);
  506. target_spec.accessors.push(Accessor {
  507. type_id: target_id,
  508. index,
  509. name: Some(target_name.to_owned()),
  510. });
  511. return Ok(Some(target_member.btf_type));
  512. } else {
  513. return Ok(None);
  514. }
  515. }
  516. }
  517. Ok(None)
  518. }
  519. #[derive(Debug)]
  520. struct AccessSpec<'a> {
  521. btf: &'a Btf,
  522. root_type_id: u32,
  523. parts: Vec<usize>,
  524. accessors: Vec<Accessor>,
  525. relocation: Relocation,
  526. bit_offset: usize,
  527. }
  528. impl<'a> AccessSpec<'a> {
  529. fn new(
  530. btf: &'a Btf,
  531. root_type_id: u32,
  532. spec: &str,
  533. relocation: Relocation,
  534. ) -> Result<AccessSpec<'a>, RelocationError> {
  535. let parts = spec
  536. .split(':')
  537. .map(|s| s.parse::<usize>())
  538. .collect::<Result<Vec<_>, _>>()
  539. .map_err(|_| RelocationError::InvalidAccessString {
  540. access_str: spec.to_string(),
  541. })?;
  542. let mut type_id = btf.resolve_type(root_type_id)?;
  543. let ty = btf.type_by_id(type_id)?;
  544. let spec = match relocation.kind {
  545. RelocationKind::TypeIdLocal
  546. | RelocationKind::TypeIdTarget
  547. | RelocationKind::TypeExists
  548. | RelocationKind::TypeSize => {
  549. if parts != [0] {
  550. return Err(RelocationError::InvalidAccessString {
  551. access_str: spec.to_string(),
  552. });
  553. }
  554. AccessSpec {
  555. btf,
  556. root_type_id,
  557. relocation,
  558. parts,
  559. accessors: Vec::new(),
  560. bit_offset: 0,
  561. }
  562. }
  563. RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => match ty {
  564. BtfType::Enum(en) => {
  565. if parts.len() != 1 {
  566. return Err(RelocationError::InvalidAccessString {
  567. access_str: spec.to_string(),
  568. });
  569. }
  570. let index = parts[0];
  571. if index >= en.variants.len() {
  572. return Err(RelocationError::InvalidAccessIndex {
  573. type_name: btf.err_type_name(ty),
  574. spec: spec.to_string(),
  575. index,
  576. max_index: en.variants.len(),
  577. error: "tried to access nonexistant enum variant".to_string(),
  578. });
  579. }
  580. let accessors = vec![Accessor {
  581. type_id,
  582. index,
  583. name: Some(
  584. btf.string_at(en.variants.get(index).unwrap().name_offset)?
  585. .to_string(),
  586. ),
  587. }];
  588. AccessSpec {
  589. btf,
  590. root_type_id,
  591. relocation,
  592. parts,
  593. accessors,
  594. bit_offset: 0,
  595. }
  596. }
  597. _ => {
  598. return Err(RelocationError::InvalidRelocationKindForType {
  599. relocation_number: relocation.number,
  600. relocation_kind: format!("{:?}", relocation.kind),
  601. type_kind: format!("{:?}", ty.kind()),
  602. error: "enum relocation on non-enum type".to_string(),
  603. })
  604. }
  605. },
  606. RelocationKind::FieldByteOffset
  607. | RelocationKind::FieldByteSize
  608. | RelocationKind::FieldExists
  609. | RelocationKind::FieldSigned
  610. | RelocationKind::FieldLShift64
  611. | RelocationKind::FieldRShift64 => {
  612. let mut accessors = vec![Accessor {
  613. type_id,
  614. index: parts[0],
  615. name: None,
  616. }];
  617. let mut bit_offset = accessors[0].index * btf.type_size(type_id)?;
  618. for index in parts.iter().skip(1).cloned() {
  619. type_id = btf.resolve_type(type_id)?;
  620. let ty = btf.type_by_id(type_id)?;
  621. match ty {
  622. BtfType::Struct(Struct { members, .. })
  623. | BtfType::Union(Union { members, .. }) => {
  624. if index >= members.len() {
  625. return Err(RelocationError::InvalidAccessIndex {
  626. type_name: btf.err_type_name(ty),
  627. spec: spec.to_string(),
  628. index,
  629. max_index: members.len(),
  630. error: "out of bounds struct or union access".to_string(),
  631. });
  632. }
  633. let member = &members[index];
  634. bit_offset += ty.member_bit_offset(member).unwrap();
  635. if member.name_offset != 0 {
  636. accessors.push(Accessor {
  637. type_id,
  638. index,
  639. name: Some(btf.string_at(member.name_offset)?.to_string()),
  640. });
  641. }
  642. type_id = member.btf_type;
  643. }
  644. BtfType::Array(Array { array, .. }) => {
  645. type_id = btf.resolve_type(array.element_type)?;
  646. let var_len = array.len == 0 && {
  647. // an array is potentially variable length if it's the last field
  648. // of the parent struct and has 0 elements
  649. let parent = accessors.last().unwrap();
  650. let parent_ty = btf.type_by_id(parent.type_id)?;
  651. match parent_ty {
  652. BtfType::Struct(s) => index == s.members.len() - 1,
  653. _ => false,
  654. }
  655. };
  656. if !var_len && index >= array.len as usize {
  657. return Err(RelocationError::InvalidAccessIndex {
  658. type_name: btf.err_type_name(ty),
  659. spec: spec.to_string(),
  660. index,
  661. max_index: array.len as usize,
  662. error: "array index out of bounds".to_string(),
  663. });
  664. }
  665. accessors.push(Accessor {
  666. type_id,
  667. index,
  668. name: None,
  669. });
  670. let size = btf.type_size(type_id)?;
  671. bit_offset += index * size * 8;
  672. }
  673. rel_kind => {
  674. return Err(RelocationError::InvalidRelocationKindForType {
  675. relocation_number: relocation.number,
  676. relocation_kind: format!("{rel_kind:?}"),
  677. type_kind: format!("{:?}", ty.kind()),
  678. error: "field relocation on a type that doesn't have fields"
  679. .to_string(),
  680. });
  681. }
  682. };
  683. }
  684. AccessSpec {
  685. btf,
  686. root_type_id,
  687. parts,
  688. accessors,
  689. relocation,
  690. bit_offset,
  691. }
  692. }
  693. };
  694. Ok(spec)
  695. }
  696. }
  697. #[derive(Debug)]
  698. struct Accessor {
  699. type_id: u32,
  700. index: usize,
  701. name: Option<String>,
  702. }
  703. #[derive(Debug)]
  704. struct Candidate<'a> {
  705. name: String,
  706. btf: &'a Btf,
  707. _ty: &'a BtfType,
  708. type_id: u32,
  709. }
  710. #[derive(Debug)]
  711. struct ComputedRelocation {
  712. local: ComputedRelocationValue,
  713. target: ComputedRelocationValue,
  714. }
  715. #[derive(Debug)]
  716. struct ComputedRelocationValue {
  717. value: u64,
  718. size: u32,
  719. type_id: Option<u32>,
  720. }
  721. impl ComputedRelocation {
  722. fn new(
  723. rel: &Relocation,
  724. local_spec: &AccessSpec,
  725. target_spec: Option<&AccessSpec>,
  726. ) -> Result<ComputedRelocation, RelocationError> {
  727. use RelocationKind::*;
  728. let ret = match rel.kind {
  729. FieldByteOffset | FieldByteSize | FieldExists | FieldSigned | FieldLShift64
  730. | FieldRShift64 => ComputedRelocation {
  731. local: Self::compute_field_relocation(rel, Some(local_spec))?,
  732. target: Self::compute_field_relocation(rel, target_spec)?,
  733. },
  734. TypeIdLocal | TypeIdTarget | TypeExists | TypeSize => ComputedRelocation {
  735. local: Self::compute_type_relocation(rel, local_spec, target_spec)?,
  736. target: Self::compute_type_relocation(rel, local_spec, target_spec)?,
  737. },
  738. EnumVariantExists | EnumVariantValue => ComputedRelocation {
  739. local: Self::compute_enum_relocation(rel, Some(local_spec))?,
  740. target: Self::compute_enum_relocation(rel, target_spec)?,
  741. },
  742. };
  743. Ok(ret)
  744. }
  745. fn apply(
  746. &self,
  747. program: &mut Program,
  748. rel: &Relocation,
  749. local_btf: &Btf,
  750. target_btf: &Btf,
  751. ) -> Result<(), RelocationError> {
  752. let instructions = &mut program.function.instructions;
  753. let num_instructions = instructions.len();
  754. let ins_index = rel.ins_offset / mem::size_of::<bpf_insn>();
  755. let mut ins =
  756. instructions
  757. .get_mut(ins_index)
  758. .ok_or(RelocationError::InvalidInstructionIndex {
  759. index: rel.ins_offset,
  760. num_instructions,
  761. relocation_number: rel.number,
  762. })?;
  763. let class = (ins.code & 0x07) as u32;
  764. let target_value = self.target.value;
  765. match class {
  766. BPF_ALU | BPF_ALU64 => {
  767. let src_reg = ins.src_reg();
  768. if src_reg != BPF_K as u8 {
  769. return Err(RelocationError::InvalidInstruction {
  770. relocation_number: rel.number,
  771. index: ins_index,
  772. error: format!("invalid src_reg={src_reg:x} expected {BPF_K:x}"),
  773. });
  774. }
  775. ins.imm = target_value as i32;
  776. }
  777. BPF_LDX | BPF_ST | BPF_STX => {
  778. if target_value > i16::MAX as u64 {
  779. return Err(RelocationError::InvalidInstruction {
  780. relocation_number: rel.number,
  781. index: ins_index,
  782. error: format!("value `{target_value}` overflows 16 bits offset field"),
  783. });
  784. }
  785. ins.off = target_value as i16;
  786. if self.local.size != self.target.size {
  787. let local_ty = local_btf.type_by_id(self.local.type_id.unwrap())?;
  788. let target_ty = target_btf.type_by_id(self.target.type_id.unwrap())?;
  789. let unsigned = |info: u32| ((info >> 24) & 0x0F) & BTF_INT_SIGNED == 0;
  790. use BtfType::*;
  791. match (local_ty, target_ty) {
  792. (Ptr(_), Ptr(_)) => {}
  793. (Int(local), Int(target))
  794. if unsigned(local.data) && unsigned(target.data) => {}
  795. _ => {
  796. return Err(RelocationError::InvalidInstruction {
  797. relocation_number: rel.number,
  798. index: ins_index,
  799. error: format!(
  800. "original type {} has size {} but target type {} has size {}",
  801. err_type_name(&local_btf.err_type_name(local_ty)),
  802. self.local.size,
  803. err_type_name(&target_btf.err_type_name(target_ty)),
  804. self.target.size,
  805. ),
  806. })
  807. }
  808. }
  809. let size = match self.target.size {
  810. 8 => BPF_DW,
  811. 4 => BPF_W,
  812. 2 => BPF_H,
  813. 1 => BPF_B,
  814. size => {
  815. return Err(RelocationError::InvalidInstruction {
  816. relocation_number: rel.number,
  817. index: ins_index,
  818. error: format!("invalid target size {size}"),
  819. })
  820. }
  821. } as u8;
  822. ins.code = ins.code & 0xE0 | size | ins.code & 0x07;
  823. }
  824. }
  825. BPF_LD => {
  826. ins.imm = target_value as i32;
  827. let mut next_ins = instructions.get_mut(ins_index + 1).ok_or(
  828. RelocationError::InvalidInstructionIndex {
  829. index: ins_index + 1,
  830. num_instructions,
  831. relocation_number: rel.number,
  832. },
  833. )?;
  834. next_ins.imm = (target_value >> 32) as i32;
  835. }
  836. class => {
  837. return Err(RelocationError::InvalidInstruction {
  838. relocation_number: rel.number,
  839. index: ins_index,
  840. error: format!("invalid instruction class {class:x}"),
  841. })
  842. }
  843. };
  844. Ok(())
  845. }
  846. fn compute_enum_relocation(
  847. rel: &Relocation,
  848. spec: Option<&AccessSpec>,
  849. ) -> Result<ComputedRelocationValue, RelocationError> {
  850. use RelocationKind::*;
  851. let value = match (rel.kind, spec) {
  852. (EnumVariantExists, spec) => spec.is_some() as u64,
  853. (EnumVariantValue, Some(spec)) => {
  854. let accessor = &spec.accessors[0];
  855. match spec.btf.type_by_id(accessor.type_id)? {
  856. BtfType::Enum(en) => en.variants[accessor.index].value as u64,
  857. // candidate selection ensures that rel_kind == local_kind == target_kind
  858. _ => unreachable!(),
  859. }
  860. }
  861. _ => {
  862. return Err(RelocationError::MissingTargetDefinition {
  863. kind: rel.kind,
  864. type_id: rel.type_id,
  865. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  866. })?;
  867. }
  868. };
  869. Ok(ComputedRelocationValue {
  870. value,
  871. size: 0,
  872. type_id: None,
  873. })
  874. }
  875. fn compute_field_relocation(
  876. rel: &Relocation,
  877. spec: Option<&AccessSpec>,
  878. ) -> Result<ComputedRelocationValue, RelocationError> {
  879. use RelocationKind::*;
  880. if let FieldExists = rel.kind {
  881. // this is the bpf_preserve_field_info(member_access, FIELD_EXISTENCE) case. If we
  882. // managed to build a spec, it means the field exists.
  883. return Ok(ComputedRelocationValue {
  884. value: spec.is_some() as u64,
  885. size: 0,
  886. type_id: None,
  887. });
  888. }
  889. let spec = match spec {
  890. Some(spec) => spec,
  891. None => {
  892. return Err(RelocationError::MissingTargetDefinition {
  893. kind: rel.kind,
  894. type_id: rel.type_id,
  895. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  896. })?;
  897. }
  898. };
  899. let accessor = spec.accessors.last().unwrap();
  900. if accessor.name.is_none() {
  901. // the last accessor is unnamed, meaning that this is an array access
  902. return match rel.kind {
  903. FieldByteOffset => Ok(ComputedRelocationValue {
  904. value: (spec.bit_offset / 8) as u64,
  905. size: spec.btf.type_size(accessor.type_id)? as u32,
  906. type_id: Some(accessor.type_id),
  907. }),
  908. FieldByteSize => Ok(ComputedRelocationValue {
  909. value: spec.btf.type_size(accessor.type_id)? as u64,
  910. size: 0,
  911. type_id: Some(accessor.type_id),
  912. }),
  913. rel_kind => {
  914. let ty = spec.btf.type_by_id(accessor.type_id)?;
  915. return Err(RelocationError::InvalidRelocationKindForType {
  916. relocation_number: rel.number,
  917. relocation_kind: format!("{rel_kind:?}"),
  918. type_kind: format!("{:?}", ty.kind()),
  919. error: "invalid relocation kind for array type".to_string(),
  920. });
  921. }
  922. };
  923. }
  924. let ty = spec.btf.type_by_id(accessor.type_id)?;
  925. let (ll_ty, member) = match ty {
  926. BtfType::Struct(t) => (ty, t.members.get(accessor.index).unwrap()),
  927. BtfType::Union(t) => (ty, t.members.get(accessor.index).unwrap()),
  928. _ => {
  929. return Err(RelocationError::InvalidRelocationKindForType {
  930. relocation_number: rel.number,
  931. relocation_kind: format!("{:?}", rel.kind),
  932. type_kind: format!("{:?}", ty.kind()),
  933. error: "field relocation on a type that doesn't have fields".to_string(),
  934. });
  935. }
  936. };
  937. let bit_off = spec.bit_offset as u32;
  938. let member_type_id = spec.btf.resolve_type(member.btf_type)?;
  939. let member_ty = spec.btf.type_by_id(member_type_id)?;
  940. let mut byte_size;
  941. let mut byte_off;
  942. let mut bit_size = ll_ty.member_bit_field_size(member).unwrap() as u32;
  943. let is_bitfield = bit_size > 0;
  944. if is_bitfield {
  945. // find out the smallest int size to load the bitfield
  946. byte_size = member_ty.size().unwrap();
  947. byte_off = bit_off / 8 / byte_size * byte_size;
  948. while bit_off + bit_size - byte_off * 8 > byte_size * 8 {
  949. if byte_size >= 8 {
  950. // the bitfield is larger than 8 bytes!?
  951. return Err(BtfError::InvalidTypeInfo.into());
  952. }
  953. byte_size *= 2;
  954. byte_off = bit_off / 8 / byte_size * byte_size;
  955. }
  956. } else {
  957. byte_size = spec.btf.type_size(member_type_id)? as u32;
  958. bit_size = byte_size * 8;
  959. byte_off = spec.bit_offset as u32 / 8;
  960. }
  961. let mut value = ComputedRelocationValue {
  962. value: 0,
  963. size: 0,
  964. type_id: None,
  965. };
  966. #[allow(clippy::wildcard_in_or_patterns)]
  967. match rel.kind {
  968. FieldByteOffset => {
  969. value.value = byte_off as u64;
  970. if !is_bitfield {
  971. value.size = byte_size;
  972. value.type_id = Some(member_type_id);
  973. }
  974. }
  975. FieldByteSize => {
  976. value.value = byte_size as u64;
  977. }
  978. FieldSigned => match member_ty {
  979. BtfType::Enum(_) => value.value = 1,
  980. BtfType::Int(i) => value.value = i.encoding() as u64 & IntEncoding::Signed as u64,
  981. _ => (),
  982. },
  983. #[cfg(target_endian = "little")]
  984. FieldLShift64 => {
  985. value.value = 64 - (bit_off + bit_size - byte_off * 8) as u64;
  986. }
  987. #[cfg(target_endian = "big")]
  988. FieldLShift64 => {
  989. value.value = (8 - byte_size) * 8 + (bit_off - byte_off * 8);
  990. }
  991. FieldRShift64 => {
  992. value.value = 64 - bit_size as u64;
  993. }
  994. FieldExists // this is handled at the start of the function
  995. | _ => panic!("bug! this should not be reached"),
  996. }
  997. Ok(value)
  998. }
  999. fn compute_type_relocation(
  1000. rel: &Relocation,
  1001. local_spec: &AccessSpec,
  1002. target_spec: Option<&AccessSpec>,
  1003. ) -> Result<ComputedRelocationValue, RelocationError> {
  1004. use RelocationKind::*;
  1005. let value = match (rel.kind, target_spec) {
  1006. (TypeIdLocal, _) => local_spec.root_type_id as u64,
  1007. (TypeIdTarget, Some(target_spec)) => target_spec.root_type_id as u64,
  1008. (TypeExists, target_spec) => target_spec.is_some() as u64,
  1009. (TypeSize, Some(target_spec)) => {
  1010. target_spec.btf.type_size(target_spec.root_type_id)? as u64
  1011. }
  1012. _ => {
  1013. return Err(RelocationError::MissingTargetDefinition {
  1014. kind: rel.kind,
  1015. type_id: rel.type_id,
  1016. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  1017. })?;
  1018. }
  1019. };
  1020. Ok(ComputedRelocationValue {
  1021. value,
  1022. size: 0,
  1023. type_id: None,
  1024. })
  1025. }
  1026. }