relocation.rs 44 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213
  1. use core::{mem, ops::Bound::Included, ptr};
  2. use alloc::{
  3. borrow::ToOwned,
  4. collections::BTreeMap,
  5. format,
  6. string::{String, ToString},
  7. vec,
  8. vec::Vec,
  9. };
  10. use object::SectionIndex;
  11. use crate::{
  12. btf::{
  13. fields_are_compatible, types_are_compatible, Array, Btf, BtfError, BtfMember, BtfType,
  14. IntEncoding, Struct, Union, MAX_SPEC_LEN,
  15. },
  16. generated::{
  17. bpf_core_relo, bpf_core_relo_kind::*, bpf_insn, BPF_ALU, BPF_ALU64, BPF_B, BPF_DW, BPF_H,
  18. BPF_K, BPF_LD, BPF_LDX, BPF_ST, BPF_STX, BPF_W, BTF_INT_SIGNED,
  19. },
  20. util::HashMap,
  21. Function, Object,
  22. };
  23. #[cfg(not(feature = "std"))]
  24. use crate::std;
  25. /// The error type returned by [`Object::relocate_btf`].
  26. #[derive(thiserror::Error, Debug)]
  27. #[error("error relocating `{section}`")]
  28. pub struct BtfRelocationError {
  29. /// The function name
  30. pub section: String,
  31. #[source]
  32. /// The original error
  33. error: RelocationError,
  34. }
  35. /// Relocation failures
  36. #[derive(thiserror::Error, Debug)]
  37. enum RelocationError {
  38. #[cfg(feature = "std")]
  39. /// I/O error
  40. #[error(transparent)]
  41. IOError(#[from] std::io::Error),
  42. /// Section not found
  43. #[error("section not found")]
  44. SectionNotFound,
  45. /// Function not found
  46. #[error("function not found")]
  47. FunctionNotFound,
  48. /// Invalid relocation access string
  49. #[error("invalid relocation access string {access_str}")]
  50. InvalidAccessString {
  51. /// The access string
  52. access_str: String,
  53. },
  54. /// Invalid instruction index referenced by relocation
  55. #[error("invalid instruction index #{index} referenced by relocation #{relocation_number}, the program contains {num_instructions} instructions")]
  56. InvalidInstructionIndex {
  57. /// The invalid instruction index
  58. index: usize,
  59. /// Number of instructions in the program
  60. num_instructions: usize,
  61. /// The relocation number
  62. relocation_number: usize,
  63. },
  64. /// Multiple candidate target types found with different memory layouts
  65. #[error("error relocating {type_name}, multiple candidate target types found with different memory layouts: {candidates:?}")]
  66. ConflictingCandidates {
  67. /// The type name
  68. type_name: String,
  69. /// The candidates
  70. candidates: Vec<String>,
  71. },
  72. /// Maximum nesting level reached evaluating candidate type
  73. #[error("maximum nesting level reached evaluating candidate type `{}`", err_type_name(.type_name))]
  74. MaximumNestingLevelReached {
  75. /// The type name
  76. type_name: Option<String>,
  77. },
  78. /// Invalid access string
  79. #[error("invalid access string `{spec}` for type `{}`: {error}", err_type_name(.type_name))]
  80. InvalidAccessIndex {
  81. /// The type name
  82. type_name: Option<String>,
  83. /// The access string
  84. spec: String,
  85. /// The index
  86. index: usize,
  87. /// The max index
  88. max_index: usize,
  89. /// The error message
  90. error: String,
  91. },
  92. /// Relocation not valid for type
  93. #[error(
  94. "relocation #{relocation_number} of kind `{relocation_kind}` not valid for type `{type_kind}`: {error}"
  95. )]
  96. InvalidRelocationKindForType {
  97. /// The relocation number
  98. relocation_number: usize,
  99. /// The relocation kind
  100. relocation_kind: String,
  101. /// The type kind
  102. type_kind: String,
  103. /// The error message
  104. error: String,
  105. },
  106. /// Invalid instruction referenced by relocation
  107. #[error(
  108. "instruction #{index} referenced by relocation #{relocation_number} is invalid: {error}"
  109. )]
  110. InvalidInstruction {
  111. /// The relocation number
  112. relocation_number: usize,
  113. /// The instruction index
  114. index: usize,
  115. /// The error message
  116. error: String,
  117. },
  118. #[error("applying relocation `{kind:?}` missing target BTF info for type `{type_id}` at instruction #{ins_index}")]
  119. MissingTargetDefinition {
  120. kind: RelocationKind,
  121. type_id: u32,
  122. ins_index: usize,
  123. },
  124. /// BTF error
  125. #[error("invalid BTF")]
  126. BtfError(#[from] BtfError),
  127. }
  128. fn err_type_name(name: &Option<String>) -> String {
  129. name.clone().unwrap_or_else(|| "[unknown name]".to_string())
  130. }
  131. #[derive(Copy, Clone, Debug)]
  132. #[repr(u32)]
  133. enum RelocationKind {
  134. FieldByteOffset = BPF_CORE_FIELD_BYTE_OFFSET,
  135. FieldByteSize = BPF_CORE_FIELD_BYTE_SIZE,
  136. FieldExists = BPF_CORE_FIELD_EXISTS,
  137. FieldSigned = BPF_CORE_FIELD_SIGNED,
  138. FieldLShift64 = BPF_CORE_FIELD_LSHIFT_U64,
  139. FieldRShift64 = BPF_CORE_FIELD_RSHIFT_U64,
  140. TypeIdLocal = BPF_CORE_TYPE_ID_LOCAL,
  141. TypeIdTarget = BPF_CORE_TYPE_ID_TARGET,
  142. TypeExists = BPF_CORE_TYPE_EXISTS,
  143. TypeSize = BPF_CORE_TYPE_SIZE,
  144. EnumVariantExists = BPF_CORE_ENUMVAL_EXISTS,
  145. EnumVariantValue = BPF_CORE_ENUMVAL_VALUE,
  146. }
  147. impl TryFrom<u32> for RelocationKind {
  148. type Error = BtfError;
  149. fn try_from(v: u32) -> Result<Self, Self::Error> {
  150. use RelocationKind::*;
  151. Ok(match v {
  152. BPF_CORE_FIELD_BYTE_OFFSET => FieldByteOffset,
  153. BPF_CORE_FIELD_BYTE_SIZE => FieldByteSize,
  154. BPF_CORE_FIELD_EXISTS => FieldExists,
  155. BPF_CORE_FIELD_SIGNED => FieldSigned,
  156. BPF_CORE_FIELD_LSHIFT_U64 => FieldLShift64,
  157. BPF_CORE_FIELD_RSHIFT_U64 => FieldRShift64,
  158. BPF_CORE_TYPE_ID_LOCAL => TypeIdLocal,
  159. BPF_CORE_TYPE_ID_TARGET => TypeIdTarget,
  160. BPF_CORE_TYPE_EXISTS => TypeExists,
  161. BPF_CORE_TYPE_SIZE => TypeSize,
  162. BPF_CORE_ENUMVAL_EXISTS => EnumVariantExists,
  163. BPF_CORE_ENUMVAL_VALUE => EnumVariantValue,
  164. kind => return Err(BtfError::InvalidRelocationKind { kind }),
  165. })
  166. }
  167. }
  168. #[derive(Debug, Copy, Clone)]
  169. pub(crate) struct Relocation {
  170. kind: RelocationKind,
  171. ins_offset: usize,
  172. type_id: u32,
  173. access_str_offset: u32,
  174. number: usize,
  175. }
  176. impl Relocation {
  177. #[allow(unused_unsafe)]
  178. pub(crate) unsafe fn parse(data: &[u8], number: usize) -> Result<Relocation, BtfError> {
  179. if mem::size_of::<bpf_core_relo>() > data.len() {
  180. return Err(BtfError::InvalidRelocationInfo);
  181. }
  182. let rel = unsafe { ptr::read_unaligned::<bpf_core_relo>(data.as_ptr() as *const _) };
  183. Ok(Relocation {
  184. kind: rel.kind.try_into()?,
  185. ins_offset: rel.insn_off as usize,
  186. type_id: rel.type_id,
  187. access_str_offset: rel.access_str_off,
  188. number,
  189. })
  190. }
  191. }
  192. impl Object {
  193. /// Relocates programs inside this object file with loaded BTF info.
  194. pub fn relocate_btf(&mut self, target_btf: &Btf) -> Result<(), BtfRelocationError> {
  195. let (local_btf, btf_ext) = match (&self.btf, &self.btf_ext) {
  196. (Some(btf), Some(btf_ext)) => (btf, btf_ext),
  197. _ => return Ok(()),
  198. };
  199. let mut candidates_cache = HashMap::<u32, Vec<Candidate>>::new();
  200. for (sec_name_off, relos) in btf_ext.relocations() {
  201. let section_name =
  202. local_btf
  203. .string_at(*sec_name_off)
  204. .map_err(|e| BtfRelocationError {
  205. section: format!("section@{sec_name_off}"),
  206. error: RelocationError::BtfError(e),
  207. })?;
  208. let (section_index, _) = self
  209. .section_infos
  210. .get(&section_name.to_string())
  211. .ok_or_else(|| BtfRelocationError {
  212. section: section_name.to_string(),
  213. error: RelocationError::SectionNotFound,
  214. })?;
  215. match relocate_btf_functions(
  216. section_index,
  217. &mut self.functions,
  218. relos,
  219. local_btf,
  220. target_btf,
  221. &mut candidates_cache,
  222. ) {
  223. Ok(_) => {}
  224. Err(error) => {
  225. return Err(BtfRelocationError {
  226. section: section_name.to_string(),
  227. error,
  228. })
  229. }
  230. }
  231. }
  232. Ok(())
  233. }
  234. }
  235. fn is_relocation_inside_function(
  236. section_index: &SectionIndex,
  237. func: &Function,
  238. rel: &Relocation,
  239. ) -> bool {
  240. if section_index.0 != func.section_index.0 {
  241. return false;
  242. }
  243. let ins_offset = rel.ins_offset / mem::size_of::<bpf_insn>();
  244. let func_offset = func.section_offset / mem::size_of::<bpf_insn>();
  245. let func_size = func.instructions.len();
  246. (func_offset..func_offset + func_size).contains(&ins_offset)
  247. }
  248. fn function_by_relocation<'a>(
  249. section_index: &SectionIndex,
  250. functions: &'a mut BTreeMap<(usize, u64), Function>,
  251. rel: &Relocation,
  252. ) -> Option<&'a mut Function> {
  253. functions
  254. .range_mut((
  255. Included(&(section_index.0, 0)),
  256. Included(&(section_index.0, u64::MAX)),
  257. ))
  258. .map(|(_, func)| func)
  259. .find(|func| is_relocation_inside_function(section_index, func, rel))
  260. }
  261. fn relocate_btf_functions<'target>(
  262. section_index: &SectionIndex,
  263. functions: &mut BTreeMap<(usize, u64), Function>,
  264. relos: &[Relocation],
  265. local_btf: &Btf,
  266. target_btf: &'target Btf,
  267. candidates_cache: &mut HashMap<u32, Vec<Candidate<'target>>>,
  268. ) -> Result<(), RelocationError> {
  269. let mut last_function_opt: Option<&mut Function> = None;
  270. for rel in relos {
  271. let function = match last_function_opt.take() {
  272. Some(func) if is_relocation_inside_function(section_index, func, rel) => func,
  273. _ => function_by_relocation(section_index, functions, rel)
  274. .ok_or(RelocationError::FunctionNotFound)?,
  275. };
  276. let instructions = &mut function.instructions;
  277. let ins_index = (rel.ins_offset - function.section_offset) / mem::size_of::<bpf_insn>();
  278. if ins_index >= instructions.len() {
  279. return Err(RelocationError::InvalidInstructionIndex {
  280. index: ins_index,
  281. num_instructions: instructions.len(),
  282. relocation_number: rel.number,
  283. });
  284. }
  285. let local_ty = local_btf.type_by_id(rel.type_id)?;
  286. let local_name = &*local_btf.type_name(local_ty)?;
  287. let access_str = &*local_btf.string_at(rel.access_str_offset)?;
  288. let local_spec = AccessSpec::new(local_btf, rel.type_id, access_str, *rel)?;
  289. let matches = match rel.kind {
  290. RelocationKind::TypeIdLocal => Vec::new(), // we don't need to look at target types to relocate this value
  291. _ => {
  292. let candidates = match candidates_cache.get(&rel.type_id) {
  293. Some(cands) => cands,
  294. None => {
  295. candidates_cache.insert(
  296. rel.type_id,
  297. find_candidates(local_ty, local_name, target_btf)?,
  298. );
  299. candidates_cache.get(&rel.type_id).unwrap()
  300. }
  301. };
  302. let mut matches = Vec::new();
  303. for candidate in candidates {
  304. if let Some(candidate_spec) = match_candidate(&local_spec, candidate)? {
  305. let comp_rel =
  306. ComputedRelocation::new(rel, &local_spec, Some(&candidate_spec))?;
  307. matches.push((candidate.name.clone(), candidate_spec, comp_rel));
  308. }
  309. }
  310. matches
  311. }
  312. };
  313. let comp_rel = if !matches.is_empty() {
  314. let mut matches = matches.into_iter();
  315. let (_, target_spec, target_comp_rel) = matches.next().unwrap();
  316. // if there's more than one candidate, make sure that they all resolve to the
  317. // same value, else the relocation is ambiguous and can't be applied
  318. let conflicts = matches
  319. .filter_map(|(cand_name, cand_spec, cand_comp_rel)| {
  320. if cand_spec.bit_offset != target_spec.bit_offset
  321. || cand_comp_rel.target.value != target_comp_rel.target.value
  322. {
  323. Some(cand_name)
  324. } else {
  325. None
  326. }
  327. })
  328. .collect::<Vec<_>>();
  329. if !conflicts.is_empty() {
  330. return Err(RelocationError::ConflictingCandidates {
  331. type_name: local_name.to_string(),
  332. candidates: conflicts,
  333. });
  334. }
  335. target_comp_rel
  336. } else {
  337. // there are no candidate matches and therefore no target_spec. This might mean
  338. // that matching failed, or that the relocation can be applied looking at local
  339. // types only (eg with EnumVariantExists, FieldExists etc)
  340. ComputedRelocation::new(rel, &local_spec, None)?
  341. };
  342. comp_rel.apply(function, rel, local_btf, target_btf)?;
  343. last_function_opt = Some(function);
  344. }
  345. Ok(())
  346. }
  347. fn flavorless_name(name: &str) -> &str {
  348. name.split_once("___").map_or(name, |x| x.0)
  349. }
  350. fn find_candidates<'target>(
  351. local_ty: &BtfType,
  352. local_name: &str,
  353. target_btf: &'target Btf,
  354. ) -> Result<Vec<Candidate<'target>>, BtfError> {
  355. let mut candidates = Vec::new();
  356. let local_name = flavorless_name(local_name);
  357. for (type_id, ty) in target_btf.types().enumerate() {
  358. if local_ty.kind() != ty.kind() {
  359. continue;
  360. }
  361. let name = &*target_btf.type_name(ty)?;
  362. if local_name != flavorless_name(name) {
  363. continue;
  364. }
  365. candidates.push(Candidate {
  366. name: name.to_owned(),
  367. btf: target_btf,
  368. _ty: ty,
  369. type_id: type_id as u32,
  370. });
  371. }
  372. Ok(candidates)
  373. }
  374. fn match_candidate<'target>(
  375. local_spec: &AccessSpec,
  376. candidate: &'target Candidate,
  377. ) -> Result<Option<AccessSpec<'target>>, RelocationError> {
  378. let mut target_spec = AccessSpec {
  379. btf: candidate.btf,
  380. root_type_id: candidate.type_id,
  381. relocation: local_spec.relocation,
  382. parts: Vec::new(),
  383. accessors: Vec::new(),
  384. bit_offset: 0,
  385. };
  386. match local_spec.relocation.kind {
  387. RelocationKind::TypeIdLocal
  388. | RelocationKind::TypeIdTarget
  389. | RelocationKind::TypeExists
  390. | RelocationKind::TypeSize => {
  391. if types_are_compatible(
  392. local_spec.btf,
  393. local_spec.root_type_id,
  394. candidate.btf,
  395. candidate.type_id,
  396. )? {
  397. return Ok(Some(target_spec));
  398. } else {
  399. return Ok(None);
  400. }
  401. }
  402. RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => {
  403. let target_id = candidate.btf.resolve_type(candidate.type_id)?;
  404. let target_ty = candidate.btf.type_by_id(target_id)?;
  405. // the first accessor is guaranteed to have a name by construction
  406. let local_variant_name = local_spec.accessors[0].name.as_ref().unwrap();
  407. let match_enum =
  408. |name_offset, index, target_spec: &mut AccessSpec| -> Result<_, BtfError> {
  409. let target_variant_name = candidate.btf.string_at(name_offset)?;
  410. if flavorless_name(local_variant_name) == flavorless_name(&target_variant_name)
  411. {
  412. target_spec.parts.push(index);
  413. target_spec.accessors.push(Accessor {
  414. index,
  415. type_id: target_id,
  416. name: None,
  417. });
  418. Ok(Some(()))
  419. } else {
  420. Ok(None)
  421. }
  422. };
  423. match target_ty {
  424. BtfType::Enum(en) => {
  425. for (index, member) in en.variants.iter().enumerate() {
  426. if let Ok(Some(_)) = match_enum(member.name_offset, index, &mut target_spec)
  427. {
  428. return Ok(Some(target_spec));
  429. }
  430. }
  431. }
  432. BtfType::Enum64(en) => {
  433. for (index, member) in en.variants.iter().enumerate() {
  434. if let Ok(Some(_)) = match_enum(member.name_offset, index, &mut target_spec)
  435. {
  436. return Ok(Some(target_spec));
  437. }
  438. }
  439. }
  440. _ => return Ok(None),
  441. }
  442. }
  443. RelocationKind::FieldByteOffset
  444. | RelocationKind::FieldByteSize
  445. | RelocationKind::FieldExists
  446. | RelocationKind::FieldSigned
  447. | RelocationKind::FieldLShift64
  448. | RelocationKind::FieldRShift64 => {
  449. let mut target_id = candidate.type_id;
  450. for (i, accessor) in local_spec.accessors.iter().enumerate() {
  451. target_id = candidate.btf.resolve_type(target_id)?;
  452. if accessor.name.is_some() {
  453. if let Some(next_id) = match_member(
  454. local_spec.btf,
  455. local_spec,
  456. accessor,
  457. candidate.btf,
  458. target_id,
  459. &mut target_spec,
  460. )? {
  461. target_id = next_id;
  462. } else {
  463. return Ok(None);
  464. }
  465. } else {
  466. // i = 0 is the base struct. for i > 0, we need to potentially do bounds checking
  467. if i > 0 {
  468. let target_ty = candidate.btf.type_by_id(target_id)?;
  469. let array = match target_ty {
  470. BtfType::Array(Array { array, .. }) => array,
  471. _ => return Ok(None),
  472. };
  473. let var_len = array.len == 0 && {
  474. // an array is potentially variable length if it's the last field
  475. // of the parent struct and has 0 elements
  476. let parent = target_spec.accessors.last().unwrap();
  477. let parent_ty = candidate.btf.type_by_id(parent.type_id)?;
  478. match parent_ty {
  479. BtfType::Struct(s) => parent.index == s.members.len() - 1,
  480. _ => false,
  481. }
  482. };
  483. if !var_len && accessor.index >= array.len as usize {
  484. return Ok(None);
  485. }
  486. target_id = candidate.btf.resolve_type(array.element_type)?;
  487. }
  488. if target_spec.parts.len() == MAX_SPEC_LEN {
  489. return Err(RelocationError::MaximumNestingLevelReached {
  490. type_name: Some(candidate.name.clone()),
  491. });
  492. }
  493. target_spec.parts.push(accessor.index);
  494. target_spec.accessors.push(Accessor {
  495. index: accessor.index,
  496. type_id: target_id,
  497. name: None,
  498. });
  499. target_spec.bit_offset +=
  500. accessor.index * candidate.btf.type_size(target_id)? * 8;
  501. }
  502. }
  503. }
  504. };
  505. Ok(Some(target_spec))
  506. }
  507. fn match_member<'target>(
  508. local_btf: &Btf,
  509. local_spec: &AccessSpec<'_>,
  510. local_accessor: &Accessor,
  511. target_btf: &'target Btf,
  512. target_id: u32,
  513. target_spec: &mut AccessSpec<'target>,
  514. ) -> Result<Option<u32>, RelocationError> {
  515. let local_ty = local_btf.type_by_id(local_accessor.type_id)?;
  516. let local_member = match local_ty {
  517. // this won't panic, bounds are checked when local_spec is built in AccessSpec::new
  518. BtfType::Struct(s) => s.members.get(local_accessor.index).unwrap(),
  519. BtfType::Union(u) => u.members.get(local_accessor.index).unwrap(),
  520. _ => panic!("bug! this should only be called for structs and unions"),
  521. };
  522. let local_name = &*local_btf.string_at(local_member.name_offset)?;
  523. let target_id = target_btf.resolve_type(target_id)?;
  524. let target_ty = target_btf.type_by_id(target_id)?;
  525. let target_members: Vec<&BtfMember> = match target_ty.members() {
  526. Some(members) => members.collect(),
  527. // not a fields type, no match
  528. None => return Ok(None),
  529. };
  530. for (index, target_member) in target_members.iter().enumerate() {
  531. if target_spec.parts.len() == MAX_SPEC_LEN {
  532. let root_ty = target_spec.btf.type_by_id(target_spec.root_type_id)?;
  533. return Err(RelocationError::MaximumNestingLevelReached {
  534. type_name: target_spec.btf.err_type_name(root_ty),
  535. });
  536. }
  537. // this will not panic as we've already established these are fields types
  538. let bit_offset = target_ty.member_bit_offset(target_member).unwrap();
  539. let target_name = &*target_btf.string_at(target_member.name_offset)?;
  540. if target_name.is_empty() {
  541. let ret = match_member(
  542. local_btf,
  543. local_spec,
  544. local_accessor,
  545. target_btf,
  546. target_member.btf_type,
  547. target_spec,
  548. )?;
  549. if ret.is_some() {
  550. target_spec.bit_offset += bit_offset;
  551. target_spec.parts.push(index);
  552. return Ok(ret);
  553. }
  554. } else if local_name == target_name {
  555. if fields_are_compatible(
  556. local_spec.btf,
  557. local_member.btf_type,
  558. target_btf,
  559. target_member.btf_type,
  560. )? {
  561. target_spec.bit_offset += bit_offset;
  562. target_spec.parts.push(index);
  563. target_spec.accessors.push(Accessor {
  564. type_id: target_id,
  565. index,
  566. name: Some(target_name.to_owned()),
  567. });
  568. return Ok(Some(target_member.btf_type));
  569. } else {
  570. return Ok(None);
  571. }
  572. }
  573. }
  574. Ok(None)
  575. }
  576. #[derive(Debug)]
  577. struct AccessSpec<'a> {
  578. btf: &'a Btf,
  579. root_type_id: u32,
  580. parts: Vec<usize>,
  581. accessors: Vec<Accessor>,
  582. relocation: Relocation,
  583. bit_offset: usize,
  584. }
  585. impl<'a> AccessSpec<'a> {
  586. fn new(
  587. btf: &'a Btf,
  588. root_type_id: u32,
  589. spec: &str,
  590. relocation: Relocation,
  591. ) -> Result<AccessSpec<'a>, RelocationError> {
  592. let parts = spec
  593. .split(':')
  594. .map(|s| s.parse::<usize>())
  595. .collect::<Result<Vec<_>, _>>()
  596. .map_err(|_| RelocationError::InvalidAccessString {
  597. access_str: spec.to_string(),
  598. })?;
  599. let mut type_id = btf.resolve_type(root_type_id)?;
  600. let ty = btf.type_by_id(type_id)?;
  601. let spec = match relocation.kind {
  602. RelocationKind::TypeIdLocal
  603. | RelocationKind::TypeIdTarget
  604. | RelocationKind::TypeExists
  605. | RelocationKind::TypeSize => {
  606. if parts != [0] {
  607. return Err(RelocationError::InvalidAccessString {
  608. access_str: spec.to_string(),
  609. });
  610. }
  611. AccessSpec {
  612. btf,
  613. root_type_id,
  614. relocation,
  615. parts,
  616. accessors: Vec::new(),
  617. bit_offset: 0,
  618. }
  619. }
  620. RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => match ty {
  621. BtfType::Enum(_) | BtfType::Enum64(_) => {
  622. if parts.len() != 1 {
  623. return Err(RelocationError::InvalidAccessString {
  624. access_str: spec.to_string(),
  625. });
  626. }
  627. let index = parts[0];
  628. let (n_variants, name_offset) = match ty {
  629. BtfType::Enum(en) => (
  630. en.variants.len(),
  631. en.variants.get(index).map(|v| v.name_offset),
  632. ),
  633. BtfType::Enum64(en) => (
  634. en.variants.len(),
  635. en.variants.get(index).map(|v| v.name_offset),
  636. ),
  637. _ => unreachable!(),
  638. };
  639. if name_offset.is_none() {
  640. return Err(RelocationError::InvalidAccessIndex {
  641. type_name: btf.err_type_name(ty),
  642. spec: spec.to_string(),
  643. index,
  644. max_index: n_variants,
  645. error: "tried to access nonexistant enum variant".to_string(),
  646. });
  647. }
  648. let accessors = vec![Accessor {
  649. type_id,
  650. index,
  651. name: Some(btf.string_at(name_offset.unwrap())?.to_string()),
  652. }];
  653. AccessSpec {
  654. btf,
  655. root_type_id,
  656. relocation,
  657. parts,
  658. accessors,
  659. bit_offset: 0,
  660. }
  661. }
  662. _ => {
  663. return Err(RelocationError::InvalidRelocationKindForType {
  664. relocation_number: relocation.number,
  665. relocation_kind: format!("{:?}", relocation.kind),
  666. type_kind: format!("{:?}", ty.kind()),
  667. error: "enum relocation on non-enum type".to_string(),
  668. })
  669. }
  670. },
  671. RelocationKind::FieldByteOffset
  672. | RelocationKind::FieldByteSize
  673. | RelocationKind::FieldExists
  674. | RelocationKind::FieldSigned
  675. | RelocationKind::FieldLShift64
  676. | RelocationKind::FieldRShift64 => {
  677. let mut accessors = vec![Accessor {
  678. type_id,
  679. index: parts[0],
  680. name: None,
  681. }];
  682. let mut bit_offset = accessors[0].index * btf.type_size(type_id)?;
  683. for index in parts.iter().skip(1).cloned() {
  684. type_id = btf.resolve_type(type_id)?;
  685. let ty = btf.type_by_id(type_id)?;
  686. match ty {
  687. BtfType::Struct(Struct { members, .. })
  688. | BtfType::Union(Union { members, .. }) => {
  689. if index >= members.len() {
  690. return Err(RelocationError::InvalidAccessIndex {
  691. type_name: btf.err_type_name(ty),
  692. spec: spec.to_string(),
  693. index,
  694. max_index: members.len(),
  695. error: "out of bounds struct or union access".to_string(),
  696. });
  697. }
  698. let member = &members[index];
  699. bit_offset += ty.member_bit_offset(member).unwrap();
  700. if member.name_offset != 0 {
  701. accessors.push(Accessor {
  702. type_id,
  703. index,
  704. name: Some(btf.string_at(member.name_offset)?.to_string()),
  705. });
  706. }
  707. type_id = member.btf_type;
  708. }
  709. BtfType::Array(Array { array, .. }) => {
  710. type_id = btf.resolve_type(array.element_type)?;
  711. let var_len = array.len == 0 && {
  712. // an array is potentially variable length if it's the last field
  713. // of the parent struct and has 0 elements
  714. let parent = accessors.last().unwrap();
  715. let parent_ty = btf.type_by_id(parent.type_id)?;
  716. match parent_ty {
  717. BtfType::Struct(s) => index == s.members.len() - 1,
  718. _ => false,
  719. }
  720. };
  721. if !var_len && index >= array.len as usize {
  722. return Err(RelocationError::InvalidAccessIndex {
  723. type_name: btf.err_type_name(ty),
  724. spec: spec.to_string(),
  725. index,
  726. max_index: array.len as usize,
  727. error: "array index out of bounds".to_string(),
  728. });
  729. }
  730. accessors.push(Accessor {
  731. type_id,
  732. index,
  733. name: None,
  734. });
  735. let size = btf.type_size(type_id)?;
  736. bit_offset += index * size * 8;
  737. }
  738. rel_kind => {
  739. return Err(RelocationError::InvalidRelocationKindForType {
  740. relocation_number: relocation.number,
  741. relocation_kind: format!("{rel_kind:?}"),
  742. type_kind: format!("{:?}", ty.kind()),
  743. error: "field relocation on a type that doesn't have fields"
  744. .to_string(),
  745. });
  746. }
  747. };
  748. }
  749. AccessSpec {
  750. btf,
  751. root_type_id,
  752. parts,
  753. accessors,
  754. relocation,
  755. bit_offset,
  756. }
  757. }
  758. };
  759. Ok(spec)
  760. }
  761. }
  762. #[derive(Debug)]
  763. struct Accessor {
  764. type_id: u32,
  765. index: usize,
  766. name: Option<String>,
  767. }
  768. #[derive(Debug)]
  769. struct Candidate<'a> {
  770. name: String,
  771. btf: &'a Btf,
  772. _ty: &'a BtfType,
  773. type_id: u32,
  774. }
  775. #[derive(Debug)]
  776. struct ComputedRelocation {
  777. local: ComputedRelocationValue,
  778. target: ComputedRelocationValue,
  779. }
  780. #[derive(Debug)]
  781. struct ComputedRelocationValue {
  782. value: u64,
  783. size: u32,
  784. type_id: Option<u32>,
  785. }
  786. impl ComputedRelocation {
  787. fn new(
  788. rel: &Relocation,
  789. local_spec: &AccessSpec,
  790. target_spec: Option<&AccessSpec>,
  791. ) -> Result<ComputedRelocation, RelocationError> {
  792. use RelocationKind::*;
  793. let ret = match rel.kind {
  794. FieldByteOffset | FieldByteSize | FieldExists | FieldSigned | FieldLShift64
  795. | FieldRShift64 => ComputedRelocation {
  796. local: Self::compute_field_relocation(rel, Some(local_spec))?,
  797. target: Self::compute_field_relocation(rel, target_spec)?,
  798. },
  799. TypeIdLocal | TypeIdTarget | TypeExists | TypeSize => ComputedRelocation {
  800. local: Self::compute_type_relocation(rel, local_spec, target_spec)?,
  801. target: Self::compute_type_relocation(rel, local_spec, target_spec)?,
  802. },
  803. EnumVariantExists | EnumVariantValue => ComputedRelocation {
  804. local: Self::compute_enum_relocation(rel, Some(local_spec))?,
  805. target: Self::compute_enum_relocation(rel, target_spec)?,
  806. },
  807. };
  808. Ok(ret)
  809. }
  810. fn apply(
  811. &self,
  812. function: &mut Function,
  813. rel: &Relocation,
  814. local_btf: &Btf,
  815. target_btf: &Btf,
  816. ) -> Result<(), RelocationError> {
  817. let instructions = &mut function.instructions;
  818. let num_instructions = instructions.len();
  819. let ins_index = (rel.ins_offset - function.section_offset) / mem::size_of::<bpf_insn>();
  820. let ins =
  821. instructions
  822. .get_mut(ins_index)
  823. .ok_or(RelocationError::InvalidInstructionIndex {
  824. index: rel.ins_offset,
  825. num_instructions,
  826. relocation_number: rel.number,
  827. })?;
  828. let class = (ins.code & 0x07) as u32;
  829. let target_value = self.target.value;
  830. match class {
  831. BPF_ALU | BPF_ALU64 => {
  832. let src_reg = ins.src_reg();
  833. if src_reg != BPF_K as u8 {
  834. return Err(RelocationError::InvalidInstruction {
  835. relocation_number: rel.number,
  836. index: ins_index,
  837. error: format!("invalid src_reg={src_reg:x} expected {BPF_K:x}"),
  838. });
  839. }
  840. ins.imm = target_value as i32;
  841. }
  842. BPF_LDX | BPF_ST | BPF_STX => {
  843. if target_value > i16::MAX as u64 {
  844. return Err(RelocationError::InvalidInstruction {
  845. relocation_number: rel.number,
  846. index: ins_index,
  847. error: format!("value `{target_value}` overflows 16 bits offset field"),
  848. });
  849. }
  850. ins.off = target_value as i16;
  851. if self.local.size != self.target.size {
  852. let local_ty = local_btf.type_by_id(self.local.type_id.unwrap())?;
  853. let target_ty = target_btf.type_by_id(self.target.type_id.unwrap())?;
  854. let unsigned = |info: u32| ((info >> 24) & 0x0F) & BTF_INT_SIGNED == 0;
  855. use BtfType::*;
  856. match (local_ty, target_ty) {
  857. (Ptr(_), Ptr(_)) => {}
  858. (Int(local), Int(target))
  859. if unsigned(local.data) && unsigned(target.data) => {}
  860. _ => {
  861. return Err(RelocationError::InvalidInstruction {
  862. relocation_number: rel.number,
  863. index: ins_index,
  864. error: format!(
  865. "original type {} has size {} but target type {} has size {}",
  866. err_type_name(&local_btf.err_type_name(local_ty)),
  867. self.local.size,
  868. err_type_name(&target_btf.err_type_name(target_ty)),
  869. self.target.size,
  870. ),
  871. })
  872. }
  873. }
  874. let size = match self.target.size {
  875. 8 => BPF_DW,
  876. 4 => BPF_W,
  877. 2 => BPF_H,
  878. 1 => BPF_B,
  879. size => {
  880. return Err(RelocationError::InvalidInstruction {
  881. relocation_number: rel.number,
  882. index: ins_index,
  883. error: format!("invalid target size {size}"),
  884. })
  885. }
  886. } as u8;
  887. ins.code = ins.code & 0xE0 | size | ins.code & 0x07;
  888. }
  889. }
  890. BPF_LD => {
  891. ins.imm = target_value as i32;
  892. let next_ins = instructions.get_mut(ins_index + 1).ok_or(
  893. RelocationError::InvalidInstructionIndex {
  894. index: ins_index + 1,
  895. num_instructions,
  896. relocation_number: rel.number,
  897. },
  898. )?;
  899. next_ins.imm = (target_value >> 32) as i32;
  900. }
  901. class => {
  902. return Err(RelocationError::InvalidInstruction {
  903. relocation_number: rel.number,
  904. index: ins_index,
  905. error: format!("invalid instruction class {class:x}"),
  906. })
  907. }
  908. };
  909. Ok(())
  910. }
  911. fn compute_enum_relocation(
  912. rel: &Relocation,
  913. spec: Option<&AccessSpec>,
  914. ) -> Result<ComputedRelocationValue, RelocationError> {
  915. use RelocationKind::*;
  916. let value = match (rel.kind, spec) {
  917. (EnumVariantExists, spec) => spec.is_some() as u64,
  918. (EnumVariantValue, Some(spec)) => {
  919. let accessor = &spec.accessors[0];
  920. match spec.btf.type_by_id(accessor.type_id)? {
  921. BtfType::Enum(en) => {
  922. let value = en.variants[accessor.index].value;
  923. if en.is_signed() {
  924. value as i32 as u64
  925. } else {
  926. value as u64
  927. }
  928. }
  929. BtfType::Enum64(en) => {
  930. let variant = &en.variants[accessor.index];
  931. (variant.value_high as u64) << 32 | variant.value_low as u64
  932. }
  933. // candidate selection ensures that rel_kind == local_kind == target_kind
  934. _ => unreachable!(),
  935. }
  936. }
  937. _ => {
  938. return Err(RelocationError::MissingTargetDefinition {
  939. kind: rel.kind,
  940. type_id: rel.type_id,
  941. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  942. })?;
  943. }
  944. };
  945. Ok(ComputedRelocationValue {
  946. value,
  947. size: 0,
  948. type_id: None,
  949. })
  950. }
  951. fn compute_field_relocation(
  952. rel: &Relocation,
  953. spec: Option<&AccessSpec>,
  954. ) -> Result<ComputedRelocationValue, RelocationError> {
  955. use RelocationKind::*;
  956. if let FieldExists = rel.kind {
  957. // this is the bpf_preserve_field_info(member_access, FIELD_EXISTENCE) case. If we
  958. // managed to build a spec, it means the field exists.
  959. return Ok(ComputedRelocationValue {
  960. value: spec.is_some() as u64,
  961. size: 0,
  962. type_id: None,
  963. });
  964. }
  965. let spec = match spec {
  966. Some(spec) => spec,
  967. None => {
  968. return Err(RelocationError::MissingTargetDefinition {
  969. kind: rel.kind,
  970. type_id: rel.type_id,
  971. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  972. })?;
  973. }
  974. };
  975. let accessor = spec.accessors.last().unwrap();
  976. if accessor.name.is_none() {
  977. // the last accessor is unnamed, meaning that this is an array access
  978. return match rel.kind {
  979. FieldByteOffset => Ok(ComputedRelocationValue {
  980. value: (spec.bit_offset / 8) as u64,
  981. size: spec.btf.type_size(accessor.type_id)? as u32,
  982. type_id: Some(accessor.type_id),
  983. }),
  984. FieldByteSize => Ok(ComputedRelocationValue {
  985. value: spec.btf.type_size(accessor.type_id)? as u64,
  986. size: 0,
  987. type_id: Some(accessor.type_id),
  988. }),
  989. rel_kind => {
  990. let ty = spec.btf.type_by_id(accessor.type_id)?;
  991. return Err(RelocationError::InvalidRelocationKindForType {
  992. relocation_number: rel.number,
  993. relocation_kind: format!("{rel_kind:?}"),
  994. type_kind: format!("{:?}", ty.kind()),
  995. error: "invalid relocation kind for array type".to_string(),
  996. });
  997. }
  998. };
  999. }
  1000. let ty = spec.btf.type_by_id(accessor.type_id)?;
  1001. let (ll_ty, member) = match ty {
  1002. BtfType::Struct(t) => (ty, t.members.get(accessor.index).unwrap()),
  1003. BtfType::Union(t) => (ty, t.members.get(accessor.index).unwrap()),
  1004. _ => {
  1005. return Err(RelocationError::InvalidRelocationKindForType {
  1006. relocation_number: rel.number,
  1007. relocation_kind: format!("{:?}", rel.kind),
  1008. type_kind: format!("{:?}", ty.kind()),
  1009. error: "field relocation on a type that doesn't have fields".to_string(),
  1010. });
  1011. }
  1012. };
  1013. let bit_off = spec.bit_offset as u32;
  1014. let member_type_id = spec.btf.resolve_type(member.btf_type)?;
  1015. let member_ty = spec.btf.type_by_id(member_type_id)?;
  1016. let mut byte_size;
  1017. let mut byte_off;
  1018. let mut bit_size = ll_ty.member_bit_field_size(member).unwrap() as u32;
  1019. let is_bitfield = bit_size > 0;
  1020. if is_bitfield {
  1021. // find out the smallest int size to load the bitfield
  1022. byte_size = member_ty.size().unwrap();
  1023. byte_off = bit_off / 8 / byte_size * byte_size;
  1024. while bit_off + bit_size - byte_off * 8 > byte_size * 8 {
  1025. if byte_size >= 8 {
  1026. // the bitfield is larger than 8 bytes!?
  1027. return Err(BtfError::InvalidTypeInfo.into());
  1028. }
  1029. byte_size *= 2;
  1030. byte_off = bit_off / 8 / byte_size * byte_size;
  1031. }
  1032. } else {
  1033. byte_size = spec.btf.type_size(member_type_id)? as u32;
  1034. bit_size = byte_size * 8;
  1035. byte_off = spec.bit_offset as u32 / 8;
  1036. }
  1037. let mut value = ComputedRelocationValue {
  1038. value: 0,
  1039. size: 0,
  1040. type_id: None,
  1041. };
  1042. #[allow(clippy::wildcard_in_or_patterns)]
  1043. match rel.kind {
  1044. FieldByteOffset => {
  1045. value.value = byte_off as u64;
  1046. if !is_bitfield {
  1047. value.size = byte_size;
  1048. value.type_id = Some(member_type_id);
  1049. }
  1050. }
  1051. FieldByteSize => {
  1052. value.value = byte_size as u64;
  1053. }
  1054. FieldSigned => match member_ty {
  1055. BtfType::Enum(en) => value.value = en.is_signed() as u64,
  1056. BtfType::Enum64(en) => value.value = en.is_signed() as u64,
  1057. BtfType::Int(i) => value.value = i.encoding() as u64 & IntEncoding::Signed as u64,
  1058. _ => (),
  1059. },
  1060. #[cfg(target_endian = "little")]
  1061. FieldLShift64 => {
  1062. value.value = 64 - (bit_off + bit_size - byte_off * 8) as u64;
  1063. }
  1064. #[cfg(target_endian = "big")]
  1065. FieldLShift64 => {
  1066. value.value = (8 - byte_size) * 8 + (bit_off - byte_off * 8);
  1067. }
  1068. FieldRShift64 => {
  1069. value.value = 64 - bit_size as u64;
  1070. }
  1071. FieldExists // this is handled at the start of the function
  1072. | _ => panic!("bug! this should not be reached"),
  1073. }
  1074. Ok(value)
  1075. }
  1076. fn compute_type_relocation(
  1077. rel: &Relocation,
  1078. local_spec: &AccessSpec,
  1079. target_spec: Option<&AccessSpec>,
  1080. ) -> Result<ComputedRelocationValue, RelocationError> {
  1081. use RelocationKind::*;
  1082. let value = match (rel.kind, target_spec) {
  1083. (TypeIdLocal, _) => local_spec.root_type_id as u64,
  1084. (TypeIdTarget, Some(target_spec)) => target_spec.root_type_id as u64,
  1085. (TypeExists, target_spec) => target_spec.is_some() as u64,
  1086. (TypeSize, Some(target_spec)) => {
  1087. target_spec.btf.type_size(target_spec.root_type_id)? as u64
  1088. }
  1089. _ => {
  1090. return Err(RelocationError::MissingTargetDefinition {
  1091. kind: rel.kind,
  1092. type_id: rel.type_id,
  1093. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  1094. })?;
  1095. }
  1096. };
  1097. Ok(ComputedRelocationValue {
  1098. value,
  1099. size: 0,
  1100. type_id: None,
  1101. })
  1102. }
  1103. }