relocation.rs 45 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250
  1. use alloc::{
  2. borrow::{Cow, ToOwned as _},
  3. collections::BTreeMap,
  4. format,
  5. string::{String, ToString},
  6. vec,
  7. vec::Vec,
  8. };
  9. use core::{mem, ops::Bound::Included, ptr};
  10. use object::SectionIndex;
  11. #[cfg(not(feature = "std"))]
  12. use crate::std;
  13. use crate::{
  14. btf::{
  15. fields_are_compatible, types_are_compatible, Array, Btf, BtfError, BtfMember, BtfType,
  16. IntEncoding, Struct, Union, MAX_SPEC_LEN,
  17. },
  18. generated::{
  19. bpf_core_relo, bpf_core_relo_kind::*, bpf_insn, BPF_ALU, BPF_ALU64, BPF_B, BPF_CALL,
  20. BPF_DW, BPF_H, BPF_JMP, BPF_K, BPF_LD, BPF_LDX, BPF_ST, BPF_STX, BPF_W, BTF_INT_SIGNED,
  21. },
  22. util::HashMap,
  23. Function, Object,
  24. };
  25. /// The error type returned by [`Object::relocate_btf`].
  26. #[derive(thiserror::Error, Debug)]
  27. #[error("error relocating `{section}`")]
  28. pub struct BtfRelocationError {
  29. /// The function name
  30. pub section: String,
  31. #[source]
  32. /// The original error
  33. error: RelocationError,
  34. }
  35. /// Relocation failures
  36. #[derive(thiserror::Error, Debug)]
  37. enum RelocationError {
  38. #[cfg(feature = "std")]
  39. /// I/O error
  40. #[error(transparent)]
  41. IOError(#[from] std::io::Error),
  42. /// Section not found
  43. #[error("section not found")]
  44. SectionNotFound,
  45. /// Function not found
  46. #[error("function not found")]
  47. FunctionNotFound,
  48. /// Invalid relocation access string
  49. #[error("invalid relocation access string {access_str}")]
  50. InvalidAccessString {
  51. /// The access string
  52. access_str: String,
  53. },
  54. /// Invalid instruction index referenced by relocation
  55. #[error("invalid instruction index #{index} referenced by relocation #{relocation_number}, the program contains {num_instructions} instructions")]
  56. InvalidInstructionIndex {
  57. /// The invalid instruction index
  58. index: usize,
  59. /// Number of instructions in the program
  60. num_instructions: usize,
  61. /// The relocation number
  62. relocation_number: usize,
  63. },
  64. /// Multiple candidate target types found with different memory layouts
  65. #[error("error relocating {type_name}, multiple candidate target types found with different memory layouts: {candidates:?}")]
  66. ConflictingCandidates {
  67. /// The type name
  68. type_name: String,
  69. /// The candidates
  70. candidates: Vec<String>,
  71. },
  72. /// Maximum nesting level reached evaluating candidate type
  73. #[error("maximum nesting level reached evaluating candidate type `{}`", err_type_name(.type_name))]
  74. MaximumNestingLevelReached {
  75. /// The type name
  76. type_name: Option<String>,
  77. },
  78. /// Invalid access string
  79. #[error("invalid access string `{spec}` for type `{}`: {error}", err_type_name(.type_name))]
  80. InvalidAccessIndex {
  81. /// The type name
  82. type_name: Option<String>,
  83. /// The access string
  84. spec: String,
  85. /// The index
  86. index: usize,
  87. /// The max index
  88. max_index: usize,
  89. /// The error message
  90. error: &'static str,
  91. },
  92. /// Relocation not valid for type
  93. #[error(
  94. "relocation #{relocation_number} of kind `{relocation_kind}` not valid for type `{type_kind}`: {error}"
  95. )]
  96. InvalidRelocationKindForType {
  97. /// The relocation number
  98. relocation_number: usize,
  99. /// The relocation kind
  100. relocation_kind: String,
  101. /// The type kind
  102. type_kind: String,
  103. /// The error message
  104. error: &'static str,
  105. },
  106. /// Invalid instruction referenced by relocation
  107. #[error(
  108. "instruction #{index} referenced by relocation #{relocation_number} is invalid: {error}"
  109. )]
  110. InvalidInstruction {
  111. /// The relocation number
  112. relocation_number: usize,
  113. /// The instruction index
  114. index: usize,
  115. /// The error message
  116. error: Cow<'static, str>,
  117. },
  118. #[error("applying relocation `{kind:?}` missing target BTF info for type `{type_id}` at instruction #{ins_index}")]
  119. MissingTargetDefinition {
  120. kind: RelocationKind,
  121. type_id: u32,
  122. ins_index: usize,
  123. },
  124. /// BTF error
  125. #[error("invalid BTF")]
  126. BtfError(#[from] BtfError),
  127. }
  128. fn err_type_name(name: &Option<String>) -> &str {
  129. name.as_deref().unwrap_or("[unknown name]")
  130. }
  131. #[derive(Copy, Clone, Debug)]
  132. #[repr(u32)]
  133. enum RelocationKind {
  134. FieldByteOffset = BPF_CORE_FIELD_BYTE_OFFSET,
  135. FieldByteSize = BPF_CORE_FIELD_BYTE_SIZE,
  136. FieldExists = BPF_CORE_FIELD_EXISTS,
  137. FieldSigned = BPF_CORE_FIELD_SIGNED,
  138. FieldLShift64 = BPF_CORE_FIELD_LSHIFT_U64,
  139. FieldRShift64 = BPF_CORE_FIELD_RSHIFT_U64,
  140. TypeIdLocal = BPF_CORE_TYPE_ID_LOCAL,
  141. TypeIdTarget = BPF_CORE_TYPE_ID_TARGET,
  142. TypeExists = BPF_CORE_TYPE_EXISTS,
  143. TypeSize = BPF_CORE_TYPE_SIZE,
  144. EnumVariantExists = BPF_CORE_ENUMVAL_EXISTS,
  145. EnumVariantValue = BPF_CORE_ENUMVAL_VALUE,
  146. }
  147. impl TryFrom<u32> for RelocationKind {
  148. type Error = BtfError;
  149. fn try_from(v: u32) -> Result<Self, Self::Error> {
  150. use RelocationKind::*;
  151. Ok(match v {
  152. BPF_CORE_FIELD_BYTE_OFFSET => FieldByteOffset,
  153. BPF_CORE_FIELD_BYTE_SIZE => FieldByteSize,
  154. BPF_CORE_FIELD_EXISTS => FieldExists,
  155. BPF_CORE_FIELD_SIGNED => FieldSigned,
  156. BPF_CORE_FIELD_LSHIFT_U64 => FieldLShift64,
  157. BPF_CORE_FIELD_RSHIFT_U64 => FieldRShift64,
  158. BPF_CORE_TYPE_ID_LOCAL => TypeIdLocal,
  159. BPF_CORE_TYPE_ID_TARGET => TypeIdTarget,
  160. BPF_CORE_TYPE_EXISTS => TypeExists,
  161. BPF_CORE_TYPE_SIZE => TypeSize,
  162. BPF_CORE_ENUMVAL_EXISTS => EnumVariantExists,
  163. BPF_CORE_ENUMVAL_VALUE => EnumVariantValue,
  164. kind => return Err(BtfError::InvalidRelocationKind { kind }),
  165. })
  166. }
  167. }
  168. #[derive(Debug, Copy, Clone)]
  169. pub(crate) struct Relocation {
  170. kind: RelocationKind,
  171. ins_offset: usize,
  172. type_id: u32,
  173. access_str_offset: u32,
  174. number: usize,
  175. }
  176. impl Relocation {
  177. #[allow(unused_unsafe)]
  178. pub(crate) unsafe fn parse(data: &[u8], number: usize) -> Result<Relocation, BtfError> {
  179. if mem::size_of::<bpf_core_relo>() > data.len() {
  180. return Err(BtfError::InvalidRelocationInfo);
  181. }
  182. let rel = unsafe { ptr::read_unaligned::<bpf_core_relo>(data.as_ptr() as *const _) };
  183. Ok(Relocation {
  184. kind: rel.kind.try_into()?,
  185. ins_offset: rel.insn_off as usize,
  186. type_id: rel.type_id,
  187. access_str_offset: rel.access_str_off,
  188. number,
  189. })
  190. }
  191. }
  192. impl Object {
  193. /// Relocates programs inside this object file with loaded BTF info.
  194. pub fn relocate_btf(&mut self, target_btf: &Btf) -> Result<(), BtfRelocationError> {
  195. let (local_btf, btf_ext) = match (&self.btf, &self.btf_ext) {
  196. (Some(btf), Some(btf_ext)) => (btf, btf_ext),
  197. _ => return Ok(()),
  198. };
  199. let mut candidates_cache = HashMap::<u32, Vec<Candidate>>::new();
  200. for (sec_name_off, relos) in btf_ext.relocations() {
  201. let section_name =
  202. local_btf
  203. .string_at(*sec_name_off)
  204. .map_err(|e| BtfRelocationError {
  205. section: format!("section@{sec_name_off}"),
  206. error: RelocationError::BtfError(e),
  207. })?;
  208. let (section_index, _) = self
  209. .section_infos
  210. .get(&section_name.to_string())
  211. .ok_or_else(|| BtfRelocationError {
  212. section: section_name.to_string(),
  213. error: RelocationError::SectionNotFound,
  214. })?;
  215. match relocate_btf_functions(
  216. section_index,
  217. &mut self.functions,
  218. relos,
  219. local_btf,
  220. target_btf,
  221. &mut candidates_cache,
  222. ) {
  223. Ok(_) => {}
  224. Err(error) => {
  225. return Err(BtfRelocationError {
  226. section: section_name.to_string(),
  227. error,
  228. })
  229. }
  230. }
  231. }
  232. Ok(())
  233. }
  234. }
  235. fn is_relocation_inside_function(
  236. section_index: &SectionIndex,
  237. func: &Function,
  238. rel: &Relocation,
  239. ) -> bool {
  240. if section_index.0 != func.section_index.0 {
  241. return false;
  242. }
  243. let ins_offset = rel.ins_offset / mem::size_of::<bpf_insn>();
  244. let func_offset = func.section_offset / mem::size_of::<bpf_insn>();
  245. let func_size = func.instructions.len();
  246. (func_offset..func_offset + func_size).contains(&ins_offset)
  247. }
  248. fn function_by_relocation<'a>(
  249. section_index: &SectionIndex,
  250. functions: &'a mut BTreeMap<(usize, u64), Function>,
  251. rel: &Relocation,
  252. ) -> Option<&'a mut Function> {
  253. functions
  254. .range_mut((
  255. Included(&(section_index.0, 0)),
  256. Included(&(section_index.0, u64::MAX)),
  257. ))
  258. .map(|(_, func)| func)
  259. .find(|func| is_relocation_inside_function(section_index, func, rel))
  260. }
  261. fn relocate_btf_functions<'target>(
  262. section_index: &SectionIndex,
  263. functions: &mut BTreeMap<(usize, u64), Function>,
  264. relos: &[Relocation],
  265. local_btf: &Btf,
  266. target_btf: &'target Btf,
  267. candidates_cache: &mut HashMap<u32, Vec<Candidate<'target>>>,
  268. ) -> Result<(), RelocationError> {
  269. let mut last_function_opt: Option<&mut Function> = None;
  270. for rel in relos {
  271. let function = match last_function_opt.take() {
  272. Some(func) if is_relocation_inside_function(section_index, func, rel) => func,
  273. _ => function_by_relocation(section_index, functions, rel)
  274. .ok_or(RelocationError::FunctionNotFound)?,
  275. };
  276. let instructions = &mut function.instructions;
  277. let ins_index = (rel.ins_offset - function.section_offset) / mem::size_of::<bpf_insn>();
  278. if ins_index >= instructions.len() {
  279. return Err(RelocationError::InvalidInstructionIndex {
  280. index: ins_index,
  281. num_instructions: instructions.len(),
  282. relocation_number: rel.number,
  283. });
  284. }
  285. let local_ty = local_btf.type_by_id(rel.type_id)?;
  286. let local_name = &*local_btf.type_name(local_ty)?;
  287. let access_str = &*local_btf.string_at(rel.access_str_offset)?;
  288. let local_spec = AccessSpec::new(local_btf, rel.type_id, access_str, *rel)?;
  289. let matches = match rel.kind {
  290. RelocationKind::TypeIdLocal => Vec::new(), // we don't need to look at target types to relocate this value
  291. _ => {
  292. let candidates = match candidates_cache.get(&rel.type_id) {
  293. Some(cands) => cands,
  294. None => {
  295. candidates_cache.insert(
  296. rel.type_id,
  297. find_candidates(local_ty, local_name, target_btf)?,
  298. );
  299. candidates_cache.get(&rel.type_id).unwrap()
  300. }
  301. };
  302. let mut matches = Vec::new();
  303. for candidate in candidates {
  304. if let Some(candidate_spec) = match_candidate(&local_spec, candidate)? {
  305. let comp_rel =
  306. ComputedRelocation::new(rel, &local_spec, Some(&candidate_spec))?;
  307. matches.push((candidate.name.clone(), candidate_spec, comp_rel));
  308. }
  309. }
  310. matches
  311. }
  312. };
  313. let comp_rel = if !matches.is_empty() {
  314. let mut matches = matches.into_iter();
  315. let (_, target_spec, target_comp_rel) = matches.next().unwrap();
  316. // if there's more than one candidate, make sure that they all resolve to the
  317. // same value, else the relocation is ambiguous and can't be applied
  318. let conflicts = matches
  319. .filter_map(|(cand_name, cand_spec, cand_comp_rel)| {
  320. if cand_spec.bit_offset != target_spec.bit_offset {
  321. return Some(cand_name);
  322. } else if let (Some(cand_comp_rel_target), Some(target_comp_rel_target)) = (
  323. cand_comp_rel.target.as_ref(),
  324. target_comp_rel.target.as_ref(),
  325. ) {
  326. if cand_comp_rel_target.value != target_comp_rel_target.value {
  327. return Some(cand_name);
  328. }
  329. }
  330. None
  331. })
  332. .collect::<Vec<_>>();
  333. if !conflicts.is_empty() {
  334. return Err(RelocationError::ConflictingCandidates {
  335. type_name: local_name.to_string(),
  336. candidates: conflicts,
  337. });
  338. }
  339. target_comp_rel
  340. } else {
  341. // there are no candidate matches and therefore no target_spec. This might mean
  342. // that matching failed, or that the relocation can be applied looking at local
  343. // types only (eg with EnumVariantExists, FieldExists etc)
  344. ComputedRelocation::new(rel, &local_spec, None)?
  345. };
  346. comp_rel.apply(function, rel, local_btf, target_btf)?;
  347. last_function_opt = Some(function);
  348. }
  349. Ok(())
  350. }
  351. fn flavorless_name(name: &str) -> &str {
  352. name.split_once("___").map_or(name, |x| x.0)
  353. }
  354. fn find_candidates<'target>(
  355. local_ty: &BtfType,
  356. local_name: &str,
  357. target_btf: &'target Btf,
  358. ) -> Result<Vec<Candidate<'target>>, BtfError> {
  359. let mut candidates = Vec::new();
  360. let local_name = flavorless_name(local_name);
  361. for (type_id, ty) in target_btf.types().enumerate() {
  362. if local_ty.kind() != ty.kind() {
  363. continue;
  364. }
  365. let name = &*target_btf.type_name(ty)?;
  366. if local_name != flavorless_name(name) {
  367. continue;
  368. }
  369. candidates.push(Candidate {
  370. name: name.to_owned(),
  371. btf: target_btf,
  372. _ty: ty,
  373. type_id: type_id as u32,
  374. });
  375. }
  376. Ok(candidates)
  377. }
  378. fn match_candidate<'target>(
  379. local_spec: &AccessSpec,
  380. candidate: &'target Candidate,
  381. ) -> Result<Option<AccessSpec<'target>>, RelocationError> {
  382. let mut target_spec = AccessSpec {
  383. btf: candidate.btf,
  384. root_type_id: candidate.type_id,
  385. relocation: local_spec.relocation,
  386. parts: Vec::new(),
  387. accessors: Vec::new(),
  388. bit_offset: 0,
  389. };
  390. match local_spec.relocation.kind {
  391. RelocationKind::TypeIdLocal
  392. | RelocationKind::TypeIdTarget
  393. | RelocationKind::TypeExists
  394. | RelocationKind::TypeSize => {
  395. if types_are_compatible(
  396. local_spec.btf,
  397. local_spec.root_type_id,
  398. candidate.btf,
  399. candidate.type_id,
  400. )? {
  401. return Ok(Some(target_spec));
  402. } else {
  403. return Ok(None);
  404. }
  405. }
  406. RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => {
  407. let target_id = candidate.btf.resolve_type(candidate.type_id)?;
  408. let target_ty = candidate.btf.type_by_id(target_id)?;
  409. // the first accessor is guaranteed to have a name by construction
  410. let local_variant_name = local_spec.accessors[0].name.as_ref().unwrap();
  411. let match_enum =
  412. |name_offset, index, target_spec: &mut AccessSpec| -> Result<_, BtfError> {
  413. let target_variant_name = candidate.btf.string_at(name_offset)?;
  414. if flavorless_name(local_variant_name) == flavorless_name(&target_variant_name)
  415. {
  416. target_spec.parts.push(index);
  417. target_spec.accessors.push(Accessor {
  418. index,
  419. type_id: target_id,
  420. name: None,
  421. });
  422. Ok(Some(()))
  423. } else {
  424. Ok(None)
  425. }
  426. };
  427. match target_ty {
  428. BtfType::Enum(en) => {
  429. for (index, member) in en.variants.iter().enumerate() {
  430. if let Ok(Some(_)) = match_enum(member.name_offset, index, &mut target_spec)
  431. {
  432. return Ok(Some(target_spec));
  433. }
  434. }
  435. }
  436. BtfType::Enum64(en) => {
  437. for (index, member) in en.variants.iter().enumerate() {
  438. if let Ok(Some(_)) = match_enum(member.name_offset, index, &mut target_spec)
  439. {
  440. return Ok(Some(target_spec));
  441. }
  442. }
  443. }
  444. _ => return Ok(None),
  445. }
  446. }
  447. RelocationKind::FieldByteOffset
  448. | RelocationKind::FieldByteSize
  449. | RelocationKind::FieldExists
  450. | RelocationKind::FieldSigned
  451. | RelocationKind::FieldLShift64
  452. | RelocationKind::FieldRShift64 => {
  453. let mut target_id = candidate.type_id;
  454. for (i, accessor) in local_spec.accessors.iter().enumerate() {
  455. target_id = candidate.btf.resolve_type(target_id)?;
  456. if accessor.name.is_some() {
  457. if let Some(next_id) = match_member(
  458. local_spec.btf,
  459. local_spec,
  460. accessor,
  461. candidate.btf,
  462. target_id,
  463. &mut target_spec,
  464. )? {
  465. target_id = next_id;
  466. } else {
  467. return Ok(None);
  468. }
  469. } else {
  470. // i = 0 is the base struct. for i > 0, we need to potentially do bounds checking
  471. if i > 0 {
  472. let target_ty = candidate.btf.type_by_id(target_id)?;
  473. let array = match target_ty {
  474. BtfType::Array(Array { array, .. }) => array,
  475. _ => return Ok(None),
  476. };
  477. let var_len = array.len == 0 && {
  478. // an array is potentially variable length if it's the last field
  479. // of the parent struct and has 0 elements
  480. let parent = target_spec.accessors.last().unwrap();
  481. let parent_ty = candidate.btf.type_by_id(parent.type_id)?;
  482. match parent_ty {
  483. BtfType::Struct(s) => parent.index == s.members.len() - 1,
  484. _ => false,
  485. }
  486. };
  487. if !var_len && accessor.index >= array.len as usize {
  488. return Ok(None);
  489. }
  490. target_id = candidate.btf.resolve_type(array.element_type)?;
  491. }
  492. if target_spec.parts.len() == MAX_SPEC_LEN {
  493. return Err(RelocationError::MaximumNestingLevelReached {
  494. type_name: Some(candidate.name.clone()),
  495. });
  496. }
  497. target_spec.parts.push(accessor.index);
  498. target_spec.accessors.push(Accessor {
  499. index: accessor.index,
  500. type_id: target_id,
  501. name: None,
  502. });
  503. target_spec.bit_offset +=
  504. accessor.index * candidate.btf.type_size(target_id)? * 8;
  505. }
  506. }
  507. }
  508. };
  509. Ok(Some(target_spec))
  510. }
  511. fn match_member<'target>(
  512. local_btf: &Btf,
  513. local_spec: &AccessSpec<'_>,
  514. local_accessor: &Accessor,
  515. target_btf: &'target Btf,
  516. target_id: u32,
  517. target_spec: &mut AccessSpec<'target>,
  518. ) -> Result<Option<u32>, RelocationError> {
  519. let local_ty = local_btf.type_by_id(local_accessor.type_id)?;
  520. let local_member = match local_ty {
  521. // this won't panic, bounds are checked when local_spec is built in AccessSpec::new
  522. BtfType::Struct(s) => s.members.get(local_accessor.index).unwrap(),
  523. BtfType::Union(u) => u.members.get(local_accessor.index).unwrap(),
  524. local_ty => panic!("unexpected type {:?}", local_ty),
  525. };
  526. let local_name = &*local_btf.string_at(local_member.name_offset)?;
  527. let target_id = target_btf.resolve_type(target_id)?;
  528. let target_ty = target_btf.type_by_id(target_id)?;
  529. let target_members: Vec<&BtfMember> = match target_ty.members() {
  530. Some(members) => members.collect(),
  531. // not a fields type, no match
  532. None => return Ok(None),
  533. };
  534. for (index, target_member) in target_members.iter().enumerate() {
  535. if target_spec.parts.len() == MAX_SPEC_LEN {
  536. let root_ty = target_spec.btf.type_by_id(target_spec.root_type_id)?;
  537. return Err(RelocationError::MaximumNestingLevelReached {
  538. type_name: target_spec.btf.err_type_name(root_ty),
  539. });
  540. }
  541. // this will not panic as we've already established these are fields types
  542. let bit_offset = target_ty.member_bit_offset(target_member).unwrap();
  543. let target_name = &*target_btf.string_at(target_member.name_offset)?;
  544. if target_name.is_empty() {
  545. let ret = match_member(
  546. local_btf,
  547. local_spec,
  548. local_accessor,
  549. target_btf,
  550. target_member.btf_type,
  551. target_spec,
  552. )?;
  553. if ret.is_some() {
  554. target_spec.bit_offset += bit_offset;
  555. target_spec.parts.push(index);
  556. return Ok(ret);
  557. }
  558. } else if local_name == target_name {
  559. if fields_are_compatible(
  560. local_spec.btf,
  561. local_member.btf_type,
  562. target_btf,
  563. target_member.btf_type,
  564. )? {
  565. target_spec.bit_offset += bit_offset;
  566. target_spec.parts.push(index);
  567. target_spec.accessors.push(Accessor {
  568. type_id: target_id,
  569. index,
  570. name: Some(target_name.to_owned()),
  571. });
  572. return Ok(Some(target_member.btf_type));
  573. } else {
  574. return Ok(None);
  575. }
  576. }
  577. }
  578. Ok(None)
  579. }
  580. #[derive(Debug)]
  581. struct AccessSpec<'a> {
  582. btf: &'a Btf,
  583. root_type_id: u32,
  584. parts: Vec<usize>,
  585. accessors: Vec<Accessor>,
  586. relocation: Relocation,
  587. bit_offset: usize,
  588. }
  589. impl<'a> AccessSpec<'a> {
  590. fn new(
  591. btf: &'a Btf,
  592. root_type_id: u32,
  593. spec: &str,
  594. relocation: Relocation,
  595. ) -> Result<AccessSpec<'a>, RelocationError> {
  596. let parts = spec
  597. .split(':')
  598. .map(|s| s.parse::<usize>())
  599. .collect::<Result<Vec<_>, _>>()
  600. .map_err(|_| RelocationError::InvalidAccessString {
  601. access_str: spec.to_string(),
  602. })?;
  603. let mut type_id = btf.resolve_type(root_type_id)?;
  604. let ty = btf.type_by_id(type_id)?;
  605. let spec = match relocation.kind {
  606. RelocationKind::TypeIdLocal
  607. | RelocationKind::TypeIdTarget
  608. | RelocationKind::TypeExists
  609. | RelocationKind::TypeSize => {
  610. if parts != [0] {
  611. return Err(RelocationError::InvalidAccessString {
  612. access_str: spec.to_string(),
  613. });
  614. }
  615. AccessSpec {
  616. btf,
  617. root_type_id,
  618. relocation,
  619. parts,
  620. accessors: Vec::new(),
  621. bit_offset: 0,
  622. }
  623. }
  624. RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => match ty {
  625. BtfType::Enum(_) | BtfType::Enum64(_) => {
  626. if parts.len() != 1 {
  627. return Err(RelocationError::InvalidAccessString {
  628. access_str: spec.to_string(),
  629. });
  630. }
  631. let index = parts[0];
  632. let (n_variants, name_offset) = match ty {
  633. BtfType::Enum(en) => (
  634. en.variants.len(),
  635. en.variants.get(index).map(|v| v.name_offset),
  636. ),
  637. BtfType::Enum64(en) => (
  638. en.variants.len(),
  639. en.variants.get(index).map(|v| v.name_offset),
  640. ),
  641. _ => unreachable!(),
  642. };
  643. if name_offset.is_none() {
  644. return Err(RelocationError::InvalidAccessIndex {
  645. type_name: btf.err_type_name(ty),
  646. spec: spec.to_string(),
  647. index,
  648. max_index: n_variants,
  649. error: "tried to access nonexistant enum variant",
  650. });
  651. }
  652. let accessors = vec![Accessor {
  653. type_id,
  654. index,
  655. name: Some(btf.string_at(name_offset.unwrap())?.to_string()),
  656. }];
  657. AccessSpec {
  658. btf,
  659. root_type_id,
  660. relocation,
  661. parts,
  662. accessors,
  663. bit_offset: 0,
  664. }
  665. }
  666. _ => {
  667. return Err(RelocationError::InvalidRelocationKindForType {
  668. relocation_number: relocation.number,
  669. relocation_kind: format!("{:?}", relocation.kind),
  670. type_kind: format!("{:?}", ty.kind()),
  671. error: "enum relocation on non-enum type",
  672. })
  673. }
  674. },
  675. RelocationKind::FieldByteOffset
  676. | RelocationKind::FieldByteSize
  677. | RelocationKind::FieldExists
  678. | RelocationKind::FieldSigned
  679. | RelocationKind::FieldLShift64
  680. | RelocationKind::FieldRShift64 => {
  681. let mut accessors = vec![Accessor {
  682. type_id,
  683. index: parts[0],
  684. name: None,
  685. }];
  686. let mut bit_offset = accessors[0].index * btf.type_size(type_id)?;
  687. for index in parts.iter().skip(1).cloned() {
  688. type_id = btf.resolve_type(type_id)?;
  689. let ty = btf.type_by_id(type_id)?;
  690. match ty {
  691. BtfType::Struct(Struct { members, .. })
  692. | BtfType::Union(Union { members, .. }) => {
  693. if index >= members.len() {
  694. return Err(RelocationError::InvalidAccessIndex {
  695. type_name: btf.err_type_name(ty),
  696. spec: spec.to_string(),
  697. index,
  698. max_index: members.len(),
  699. error: "out of bounds struct or union access",
  700. });
  701. }
  702. let member = &members[index];
  703. bit_offset += ty.member_bit_offset(member).unwrap();
  704. if member.name_offset != 0 {
  705. accessors.push(Accessor {
  706. type_id,
  707. index,
  708. name: Some(btf.string_at(member.name_offset)?.to_string()),
  709. });
  710. }
  711. type_id = member.btf_type;
  712. }
  713. BtfType::Array(Array { array, .. }) => {
  714. type_id = btf.resolve_type(array.element_type)?;
  715. let var_len = array.len == 0 && {
  716. // an array is potentially variable length if it's the last field
  717. // of the parent struct and has 0 elements
  718. let parent = accessors.last().unwrap();
  719. let parent_ty = btf.type_by_id(parent.type_id)?;
  720. match parent_ty {
  721. BtfType::Struct(s) => index == s.members.len() - 1,
  722. _ => false,
  723. }
  724. };
  725. if !var_len && index >= array.len as usize {
  726. return Err(RelocationError::InvalidAccessIndex {
  727. type_name: btf.err_type_name(ty),
  728. spec: spec.to_string(),
  729. index,
  730. max_index: array.len as usize,
  731. error: "array index out of bounds",
  732. });
  733. }
  734. accessors.push(Accessor {
  735. type_id,
  736. index,
  737. name: None,
  738. });
  739. let size = btf.type_size(type_id)?;
  740. bit_offset += index * size * 8;
  741. }
  742. rel_kind => {
  743. return Err(RelocationError::InvalidRelocationKindForType {
  744. relocation_number: relocation.number,
  745. relocation_kind: format!("{rel_kind:?}"),
  746. type_kind: format!("{:?}", ty.kind()),
  747. error: "field relocation on a type that doesn't have fields",
  748. });
  749. }
  750. };
  751. }
  752. AccessSpec {
  753. btf,
  754. root_type_id,
  755. parts,
  756. accessors,
  757. relocation,
  758. bit_offset,
  759. }
  760. }
  761. };
  762. Ok(spec)
  763. }
  764. }
  765. #[derive(Debug)]
  766. struct Accessor {
  767. type_id: u32,
  768. index: usize,
  769. name: Option<String>,
  770. }
  771. #[derive(Debug)]
  772. struct Candidate<'a> {
  773. name: String,
  774. btf: &'a Btf,
  775. _ty: &'a BtfType,
  776. type_id: u32,
  777. }
  778. #[derive(Debug)]
  779. struct ComputedRelocation {
  780. local: ComputedRelocationValue,
  781. target: Option<ComputedRelocationValue>,
  782. }
  783. #[derive(Debug)]
  784. struct ComputedRelocationValue {
  785. value: u64,
  786. size: u32,
  787. type_id: Option<u32>,
  788. }
  789. fn poison_insn(ins: &mut bpf_insn) {
  790. ins.code = (BPF_JMP | BPF_CALL) as u8;
  791. ins.set_dst_reg(0);
  792. ins.set_src_reg(0);
  793. ins.off = 0;
  794. ins.imm = 0xBAD2310;
  795. }
  796. impl ComputedRelocation {
  797. fn new(
  798. rel: &Relocation,
  799. local_spec: &AccessSpec,
  800. target_spec: Option<&AccessSpec>,
  801. ) -> Result<ComputedRelocation, RelocationError> {
  802. use RelocationKind::*;
  803. let ret = match rel.kind {
  804. FieldByteOffset | FieldByteSize | FieldExists | FieldSigned | FieldLShift64
  805. | FieldRShift64 => ComputedRelocation {
  806. local: Self::compute_field_relocation(rel, Some(local_spec))?,
  807. target: Self::compute_field_relocation(rel, target_spec).ok(),
  808. },
  809. TypeIdLocal | TypeIdTarget | TypeExists | TypeSize => ComputedRelocation {
  810. local: Self::compute_type_relocation(rel, local_spec, target_spec)?,
  811. target: Self::compute_type_relocation(rel, local_spec, target_spec).ok(),
  812. },
  813. EnumVariantExists | EnumVariantValue => ComputedRelocation {
  814. local: Self::compute_enum_relocation(rel, Some(local_spec))?,
  815. target: Self::compute_enum_relocation(rel, target_spec).ok(),
  816. },
  817. };
  818. Ok(ret)
  819. }
  820. fn apply(
  821. &self,
  822. function: &mut Function,
  823. rel: &Relocation,
  824. local_btf: &Btf,
  825. target_btf: &Btf,
  826. ) -> Result<(), RelocationError> {
  827. let instructions = &mut function.instructions;
  828. let num_instructions = instructions.len();
  829. let ins_index = (rel.ins_offset - function.section_offset) / mem::size_of::<bpf_insn>();
  830. let ins =
  831. instructions
  832. .get_mut(ins_index)
  833. .ok_or(RelocationError::InvalidInstructionIndex {
  834. index: rel.ins_offset,
  835. num_instructions,
  836. relocation_number: rel.number,
  837. })?;
  838. let target = if let Some(target) = self.target.as_ref() {
  839. target
  840. } else {
  841. let is_ld_imm64 = ins.code == (BPF_LD | BPF_DW) as u8;
  842. poison_insn(ins);
  843. if is_ld_imm64 {
  844. let next_ins = instructions.get_mut(ins_index + 1).ok_or(
  845. RelocationError::InvalidInstructionIndex {
  846. index: (ins_index + 1) * mem::size_of::<bpf_insn>(),
  847. num_instructions,
  848. relocation_number: rel.number,
  849. },
  850. )?;
  851. poison_insn(next_ins);
  852. }
  853. return Ok(());
  854. };
  855. let class = (ins.code & 0x07) as u32;
  856. let target_value = target.value;
  857. match class {
  858. BPF_ALU | BPF_ALU64 => {
  859. let src_reg = ins.src_reg();
  860. if src_reg != BPF_K as u8 {
  861. return Err(RelocationError::InvalidInstruction {
  862. relocation_number: rel.number,
  863. index: ins_index,
  864. error: format!("invalid src_reg={src_reg:x} expected {BPF_K:x}").into(),
  865. });
  866. }
  867. ins.imm = target_value as i32;
  868. }
  869. BPF_LDX | BPF_ST | BPF_STX => {
  870. if target_value > i16::MAX as u64 {
  871. return Err(RelocationError::InvalidInstruction {
  872. relocation_number: rel.number,
  873. index: ins_index,
  874. error: format!("value `{target_value}` overflows 16 bits offset field")
  875. .into(),
  876. });
  877. }
  878. ins.off = target_value as i16;
  879. if self.local.size != target.size {
  880. let local_ty = local_btf.type_by_id(self.local.type_id.unwrap())?;
  881. let target_ty = target_btf.type_by_id(target.type_id.unwrap())?;
  882. let unsigned = |info: u32| ((info >> 24) & 0x0F) & BTF_INT_SIGNED == 0;
  883. use BtfType::*;
  884. match (local_ty, target_ty) {
  885. (Ptr(_), Ptr(_)) => {}
  886. (Int(local), Int(target))
  887. if unsigned(local.data) && unsigned(target.data) => {}
  888. _ => {
  889. return Err(RelocationError::InvalidInstruction {
  890. relocation_number: rel.number,
  891. index: ins_index,
  892. error: format!(
  893. "original type {} has size {} but target type {} has size {}",
  894. err_type_name(&local_btf.err_type_name(local_ty)),
  895. self.local.size,
  896. err_type_name(&target_btf.err_type_name(target_ty)),
  897. target.size,
  898. )
  899. .into(),
  900. })
  901. }
  902. }
  903. let size = match target.size {
  904. 8 => BPF_DW,
  905. 4 => BPF_W,
  906. 2 => BPF_H,
  907. 1 => BPF_B,
  908. size => {
  909. return Err(RelocationError::InvalidInstruction {
  910. relocation_number: rel.number,
  911. index: ins_index,
  912. error: format!("invalid target size {size}").into(),
  913. })
  914. }
  915. } as u8;
  916. ins.code = ins.code & 0xE0 | size | ins.code & 0x07;
  917. }
  918. }
  919. BPF_LD => {
  920. ins.imm = target_value as i32;
  921. let next_ins = instructions.get_mut(ins_index + 1).ok_or(
  922. RelocationError::InvalidInstructionIndex {
  923. index: ins_index + 1,
  924. num_instructions,
  925. relocation_number: rel.number,
  926. },
  927. )?;
  928. next_ins.imm = (target_value >> 32) as i32;
  929. }
  930. class => {
  931. return Err(RelocationError::InvalidInstruction {
  932. relocation_number: rel.number,
  933. index: ins_index,
  934. error: format!("invalid instruction class {class:x}").into(),
  935. })
  936. }
  937. };
  938. Ok(())
  939. }
  940. fn compute_enum_relocation(
  941. rel: &Relocation,
  942. spec: Option<&AccessSpec>,
  943. ) -> Result<ComputedRelocationValue, RelocationError> {
  944. use RelocationKind::*;
  945. let value = match (rel.kind, spec) {
  946. (EnumVariantExists, spec) => spec.is_some() as u64,
  947. (EnumVariantValue, Some(spec)) => {
  948. let accessor = &spec.accessors[0];
  949. match spec.btf.type_by_id(accessor.type_id)? {
  950. BtfType::Enum(en) => {
  951. let value = en.variants[accessor.index].value;
  952. if en.is_signed() {
  953. value as i32 as u64
  954. } else {
  955. value as u64
  956. }
  957. }
  958. BtfType::Enum64(en) => {
  959. let variant = &en.variants[accessor.index];
  960. (variant.value_high as u64) << 32 | variant.value_low as u64
  961. }
  962. // candidate selection ensures that rel_kind == local_kind == target_kind
  963. _ => unreachable!(),
  964. }
  965. }
  966. _ => {
  967. return Err(RelocationError::MissingTargetDefinition {
  968. kind: rel.kind,
  969. type_id: rel.type_id,
  970. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  971. })?;
  972. }
  973. };
  974. Ok(ComputedRelocationValue {
  975. value,
  976. size: 0,
  977. type_id: None,
  978. })
  979. }
  980. fn compute_field_relocation(
  981. rel: &Relocation,
  982. spec: Option<&AccessSpec>,
  983. ) -> Result<ComputedRelocationValue, RelocationError> {
  984. use RelocationKind::*;
  985. if let FieldExists = rel.kind {
  986. // this is the bpf_preserve_field_info(member_access, FIELD_EXISTENCE) case. If we
  987. // managed to build a spec, it means the field exists.
  988. return Ok(ComputedRelocationValue {
  989. value: spec.is_some() as u64,
  990. size: 0,
  991. type_id: None,
  992. });
  993. }
  994. let spec = match spec {
  995. Some(spec) => spec,
  996. None => {
  997. return Err(RelocationError::MissingTargetDefinition {
  998. kind: rel.kind,
  999. type_id: rel.type_id,
  1000. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  1001. })?;
  1002. }
  1003. };
  1004. let accessor = spec.accessors.last().unwrap();
  1005. if accessor.name.is_none() {
  1006. // the last accessor is unnamed, meaning that this is an array access
  1007. return match rel.kind {
  1008. FieldByteOffset => Ok(ComputedRelocationValue {
  1009. value: (spec.bit_offset / 8) as u64,
  1010. size: spec.btf.type_size(accessor.type_id)? as u32,
  1011. type_id: Some(accessor.type_id),
  1012. }),
  1013. FieldByteSize => Ok(ComputedRelocationValue {
  1014. value: spec.btf.type_size(accessor.type_id)? as u64,
  1015. size: 0,
  1016. type_id: Some(accessor.type_id),
  1017. }),
  1018. rel_kind => {
  1019. let ty = spec.btf.type_by_id(accessor.type_id)?;
  1020. return Err(RelocationError::InvalidRelocationKindForType {
  1021. relocation_number: rel.number,
  1022. relocation_kind: format!("{rel_kind:?}"),
  1023. type_kind: format!("{:?}", ty.kind()),
  1024. error: "invalid relocation kind for array type",
  1025. });
  1026. }
  1027. };
  1028. }
  1029. let ty = spec.btf.type_by_id(accessor.type_id)?;
  1030. let (ll_ty, member) = match ty {
  1031. BtfType::Struct(t) => (ty, t.members.get(accessor.index).unwrap()),
  1032. BtfType::Union(t) => (ty, t.members.get(accessor.index).unwrap()),
  1033. _ => {
  1034. return Err(RelocationError::InvalidRelocationKindForType {
  1035. relocation_number: rel.number,
  1036. relocation_kind: format!("{:?}", rel.kind),
  1037. type_kind: format!("{:?}", ty.kind()),
  1038. error: "field relocation on a type that doesn't have fields",
  1039. });
  1040. }
  1041. };
  1042. let bit_off = spec.bit_offset as u32;
  1043. let member_type_id = spec.btf.resolve_type(member.btf_type)?;
  1044. let member_ty = spec.btf.type_by_id(member_type_id)?;
  1045. let mut byte_size;
  1046. let mut byte_off;
  1047. let mut bit_size = ll_ty.member_bit_field_size(member).unwrap() as u32;
  1048. let is_bitfield = bit_size > 0;
  1049. if is_bitfield {
  1050. // find out the smallest int size to load the bitfield
  1051. byte_size = member_ty.size().unwrap();
  1052. byte_off = bit_off / 8 / byte_size * byte_size;
  1053. while bit_off + bit_size - byte_off * 8 > byte_size * 8 {
  1054. if byte_size >= 8 {
  1055. // the bitfield is larger than 8 bytes!?
  1056. return Err(BtfError::InvalidTypeInfo.into());
  1057. }
  1058. byte_size *= 2;
  1059. byte_off = bit_off / 8 / byte_size * byte_size;
  1060. }
  1061. } else {
  1062. byte_size = spec.btf.type_size(member_type_id)? as u32;
  1063. bit_size = byte_size * 8;
  1064. byte_off = spec.bit_offset as u32 / 8;
  1065. }
  1066. let mut value = ComputedRelocationValue {
  1067. value: 0,
  1068. size: 0,
  1069. type_id: None,
  1070. };
  1071. #[allow(clippy::wildcard_in_or_patterns)]
  1072. match rel.kind {
  1073. FieldByteOffset => {
  1074. value.value = byte_off as u64;
  1075. if !is_bitfield {
  1076. value.size = byte_size;
  1077. value.type_id = Some(member_type_id);
  1078. }
  1079. }
  1080. FieldByteSize => {
  1081. value.value = byte_size as u64;
  1082. }
  1083. FieldSigned => match member_ty {
  1084. BtfType::Enum(en) => value.value = en.is_signed() as u64,
  1085. BtfType::Enum64(en) => value.value = en.is_signed() as u64,
  1086. BtfType::Int(i) => value.value = i.encoding() as u64 & IntEncoding::Signed as u64,
  1087. _ => (),
  1088. },
  1089. #[cfg(target_endian = "little")]
  1090. FieldLShift64 => {
  1091. value.value = 64 - (bit_off + bit_size - byte_off * 8) as u64;
  1092. }
  1093. #[cfg(target_endian = "big")]
  1094. FieldLShift64 => {
  1095. value.value = (8 - byte_size) * 8 + (bit_off - byte_off * 8);
  1096. }
  1097. FieldRShift64 => {
  1098. value.value = 64 - bit_size as u64;
  1099. }
  1100. kind @ (FieldExists | TypeIdLocal | TypeIdTarget | TypeExists | TypeSize
  1101. | EnumVariantExists | EnumVariantValue) => {
  1102. panic!("unexpected relocation kind {:?}", kind)
  1103. }
  1104. }
  1105. Ok(value)
  1106. }
  1107. fn compute_type_relocation(
  1108. rel: &Relocation,
  1109. local_spec: &AccessSpec,
  1110. target_spec: Option<&AccessSpec>,
  1111. ) -> Result<ComputedRelocationValue, RelocationError> {
  1112. use RelocationKind::*;
  1113. let value = match (rel.kind, target_spec) {
  1114. (TypeIdLocal, _) => local_spec.root_type_id as u64,
  1115. (TypeIdTarget, Some(target_spec)) => target_spec.root_type_id as u64,
  1116. (TypeExists, target_spec) => target_spec.is_some() as u64,
  1117. (TypeSize, Some(target_spec)) => {
  1118. target_spec.btf.type_size(target_spec.root_type_id)? as u64
  1119. }
  1120. _ => {
  1121. return Err(RelocationError::MissingTargetDefinition {
  1122. kind: rel.kind,
  1123. type_id: rel.type_id,
  1124. ins_index: rel.ins_offset / mem::size_of::<bpf_insn>(),
  1125. })?;
  1126. }
  1127. };
  1128. Ok(ComputedRelocationValue {
  1129. value,
  1130. size: 0,
  1131. type_id: None,
  1132. })
  1133. }
  1134. }