1 /* $NetBSD: subr_kobj.c,v 1.80 2026/07/17 14:37:18 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software developed for The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1998-2000 Doug Rabson 34 * Copyright (c) 2004 Peter Wemm 35 * All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 46 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 49 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 56 * SUCH DAMAGE. 57 */ 58 59 /* 60 * Kernel loader for ELF objects. 61 * 62 * TODO: adjust kmem_alloc() calls to avoid needless fragmentation. 63 */ 64 65 #include <sys/cdefs.h> 66 __KERNEL_RCSID(0, "$NetBSD: subr_kobj.c,v 1.80 2026/07/17 14:37:18 thorpej Exp $"); 67 68 #ifdef _KERNEL_OPT 69 #include "opt_modular.h" 70 #endif 71 72 #include <sys/kobj_impl.h> 73 #include <sys/sdt.h> 74 75 #ifdef MODULAR 76 77 #include <sys/param.h> 78 79 #include <sys/kernel.h> 80 #include <sys/kmem.h> 81 #include <sys/ksyms.h> 82 #include <sys/module.h> 83 #include <sys/proc.h> 84 85 #include <uvm/uvm_extern.h> 86 87 #define kobj_error(_kobj, ...) \ 88 kobj_out(__func__, __LINE__, _kobj, __VA_ARGS__) 89 90 static int kobj_relocate(kobj_t, bool); 91 static int kobj_checksyms(kobj_t, bool); 92 static void kobj_out(const char *, int, kobj_t, const char *, ...) 93 __printflike(4, 5); 94 static void kobj_jettison(kobj_t); 95 static void kobj_free(kobj_t, void *, size_t); 96 static void kobj_close(kobj_t); 97 static int kobj_read_mem(kobj_t, void **, size_t, off_t, bool); 98 static void kobj_close_mem(kobj_t); 99 100 /* 101 * kobj_load_mem: 102 * 103 * Load an object already resident in memory. If size is not -1, 104 * the complete size of the object is known. 105 */ 106 int 107 kobj_load_mem(kobj_t *kop, const char *name, void *base, ssize_t size) 108 { 109 kobj_t ko; 110 111 ko = kmem_zalloc(sizeof(*ko), KM_SLEEP); 112 ko->ko_type = KT_MEMORY; 113 kobj_setname(ko, name); 114 ko->ko_source = base; 115 ko->ko_memsize = size; 116 ko->ko_read = kobj_read_mem; 117 ko->ko_close = kobj_close_mem; 118 119 *kop = ko; 120 return kobj_load(ko); 121 } 122 123 /* 124 * kobj_close: 125 * 126 * Close an open ELF object. 127 */ 128 static void 129 kobj_close(kobj_t ko) 130 { 131 132 if (ko->ko_source == NULL) { 133 return; 134 } 135 136 ko->ko_close(ko); 137 ko->ko_source = NULL; 138 } 139 140 static void 141 kobj_close_mem(kobj_t ko) 142 { 143 144 return; 145 } 146 147 /* 148 * kobj_load: 149 * 150 * Load an ELF object and prepare to link into the running kernel 151 * image. 152 */ 153 int 154 kobj_load(kobj_t ko) 155 { 156 Elf_Ehdr *hdr; 157 Elf_Shdr *shdr; 158 Elf_Sym *es; 159 vaddr_t map_text_base; 160 vaddr_t map_data_base; 161 vaddr_t map_rodata_base; 162 size_t map_text_size; 163 size_t map_data_size; 164 size_t map_rodata_size; 165 int error; 166 int symtabindex; 167 int symstrindex; 168 int nsym; 169 int pb, rl, ra; 170 int alignmask; 171 int i, j; 172 void *addr; 173 174 KASSERT(ko->ko_type != KT_UNSET); 175 KASSERT(ko->ko_source != NULL); 176 177 shdr = NULL; 178 error = 0; 179 hdr = NULL; 180 181 /* 182 * Read the elf header from the file. 183 */ 184 error = ko->ko_read(ko, (void **)&hdr, sizeof(*hdr), 0, true); 185 if (error != 0) { 186 kobj_error(ko, "read failed %d", error); 187 goto out; 188 } 189 if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0) { 190 kobj_error(ko, "not an ELF object"); 191 error = SET_ERROR(ENOEXEC); 192 goto out; 193 } 194 195 if (hdr->e_ident[EI_VERSION] != EV_CURRENT || 196 hdr->e_version != EV_CURRENT) { 197 kobj_error(ko, "unsupported file version %d", 198 hdr->e_ident[EI_VERSION]); 199 error = SET_ERROR(ENOEXEC); 200 goto out; 201 } 202 if (hdr->e_type != ET_REL) { 203 kobj_error(ko, "unsupported file type %d", hdr->e_type); 204 error = SET_ERROR(ENOEXEC); 205 goto out; 206 } 207 switch (hdr->e_machine) { 208 #if ELFSIZE == 32 209 ELF32_MACHDEP_ID_CASES 210 #elif ELFSIZE == 64 211 ELF64_MACHDEP_ID_CASES 212 #else 213 #error not defined 214 #endif 215 default: 216 kobj_error(ko, "unsupported machine %d", hdr->e_machine); 217 error = SET_ERROR(ENOEXEC); 218 goto out; 219 } 220 221 /* 222 * On some architectures, we may need to filter objects based 223 * on their flags. 224 */ 225 #if (ELFSIZE == 32) && defined(ELF32_MACHDEP_FLAGS_OK) 226 #define FLAGS_OK(x) ELF32_MACHDEP_FLAGS_OK(x) 227 #elif (ELFSIZE == 64) && defined(ELF64_MACHDEP_FLAGS_OK) 228 #define FLAGS_OK(x) ELF64_MACHDEP_FLAGS_OK(x) 229 #endif 230 #ifdef FLAGS_OK 231 if (! FLAGS_OK(hdr->e_flags)) { 232 kobj_error(ko, "unsupported flags 0x%"PRIxMAX, 233 (uintmax_t)hdr->e_flags); 234 error = SET_ERROR(ENOEXEC); 235 goto out; 236 } 237 #endif 238 #undef FLAGS_OK 239 240 ko->ko_nprogtab = 0; 241 ko->ko_shdr = 0; 242 ko->ko_nrel = 0; 243 ko->ko_nrela = 0; 244 245 /* 246 * Allocate and read in the section header. 247 */ 248 if (hdr->e_shnum == 0 || hdr->e_shnum > ELF_MAXSHNUM || 249 hdr->e_shoff == 0 || hdr->e_shentsize != sizeof(Elf_Shdr)) { 250 kobj_error(ko, "bad sizes"); 251 error = SET_ERROR(ENOEXEC); 252 goto out; 253 } 254 ko->ko_shdrsz = hdr->e_shnum * sizeof(Elf_Shdr); 255 error = ko->ko_read(ko, (void **)&shdr, ko->ko_shdrsz, hdr->e_shoff, 256 true); 257 if (error != 0) { 258 kobj_error(ko, "read failed %d", error); 259 goto out; 260 } 261 ko->ko_shdr = shdr; 262 263 /* 264 * Scan the section header for information and table sizing. 265 */ 266 nsym = 0; 267 symtabindex = symstrindex = -1; 268 for (i = 0; i < hdr->e_shnum; i++) { 269 switch (shdr[i].sh_type) { 270 case SHT_PROGBITS: 271 case SHT_NOBITS: 272 ko->ko_nprogtab++; 273 break; 274 case SHT_SYMTAB: 275 nsym++; 276 symtabindex = i; 277 symstrindex = shdr[i].sh_link; 278 break; 279 case SHT_REL: 280 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 281 continue; 282 ko->ko_nrel++; 283 break; 284 case SHT_RELA: 285 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 286 continue; 287 ko->ko_nrela++; 288 break; 289 case SHT_STRTAB: 290 break; 291 } 292 } 293 if (ko->ko_nprogtab == 0) { 294 kobj_error(ko, "file has no contents"); 295 error = SET_ERROR(ENOEXEC); 296 goto out; 297 } 298 if (nsym != 1) { 299 /* Only allow one symbol table for now */ 300 kobj_error(ko, "file has no valid symbol table"); 301 error = SET_ERROR(ENOEXEC); 302 goto out; 303 } 304 KASSERT(symtabindex != -1); 305 KASSERT(symstrindex != -1); 306 307 if (symstrindex == SHN_UNDEF || symstrindex >= hdr->e_shnum || 308 shdr[symstrindex].sh_type != SHT_STRTAB) { 309 kobj_error(ko, "file has invalid symbol strings"); 310 error = SET_ERROR(ENOEXEC); 311 goto out; 312 } 313 314 /* 315 * Allocate space for tracking the load chunks. 316 */ 317 if (ko->ko_nprogtab != 0) { 318 ko->ko_progtab = kmem_zalloc(ko->ko_nprogtab * 319 sizeof(*ko->ko_progtab), KM_SLEEP); 320 if (ko->ko_progtab == NULL) { 321 error = SET_ERROR(ENOMEM); 322 kobj_error(ko, "out of memory"); 323 goto out; 324 } 325 } 326 if (ko->ko_nrel != 0) { 327 ko->ko_reltab = kmem_zalloc(ko->ko_nrel * 328 sizeof(*ko->ko_reltab), KM_SLEEP); 329 if (ko->ko_reltab == NULL) { 330 error = SET_ERROR(ENOMEM); 331 kobj_error(ko, "out of memory"); 332 goto out; 333 } 334 } 335 if (ko->ko_nrela != 0) { 336 ko->ko_relatab = kmem_zalloc(ko->ko_nrela * 337 sizeof(*ko->ko_relatab), KM_SLEEP); 338 if (ko->ko_relatab == NULL) { 339 error = SET_ERROR(ENOMEM); 340 kobj_error(ko, "out of memory"); 341 goto out; 342 } 343 } 344 345 /* 346 * Allocate space for and load the symbol table. 347 */ 348 ko->ko_symcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym); 349 if (ko->ko_symcnt == 0) { 350 kobj_error(ko, "no symbol table"); 351 error = SET_ERROR(ENOEXEC); 352 goto out; 353 } 354 error = ko->ko_read(ko, (void **)&ko->ko_symtab, 355 ko->ko_symcnt * sizeof(Elf_Sym), 356 shdr[symtabindex].sh_offset, true); 357 if (error != 0) { 358 kobj_error(ko, "read failed %d", error); 359 goto out; 360 } 361 362 /* 363 * Allocate space for and load the symbol strings. 364 */ 365 ko->ko_strtabsz = shdr[symstrindex].sh_size; 366 if (ko->ko_strtabsz == 0) { 367 kobj_error(ko, "no symbol strings"); 368 error = SET_ERROR(ENOEXEC); 369 goto out; 370 } 371 error = ko->ko_read(ko, (void *)&ko->ko_strtab, ko->ko_strtabsz, 372 shdr[symstrindex].sh_offset, true); 373 if (error != 0) { 374 kobj_error(ko, "read failed %d", error); 375 goto out; 376 } 377 378 /* 379 * Adjust module symbol namespace, if necessary (e.g. with rump) 380 */ 381 error = kobj_renamespace(ko->ko_symtab, ko->ko_symcnt, 382 &ko->ko_strtab, &ko->ko_strtabsz); 383 if (error != 0) { 384 kobj_error(ko, "renamespace failed %d", error); 385 goto out; 386 } 387 388 /* 389 * Do we have a string table for the section names? 390 */ 391 if (hdr->e_shstrndx != SHN_UNDEF) { 392 if (hdr->e_shstrndx >= hdr->e_shnum) { 393 kobj_error(ko, "bad shstrndx"); 394 error = SET_ERROR(ENOEXEC); 395 goto out; 396 } 397 if (shdr[hdr->e_shstrndx].sh_size != 0 && 398 shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) { 399 ko->ko_shstrtabsz = shdr[hdr->e_shstrndx].sh_size; 400 error = ko->ko_read(ko, (void **)&ko->ko_shstrtab, 401 shdr[hdr->e_shstrndx].sh_size, 402 shdr[hdr->e_shstrndx].sh_offset, true); 403 if (error != 0) { 404 kobj_error(ko, "read failed %d", error); 405 goto out; 406 } 407 } 408 } 409 410 /* 411 * Size up code/data(progbits) and bss(nobits). 412 */ 413 alignmask = 0; 414 map_text_size = 0; 415 map_data_size = 0; 416 map_rodata_size = 0; 417 for (i = 0; i < hdr->e_shnum; i++) { 418 if (shdr[i].sh_type != SHT_PROGBITS && 419 shdr[i].sh_type != SHT_NOBITS) 420 continue; 421 alignmask = shdr[i].sh_addralign - 1; 422 if ((shdr[i].sh_flags & SHF_EXECINSTR)) { 423 map_text_size += alignmask; 424 map_text_size &= ~alignmask; 425 map_text_size += shdr[i].sh_size; 426 } else if (!(shdr[i].sh_flags & SHF_WRITE)) { 427 map_rodata_size += alignmask; 428 map_rodata_size &= ~alignmask; 429 map_rodata_size += shdr[i].sh_size; 430 } else { 431 map_data_size += alignmask; 432 map_data_size &= ~alignmask; 433 map_data_size += shdr[i].sh_size; 434 } 435 } 436 437 if (map_text_size == 0) { 438 kobj_error(ko, "no text"); 439 error = SET_ERROR(ENOEXEC); 440 goto out; 441 } 442 443 if (map_data_size != 0) { 444 map_data_base = uvm_km_alloc(module_map, round_page(map_data_size), 445 0, UVM_KMF_WIRED); 446 if (map_data_base == 0) { 447 kobj_error(ko, "out of memory"); 448 error = SET_ERROR(ENOMEM); 449 goto out; 450 } 451 ko->ko_data_address = map_data_base; 452 ko->ko_data_size = map_data_size; 453 } else { 454 map_data_base = 0; 455 ko->ko_data_address = 0; 456 ko->ko_data_size = 0; 457 } 458 459 if (map_rodata_size != 0) { 460 map_rodata_base = uvm_km_alloc(module_map, round_page(map_rodata_size), 461 0, UVM_KMF_WIRED); 462 if (map_rodata_base == 0) { 463 kobj_error(ko, "out of memory"); 464 error = SET_ERROR(ENOMEM); 465 goto out; 466 } 467 ko->ko_rodata_address = map_rodata_base; 468 ko->ko_rodata_size = map_rodata_size; 469 } else { 470 map_rodata_base = 0; 471 ko->ko_rodata_address = 0; 472 ko->ko_rodata_size = 0; 473 } 474 475 map_text_base = uvm_km_alloc(module_map, round_page(map_text_size), 476 0, UVM_KMF_WIRED | UVM_KMF_EXEC); 477 if (map_text_base == 0) { 478 kobj_error(ko, "out of memory"); 479 error = SET_ERROR(ENOMEM); 480 goto out; 481 } 482 ko->ko_text_address = map_text_base; 483 ko->ko_text_size = map_text_size; 484 485 /* 486 * Now load code/data(progbits), zero bss(nobits), allocate space 487 * for and load relocs 488 */ 489 pb = 0; 490 rl = 0; 491 ra = 0; 492 alignmask = 0; 493 for (i = 0; i < hdr->e_shnum; i++) { 494 switch (shdr[i].sh_type) { 495 case SHT_PROGBITS: 496 case SHT_NOBITS: 497 alignmask = shdr[i].sh_addralign - 1; 498 if ((shdr[i].sh_flags & SHF_EXECINSTR)) { 499 map_text_base += alignmask; 500 map_text_base &= ~alignmask; 501 addr = (void *)map_text_base; 502 map_text_base += shdr[i].sh_size; 503 } else if (!(shdr[i].sh_flags & SHF_WRITE)) { 504 map_rodata_base += alignmask; 505 map_rodata_base &= ~alignmask; 506 addr = (void *)map_rodata_base; 507 map_rodata_base += shdr[i].sh_size; 508 } else { 509 map_data_base += alignmask; 510 map_data_base &= ~alignmask; 511 addr = (void *)map_data_base; 512 map_data_base += shdr[i].sh_size; 513 } 514 515 ko->ko_progtab[pb].addr = addr; 516 if (shdr[i].sh_type == SHT_PROGBITS) { 517 ko->ko_progtab[pb].name = "<<PROGBITS>>"; 518 error = ko->ko_read(ko, &addr, 519 shdr[i].sh_size, shdr[i].sh_offset, false); 520 if (error != 0) { 521 kobj_error(ko, "read failed %d", error); 522 goto out; 523 } 524 } else { /* SHT_NOBITS */ 525 ko->ko_progtab[pb].name = "<<NOBITS>>"; 526 memset(addr, 0, shdr[i].sh_size); 527 } 528 529 ko->ko_progtab[pb].size = shdr[i].sh_size; 530 ko->ko_progtab[pb].sec = i; 531 if (ko->ko_shstrtab != NULL && shdr[i].sh_name != 0) { 532 ko->ko_progtab[pb].name = 533 ko->ko_shstrtab + shdr[i].sh_name; 534 } 535 536 /* Update all symbol values with the offset. */ 537 for (j = 0; j < ko->ko_symcnt; j++) { 538 es = &ko->ko_symtab[j]; 539 if (es->st_shndx != i) { 540 continue; 541 } 542 es->st_value += (Elf_Addr)addr; 543 } 544 pb++; 545 break; 546 case SHT_REL: 547 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 548 break; 549 ko->ko_reltab[rl].size = shdr[i].sh_size; 550 ko->ko_reltab[rl].size -= 551 shdr[i].sh_size % sizeof(Elf_Rel); 552 if (ko->ko_reltab[rl].size != 0) { 553 ko->ko_reltab[rl].nrel = 554 shdr[i].sh_size / sizeof(Elf_Rel); 555 ko->ko_reltab[rl].sec = shdr[i].sh_info; 556 error = ko->ko_read(ko, 557 (void **)&ko->ko_reltab[rl].rel, 558 ko->ko_reltab[rl].size, 559 shdr[i].sh_offset, true); 560 if (error != 0) { 561 kobj_error(ko, "read failed %d", 562 error); 563 goto out; 564 } 565 } 566 rl++; 567 break; 568 case SHT_RELA: 569 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 570 break; 571 ko->ko_relatab[ra].size = shdr[i].sh_size; 572 ko->ko_relatab[ra].size -= 573 shdr[i].sh_size % sizeof(Elf_Rela); 574 if (ko->ko_relatab[ra].size != 0) { 575 ko->ko_relatab[ra].nrela = 576 shdr[i].sh_size / sizeof(Elf_Rela); 577 ko->ko_relatab[ra].sec = shdr[i].sh_info; 578 error = ko->ko_read(ko, 579 (void **)&ko->ko_relatab[ra].rela, 580 shdr[i].sh_size, 581 shdr[i].sh_offset, true); 582 if (error != 0) { 583 kobj_error(ko, "read failed %d", error); 584 goto out; 585 } 586 } 587 ra++; 588 break; 589 default: 590 break; 591 } 592 } 593 if (pb != ko->ko_nprogtab) { 594 panic("%s:%d: %s: lost progbits", __func__, __LINE__, 595 ko->ko_name); 596 } 597 if (rl != ko->ko_nrel) { 598 panic("%s:%d: %s: lost rel", __func__, __LINE__, 599 ko->ko_name); 600 } 601 if (ra != ko->ko_nrela) { 602 panic("%s:%d: %s: lost rela", __func__, __LINE__, 603 ko->ko_name); 604 } 605 if (map_text_base != ko->ko_text_address + map_text_size) { 606 panic("%s:%d: %s: map_text_base 0x%lx != address %lx " 607 "+ map_text_size %ld (0x%lx)\n", 608 __func__, __LINE__, ko->ko_name, (long)map_text_base, 609 (long)ko->ko_text_address, (long)map_text_size, 610 (long)ko->ko_text_address + map_text_size); 611 } 612 if (map_data_base != ko->ko_data_address + map_data_size) { 613 panic("%s:%d: %s: map_data_base 0x%lx != address %lx " 614 "+ map_data_size %ld (0x%lx)\n", 615 __func__, __LINE__, ko->ko_name, (long)map_data_base, 616 (long)ko->ko_data_address, (long)map_data_size, 617 (long)ko->ko_data_address + map_data_size); 618 } 619 if (map_rodata_base != ko->ko_rodata_address + map_rodata_size) { 620 panic("%s:%d: %s: map_rodata_base 0x%lx != address %lx " 621 "+ map_rodata_size %ld (0x%lx)\n", 622 __func__, __LINE__, ko->ko_name, (long)map_rodata_base, 623 (long)ko->ko_rodata_address, (long)map_rodata_size, 624 (long)ko->ko_rodata_address + map_rodata_size); 625 } 626 627 /* 628 * Perform local relocations only. Relocations relating to global 629 * symbols will be done by kobj_affix(). 630 */ 631 error = kobj_checksyms(ko, false); 632 if (error) 633 goto out; 634 635 error = kobj_relocate(ko, true); 636 if (error) 637 goto out; 638 out: 639 if (hdr != NULL) { 640 kobj_free(ko, hdr, sizeof(*hdr)); 641 } 642 kobj_close(ko); 643 if (error != 0) { 644 kobj_unload(ko); 645 } 646 647 return error; 648 } 649 650 static void 651 kobj_unload_notify(kobj_t ko, vaddr_t addr, size_t size, const char *note) 652 { 653 if (addr == 0) 654 return; 655 656 int error = kobj_machdep(ko, (void *)addr, size, false); 657 if (error) 658 kobj_error(ko, "machine dependent deinit failed (%s) %d", 659 note, error); 660 } 661 662 #define KOBJ_SEGMENT_NOTIFY(ko, what) \ 663 kobj_unload_notify(ko, (ko)->ko_ ## what ## _address, \ 664 (ko)->ko_ ## what ## _size, # what); 665 666 #define KOBJ_SEGMENT_FREE(ko, what) \ 667 do \ 668 if ((ko)->ko_ ## what ## _address != 0) \ 669 uvm_km_free(module_map, (ko)->ko_ ## what ## _address, \ 670 round_page((ko)->ko_ ## what ## _size), UVM_KMF_WIRED); \ 671 while (/*CONSTCOND*/ 0) 672 673 /* 674 * kobj_unload: 675 * 676 * Unload an object previously loaded by kobj_load(). 677 */ 678 void 679 kobj_unload(kobj_t ko) 680 { 681 kobj_close(ko); 682 kobj_jettison(ko); 683 684 685 /* 686 * Notify MD code that a module has been unloaded. 687 */ 688 if (ko->ko_loaded) { 689 KOBJ_SEGMENT_NOTIFY(ko, text); 690 KOBJ_SEGMENT_NOTIFY(ko, data); 691 KOBJ_SEGMENT_NOTIFY(ko, rodata); 692 } 693 694 KOBJ_SEGMENT_FREE(ko, text); 695 KOBJ_SEGMENT_FREE(ko, data); 696 KOBJ_SEGMENT_FREE(ko, rodata); 697 698 if (ko->ko_ksyms == true) { 699 ksyms_modunload(ko->ko_name); 700 } 701 if (ko->ko_symtab != NULL) { 702 kobj_free(ko, ko->ko_symtab, ko->ko_symcnt * sizeof(Elf_Sym)); 703 } 704 if (ko->ko_strtab != NULL) { 705 kobj_free(ko, ko->ko_strtab, ko->ko_strtabsz); 706 } 707 if (ko->ko_progtab != NULL) { 708 kobj_free(ko, ko->ko_progtab, ko->ko_nprogtab * 709 sizeof(*ko->ko_progtab)); 710 ko->ko_progtab = NULL; 711 } 712 if (ko->ko_shstrtab) { 713 kobj_free(ko, ko->ko_shstrtab, ko->ko_shstrtabsz); 714 ko->ko_shstrtab = NULL; 715 } 716 717 kmem_free(ko, sizeof(*ko)); 718 } 719 720 /* 721 * kobj_stat: 722 * 723 * Return size and load address of an object. 724 */ 725 int 726 kobj_stat(kobj_t ko, vaddr_t *address, size_t *size) 727 { 728 729 if (address != NULL) { 730 *address = ko->ko_text_address; 731 } 732 if (size != NULL) { 733 *size = ko->ko_text_size; 734 } 735 return 0; 736 } 737 738 /* 739 * kobj_affix: 740 * 741 * Set an object's name and perform global relocs. May only be 742 * called after the module and any requisite modules are loaded. 743 */ 744 int 745 kobj_affix(kobj_t ko, const char *name) 746 { 747 int error; 748 749 KASSERT(ko->ko_ksyms == false); 750 KASSERT(ko->ko_loaded == false); 751 752 kobj_setname(ko, name); 753 754 /* Cache addresses of undefined symbols. */ 755 error = kobj_checksyms(ko, true); 756 if (error) 757 goto out; 758 759 /* Now do global relocations. */ 760 error = kobj_relocate(ko, false); 761 if (error) 762 goto out; 763 764 /* 765 * Now that we know the name, register the symbol table. 766 * Do after global relocations because ksyms will pack 767 * the table. 768 */ 769 ksyms_modload(ko->ko_name, ko->ko_symtab, 770 ko->ko_symcnt * sizeof(Elf_Sym), ko->ko_strtab, ko->ko_strtabsz); 771 ko->ko_ksyms = true; 772 773 /* Jettison unneeded memory post-link. */ 774 kobj_jettison(ko); 775 776 /* 777 * Notify MD code that a module has been loaded. 778 * 779 * Most architectures use this opportunity to flush their caches. 780 */ 781 if (ko->ko_text_address != 0) { 782 error = kobj_machdep(ko, (void *)ko->ko_text_address, 783 ko->ko_text_size, true); 784 if (error) { 785 kobj_error(ko, "machine dependent init failed (text)" 786 " %d", error); 787 goto out; 788 } 789 } 790 791 if (ko->ko_data_address != 0) { 792 error = kobj_machdep(ko, (void *)ko->ko_data_address, 793 ko->ko_data_size, true); 794 if (error) { 795 kobj_error(ko, "machine dependent init failed (data)" 796 " %d", error); 797 goto out; 798 } 799 } 800 801 if (ko->ko_rodata_address != 0) { 802 error = kobj_machdep(ko, (void *)ko->ko_rodata_address, 803 ko->ko_rodata_size, true); 804 if (error) { 805 kobj_error(ko, "machine dependent init failed (rodata)" 806 " %d", error); 807 goto out; 808 } 809 } 810 811 ko->ko_loaded = true; 812 813 /* Change the memory protections, when needed. */ 814 if (ko->ko_text_address != 0) { 815 uvm_km_protect(module_map, ko->ko_text_address, 816 ko->ko_text_size, VM_PROT_READ|VM_PROT_EXECUTE); 817 } 818 if (ko->ko_rodata_address != 0) { 819 uvm_km_protect(module_map, ko->ko_rodata_address, 820 ko->ko_rodata_size, VM_PROT_READ); 821 } 822 823 /* Success! */ 824 error = 0; 825 826 out: if (error) { 827 /* If there was an error, destroy the whole object. */ 828 kobj_unload(ko); 829 } 830 return error; 831 } 832 833 /* 834 * kobj_find_section: 835 * 836 * Given a section name, search the loaded object and return 837 * virtual address if present and loaded. 838 */ 839 int 840 kobj_find_section(kobj_t ko, const char *name, void **addr, size_t *size) 841 { 842 int i; 843 844 KASSERT(ko->ko_progtab != NULL); 845 846 for (i = 0; i < ko->ko_nprogtab; i++) { 847 if (strcmp(ko->ko_progtab[i].name, name) == 0) { 848 if (addr != NULL) { 849 *addr = ko->ko_progtab[i].addr; 850 } 851 if (size != NULL) { 852 *size = ko->ko_progtab[i].size; 853 } 854 return 0; 855 } 856 } 857 858 return SET_ERROR(ENOENT); 859 } 860 861 /* 862 * kobj_jettison: 863 * 864 * Release object data not needed after performing relocations. 865 */ 866 static void 867 kobj_jettison(kobj_t ko) 868 { 869 int i; 870 871 if (ko->ko_reltab != NULL) { 872 for (i = 0; i < ko->ko_nrel; i++) { 873 if (ko->ko_reltab[i].rel) { 874 kobj_free(ko, ko->ko_reltab[i].rel, 875 ko->ko_reltab[i].size); 876 } 877 } 878 kobj_free(ko, ko->ko_reltab, ko->ko_nrel * 879 sizeof(*ko->ko_reltab)); 880 ko->ko_reltab = NULL; 881 ko->ko_nrel = 0; 882 } 883 if (ko->ko_relatab != NULL) { 884 for (i = 0; i < ko->ko_nrela; i++) { 885 if (ko->ko_relatab[i].rela) { 886 kobj_free(ko, ko->ko_relatab[i].rela, 887 ko->ko_relatab[i].size); 888 } 889 } 890 kobj_free(ko, ko->ko_relatab, ko->ko_nrela * 891 sizeof(*ko->ko_relatab)); 892 ko->ko_relatab = NULL; 893 ko->ko_nrela = 0; 894 } 895 if (ko->ko_shdr != NULL) { 896 kobj_free(ko, ko->ko_shdr, ko->ko_shdrsz); 897 ko->ko_shdr = NULL; 898 } 899 } 900 901 const Elf_Sym * 902 kobj_symbol(kobj_t ko, uintptr_t symidx) 903 { 904 905 return ko->ko_symtab + symidx; 906 } 907 908 909 /* 910 * kobj_sym_lookup: 911 * 912 * Symbol lookup function to be used when the symbol index 913 * is known (ie during relocation). 914 */ 915 int 916 kobj_sym_lookup(kobj_t ko, uintptr_t symidx, Elf_Addr *val) 917 { 918 const Elf_Sym *sym; 919 const char *symbol; 920 921 sym = ko->ko_symtab + symidx; 922 923 if (symidx == SHN_ABS || symidx == 0) { 924 *val = (uintptr_t)sym->st_value; 925 return 0; 926 } else if (symidx >= ko->ko_symcnt) { 927 /* 928 * Don't even try to lookup the symbol if the index is 929 * bogus. 930 */ 931 kobj_error(ko, "symbol index %ju out of range", 932 (uintmax_t)symidx); 933 return SET_ERROR(EINVAL); 934 } 935 936 /* Quick answer if there is a definition included. */ 937 if (sym->st_shndx != SHN_UNDEF) { 938 *val = (uintptr_t)sym->st_value; 939 return 0; 940 } 941 942 /* If we get here, then it is undefined and needs a lookup. */ 943 switch (ELF_ST_BIND(sym->st_info)) { 944 case STB_LOCAL: 945 /* Local, but undefined? huh? */ 946 kobj_error(ko, "local symbol @%ju undefined", 947 (uintmax_t)symidx); 948 return SET_ERROR(EINVAL); 949 950 case STB_GLOBAL: 951 /* Relative to Data or Function name */ 952 symbol = ko->ko_strtab + sym->st_name; 953 954 /* Force a lookup failure if the symbol name is bogus. */ 955 if (*symbol == 0) { 956 kobj_error(ko, "bad symbol @%ju name", 957 (uintmax_t)symidx); 958 return SET_ERROR(EINVAL); 959 } 960 if (sym->st_value == 0) { 961 kobj_error(ko, "%s @%ju: bad value", symbol, 962 (uintmax_t)symidx); 963 return SET_ERROR(EINVAL); 964 } 965 966 *val = (uintptr_t)sym->st_value; 967 return 0; 968 969 case STB_WEAK: 970 kobj_error(ko, "weak symbol @%ju not supported", 971 (uintmax_t)symidx); 972 return SET_ERROR(EINVAL); 973 974 default: 975 kobj_error(ko, "bad binding %#x for symbol @%ju", 976 ELF_ST_BIND(sym->st_info), (uintmax_t)symidx); 977 return SET_ERROR(EINVAL); 978 } 979 } 980 981 /* 982 * kobj_findbase: 983 * 984 * Return base address of the given section. 985 */ 986 static uintptr_t 987 kobj_findbase(kobj_t ko, int sec) 988 { 989 int i; 990 991 for (i = 0; i < ko->ko_nprogtab; i++) { 992 if (sec == ko->ko_progtab[i].sec) { 993 return (uintptr_t)ko->ko_progtab[i].addr; 994 } 995 } 996 return 0; 997 } 998 999 /* 1000 * kobj_checksyms: 1001 * 1002 * Scan symbol table for duplicates or resolve references to 1003 * external symbols. 1004 */ 1005 static int 1006 kobj_checksyms(kobj_t ko, bool undefined) 1007 { 1008 unsigned long rval; 1009 Elf_Sym *sym, *ksym, *ms; 1010 const char *name; 1011 int error; 1012 1013 error = 0; 1014 1015 for (ms = (sym = ko->ko_symtab) + ko->ko_symcnt; sym < ms; sym++) { 1016 /* Check validity of the symbol. */ 1017 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL || 1018 sym->st_name == 0) 1019 continue; 1020 if (undefined != (sym->st_shndx == SHN_UNDEF)) { 1021 continue; 1022 } 1023 1024 /* 1025 * Look it up. Don't need to lock, as it is known that 1026 * the symbol tables aren't going to change (we hold 1027 * module_lock). 1028 */ 1029 name = ko->ko_strtab + sym->st_name; 1030 if (ksyms_getval_unlocked(NULL, name, &ksym, &rval, 1031 KSYMS_EXTERN) != 0) { 1032 if (undefined) { 1033 kobj_error(ko, "symbol `%s' not found", 1034 name); 1035 error = SET_ERROR(ENOEXEC); 1036 } 1037 continue; 1038 } 1039 1040 /* Save values of undefined globals. */ 1041 if (undefined) { 1042 if (ksym->st_shndx == SHN_ABS) { 1043 sym->st_shndx = SHN_ABS; 1044 } 1045 sym->st_value = (Elf_Addr)rval; 1046 continue; 1047 } 1048 1049 /* Check (and complain) about differing values. */ 1050 if (sym->st_value == rval) { 1051 continue; 1052 } 1053 if (strcmp(name, "_bss_start") == 0 || 1054 strcmp(name, "__bss_start") == 0 || 1055 strcmp(name, "_bss_end__") == 0 || 1056 strcmp(name, "__bss_end__") == 0 || 1057 strcmp(name, "_edata") == 0 || 1058 strcmp(name, "_end") == 0 || 1059 strcmp(name, "__end") == 0 || 1060 strcmp(name, "__end__") == 0 || 1061 strncmp(name, "__start_link_set_", 17) == 0 || 1062 strncmp(name, "__stop_link_set_", 16) == 0) { 1063 continue; 1064 } 1065 kobj_error(ko, "global symbol `%s' redefined", 1066 name); 1067 error = SET_ERROR(ENOEXEC); 1068 } 1069 1070 return error; 1071 } 1072 1073 /* 1074 * kobj_relocate: 1075 * 1076 * Resolve relocations for the loaded object. 1077 */ 1078 static int 1079 kobj_relocate(kobj_t ko, bool local) 1080 { 1081 const Elf_Rel *rellim; 1082 const Elf_Rel *rel; 1083 const Elf_Rela *relalim; 1084 const Elf_Rela *rela; 1085 const Elf_Sym *sym; 1086 uintptr_t base; 1087 int i, error; 1088 uintptr_t symidx; 1089 1090 /* 1091 * Perform relocations without addend if there are any. 1092 */ 1093 for (i = 0; i < ko->ko_nrel; i++) { 1094 rel = ko->ko_reltab[i].rel; 1095 if (rel == NULL) { 1096 continue; 1097 } 1098 rellim = rel + ko->ko_reltab[i].nrel; 1099 base = kobj_findbase(ko, ko->ko_reltab[i].sec); 1100 if (base == 0) { 1101 panic("%s:%d: %s: lost base for e_reltab[%d] sec %d", 1102 __func__, __LINE__, ko->ko_name, i, 1103 ko->ko_reltab[i].sec); 1104 } 1105 for (; rel < rellim; rel++) { 1106 symidx = ELF_R_SYM(rel->r_info); 1107 if (symidx >= ko->ko_symcnt) { 1108 continue; 1109 } 1110 sym = ko->ko_symtab + symidx; 1111 /* Skip non-local symbols in the first pass (local == TRUE) */ 1112 if (local && (ELF_ST_BIND(sym->st_info) != STB_LOCAL)) { 1113 continue; 1114 } 1115 error = kobj_reloc(ko, base, rel, false, local); 1116 if (error != 0) { 1117 kobj_error(ko, "unresolved rel relocation " 1118 "@%#jx type=%d symidx=%d", 1119 (intmax_t)rel->r_offset, 1120 (int)ELF_R_TYPE(rel->r_info), 1121 (int)ELF_R_SYM(rel->r_info)); 1122 return SET_ERROR(ENOEXEC); 1123 } 1124 } 1125 } 1126 1127 /* 1128 * Perform relocations with addend if there are any. 1129 */ 1130 for (i = 0; i < ko->ko_nrela; i++) { 1131 rela = ko->ko_relatab[i].rela; 1132 if (rela == NULL) { 1133 continue; 1134 } 1135 relalim = rela + ko->ko_relatab[i].nrela; 1136 base = kobj_findbase(ko, ko->ko_relatab[i].sec); 1137 if (base == 0) { 1138 panic("%s:%d: %s: lost base for e_relatab[%d] sec %d", 1139 __func__, __LINE__, ko->ko_name, i, 1140 ko->ko_relatab[i].sec); 1141 } 1142 for (; rela < relalim; rela++) { 1143 symidx = ELF_R_SYM(rela->r_info); 1144 if (symidx >= ko->ko_symcnt) { 1145 continue; 1146 } 1147 sym = ko->ko_symtab + symidx; 1148 /* Skip non-local symbols in the first pass (local == TRUE) */ 1149 if (local && (ELF_ST_BIND(sym->st_info) != STB_LOCAL)) { 1150 continue; 1151 } 1152 error = kobj_reloc(ko, base, rela, true, local); 1153 if (error != 0) { 1154 kobj_error(ko, "unresolved rela relocation " 1155 "@%#jx type=%d symidx=%d", 1156 (intmax_t)rela->r_offset, 1157 (int)ELF_R_TYPE(rela->r_info), 1158 (int)ELF_R_SYM(rela->r_info)); 1159 return SET_ERROR(ENOEXEC); 1160 } 1161 } 1162 } 1163 1164 return 0; 1165 } 1166 1167 /* 1168 * kobj_out: 1169 * 1170 * Utility function: log an error. 1171 */ 1172 static void 1173 kobj_out(const char *fname, int lnum, kobj_t ko, const char *fmt, ...) 1174 { 1175 va_list ap; 1176 1177 printf("%s, %d: [%s]: linker error: ", fname, lnum, ko->ko_name); 1178 va_start(ap, fmt); 1179 vprintf(fmt, ap); 1180 va_end(ap); 1181 printf("\n"); 1182 } 1183 1184 static int 1185 kobj_read_mem(kobj_t ko, void **basep, size_t size, off_t off, 1186 bool allocate) 1187 { 1188 void *base = *basep; 1189 int error = 0; 1190 1191 KASSERT(ko->ko_source != NULL); 1192 1193 if (off < 0) { 1194 kobj_error(ko, "negative offset %lld", 1195 (unsigned long long)off); 1196 error = SET_ERROR(EINVAL); 1197 base = NULL; 1198 goto out; 1199 } else if (ko->ko_memsize != -1 && 1200 (size > ko->ko_memsize || off > ko->ko_memsize - size)) { 1201 kobj_error(ko, "preloaded object short"); 1202 error = SET_ERROR(EINVAL); 1203 base = NULL; 1204 goto out; 1205 } 1206 1207 if (allocate) 1208 base = kmem_alloc(size, KM_SLEEP); 1209 1210 /* Copy the section */ 1211 memcpy(base, (uint8_t *)ko->ko_source + off, size); 1212 1213 out: if (allocate) 1214 *basep = base; 1215 return error; 1216 } 1217 1218 /* 1219 * kobj_free: 1220 * 1221 * Utility function: free memory if it was allocated from the heap. 1222 */ 1223 static void 1224 kobj_free(kobj_t ko, void *base, size_t size) 1225 { 1226 1227 kmem_free(base, size); 1228 } 1229 1230 void 1231 kobj_setname(kobj_t ko, const char *name) 1232 { 1233 const char *d = name, *dots = ""; 1234 size_t len, dlen; 1235 1236 for (char *s = module_base; *d == *s; d++, s++) 1237 continue; 1238 1239 if (d == name) 1240 name = ""; 1241 else 1242 name = "%M"; 1243 dlen = strlen(d); 1244 len = dlen + strlen(name); 1245 if (len >= sizeof(ko->ko_name)) { 1246 len = (len - sizeof(ko->ko_name)) + 5; /* dots + NUL */ 1247 if (dlen >= len) { 1248 d += len; 1249 dots = "/..."; 1250 } 1251 } 1252 snprintf(ko->ko_name, sizeof(ko->ko_name), "%s%s%s", name, dots, d); 1253 } 1254 1255 #else /* MODULAR */ 1256 1257 int 1258 kobj_load_mem(kobj_t *kop, const char *name, void *base, ssize_t size) 1259 { 1260 1261 return SET_ERROR(ENOSYS); 1262 } 1263 1264 void 1265 kobj_unload(kobj_t ko) 1266 { 1267 1268 panic("not modular"); 1269 } 1270 1271 int 1272 kobj_stat(kobj_t ko, vaddr_t *base, size_t *size) 1273 { 1274 1275 return SET_ERROR(ENOSYS); 1276 } 1277 1278 int 1279 kobj_affix(kobj_t ko, const char *name) 1280 { 1281 1282 panic("not modular"); 1283 } 1284 1285 int 1286 kobj_find_section(kobj_t ko, const char *name, void **addr, size_t *size) 1287 { 1288 1289 panic("not modular"); 1290 } 1291 1292 void 1293 kobj_setname(kobj_t ko, const char *name) 1294 { 1295 1296 panic("not modular"); 1297 } 1298 1299 #endif /* MODULAR */ 1300
Indexes created Wed Aug 05 00:25:39 UTC 2026