Home | History | Annotate | Line # | Download | only in xray
      1 //===-- xray_interface.cpp --------------------------------------*- C++ -*-===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file is a part of XRay, a dynamic runtime instrumentation system.
     11 //
     12 // Implementation of the API functions.
     13 //
     14 //===----------------------------------------------------------------------===//
     15 
     16 #include "xray_interface_internal.h"
     17 
     18 #include <cstdint>
     19 #include <cstdio>
     20 #include <errno.h>
     21 #include <limits>
     22 #include <string.h>
     23 #include <sys/mman.h>
     24 
     25 #if SANITIZER_FUCHSIA
     26 #include <zircon/process.h>
     27 #include <zircon/sanitizer.h>
     28 #include <zircon/status.h>
     29 #include <zircon/syscalls.h>
     30 #endif
     31 
     32 #include "sanitizer_common/sanitizer_addrhashmap.h"
     33 #include "sanitizer_common/sanitizer_common.h"
     34 
     35 #include "xray_defs.h"
     36 #include "xray_flags.h"
     37 
     38 extern __sanitizer::SpinMutex XRayInstrMapMutex;
     39 extern __sanitizer::atomic_uint8_t XRayInitialized;
     40 extern __xray::XRaySledMap XRayInstrMap;
     41 
     42 namespace __xray {
     43 
     44 #if defined(__x86_64__)
     45 static const int16_t cSledLength = 12;
     46 #elif defined(__aarch64__)
     47 static const int16_t cSledLength = 32;
     48 #elif defined(__arm__)
     49 static const int16_t cSledLength = 28;
     50 #elif SANITIZER_MIPS32
     51 static const int16_t cSledLength = 48;
     52 #elif SANITIZER_MIPS64
     53 static const int16_t cSledLength = 64;
     54 #elif defined(__powerpc64__)
     55 static const int16_t cSledLength = 8;
     56 #else
     57 #error "Unsupported CPU Architecture"
     58 #endif /* CPU architecture */
     59 
     60 // This is the function to call when we encounter the entry or exit sleds.
     61 atomic_uintptr_t XRayPatchedFunction{0};
     62 
     63 // This is the function to call from the arg1-enabled sleds/trampolines.
     64 atomic_uintptr_t XRayArgLogger{0};
     65 
     66 // This is the function to call when we encounter a custom event log call.
     67 atomic_uintptr_t XRayPatchedCustomEvent{0};
     68 
     69 // This is the function to call when we encounter a typed event log call.
     70 atomic_uintptr_t XRayPatchedTypedEvent{0};
     71 
     72 // This is the global status to determine whether we are currently
     73 // patching/unpatching.
     74 atomic_uint8_t XRayPatching{0};
     75 
     76 struct TypeDescription {
     77   uint32_t type_id;
     78   std::size_t description_string_length;
     79 };
     80 
     81 using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>;
     82 // An address map from immutable descriptors to type ids.
     83 TypeDescriptorMapType TypeDescriptorAddressMap{};
     84 
     85 atomic_uint32_t TypeEventDescriptorCounter{0};
     86 
     87 // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will
     88 // undo any successful mprotect(...) changes. This is used to make a page
     89 // writeable and executable, and upon destruction if it was successful in
     90 // doing so returns the page into a read-only and executable page.
     91 //
     92 // This is only used specifically for runtime-patching of the XRay
     93 // instrumentation points. This assumes that the executable pages are
     94 // originally read-and-execute only.
     95 class MProtectHelper {
     96   void *PageAlignedAddr;
     97   std::size_t MProtectLen;
     98   bool MustCleanup;
     99 
    100 public:
    101   explicit MProtectHelper(void *PageAlignedAddr,
    102                           std::size_t MProtectLen,
    103                           std::size_t PageSize) XRAY_NEVER_INSTRUMENT
    104       : PageAlignedAddr(PageAlignedAddr),
    105         MProtectLen(MProtectLen),
    106         MustCleanup(false) {
    107 #if SANITIZER_FUCHSIA
    108     MProtectLen = RoundUpTo(MProtectLen, PageSize);
    109 #endif
    110   }
    111 
    112   int MakeWriteable() XRAY_NEVER_INSTRUMENT {
    113 #if SANITIZER_FUCHSIA
    114     auto R = __sanitizer_change_code_protection(
    115         reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true);
    116     if (R != ZX_OK) {
    117       Report("XRay: cannot change code protection: %s\n",
    118              _zx_status_get_string(R));
    119       return -1;
    120     }
    121     MustCleanup = true;
    122     return 0;
    123 #else
    124     auto R = mprotect(PageAlignedAddr, MProtectLen,
    125                       PROT_READ | PROT_WRITE | PROT_EXEC);
    126     if (R != -1)
    127       MustCleanup = true;
    128     return R;
    129 #endif
    130   }
    131 
    132   ~MProtectHelper() XRAY_NEVER_INSTRUMENT {
    133     if (MustCleanup) {
    134 #if SANITIZER_FUCHSIA
    135       auto R = __sanitizer_change_code_protection(
    136           reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false);
    137       if (R != ZX_OK) {
    138         Report("XRay: cannot change code protection: %s\n",
    139                _zx_status_get_string(R));
    140       }
    141 #else
    142       mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC);
    143 #endif
    144     }
    145   }
    146 };
    147 
    148 namespace {
    149 
    150 bool patchSled(const XRaySledEntry &Sled, bool Enable,
    151                int32_t FuncId) XRAY_NEVER_INSTRUMENT {
    152   bool Success = false;
    153   switch (Sled.Kind) {
    154   case XRayEntryType::ENTRY:
    155     Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry);
    156     break;
    157   case XRayEntryType::EXIT:
    158     Success = patchFunctionExit(Enable, FuncId, Sled);
    159     break;
    160   case XRayEntryType::TAIL:
    161     Success = patchFunctionTailExit(Enable, FuncId, Sled);
    162     break;
    163   case XRayEntryType::LOG_ARGS_ENTRY:
    164     Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry);
    165     break;
    166   case XRayEntryType::CUSTOM_EVENT:
    167     Success = patchCustomEvent(Enable, FuncId, Sled);
    168     break;
    169   case XRayEntryType::TYPED_EVENT:
    170     Success = patchTypedEvent(Enable, FuncId, Sled);
    171     break;
    172   default:
    173     Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind));
    174     return false;
    175   }
    176   return Success;
    177 }
    178 
    179 XRayPatchingStatus patchFunction(int32_t FuncId,
    180                                  bool Enable) XRAY_NEVER_INSTRUMENT {
    181   if (!atomic_load(&XRayInitialized,
    182                                 memory_order_acquire))
    183     return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
    184 
    185   uint8_t NotPatching = false;
    186   if (!atomic_compare_exchange_strong(
    187           &XRayPatching, &NotPatching, true, memory_order_acq_rel))
    188     return XRayPatchingStatus::ONGOING; // Already patching.
    189 
    190   // Next, we look for the function index.
    191   XRaySledMap InstrMap;
    192   {
    193     SpinMutexLock Guard(&XRayInstrMapMutex);
    194     InstrMap = XRayInstrMap;
    195   }
    196 
    197   // If we don't have an index, we can't patch individual functions.
    198   if (InstrMap.Functions == 0)
    199     return XRayPatchingStatus::NOT_INITIALIZED;
    200 
    201   // FuncId must be a positive number, less than the number of functions
    202   // instrumented.
    203   if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
    204     Report("Invalid function id provided: %d\n", FuncId);
    205     return XRayPatchingStatus::FAILED;
    206   }
    207 
    208   // Now we patch ths sleds for this specific function.
    209   auto SledRange = InstrMap.SledsIndex[FuncId - 1];
    210   auto *f = SledRange.Begin;
    211   auto *e = SledRange.End;
    212 
    213   bool SucceedOnce = false;
    214   while (f != e)
    215     SucceedOnce |= patchSled(*f++, Enable, FuncId);
    216 
    217   atomic_store(&XRayPatching, false,
    218                             memory_order_release);
    219 
    220   if (!SucceedOnce) {
    221     Report("Failed patching any sled for function '%d'.", FuncId);
    222     return XRayPatchingStatus::FAILED;
    223   }
    224 
    225   return XRayPatchingStatus::SUCCESS;
    226 }
    227 
    228 // controlPatching implements the common internals of the patching/unpatching
    229 // implementation. |Enable| defines whether we're enabling or disabling the
    230 // runtime XRay instrumentation.
    231 XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
    232   if (!atomic_load(&XRayInitialized,
    233                                 memory_order_acquire))
    234     return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
    235 
    236   uint8_t NotPatching = false;
    237   if (!atomic_compare_exchange_strong(
    238           &XRayPatching, &NotPatching, true, memory_order_acq_rel))
    239     return XRayPatchingStatus::ONGOING; // Already patching.
    240 
    241   uint8_t PatchingSuccess = false;
    242   auto XRayPatchingStatusResetter =
    243       at_scope_exit([&PatchingSuccess] {
    244         if (!PatchingSuccess)
    245           atomic_store(&XRayPatching, false,
    246                                     memory_order_release);
    247       });
    248 
    249   XRaySledMap InstrMap;
    250   {
    251     SpinMutexLock Guard(&XRayInstrMapMutex);
    252     InstrMap = XRayInstrMap;
    253   }
    254   if (InstrMap.Entries == 0)
    255     return XRayPatchingStatus::NOT_INITIALIZED;
    256 
    257   uint32_t FuncId = 1;
    258   uint64_t CurFun = 0;
    259 
    260   // First we want to find the bounds for which we have instrumentation points,
    261   // and try to get as few calls to mprotect(...) as possible. We're assuming
    262   // that all the sleds for the instrumentation map are contiguous as a single
    263   // set of pages. When we do support dynamic shared object instrumentation,
    264   // we'll need to do this for each set of page load offsets per DSO loaded. For
    265   // now we're assuming we can mprotect the whole section of text between the
    266   // minimum sled address and the maximum sled address (+ the largest sled
    267   // size).
    268   auto MinSled = InstrMap.Sleds[0];
    269   auto MaxSled = InstrMap.Sleds[InstrMap.Entries - 1];
    270   for (std::size_t I = 0; I < InstrMap.Entries; I++) {
    271     const auto &Sled = InstrMap.Sleds[I];
    272     if (Sled.Address < MinSled.Address)
    273       MinSled = Sled;
    274     if (Sled.Address > MaxSled.Address)
    275       MaxSled = Sled;
    276   }
    277 
    278   const size_t PageSize = flags()->xray_page_size_override > 0
    279                               ? flags()->xray_page_size_override
    280                               : GetPageSizeCached();
    281   if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
    282     Report("System page size is not a power of two: %lld\n", PageSize);
    283     return XRayPatchingStatus::FAILED;
    284   }
    285 
    286   void *PageAlignedAddr =
    287       reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1));
    288   size_t MProtectLen =
    289       (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength;
    290   MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
    291   if (Protector.MakeWriteable() == -1) {
    292     Report("Failed mprotect: %d\n", errno);
    293     return XRayPatchingStatus::FAILED;
    294   }
    295 
    296   for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
    297     auto &Sled = InstrMap.Sleds[I];
    298     auto F = Sled.Function;
    299     if (CurFun == 0)
    300       CurFun = F;
    301     if (F != CurFun) {
    302       ++FuncId;
    303       CurFun = F;
    304     }
    305     patchSled(Sled, Enable, FuncId);
    306   }
    307   atomic_store(&XRayPatching, false,
    308                             memory_order_release);
    309   PatchingSuccess = true;
    310   return XRayPatchingStatus::SUCCESS;
    311 }
    312 
    313 XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
    314                                             bool Enable) XRAY_NEVER_INSTRUMENT {
    315   XRaySledMap InstrMap;
    316   {
    317     SpinMutexLock Guard(&XRayInstrMapMutex);
    318     InstrMap = XRayInstrMap;
    319   }
    320 
    321   // FuncId must be a positive number, less than the number of functions
    322   // instrumented.
    323   if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
    324     Report("Invalid function id provided: %d\n", FuncId);
    325     return XRayPatchingStatus::FAILED;
    326   }
    327 
    328   const size_t PageSize = flags()->xray_page_size_override > 0
    329                               ? flags()->xray_page_size_override
    330                               : GetPageSizeCached();
    331   if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
    332     Report("Provided page size is not a power of two: %lld\n", PageSize);
    333     return XRayPatchingStatus::FAILED;
    334   }
    335 
    336   // Here we compute the minumum sled and maximum sled associated with a
    337   // particular function ID.
    338   auto SledRange = InstrMap.SledsIndex[FuncId - 1];
    339   auto *f = SledRange.Begin;
    340   auto *e = SledRange.End;
    341   auto MinSled = *f;
    342   auto MaxSled = *(SledRange.End - 1);
    343   while (f != e) {
    344     if (f->Address < MinSled.Address)
    345       MinSled = *f;
    346     if (f->Address > MaxSled.Address)
    347       MaxSled = *f;
    348     ++f;
    349   }
    350 
    351   void *PageAlignedAddr =
    352       reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1));
    353   size_t MProtectLen =
    354       (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength;
    355   MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
    356   if (Protector.MakeWriteable() == -1) {
    357     Report("Failed mprotect: %d\n", errno);
    358     return XRayPatchingStatus::FAILED;
    359   }
    360   return patchFunction(FuncId, Enable);
    361 }
    362 
    363 } // namespace
    364 
    365 } // namespace __xray
    366 
    367 using namespace __xray;
    368 
    369 // The following functions are declared `extern "C" {...}` in the header, hence
    370 // they're defined in the global namespace.
    371 
    372 int __xray_set_handler(void (*entry)(int32_t,
    373                                      XRayEntryType)) XRAY_NEVER_INSTRUMENT {
    374   if (atomic_load(&XRayInitialized,
    375                                memory_order_acquire)) {
    376 
    377     atomic_store(&__xray::XRayPatchedFunction,
    378                               reinterpret_cast<uintptr_t>(entry),
    379                               memory_order_release);
    380     return 1;
    381   }
    382   return 0;
    383 }
    384 
    385 int __xray_set_customevent_handler(void (*entry)(void *, size_t))
    386     XRAY_NEVER_INSTRUMENT {
    387   if (atomic_load(&XRayInitialized,
    388                                memory_order_acquire)) {
    389     atomic_store(&__xray::XRayPatchedCustomEvent,
    390                               reinterpret_cast<uintptr_t>(entry),
    391                               memory_order_release);
    392     return 1;
    393   }
    394   return 0;
    395 }
    396 
    397 int __xray_set_typedevent_handler(void (*entry)(
    398     uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT {
    399   if (atomic_load(&XRayInitialized,
    400                                memory_order_acquire)) {
    401     atomic_store(&__xray::XRayPatchedTypedEvent,
    402                               reinterpret_cast<uintptr_t>(entry),
    403                               memory_order_release);
    404     return 1;
    405   }
    406   return 0;
    407 }
    408 
    409 int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
    410   return __xray_set_handler(nullptr);
    411 }
    412 
    413 int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
    414   return __xray_set_customevent_handler(nullptr);
    415 }
    416 
    417 int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT {
    418   return __xray_set_typedevent_handler(nullptr);
    419 }
    420 
    421 uint16_t __xray_register_event_type(
    422     const char *const event_type) XRAY_NEVER_INSTRUMENT {
    423   TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type);
    424   if (h.created()) {
    425     h->type_id = atomic_fetch_add(
    426         &TypeEventDescriptorCounter, 1, memory_order_acq_rel);
    427     h->description_string_length = strnlen(event_type, 1024);
    428   }
    429   return h->type_id;
    430 }
    431 
    432 XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
    433   return controlPatching(true);
    434 }
    435 
    436 XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
    437   return controlPatching(false);
    438 }
    439 
    440 XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
    441   return mprotectAndPatchFunction(FuncId, true);
    442 }
    443 
    444 XRayPatchingStatus
    445 __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
    446   return mprotectAndPatchFunction(FuncId, false);
    447 }
    448 
    449 int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
    450   if (!atomic_load(&XRayInitialized,
    451                                 memory_order_acquire))
    452     return 0;
    453 
    454   // A relaxed write might not be visible even if the current thread gets
    455   // scheduled on a different CPU/NUMA node.  We need to wait for everyone to
    456   // have this handler installed for consistency of collected data across CPUs.
    457   atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry),
    458                             memory_order_release);
    459   return 1;
    460 }
    461 
    462 int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
    463 
    464 uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
    465   SpinMutexLock Guard(&XRayInstrMapMutex);
    466   if (FuncId <= 0 || static_cast<size_t>(FuncId) > XRayInstrMap.Functions)
    467     return 0;
    468   return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Function
    469 // On PPC, function entries are always aligned to 16 bytes. The beginning of a
    470 // sled might be a local entry, which is always +8 based on the global entry.
    471 // Always return the global entry.
    472 #ifdef __PPC__
    473          & ~0xf
    474 #endif
    475       ;
    476 }
    477 
    478 size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
    479   SpinMutexLock Guard(&XRayInstrMapMutex);
    480   return XRayInstrMap.Functions;
    481 }
    482