1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of ThreadSanitizer (TSan), a race detector. 11 // 12 // Mac-specific libdispatch (GCD) support. 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_common/sanitizer_platform.h" 16 #if SANITIZER_MAC 17 18 #include "sanitizer_common/sanitizer_common.h" 19 #include "interception/interception.h" 20 #include "tsan_interceptors.h" 21 #include "tsan_platform.h" 22 #include "tsan_rtl.h" 23 24 #include <Block.h> 25 #include <dispatch/dispatch.h> 26 #include <pthread.h> 27 28 // DISPATCH_NOESCAPE is not defined prior to XCode 8. 29 #ifndef DISPATCH_NOESCAPE 30 #define DISPATCH_NOESCAPE 31 #endif 32 33 typedef long long_t; // NOLINT 34 35 namespace __tsan { 36 37 typedef struct { 38 dispatch_queue_t queue; 39 void *orig_context; 40 dispatch_function_t orig_work; 41 bool free_context_in_callback; 42 bool submitted_synchronously; 43 bool is_barrier_block; 44 uptr non_queue_sync_object; 45 } tsan_block_context_t; 46 47 // The offsets of different fields of the dispatch_queue_t structure, exported 48 // by libdispatch.dylib. 49 extern "C" struct dispatch_queue_offsets_s { 50 const uint16_t dqo_version; 51 const uint16_t dqo_label; 52 const uint16_t dqo_label_size; 53 const uint16_t dqo_flags; 54 const uint16_t dqo_flags_size; 55 const uint16_t dqo_serialnum; 56 const uint16_t dqo_serialnum_size; 57 const uint16_t dqo_width; 58 const uint16_t dqo_width_size; 59 const uint16_t dqo_running; 60 const uint16_t dqo_running_size; 61 const uint16_t dqo_suspend_cnt; 62 const uint16_t dqo_suspend_cnt_size; 63 const uint16_t dqo_target_queue; 64 const uint16_t dqo_target_queue_size; 65 const uint16_t dqo_priority; 66 const uint16_t dqo_priority_size; 67 } dispatch_queue_offsets; 68 69 static bool IsQueueSerial(dispatch_queue_t q) { 70 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); 71 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); 72 CHECK_NE(width, 0); 73 return width == 1; 74 } 75 76 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) { 77 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); 78 dispatch_queue_t tq = *( 79 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue); 80 return tq; 81 } 82 83 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { 84 dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source); 85 CHECK_NE(tq, 0); 86 return tq; 87 } 88 89 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, 90 dispatch_queue_t queue, 91 void *orig_context, 92 dispatch_function_t orig_work) { 93 tsan_block_context_t *new_context = 94 (tsan_block_context_t *)user_alloc_internal(thr, pc, 95 sizeof(tsan_block_context_t)); 96 new_context->queue = queue; 97 new_context->orig_context = orig_context; 98 new_context->orig_work = orig_work; 99 new_context->free_context_in_callback = true; 100 new_context->submitted_synchronously = false; 101 new_context->is_barrier_block = false; 102 new_context->non_queue_sync_object = 0; 103 return new_context; 104 } 105 106 #define GET_QUEUE_SYNC_VARS(context, q) \ 107 bool is_queue_serial = q && IsQueueSerial(q); \ 108 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \ 109 uptr serial_sync = (uptr)sync_ptr; \ 110 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \ 111 bool serial_task = context->is_barrier_block || is_queue_serial 112 113 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, 114 tsan_block_context_t *context) { 115 uptr submit_sync = (uptr)context; 116 Acquire(thr, pc, submit_sync); 117 118 dispatch_queue_t q = context->queue; 119 do { 120 GET_QUEUE_SYNC_VARS(context, q); 121 if (serial_sync) Acquire(thr, pc, serial_sync); 122 if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync); 123 124 if (q) q = GetTargetQueueFromQueue(q); 125 } while (q); 126 } 127 128 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, 129 tsan_block_context_t *context) { 130 uptr submit_sync = (uptr)context; 131 if (context->submitted_synchronously) Release(thr, pc, submit_sync); 132 133 dispatch_queue_t q = context->queue; 134 do { 135 GET_QUEUE_SYNC_VARS(context, q); 136 if (serial_task && serial_sync) Release(thr, pc, serial_sync); 137 if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync); 138 139 if (q) q = GetTargetQueueFromQueue(q); 140 } while (q); 141 } 142 143 static void dispatch_callback_wrap(void *param) { 144 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); 145 tsan_block_context_t *context = (tsan_block_context_t *)param; 146 147 dispatch_sync_pre_execute(thr, pc, context); 148 149 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 150 context->orig_work(context->orig_context); 151 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 152 153 dispatch_sync_post_execute(thr, pc, context); 154 155 if (context->free_context_in_callback) user_free(thr, pc, context); 156 } 157 158 static void invoke_block(void *param) { 159 dispatch_block_t block = (dispatch_block_t)param; 160 block(); 161 } 162 163 static void invoke_and_release_block(void *param) { 164 dispatch_block_t block = (dispatch_block_t)param; 165 block(); 166 Block_release(block); 167 } 168 169 #define DISPATCH_INTERCEPT_B(name, barrier) \ 170 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ 171 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 172 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 173 dispatch_block_t heap_block = Block_copy(block); \ 174 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 175 tsan_block_context_t *new_context = \ 176 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ 177 new_context->is_barrier_block = barrier; \ 178 Release(thr, pc, (uptr)new_context); \ 179 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 180 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ 181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 182 } 183 184 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ 185 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ 186 DISPATCH_NOESCAPE dispatch_block_t block) { \ 187 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 188 tsan_block_context_t new_context = { \ 189 q, block, &invoke_block, false, true, barrier, 0}; \ 190 Release(thr, pc, (uptr)&new_context); \ 191 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 192 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ 193 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 194 Acquire(thr, pc, (uptr)&new_context); \ 195 } 196 197 #define DISPATCH_INTERCEPT_F(name, barrier) \ 198 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 199 dispatch_function_t work) { \ 200 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 201 tsan_block_context_t *new_context = \ 202 AllocContext(thr, pc, q, context, work); \ 203 new_context->is_barrier_block = barrier; \ 204 Release(thr, pc, (uptr)new_context); \ 205 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 206 REAL(name)(q, new_context, dispatch_callback_wrap); \ 207 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 208 } 209 210 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ 211 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 212 dispatch_function_t work) { \ 213 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 214 tsan_block_context_t new_context = { \ 215 q, context, work, false, true, barrier, 0}; \ 216 Release(thr, pc, (uptr)&new_context); \ 217 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 218 REAL(name)(q, &new_context, dispatch_callback_wrap); \ 219 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 220 Acquire(thr, pc, (uptr)&new_context); \ 221 } 222 223 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new 224 // context, which is used to synchronize (we release the context before 225 // submitting, and the callback acquires it before executing the original 226 // callback). 227 DISPATCH_INTERCEPT_B(dispatch_async, false) 228 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true) 229 DISPATCH_INTERCEPT_F(dispatch_async_f, false) 230 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true) 231 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false) 232 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true) 233 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false) 234 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true) 235 236 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, 237 dispatch_queue_t queue, dispatch_block_t block) { 238 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); 239 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 240 dispatch_block_t heap_block = Block_copy(block); 241 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 242 tsan_block_context_t *new_context = 243 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); 244 Release(thr, pc, (uptr)new_context); 245 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 246 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); 247 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 248 } 249 250 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, 251 dispatch_queue_t queue, void *context, 252 dispatch_function_t work) { 253 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); 254 WRAP(dispatch_after)(when, queue, ^(void) { 255 work(context); 256 }); 257 } 258 259 // GCD's dispatch_once implementation has a fast path that contains a racy read 260 // and it's inlined into user's code. Furthermore, this fast path doesn't 261 // establish a proper happens-before relations between the initialization and 262 // code following the call to dispatch_once. We could deal with this in 263 // instrumented code, but there's not much we can do about it in system 264 // libraries. Let's disable the fast path (by never storing the value ~0 to 265 // predicate), so the interceptor is always called, and let's add proper release 266 // and acquire semantics. Since TSan does not see its own atomic stores, the 267 // race on predicate won't be reported - the only accesses to it that TSan sees 268 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is 269 // both a macro and a real function, we want to intercept the function, so we 270 // need to undefine the macro. 271 #undef dispatch_once 272 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, 273 DISPATCH_NOESCAPE dispatch_block_t block) { 274 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); 275 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); 276 u32 v = atomic_load(a, memory_order_acquire); 277 if (v == 0 && 278 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { 279 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 280 block(); 281 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 282 Release(thr, pc, (uptr)a); 283 atomic_store(a, 2, memory_order_release); 284 } else { 285 while (v != 2) { 286 internal_sched_yield(); 287 v = atomic_load(a, memory_order_acquire); 288 } 289 Acquire(thr, pc, (uptr)a); 290 } 291 } 292 293 #undef dispatch_once_f 294 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, 295 void *context, dispatch_function_t function) { 296 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); 297 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 298 WRAP(dispatch_once)(predicate, ^(void) { 299 function(context); 300 }); 301 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 302 } 303 304 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, 305 dispatch_semaphore_t dsema) { 306 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); 307 Release(thr, pc, (uptr)dsema); 308 return REAL(dispatch_semaphore_signal)(dsema); 309 } 310 311 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, 312 dispatch_time_t timeout) { 313 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); 314 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); 315 if (result == 0) Acquire(thr, pc, (uptr)dsema); 316 return result; 317 } 318 319 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, 320 dispatch_time_t timeout) { 321 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); 322 long_t result = REAL(dispatch_group_wait)(group, timeout); 323 if (result == 0) Acquire(thr, pc, (uptr)group); 324 return result; 325 } 326 327 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { 328 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); 329 // Acquired in the group noticifaction callback in dispatch_group_notify[_f]. 330 Release(thr, pc, (uptr)group); 331 REAL(dispatch_group_leave)(group); 332 } 333 334 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, 335 dispatch_queue_t queue, dispatch_block_t block) { 336 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); 337 dispatch_retain(group); 338 dispatch_group_enter(group); 339 __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block); 340 WRAP(dispatch_async)(queue, ^(void) { 341 block_copy(); 342 _Block_release(block_copy); 343 WRAP(dispatch_group_leave)(group); 344 dispatch_release(group); 345 }); 346 } 347 348 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, 349 dispatch_queue_t queue, void *context, 350 dispatch_function_t work) { 351 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); 352 dispatch_retain(group); 353 dispatch_group_enter(group); 354 WRAP(dispatch_async)(queue, ^(void) { 355 work(context); 356 WRAP(dispatch_group_leave)(group); 357 dispatch_release(group); 358 }); 359 } 360 361 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, 362 dispatch_queue_t q, dispatch_block_t block) { 363 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); 364 365 // To make sure the group is still available in the callback (otherwise 366 // it can be already destroyed). Will be released in the callback. 367 dispatch_retain(group); 368 369 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 370 dispatch_block_t heap_block = Block_copy(^(void) { 371 { 372 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); 373 // Released when leaving the group (dispatch_group_leave). 374 Acquire(thr, pc, (uptr)group); 375 } 376 dispatch_release(group); 377 block(); 378 }); 379 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 380 tsan_block_context_t *new_context = 381 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 382 new_context->is_barrier_block = true; 383 Release(thr, pc, (uptr)new_context); 384 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); 385 } 386 387 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, 388 dispatch_queue_t q, void *context, dispatch_function_t work) { 389 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); 390 } 391 392 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, 393 dispatch_source_t source, dispatch_block_t handler) { 394 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); 395 if (handler == nullptr) 396 return REAL(dispatch_source_set_event_handler)(source, nullptr); 397 dispatch_queue_t q = GetTargetQueueFromSource(source); 398 __block tsan_block_context_t new_context = { 399 q, handler, &invoke_block, false, false, false, 0 }; 400 dispatch_block_t new_handler = Block_copy(^(void) { 401 new_context.orig_context = handler; // To explicitly capture "handler". 402 dispatch_callback_wrap(&new_context); 403 }); 404 uptr submit_sync = (uptr)&new_context; 405 Release(thr, pc, submit_sync); 406 REAL(dispatch_source_set_event_handler)(source, new_handler); 407 Block_release(new_handler); 408 } 409 410 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, 411 dispatch_source_t source, dispatch_function_t handler) { 412 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); 413 if (handler == nullptr) 414 return REAL(dispatch_source_set_event_handler)(source, nullptr); 415 dispatch_block_t block = ^(void) { 416 handler(dispatch_get_context(source)); 417 }; 418 WRAP(dispatch_source_set_event_handler)(source, block); 419 } 420 421 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, 422 dispatch_source_t source, dispatch_block_t handler) { 423 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); 424 if (handler == nullptr) 425 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 426 dispatch_queue_t q = GetTargetQueueFromSource(source); 427 __block tsan_block_context_t new_context = { 428 q, handler, &invoke_block, false, false, false, 0}; 429 dispatch_block_t new_handler = Block_copy(^(void) { 430 new_context.orig_context = handler; // To explicitly capture "handler". 431 dispatch_callback_wrap(&new_context); 432 }); 433 uptr submit_sync = (uptr)&new_context; 434 Release(thr, pc, submit_sync); 435 REAL(dispatch_source_set_cancel_handler)(source, new_handler); 436 Block_release(new_handler); 437 } 438 439 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, 440 dispatch_source_t source, dispatch_function_t handler) { 441 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, 442 handler); 443 if (handler == nullptr) 444 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 445 dispatch_block_t block = ^(void) { 446 handler(dispatch_get_context(source)); 447 }; 448 WRAP(dispatch_source_set_cancel_handler)(source, block); 449 } 450 451 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, 452 dispatch_source_t source, dispatch_block_t handler) { 453 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, 454 handler); 455 if (handler == nullptr) 456 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 457 dispatch_queue_t q = GetTargetQueueFromSource(source); 458 __block tsan_block_context_t new_context = { 459 q, handler, &invoke_block, false, false, false, 0}; 460 dispatch_block_t new_handler = Block_copy(^(void) { 461 new_context.orig_context = handler; // To explicitly capture "handler". 462 dispatch_callback_wrap(&new_context); 463 }); 464 uptr submit_sync = (uptr)&new_context; 465 Release(thr, pc, submit_sync); 466 REAL(dispatch_source_set_registration_handler)(source, new_handler); 467 Block_release(new_handler); 468 } 469 470 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, 471 dispatch_source_t source, dispatch_function_t handler) { 472 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, 473 handler); 474 if (handler == nullptr) 475 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 476 dispatch_block_t block = ^(void) { 477 handler(dispatch_get_context(source)); 478 }; 479 WRAP(dispatch_source_set_registration_handler)(source, block); 480 } 481 482 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, 483 dispatch_queue_t queue, 484 DISPATCH_NOESCAPE void (^block)(size_t)) { 485 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); 486 487 void *parent_to_child_sync = nullptr; 488 uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync; 489 void *child_to_parent_sync = nullptr; 490 uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync; 491 492 Release(thr, pc, parent_to_child_sync_uptr); 493 void (^new_block)(size_t) = ^(size_t iteration) { 494 SCOPED_INTERCEPTOR_RAW(dispatch_apply); 495 Acquire(thr, pc, parent_to_child_sync_uptr); 496 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 497 block(iteration); 498 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 499 Release(thr, pc, child_to_parent_sync_uptr); 500 }; 501 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 502 REAL(dispatch_apply)(iterations, queue, new_block); 503 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 504 Acquire(thr, pc, child_to_parent_sync_uptr); 505 } 506 507 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, 508 dispatch_queue_t queue, void *context, 509 void (*work)(void *, size_t)) { 510 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); 511 void (^new_block)(size_t) = ^(size_t iteration) { 512 work(context, iteration); 513 }; 514 WRAP(dispatch_apply)(iterations, queue, new_block); 515 } 516 517 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) 518 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) 519 520 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, 521 size_t size, dispatch_queue_t q, dispatch_block_t destructor) { 522 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); 523 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) 524 return REAL(dispatch_data_create)(buffer, size, q, destructor); 525 526 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) 527 destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); }; 528 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) 529 destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); }; 530 531 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 532 dispatch_block_t heap_block = Block_copy(destructor); 533 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 534 tsan_block_context_t *new_context = 535 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 536 uptr submit_sync = (uptr)new_context; 537 Release(thr, pc, submit_sync); 538 return REAL(dispatch_data_create)(buffer, size, q, ^(void) { 539 dispatch_callback_wrap(new_context); 540 }); 541 } 542 543 typedef void (^fd_handler_t)(dispatch_data_t data, int error); 544 typedef void (^cleanup_handler_t)(int error); 545 546 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, 547 dispatch_queue_t q, fd_handler_t h) { 548 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); 549 __block tsan_block_context_t new_context = { 550 q, nullptr, &invoke_block, false, false, false, 0}; 551 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 552 new_context.orig_context = ^(void) { 553 h(data, error); 554 }; 555 dispatch_callback_wrap(&new_context); 556 }); 557 uptr submit_sync = (uptr)&new_context; 558 Release(thr, pc, submit_sync); 559 REAL(dispatch_read)(fd, length, q, new_h); 560 Block_release(new_h); 561 } 562 563 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, 564 dispatch_queue_t q, fd_handler_t h) { 565 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); 566 __block tsan_block_context_t new_context = { 567 q, nullptr, &invoke_block, false, false, false, 0}; 568 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 569 new_context.orig_context = ^(void) { 570 h(data, error); 571 }; 572 dispatch_callback_wrap(&new_context); 573 }); 574 uptr submit_sync = (uptr)&new_context; 575 Release(thr, pc, submit_sync); 576 REAL(dispatch_write)(fd, data, q, new_h); 577 Block_release(new_h); 578 } 579 580 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, 581 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { 582 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); 583 __block tsan_block_context_t new_context = { 584 q, nullptr, &invoke_block, false, false, false, 0}; 585 dispatch_io_handler_t new_h = 586 Block_copy(^(bool done, dispatch_data_t data, int error) { 587 new_context.orig_context = ^(void) { 588 h(done, data, error); 589 }; 590 dispatch_callback_wrap(&new_context); 591 }); 592 uptr submit_sync = (uptr)&new_context; 593 Release(thr, pc, submit_sync); 594 REAL(dispatch_io_read)(channel, offset, length, q, new_h); 595 Block_release(new_h); 596 } 597 598 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, 599 dispatch_data_t data, dispatch_queue_t q, 600 dispatch_io_handler_t h) { 601 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); 602 __block tsan_block_context_t new_context = { 603 q, nullptr, &invoke_block, false, false, false, 0}; 604 dispatch_io_handler_t new_h = 605 Block_copy(^(bool done, dispatch_data_t data, int error) { 606 new_context.orig_context = ^(void) { 607 h(done, data, error); 608 }; 609 dispatch_callback_wrap(&new_context); 610 }); 611 uptr submit_sync = (uptr)&new_context; 612 Release(thr, pc, submit_sync); 613 REAL(dispatch_io_write)(channel, offset, data, q, new_h); 614 Block_release(new_h); 615 } 616 617 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, 618 dispatch_block_t barrier) { 619 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); 620 __block tsan_block_context_t new_context = { 621 nullptr, nullptr, &invoke_block, false, false, false, 0}; 622 new_context.non_queue_sync_object = (uptr)channel; 623 new_context.is_barrier_block = true; 624 dispatch_block_t new_block = Block_copy(^(void) { 625 new_context.orig_context = ^(void) { 626 barrier(); 627 }; 628 dispatch_callback_wrap(&new_context); 629 }); 630 uptr submit_sync = (uptr)&new_context; 631 Release(thr, pc, submit_sync); 632 REAL(dispatch_io_barrier)(channel, new_block); 633 Block_release(new_block); 634 } 635 636 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, 637 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { 638 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); 639 __block dispatch_io_t new_channel = nullptr; 640 __block tsan_block_context_t new_context = { 641 q, nullptr, &invoke_block, false, false, false, 0}; 642 cleanup_handler_t new_h = Block_copy(^(int error) { 643 { 644 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 645 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 646 } 647 new_context.orig_context = ^(void) { 648 h(error); 649 }; 650 dispatch_callback_wrap(&new_context); 651 }); 652 uptr submit_sync = (uptr)&new_context; 653 Release(thr, pc, submit_sync); 654 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); 655 Block_release(new_h); 656 return new_channel; 657 } 658 659 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, 660 dispatch_io_type_t type, const char *path, int oflag, 661 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { 662 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, 663 q, h); 664 __block dispatch_io_t new_channel = nullptr; 665 __block tsan_block_context_t new_context = { 666 q, nullptr, &invoke_block, false, false, false, 0}; 667 cleanup_handler_t new_h = Block_copy(^(int error) { 668 { 669 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 670 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 671 } 672 new_context.orig_context = ^(void) { 673 h(error); 674 }; 675 dispatch_callback_wrap(&new_context); 676 }); 677 uptr submit_sync = (uptr)&new_context; 678 Release(thr, pc, submit_sync); 679 new_channel = 680 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); 681 Block_release(new_h); 682 return new_channel; 683 } 684 685 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, 686 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, 687 cleanup_handler_t h) { 688 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); 689 __block dispatch_io_t new_channel = nullptr; 690 __block tsan_block_context_t new_context = { 691 q, nullptr, &invoke_block, false, false, false, 0}; 692 cleanup_handler_t new_h = Block_copy(^(int error) { 693 { 694 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 695 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 696 } 697 new_context.orig_context = ^(void) { 698 h(error); 699 }; 700 dispatch_callback_wrap(&new_context); 701 }); 702 uptr submit_sync = (uptr)&new_context; 703 Release(thr, pc, submit_sync); 704 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); 705 Block_release(new_h); 706 return new_channel; 707 } 708 709 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, 710 dispatch_io_close_flags_t flags) { 711 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); 712 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. 713 return REAL(dispatch_io_close)(channel, flags); 714 } 715 716 // Resuming a suspended queue needs to synchronize with all subsequent 717 // executions of blocks in that queue. 718 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { 719 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o); 720 Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync 721 // in dispatch_sync_pre_execute 722 return REAL(dispatch_resume)(o); 723 } 724 725 } // namespace __tsan 726 727 #endif // SANITIZER_MAC 728
Indexes created Sat Sep 20 22:09:52 GMT 2025