1/* 2 * Copyright 2021 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22*/ 23 24#include <stdlib.h> 25#include <unistd.h> 26#include <sys/types.h> 27#include <sys/stat.h> 28#include <fcntl.h> 29#if HAVE_ALLOCA_H 30# include <alloca.h> 31#endif 32 33#include "CUnit/Basic.h" 34 35#include "amdgpu_test.h" 36#include "amdgpu_drm.h" 37#include "amdgpu_internal.h" 38#include "xf86drm.h" 39#include <pthread.h> 40 41#define GFX_COMPUTE_NOP 0xffff1000 42 43static amdgpu_device_handle device_handle; 44static uint32_t major_version; 45static uint32_t minor_version; 46static char *sysfs_remove = NULL; 47static bool do_cs; 48 49CU_BOOL suite_hotunplug_tests_enable(void) 50{ 51 CU_BOOL enable = CU_TRUE; 52 drmDevicePtr device; 53 54 if (drmGetDevice2(drm_amdgpu[0], DRM_DEVICE_GET_PCI_REVISION, &device)) { 55 printf("\n\nGPU Failed to get DRM device PCI info!\n"); 56 return CU_FALSE; 57 } 58 59 if (device->bustype != DRM_BUS_PCI) { 60 printf("\n\nGPU device is not on PCI bus!\n"); 61 amdgpu_device_deinitialize(device_handle); 62 return CU_FALSE; 63 } 64 65 if (amdgpu_device_initialize(drm_amdgpu[0], &major_version, 66 &minor_version, &device_handle)) 67 return CU_FALSE; 68 69 /* Latest tested amdgpu version to work with all the tests */ 70 if (minor_version < 46) 71 enable = false; 72 73 /* skip hotplug test on APUs */ 74 if(device_handle->dev_info.ids_flags & AMDGPU_IDS_FLAGS_FUSION) 75 enable = false; 76 77 if (amdgpu_device_deinitialize(device_handle)) 78 return CU_FALSE; 79 80 return enable; 81} 82 83int suite_hotunplug_tests_init(void) 84{ 85 /* We need to open/close device at each test manually */ 86 amdgpu_close_devices(); 87 88 return CUE_SUCCESS; 89} 90 91int suite_hotunplug_tests_clean(void) 92{ 93 94 95 return CUE_SUCCESS; 96} 97 98static int amdgpu_hotunplug_trigger(const char *pathname) 99{ 100 int fd, len; 101 102 fd = open(pathname, O_WRONLY); 103 if (fd < 0) 104 return -errno; 105 106 len = write(fd, "1", 1); 107 close(fd); 108 109 return len; 110} 111 112static int amdgpu_hotunplug_setup_test() 113{ 114 int r; 115 char *tmp_str; 116 117 if (amdgpu_open_device_on_test_index(open_render_node) < 0) { 118 printf("\n\n Failed to reopen device file!\n"); 119 return CUE_SINIT_FAILED; 120 121 122 123 } 124 125 r = amdgpu_device_initialize(drm_amdgpu[0], &major_version, 126 &minor_version, &device_handle); 127 128 if (r) { 129 if ((r == -EACCES) && (errno == EACCES)) 130 printf("\n\nError:%s. " 131 "Hint:Try to run this test program as root.", 132 strerror(errno)); 133 return CUE_SINIT_FAILED; 134 } 135 136 tmp_str = amdgpu_get_device_from_fd(drm_amdgpu[0]); 137 if (!tmp_str){ 138 printf("\n\n Device path not found!\n"); 139 return CUE_SINIT_FAILED; 140 } 141 142 sysfs_remove = realloc(tmp_str, strlen(tmp_str) * 2); 143 strcat(sysfs_remove, "/remove"); 144 145 return 0; 146} 147 148static int amdgpu_hotunplug_teardown_test() 149{ 150 if (amdgpu_device_deinitialize(device_handle)) 151 return CUE_SCLEAN_FAILED; 152 153 amdgpu_close_devices(); 154 155 if (sysfs_remove) 156 free(sysfs_remove); 157 158 return 0; 159} 160 161static inline int amdgpu_hotunplug_remove() 162{ 163 return amdgpu_hotunplug_trigger(sysfs_remove); 164} 165 166static inline int amdgpu_hotunplug_rescan() 167{ 168 return amdgpu_hotunplug_trigger("/sys/bus/pci/rescan"); 169} 170 171static int amdgpu_cs_sync(amdgpu_context_handle context, 172 unsigned int ip_type, 173 int ring, 174 unsigned int seqno) 175{ 176 struct amdgpu_cs_fence fence = { 177 .context = context, 178 .ip_type = ip_type, 179 .ring = ring, 180 .fence = seqno, 181 }; 182 uint32_t expired; 183 184 return amdgpu_cs_query_fence_status(&fence, 185 AMDGPU_TIMEOUT_INFINITE, 186 0, &expired); 187} 188 189static void *amdgpu_nop_cs() 190{ 191 amdgpu_bo_handle ib_result_handle; 192 void *ib_result_cpu; 193 uint64_t ib_result_mc_address; 194 uint32_t *ptr; 195 int i, r; 196 amdgpu_bo_list_handle bo_list; 197 amdgpu_va_handle va_handle; 198 amdgpu_context_handle context; 199 struct amdgpu_cs_request ibs_request; 200 struct amdgpu_cs_ib_info ib_info; 201 202 r = amdgpu_cs_ctx_create(device_handle, &context); 203 CU_ASSERT_EQUAL(r, 0); 204 205 r = amdgpu_bo_alloc_and_map(device_handle, 4096, 4096, 206 AMDGPU_GEM_DOMAIN_GTT, 0, 207 &ib_result_handle, &ib_result_cpu, 208 &ib_result_mc_address, &va_handle); 209 CU_ASSERT_EQUAL(r, 0); 210 211 ptr = ib_result_cpu; 212 for (i = 0; i < 16; ++i) 213 ptr[i] = GFX_COMPUTE_NOP; 214 215 r = amdgpu_bo_list_create(device_handle, 1, &ib_result_handle, NULL, &bo_list); 216 CU_ASSERT_EQUAL(r, 0); 217 218 memset(&ib_info, 0, sizeof(struct amdgpu_cs_ib_info)); 219 ib_info.ib_mc_address = ib_result_mc_address; 220 ib_info.size = 16; 221 222 memset(&ibs_request, 0, sizeof(struct amdgpu_cs_request)); 223 ibs_request.ip_type = AMDGPU_HW_IP_GFX; 224 ibs_request.ring = 0; 225 ibs_request.number_of_ibs = 1; 226 ibs_request.ibs = &ib_info; 227 ibs_request.resources = bo_list; 228 229 while (do_cs) 230 amdgpu_cs_submit(context, 0, &ibs_request, 1); 231 232 amdgpu_cs_sync(context, AMDGPU_HW_IP_GFX, 0, ibs_request.seq_no); 233 amdgpu_bo_list_destroy(bo_list); 234 amdgpu_bo_unmap_and_free(ib_result_handle, va_handle, 235 ib_result_mc_address, 4096); 236 237 amdgpu_cs_ctx_free(context); 238 239 return (void *)0; 240} 241 242static pthread_t* amdgpu_create_cs_thread() 243{ 244 int r; 245 pthread_t *thread = malloc(sizeof(*thread)); 246 if (!thread) 247 return NULL; 248 249 do_cs = true; 250 251 r = pthread_create(thread, NULL, amdgpu_nop_cs, NULL); 252 CU_ASSERT_EQUAL(r, 0); 253 254 /* Give thread enough time to start*/ 255 usleep(100000); 256 return thread; 257} 258 259static void amdgpu_destroy_cs_thread(pthread_t *thread) 260{ 261 void *status; 262 263 do_cs = false; 264 265 pthread_join(*thread, &status); 266 CU_ASSERT_EQUAL(status, 0); 267 268 free(thread); 269} 270 271 272static void amdgpu_hotunplug_test(bool with_cs) 273{ 274 int r; 275 pthread_t *thread = NULL; 276 277 r = amdgpu_hotunplug_setup_test(); 278 CU_ASSERT_EQUAL(r , 0); 279 280 if (with_cs) { 281 thread = amdgpu_create_cs_thread(); 282 CU_ASSERT_NOT_EQUAL(thread, NULL); 283 } 284 285 r = amdgpu_hotunplug_remove(); 286 CU_ASSERT_EQUAL(r > 0, 1); 287 288 if (with_cs) 289 amdgpu_destroy_cs_thread(thread); 290 291 r = amdgpu_hotunplug_teardown_test(); 292 CU_ASSERT_EQUAL(r , 0); 293 294 r = amdgpu_hotunplug_rescan(); 295 CU_ASSERT_EQUAL(r > 0, 1); 296} 297 298static void amdgpu_hotunplug_simple(void) 299{ 300 amdgpu_hotunplug_test(false); 301} 302 303static void amdgpu_hotunplug_with_cs(void) 304{ 305 amdgpu_hotunplug_test(true); 306} 307 308static void amdgpu_hotunplug_with_exported_bo(void) 309{ 310 int r; 311 uint32_t dma_buf_fd; 312 unsigned int *ptr; 313 amdgpu_bo_handle bo_handle; 314 315 struct amdgpu_bo_alloc_request request = { 316 .alloc_size = 4096, 317 .phys_alignment = 4096, 318 .preferred_heap = AMDGPU_GEM_DOMAIN_GTT, 319 .flags = 0, 320 }; 321 322 r = amdgpu_hotunplug_setup_test(); 323 CU_ASSERT_EQUAL(r , 0); 324 325 amdgpu_bo_alloc(device_handle, &request, &bo_handle); 326 CU_ASSERT_EQUAL(r, 0); 327 328 r = amdgpu_bo_export(bo_handle, amdgpu_bo_handle_type_dma_buf_fd, &dma_buf_fd); 329 CU_ASSERT_EQUAL(r, 0); 330 331 ptr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, dma_buf_fd, 0); 332 CU_ASSERT_NOT_EQUAL(ptr, MAP_FAILED); 333 334 r = amdgpu_hotunplug_remove(); 335 CU_ASSERT_EQUAL(r > 0, 1); 336 337 amdgpu_bo_free(bo_handle); 338 339 r = amdgpu_hotunplug_teardown_test(); 340 CU_ASSERT_EQUAL(r , 0); 341 342 *ptr = 0xdeafbeef; 343 344 munmap(ptr, 4096); 345 close (dma_buf_fd); 346 347 r = amdgpu_hotunplug_rescan(); 348 CU_ASSERT_EQUAL(r > 0, 1); 349} 350 351static void amdgpu_hotunplug_with_exported_fence(void) 352{ 353 amdgpu_bo_handle ib_result_handle; 354 void *ib_result_cpu; 355 uint64_t ib_result_mc_address; 356 uint32_t *ptr, sync_obj_handle, sync_obj_handle2; 357 int i, r; 358 amdgpu_bo_list_handle bo_list; 359 amdgpu_va_handle va_handle; 360 uint32_t major2, minor2; 361 amdgpu_device_handle device2; 362 amdgpu_context_handle context; 363 struct amdgpu_cs_request ibs_request; 364 struct amdgpu_cs_ib_info ib_info; 365 struct amdgpu_cs_fence fence_status = {0}; 366 int shared_fd; 367 368 r = amdgpu_hotunplug_setup_test(); 369 CU_ASSERT_EQUAL(r , 0); 370 371 r = amdgpu_device_initialize(drm_amdgpu[1], &major2, &minor2, &device2); 372 CU_ASSERT_EQUAL(r, 0); 373 374 r = amdgpu_cs_ctx_create(device_handle, &context); 375 CU_ASSERT_EQUAL(r, 0); 376 377 r = amdgpu_bo_alloc_and_map(device_handle, 4096, 4096, 378 AMDGPU_GEM_DOMAIN_GTT, 0, 379 &ib_result_handle, &ib_result_cpu, 380 &ib_result_mc_address, &va_handle); 381 CU_ASSERT_EQUAL(r, 0); 382 383 ptr = ib_result_cpu; 384 for (i = 0; i < 16; ++i) 385 ptr[i] = GFX_COMPUTE_NOP; 386 387 r = amdgpu_bo_list_create(device_handle, 1, &ib_result_handle, NULL, &bo_list); 388 CU_ASSERT_EQUAL(r, 0); 389 390 memset(&ib_info, 0, sizeof(struct amdgpu_cs_ib_info)); 391 ib_info.ib_mc_address = ib_result_mc_address; 392 ib_info.size = 16; 393 394 memset(&ibs_request, 0, sizeof(struct amdgpu_cs_request)); 395 ibs_request.ip_type = AMDGPU_HW_IP_GFX; 396 ibs_request.ring = 0; 397 ibs_request.number_of_ibs = 1; 398 ibs_request.ibs = &ib_info; 399 ibs_request.resources = bo_list; 400 401 CU_ASSERT_EQUAL(amdgpu_cs_submit(context, 0, &ibs_request, 1), 0); 402 403 fence_status.context = context; 404 fence_status.ip_type = AMDGPU_HW_IP_GFX; 405 fence_status.ip_instance = 0; 406 fence_status.fence = ibs_request.seq_no; 407 408 CU_ASSERT_EQUAL(amdgpu_cs_fence_to_handle(device_handle, &fence_status, 409 AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ, 410 &sync_obj_handle), 411 0); 412 413 CU_ASSERT_EQUAL(amdgpu_cs_export_syncobj(device_handle, sync_obj_handle, &shared_fd), 0); 414 415 CU_ASSERT_EQUAL(amdgpu_cs_import_syncobj(device2, shared_fd, &sync_obj_handle2), 0); 416 417 CU_ASSERT_EQUAL(amdgpu_cs_destroy_syncobj(device_handle, sync_obj_handle), 0); 418 419 CU_ASSERT_EQUAL(amdgpu_bo_list_destroy(bo_list), 0); 420 CU_ASSERT_EQUAL(amdgpu_bo_unmap_and_free(ib_result_handle, va_handle, 421 ib_result_mc_address, 4096), 0); 422 CU_ASSERT_EQUAL(amdgpu_cs_ctx_free(context), 0); 423 424 r = amdgpu_hotunplug_remove(); 425 CU_ASSERT_EQUAL(r > 0, 1); 426 427 CU_ASSERT_EQUAL(amdgpu_cs_syncobj_wait(device2, &sync_obj_handle2, 1, 100000000, 0, NULL), 0); 428 429 CU_ASSERT_EQUAL(amdgpu_cs_destroy_syncobj(device2, sync_obj_handle2), 0); 430 431 amdgpu_device_deinitialize(device2); 432 433 r = amdgpu_hotunplug_teardown_test(); 434 CU_ASSERT_EQUAL(r , 0); 435 436 r = amdgpu_hotunplug_rescan(); 437 CU_ASSERT_EQUAL(r > 0, 1); 438} 439 440 441CU_TestInfo hotunplug_tests[] = { 442 { "Unplug card and rescan the bus to plug it back", amdgpu_hotunplug_simple }, 443 { "Same as first test but with command submission", amdgpu_hotunplug_with_cs }, 444 { "Unplug with exported bo", amdgpu_hotunplug_with_exported_bo }, 445 { "Unplug with exported fence", amdgpu_hotunplug_with_exported_fence }, 446 CU_TEST_INFO_NULL, 447}; 448