1 /* $NetBSD: amdgpu_mxgpu_ai.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $ */ 2 3 /* 4 * Copyright 2014 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 */ 25 26 #include <sys/cdefs.h> 27 __KERNEL_RCSID(0, "$NetBSD: amdgpu_mxgpu_ai.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $"); 28 29 #include "amdgpu.h" 30 #include "nbio/nbio_6_1_offset.h" 31 #include "nbio/nbio_6_1_sh_mask.h" 32 #include "gc/gc_9_0_offset.h" 33 #include "gc/gc_9_0_sh_mask.h" 34 #include "mp/mp_9_0_offset.h" 35 #include "soc15.h" 36 #include "vega10_ih.h" 37 #include "soc15_common.h" 38 #include "mxgpu_ai.h" 39 40 static void xgpu_ai_mailbox_send_ack(struct amdgpu_device *adev) 41 { 42 WREG8(AI_MAIBOX_CONTROL_RCV_OFFSET_BYTE, 2); 43 } 44 45 static void xgpu_ai_mailbox_set_valid(struct amdgpu_device *adev, bool val) 46 { 47 WREG8(AI_MAIBOX_CONTROL_TRN_OFFSET_BYTE, val ? 1 : 0); 48 } 49 50 /* 51 * this peek_msg could *only* be called in IRQ routine becuase in IRQ routine 52 * RCV_MSG_VALID filed of BIF_BX_PF0_MAILBOX_CONTROL must already be set to 1 53 * by host. 54 * 55 * if called no in IRQ routine, this peek_msg cannot guaranteed to return the 56 * correct value since it doesn't return the RCV_DW0 under the case that 57 * RCV_MSG_VALID is set by host. 58 */ 59 static enum idh_event xgpu_ai_mailbox_peek_msg(struct amdgpu_device *adev) 60 { 61 return RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 62 mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0)); 63 } 64 65 66 static int xgpu_ai_mailbox_rcv_msg(struct amdgpu_device *adev, 67 enum idh_event event) 68 { 69 u32 reg; 70 71 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 72 mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0)); 73 if (reg != event) 74 return -ENOENT; 75 76 xgpu_ai_mailbox_send_ack(adev); 77 78 return 0; 79 } 80 81 static uint8_t xgpu_ai_peek_ack(struct amdgpu_device *adev) { 82 return RREG8(AI_MAIBOX_CONTROL_TRN_OFFSET_BYTE) & 2; 83 } 84 85 static int xgpu_ai_poll_ack(struct amdgpu_device *adev) 86 { 87 int timeout = AI_MAILBOX_POLL_ACK_TIMEDOUT; 88 u8 reg; 89 90 do { 91 reg = RREG8(AI_MAIBOX_CONTROL_TRN_OFFSET_BYTE); 92 if (reg & 2) 93 return 0; 94 95 mdelay(5); 96 timeout -= 5; 97 } while (timeout > 1); 98 99 pr_err("Doesn't get TRN_MSG_ACK from pf in %d msec\n", AI_MAILBOX_POLL_ACK_TIMEDOUT); 100 101 return -ETIME; 102 } 103 104 static int xgpu_ai_poll_msg(struct amdgpu_device *adev, enum idh_event event) 105 { 106 int r, timeout = AI_MAILBOX_POLL_MSG_TIMEDOUT; 107 108 do { 109 r = xgpu_ai_mailbox_rcv_msg(adev, event); 110 if (!r) 111 return 0; 112 113 msleep(10); 114 timeout -= 10; 115 } while (timeout > 1); 116 117 pr_err("Doesn't get msg:%d from pf, error=%d\n", event, r); 118 119 return -ETIME; 120 } 121 122 static void xgpu_ai_mailbox_trans_msg (struct amdgpu_device *adev, 123 enum idh_request req, u32 data1, u32 data2, u32 data3) { 124 u32 reg; 125 int r; 126 uint8_t trn; 127 128 /* IMPORTANT: 129 * clear TRN_MSG_VALID valid to clear host's RCV_MSG_ACK 130 * and with host's RCV_MSG_ACK cleared hw automatically clear host's RCV_MSG_ACK 131 * which lead to VF's TRN_MSG_ACK cleared, otherwise below xgpu_ai_poll_ack() 132 * will return immediatly 133 */ 134 do { 135 xgpu_ai_mailbox_set_valid(adev, false); 136 trn = xgpu_ai_peek_ack(adev); 137 if (trn) { 138 pr_err("trn=%x ACK should not assert! wait again !\n", trn); 139 msleep(1); 140 } 141 } while(trn); 142 143 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 144 mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0)); 145 reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0, 146 MSGBUF_DATA, req); 147 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0), 148 reg); 149 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1), 150 data1); 151 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2), 152 data2); 153 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3), 154 data3); 155 156 xgpu_ai_mailbox_set_valid(adev, true); 157 158 /* start to poll ack */ 159 r = xgpu_ai_poll_ack(adev); 160 if (r) 161 pr_err("Doesn't get ack from pf, continue\n"); 162 163 xgpu_ai_mailbox_set_valid(adev, false); 164 } 165 166 static int xgpu_ai_send_access_requests(struct amdgpu_device *adev, 167 enum idh_request req) 168 { 169 int r; 170 171 xgpu_ai_mailbox_trans_msg(adev, req, 0, 0, 0); 172 173 /* start to check msg if request is idh_req_gpu_init_access */ 174 if (req == IDH_REQ_GPU_INIT_ACCESS || 175 req == IDH_REQ_GPU_FINI_ACCESS || 176 req == IDH_REQ_GPU_RESET_ACCESS) { 177 r = xgpu_ai_poll_msg(adev, IDH_READY_TO_ACCESS_GPU); 178 if (r) { 179 pr_err("Doesn't get READY_TO_ACCESS_GPU from pf, give up\n"); 180 return r; 181 } 182 /* Retrieve checksum from mailbox2 */ 183 if (req == IDH_REQ_GPU_INIT_ACCESS || req == IDH_REQ_GPU_RESET_ACCESS) { 184 adev->virt.fw_reserve.checksum_key = 185 RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 186 mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2)); 187 } 188 } 189 190 return 0; 191 } 192 193 static int xgpu_ai_request_reset(struct amdgpu_device *adev) 194 { 195 return xgpu_ai_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS); 196 } 197 198 static int xgpu_ai_request_full_gpu_access(struct amdgpu_device *adev, 199 bool init) 200 { 201 enum idh_request req; 202 203 req = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS; 204 return xgpu_ai_send_access_requests(adev, req); 205 } 206 207 static int xgpu_ai_release_full_gpu_access(struct amdgpu_device *adev, 208 bool init) 209 { 210 enum idh_request req; 211 int r = 0; 212 213 req = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS; 214 r = xgpu_ai_send_access_requests(adev, req); 215 216 return r; 217 } 218 219 static int xgpu_ai_mailbox_ack_irq(struct amdgpu_device *adev, 220 struct amdgpu_irq_src *source, 221 struct amdgpu_iv_entry *entry) 222 { 223 DRM_DEBUG("get ack intr and do nothing.\n"); 224 return 0; 225 } 226 227 static int xgpu_ai_set_mailbox_ack_irq(struct amdgpu_device *adev, 228 struct amdgpu_irq_src *source, 229 unsigned type, 230 enum amdgpu_interrupt_state state) 231 { 232 u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL)); 233 234 tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, ACK_INT_EN, 235 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0); 236 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp); 237 238 return 0; 239 } 240 241 static void xgpu_ai_mailbox_flr_work(struct work_struct *work) 242 { 243 struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); 244 struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); 245 int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT; 246 int locked; 247 248 /* block amdgpu_gpu_recover till msg FLR COMPLETE received, 249 * otherwise the mailbox msg will be ruined/reseted by 250 * the VF FLR. 251 * 252 * we can unlock the lock_reset to allow "amdgpu_job_timedout" 253 * to run gpu_recover() after FLR_NOTIFICATION_CMPL received 254 * which means host side had finished this VF's FLR. 255 */ 256 locked = mutex_trylock(&adev->lock_reset); 257 if (locked) 258 adev->in_gpu_reset = true; 259 260 do { 261 if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) 262 goto flr_done; 263 264 msleep(10); 265 timeout -= 10; 266 } while (timeout > 1); 267 268 flr_done: 269 if (locked) { 270 adev->in_gpu_reset = false; 271 mutex_unlock(&adev->lock_reset); 272 } 273 274 /* Trigger recovery for world switch failure if no TDR */ 275 if (amdgpu_device_should_recover_gpu(adev) 276 && adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT) 277 amdgpu_device_gpu_recover(adev, NULL); 278 } 279 280 static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev, 281 struct amdgpu_irq_src *src, 282 unsigned type, 283 enum amdgpu_interrupt_state state) 284 { 285 u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL)); 286 287 tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, VALID_INT_EN, 288 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0); 289 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp); 290 291 return 0; 292 } 293 294 static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev, 295 struct amdgpu_irq_src *source, 296 struct amdgpu_iv_entry *entry) 297 { 298 enum idh_event event = xgpu_ai_mailbox_peek_msg(adev); 299 300 switch (event) { 301 case IDH_FLR_NOTIFICATION: 302 if (amdgpu_sriov_runtime(adev)) 303 schedule_work(&adev->virt.flr_work); 304 break; 305 case IDH_QUERY_ALIVE: 306 xgpu_ai_mailbox_send_ack(adev); 307 break; 308 /* READY_TO_ACCESS_GPU is fetched by kernel polling, IRQ can ignore 309 * it byfar since that polling thread will handle it, 310 * other msg like flr complete is not handled here. 311 */ 312 case IDH_CLR_MSG_BUF: 313 case IDH_FLR_NOTIFICATION_CMPL: 314 case IDH_READY_TO_ACCESS_GPU: 315 default: 316 break; 317 } 318 319 return 0; 320 } 321 322 static const struct amdgpu_irq_src_funcs xgpu_ai_mailbox_ack_irq_funcs = { 323 .set = xgpu_ai_set_mailbox_ack_irq, 324 .process = xgpu_ai_mailbox_ack_irq, 325 }; 326 327 static const struct amdgpu_irq_src_funcs xgpu_ai_mailbox_rcv_irq_funcs = { 328 .set = xgpu_ai_set_mailbox_rcv_irq, 329 .process = xgpu_ai_mailbox_rcv_irq, 330 }; 331 332 void xgpu_ai_mailbox_set_irq_funcs(struct amdgpu_device *adev) 333 { 334 adev->virt.ack_irq.num_types = 1; 335 adev->virt.ack_irq.funcs = &xgpu_ai_mailbox_ack_irq_funcs; 336 adev->virt.rcv_irq.num_types = 1; 337 adev->virt.rcv_irq.funcs = &xgpu_ai_mailbox_rcv_irq_funcs; 338 } 339 340 int xgpu_ai_mailbox_add_irq_id(struct amdgpu_device *adev) 341 { 342 int r; 343 344 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_BIF, 135, &adev->virt.rcv_irq); 345 if (r) 346 return r; 347 348 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_BIF, 138, &adev->virt.ack_irq); 349 if (r) { 350 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); 351 return r; 352 } 353 354 return 0; 355 } 356 357 int xgpu_ai_mailbox_get_irq(struct amdgpu_device *adev) 358 { 359 int r; 360 361 r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0); 362 if (r) 363 return r; 364 r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0); 365 if (r) { 366 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); 367 return r; 368 } 369 370 INIT_WORK(&adev->virt.flr_work, xgpu_ai_mailbox_flr_work); 371 372 return 0; 373 } 374 375 void xgpu_ai_mailbox_put_irq(struct amdgpu_device *adev) 376 { 377 amdgpu_irq_put(adev, &adev->virt.ack_irq, 0); 378 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); 379 } 380 381 const struct amdgpu_virt_ops xgpu_ai_virt_ops = { 382 .req_full_gpu = xgpu_ai_request_full_gpu_access, 383 .rel_full_gpu = xgpu_ai_release_full_gpu_access, 384 .reset_gpu = xgpu_ai_request_reset, 385 .wait_reset = NULL, 386 .trans_msg = xgpu_ai_mailbox_trans_msg, 387 }; 388
Indexes created Sun Sep 21 20:09:37 GMT 2025