1 /* $NetBSD: i915_memcpy.c,v 1.4 2021/12/19 11:33:49 riastradh Exp $ */ 2 3 /* 4 * Copyright 2016 Intel Corporation 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 */ 26 27 #include <sys/cdefs.h> 28 __KERNEL_RCSID(0, "$NetBSD: i915_memcpy.c,v 1.4 2021/12/19 11:33:49 riastradh Exp $"); 29 30 #include <linux/kernel.h> 31 #include <asm/fpu/api.h> 32 33 #include "i915_memcpy.h" 34 35 #include <linux/nbsd-namespace.h> 36 37 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) 38 #define CI_BUG_ON(expr) BUG_ON(expr) 39 #else 40 #define CI_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) 41 #endif 42 43 static DEFINE_STATIC_KEY_FALSE(has_movntdqa); 44 45 #ifdef CONFIG_AS_MOVNTDQA 46 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) 47 { 48 kernel_fpu_begin(); 49 50 while (len >= 4) { 51 asm("movntdqa (%0), %%xmm0\n" 52 "movntdqa 16(%0), %%xmm1\n" 53 "movntdqa 32(%0), %%xmm2\n" 54 "movntdqa 48(%0), %%xmm3\n" 55 "movaps %%xmm0, (%1)\n" 56 "movaps %%xmm1, 16(%1)\n" 57 "movaps %%xmm2, 32(%1)\n" 58 "movaps %%xmm3, 48(%1)\n" 59 :: "r" (src), "r" (dst) : "memory"); 60 src += 64; 61 dst += 64; 62 len -= 4; 63 } 64 while (len--) { 65 asm("movntdqa (%0), %%xmm0\n" 66 "movaps %%xmm0, (%1)\n" 67 :: "r" (src), "r" (dst) : "memory"); 68 src += 16; 69 dst += 16; 70 } 71 72 kernel_fpu_end(); 73 } 74 75 static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len) 76 { 77 kernel_fpu_begin(); 78 79 while (len >= 4) { 80 asm("movntdqa (%0), %%xmm0\n" 81 "movntdqa 16(%0), %%xmm1\n" 82 "movntdqa 32(%0), %%xmm2\n" 83 "movntdqa 48(%0), %%xmm3\n" 84 "movups %%xmm0, (%1)\n" 85 "movups %%xmm1, 16(%1)\n" 86 "movups %%xmm2, 32(%1)\n" 87 "movups %%xmm3, 48(%1)\n" 88 :: "r" (src), "r" (dst) : "memory"); 89 src += 64; 90 dst += 64; 91 len -= 4; 92 } 93 while (len--) { 94 asm("movntdqa (%0), %%xmm0\n" 95 "movups %%xmm0, (%1)\n" 96 :: "r" (src), "r" (dst) : "memory"); 97 src += 16; 98 dst += 16; 99 } 100 101 kernel_fpu_end(); 102 } 103 #else 104 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) {} 105 static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len) {} 106 #endif 107 108 /** 109 * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC 110 * @dst: destination pointer 111 * @src: source pointer 112 * @len: how many bytes to copy 113 * 114 * i915_memcpy_from_wc copies @len bytes from @src to @dst using 115 * non-temporal instructions where available. Note that all arguments 116 * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple 117 * of 16. 118 * 119 * To test whether accelerated reads from WC are supported, use 120 * i915_memcpy_from_wc(NULL, NULL, 0); 121 * 122 * Returns true if the copy was successful, false if the preconditions 123 * are not met. 124 */ 125 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len) 126 { 127 if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15)) 128 return false; 129 130 if (static_branch_likely(&has_movntdqa)) { 131 if (likely(len)) 132 __memcpy_ntdqa(dst, src, len >> 4); 133 return true; 134 } 135 136 return false; 137 } 138 139 /** 140 * i915_unaligned_memcpy_from_wc: perform a mostly accelerated read from WC 141 * @dst: destination pointer 142 * @src: source pointer 143 * @len: how many bytes to copy 144 * 145 * Like i915_memcpy_from_wc(), the unaligned variant copies @len bytes from 146 * @src to @dst using * non-temporal instructions where available, but 147 * accepts that its arguments may not be aligned, but are valid for the 148 * potential 16-byte read past the end. 149 */ 150 void i915_unaligned_memcpy_from_wc(void *dst, void *src, unsigned long len) 151 { 152 unsigned long addr; 153 154 CI_BUG_ON(!i915_has_memcpy_from_wc()); 155 156 addr = (unsigned long)src; 157 if (!IS_ALIGNED(addr, 16)) { 158 unsigned long x = min(ALIGN(addr, 16) - addr, len); 159 160 memcpy(dst, src, x); 161 162 len -= x; 163 dst += x; 164 src += x; 165 } 166 167 if (likely(len)) 168 __memcpy_ntdqu(dst, src, DIV_ROUND_UP(len, 16)); 169 } 170 171 void i915_memcpy_init_early(struct drm_i915_private *dev_priv) 172 { 173 #ifdef CONFIG_AS_MOVNTDQA 174 /* 175 * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions 176 * emulation. So don't enable movntdqa in hypervisor guest. 177 */ 178 if (static_cpu_has(X86_FEATURE_XMM4_1) && 179 !boot_cpu_has(X86_FEATURE_HYPERVISOR)) 180 static_branch_enable(&has_movntdqa); 181 #endif 182 } 183
Indexes created Sat Sep 20 22:09:52 GMT 2025