xref: /xsrc/external/mit/MesaLib/dist/src/broadcom/common/v3d_cpu_tiling.h

/*
 * Copyright © 2017 Broadcom
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/** @file v3d_cpu_tiling.h
 *
 * Contains load/store functions common to both v3d and vc4.  The utile layout
 * stayed the same, though the way utiles get laid out has changed.
 */

static inline void
v3d_load_utile(void *cpu, uint32_t cpu_stride,
               void *gpu, uint32_t gpu_stride)
{
#if defined(V3D_BUILD_NEON) && defined(PIPE_ARCH_ARM)
        if (gpu_stride == 8) {
                __asm__ volatile (
                        /* Load from the GPU in one shot, no interleave, to
                         * d0-d7.
                         */
                        "vldm %[gpu], {q0, q1, q2, q3}\n"
                        /* Store each 8-byte line to cpu-side destination,
                         * incrementing it by the stride each time.
                         */
                        "vst1.8 d0, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d1, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d2, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d3, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d4, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d5, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d6, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d7, [%[cpu]]\n"
                        : [cpu]         "+r"(cpu)
                        : [gpu]         "r"(gpu),
                          [cpu_stride]  "r"(cpu_stride)
                        : "q0", "q1", "q2", "q3");
                return;
        } else if (gpu_stride == 16) {
                void *cpu2 = cpu + 8;
                __asm__ volatile (
                        /* Load from the GPU in one shot, no interleave, to
                         * d0-d7.
                         */
                        "vldm %[gpu], {q0, q1, q2, q3};\n"
                        /* Store each 16-byte line in 2 parts to the cpu-side
                         * destination.  (vld1 can only store one d-register
                         * at a time).
                         */
                        "vst1.8 d0, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d1, [%[cpu2]],%[cpu_stride]\n"
                        "vst1.8 d2, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d3, [%[cpu2]],%[cpu_stride]\n"
                        "vst1.8 d4, [%[cpu]], %[cpu_stride]\n"
                        "vst1.8 d5, [%[cpu2]],%[cpu_stride]\n"
                        "vst1.8 d6, [%[cpu]]\n"
                        "vst1.8 d7, [%[cpu2]]\n"
                        : [cpu]         "+r"(cpu),
                          [cpu2]        "+r"(cpu2)
                        : [gpu]         "r"(gpu),
                          [cpu_stride]  "r"(cpu_stride)
                        : "q0", "q1", "q2", "q3");
                return;
        }
#elif defined (PIPE_ARCH_AARCH64)
        if (gpu_stride == 8) {
                __asm__ volatile (
                        /* Load from the GPU in one shot, no interleave, to
                         * d0-d7.
                         */
                        "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n"
                        /* Store each 8-byte line to cpu-side destination,
                         * incrementing it by the stride each time.
                         */
                        "st1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v0.D}[1], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v1.D}[1], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v2.D}[1], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v3.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v3.D}[1], [%[cpu]]\n"
                        : [cpu]         "+r"(cpu)
                        : [gpu]         "r"(gpu),
                          [cpu_stride]  "r"(cpu_stride)
                        : "v0", "v1", "v2", "v3");
                return;
        } else if (gpu_stride == 16) {
                void *cpu2 = cpu + 8;
                __asm__ volatile (
                        /* Load from the GPU in one shot, no interleave, to
                         * d0-d7.
                         */
                        "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n"
                        /* Store each 16-byte line in 2 parts to the cpu-side
                         * destination.  (vld1 can only store one d-register
                         * at a time).
                         */
                        "st1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v0.D}[1], [%[cpu2]],%[cpu_stride]\n"
                        "st1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v1.D}[1], [%[cpu2]],%[cpu_stride]\n"
                        "st1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "st1 {v2.D}[1], [%[cpu2]],%[cpu_stride]\n"
                        "st1 {v3.D}[0], [%[cpu]]\n"
                        "st1 {v3.D}[1], [%[cpu2]]\n"
                        : [cpu]         "+r"(cpu),
                          [cpu2]        "+r"(cpu2)
                        : [gpu]         "r"(gpu),
                          [cpu_stride]  "r"(cpu_stride)
                        : "v0", "v1", "v2", "v3");
                return;
        }
#endif

        for (uint32_t gpu_offset = 0; gpu_offset < 64; gpu_offset += gpu_stride) {
                memcpy(cpu, gpu + gpu_offset, gpu_stride);
                cpu += cpu_stride;
        }
}

static inline void
v3d_store_utile(void *gpu, uint32_t gpu_stride,
                void *cpu, uint32_t cpu_stride)
{
#if defined(V3D_BUILD_NEON) && defined(PIPE_ARCH_ARM)
        if (gpu_stride == 8) {
                __asm__ volatile (
                        /* Load each 8-byte line from cpu-side source,
                         * incrementing it by the stride each time.
                         */
                        "vld1.8 d0, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d1, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d2, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d3, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d4, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d5, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d6, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d7, [%[cpu]]\n"
                        /* Load from the GPU in one shot, no interleave, to
                         * d0-d7.
                         */
                        "vstm %[gpu], {q0, q1, q2, q3}\n"
                        : [cpu]         "+r"(cpu)
                        : [gpu]         "r"(gpu),
                          [cpu_stride]  "r"(cpu_stride)
                        : "q0", "q1", "q2", "q3");
                return;
        } else if (gpu_stride == 16) {
                void *cpu2 = cpu + 8;
                __asm__ volatile (
                        /* Load each 16-byte line in 2 parts from the cpu-side
                         * destination.  (vld1 can only store one d-register
                         * at a time).
                         */
                        "vld1.8 d0, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d1, [%[cpu2]],%[cpu_stride]\n"
                        "vld1.8 d2, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d3, [%[cpu2]],%[cpu_stride]\n"
                        "vld1.8 d4, [%[cpu]], %[cpu_stride]\n"
                        "vld1.8 d5, [%[cpu2]],%[cpu_stride]\n"
                        "vld1.8 d6, [%[cpu]]\n"
                        "vld1.8 d7, [%[cpu2]]\n"
                        /* Store to the GPU in one shot, no interleave. */
                        "vstm %[gpu], {q0, q1, q2, q3}\n"
                        : [cpu]         "+r"(cpu),
                          [cpu2]        "+r"(cpu2)
                        : [gpu]         "r"(gpu),
                          [cpu_stride]  "r"(cpu_stride)
                        : "q0", "q1", "q2", "q3");
                return;
        }
#elif defined (PIPE_ARCH_AARCH64)
        if (gpu_stride == 8) {
                __asm__ volatile (
                        /* Load each 8-byte line from cpu-side source,
                         * incrementing it by the stride each time.
                         */
                        "ld1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v0.D}[1], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v1.D}[1], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v2.D}[1], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v3.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v3.D}[1], [%[cpu]]\n"
                        /* Store to the GPU in one shot, no interleave. */
                        "st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n"
                        : [cpu]         "+r"(cpu)
                        : [gpu]         "r"(gpu),
                          [cpu_stride]  "r"(cpu_stride)
                        : "v0", "v1", "v2", "v3");
                return;
        } else if (gpu_stride == 16) {
                void *cpu2 = cpu + 8;
                __asm__ volatile (
                        /* Load each 16-byte line in 2 parts from the cpu-side
                         * destination.  (vld1 can only store one d-register
                         * at a time).
                         */
                        "ld1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v0.D}[1], [%[cpu2]],%[cpu_stride]\n"
                        "ld1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v1.D}[1], [%[cpu2]],%[cpu_stride]\n"
                        "ld1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n"
                        "ld1 {v2.D}[1], [%[cpu2]],%[cpu_stride]\n"
                        "ld1 {v3.D}[0], [%[cpu]]\n"
                        "ld1 {v3.D}[1], [%[cpu2]]\n"
                        /* Store to the GPU in one shot, no interleave. */
                        "st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n"
                        : [cpu]         "+r"(cpu),
                          [cpu2]        "+r"(cpu2)
                        : [gpu]         "r"(gpu),
                          [cpu_stride]  "r"(cpu_stride)
                        : "v0", "v1", "v2", "v3");
                return;
        }
#endif

        for (uint32_t gpu_offset = 0; gpu_offset < 64; gpu_offset += gpu_stride) {
                memcpy(gpu + gpu_offset, cpu, gpu_stride);
                cpu += cpu_stride;
        }
}