Home | History | Annotate | Line # | Download | only in gvt
      1 /*	$NetBSD: gvt.h,v 1.2 2021/12/18 23:45:31 riastradh Exp $	*/
      2 
      3 /*
      4  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
      5  *
      6  * Permission is hereby granted, free of charge, to any person obtaining a
      7  * copy of this software and associated documentation files (the "Software"),
      8  * to deal in the Software without restriction, including without limitation
      9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
     10  * and/or sell copies of the Software, and to permit persons to whom the
     11  * Software is furnished to do so, subject to the following conditions:
     12  *
     13  * The above copyright notice and this permission notice (including the next
     14  * paragraph) shall be included in all copies or substantial portions of the
     15  * Software.
     16  *
     17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
     23  * SOFTWARE.
     24  *
     25  * Authors:
     26  *    Kevin Tian <kevin.tian (at) intel.com>
     27  *    Eddie Dong <eddie.dong (at) intel.com>
     28  *
     29  * Contributors:
     30  *    Niu Bing <bing.niu (at) intel.com>
     31  *    Zhi Wang <zhi.a.wang (at) intel.com>
     32  *
     33  */
     34 
     35 #ifndef _GVT_H_
     36 #define _GVT_H_
     37 
     38 #include "debug.h"
     39 #include "hypercall.h"
     40 #include "mmio.h"
     41 #include "reg.h"
     42 #include "interrupt.h"
     43 #include "gtt.h"
     44 #include "display.h"
     45 #include "edid.h"
     46 #include "execlist.h"
     47 #include "scheduler.h"
     48 #include "sched_policy.h"
     49 #include "mmio_context.h"
     50 #include "cmd_parser.h"
     51 #include "fb_decoder.h"
     52 #include "dmabuf.h"
     53 #include "page_track.h"
     54 
     55 #define GVT_MAX_VGPU 8
     56 
     57 struct intel_gvt_host {
     58 	struct device *dev;
     59 	bool initialized;
     60 	int hypervisor_type;
     61 	struct intel_gvt_mpt *mpt;
     62 };
     63 
     64 extern struct intel_gvt_host intel_gvt_host;
     65 
     66 /* Describe per-platform limitations. */
     67 struct intel_gvt_device_info {
     68 	u32 max_support_vgpus;
     69 	u32 cfg_space_size;
     70 	u32 mmio_size;
     71 	u32 mmio_bar;
     72 	unsigned long msi_cap_offset;
     73 	u32 gtt_start_offset;
     74 	u32 gtt_entry_size;
     75 	u32 gtt_entry_size_shift;
     76 	int gmadr_bytes_in_cmd;
     77 	u32 max_surface_size;
     78 };
     79 
     80 /* GM resources owned by a vGPU */
     81 struct intel_vgpu_gm {
     82 	u64 aperture_sz;
     83 	u64 hidden_sz;
     84 	struct drm_mm_node low_gm_node;
     85 	struct drm_mm_node high_gm_node;
     86 };
     87 
     88 #define INTEL_GVT_MAX_NUM_FENCES 32
     89 
     90 /* Fences owned by a vGPU */
     91 struct intel_vgpu_fence {
     92 	struct i915_fence_reg *regs[INTEL_GVT_MAX_NUM_FENCES];
     93 	u32 base;
     94 	u32 size;
     95 };
     96 
     97 struct intel_vgpu_mmio {
     98 	void *vreg;
     99 };
    100 
    101 #define INTEL_GVT_MAX_BAR_NUM 4
    102 
    103 struct intel_vgpu_pci_bar {
    104 	u64 size;
    105 	bool tracked;
    106 };
    107 
    108 struct intel_vgpu_cfg_space {
    109 	unsigned char virtual_cfg_space[PCI_CFG_SPACE_EXP_SIZE];
    110 	struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM];
    111 };
    112 
    113 #define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space)
    114 
    115 struct intel_vgpu_irq {
    116 	bool irq_warn_once[INTEL_GVT_EVENT_MAX];
    117 	DECLARE_BITMAP(flip_done_event[I915_MAX_PIPES],
    118 		       INTEL_GVT_EVENT_MAX);
    119 };
    120 
    121 struct intel_vgpu_opregion {
    122 	bool mapped;
    123 	void *va;
    124 	u32 gfn[INTEL_GVT_OPREGION_PAGES];
    125 };
    126 
    127 #define vgpu_opregion(vgpu) (&(vgpu->opregion))
    128 
    129 struct intel_vgpu_display {
    130 	struct intel_vgpu_i2c_edid i2c_edid;
    131 	struct intel_vgpu_port ports[I915_MAX_PORTS];
    132 	struct intel_vgpu_sbi sbi;
    133 };
    134 
    135 struct vgpu_sched_ctl {
    136 	int weight;
    137 };
    138 
    139 enum {
    140 	INTEL_VGPU_EXECLIST_SUBMISSION = 1,
    141 	INTEL_VGPU_GUC_SUBMISSION,
    142 };
    143 
    144 struct intel_vgpu_submission_ops {
    145 	const char *name;
    146 	int (*init)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
    147 	void (*clean)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
    148 	void (*reset)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
    149 };
    150 
    151 struct intel_vgpu_submission {
    152 	struct intel_vgpu_execlist execlist[I915_NUM_ENGINES];
    153 	struct list_head workload_q_head[I915_NUM_ENGINES];
    154 	struct intel_context *shadow[I915_NUM_ENGINES];
    155 	struct kmem_cache *workloads;
    156 	atomic_t running_workload_num;
    157 	union {
    158 		u64 i915_context_pml4;
    159 		u64 i915_context_pdps[GEN8_3LVL_PDPES];
    160 	};
    161 	DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES);
    162 	DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
    163 	void *ring_scan_buffer[I915_NUM_ENGINES];
    164 	int ring_scan_buffer_size[I915_NUM_ENGINES];
    165 	const struct intel_vgpu_submission_ops *ops;
    166 	int virtual_submission_interface;
    167 	bool active;
    168 };
    169 
    170 struct intel_vgpu {
    171 	struct intel_gvt *gvt;
    172 	struct mutex vgpu_lock;
    173 	int id;
    174 	unsigned long handle; /* vGPU handle used by hypervisor MPT modules */
    175 	bool active;
    176 	bool pv_notified;
    177 	bool failsafe;
    178 	unsigned int resetting_eng;
    179 
    180 	/* Both sched_data and sched_ctl can be seen a part of the global gvt
    181 	 * scheduler structure. So below 2 vgpu data are protected
    182 	 * by sched_lock, not vgpu_lock.
    183 	 */
    184 	void *sched_data;
    185 	struct vgpu_sched_ctl sched_ctl;
    186 
    187 	struct intel_vgpu_fence fence;
    188 	struct intel_vgpu_gm gm;
    189 	struct intel_vgpu_cfg_space cfg_space;
    190 	struct intel_vgpu_mmio mmio;
    191 	struct intel_vgpu_irq irq;
    192 	struct intel_vgpu_gtt gtt;
    193 	struct intel_vgpu_opregion opregion;
    194 	struct intel_vgpu_display display;
    195 	struct intel_vgpu_submission submission;
    196 	struct radix_tree_root page_track_tree;
    197 	u32 hws_pga[I915_NUM_ENGINES];
    198 
    199 	struct dentry *debugfs;
    200 
    201 #if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
    202 	struct {
    203 		struct mdev_device *mdev;
    204 		struct vfio_region *region;
    205 		int num_regions;
    206 		struct eventfd_ctx *intx_trigger;
    207 		struct eventfd_ctx *msi_trigger;
    208 
    209 		/*
    210 		 * Two caches are used to avoid mapping duplicated pages (eg.
    211 		 * scratch pages). This help to reduce dma setup overhead.
    212 		 */
    213 		struct rb_root gfn_cache;
    214 		struct rb_root dma_addr_cache;
    215 		unsigned long nr_cache_entries;
    216 		struct mutex cache_lock;
    217 
    218 		struct notifier_block iommu_notifier;
    219 		struct notifier_block group_notifier;
    220 		struct kvm *kvm;
    221 		struct work_struct release_work;
    222 		atomic_t released;
    223 		struct vfio_device *vfio_device;
    224 	} vdev;
    225 #endif
    226 
    227 	struct list_head dmabuf_obj_list_head;
    228 	struct mutex dmabuf_lock;
    229 	struct idr object_idr;
    230 
    231 	struct completion vblank_done;
    232 
    233 	u32 scan_nonprivbb;
    234 };
    235 
    236 /* validating GM healthy status*/
    237 #define vgpu_is_vm_unhealthy(ret_val) \
    238 	(((ret_val) == -EBADRQC) || ((ret_val) == -EFAULT))
    239 
    240 struct intel_gvt_gm {
    241 	unsigned long vgpu_allocated_low_gm_size;
    242 	unsigned long vgpu_allocated_high_gm_size;
    243 };
    244 
    245 struct intel_gvt_fence {
    246 	unsigned long vgpu_allocated_fence_num;
    247 };
    248 
    249 /* Special MMIO blocks. */
    250 struct gvt_mmio_block {
    251 	unsigned int device;
    252 	i915_reg_t   offset;
    253 	unsigned int size;
    254 	gvt_mmio_func read;
    255 	gvt_mmio_func write;
    256 };
    257 
    258 #define INTEL_GVT_MMIO_HASH_BITS 11
    259 
    260 struct intel_gvt_mmio {
    261 	u8 *mmio_attribute;
    262 /* Register contains RO bits */
    263 #define F_RO		(1 << 0)
    264 /* Register contains graphics address */
    265 #define F_GMADR		(1 << 1)
    266 /* Mode mask registers with high 16 bits as the mask bits */
    267 #define F_MODE_MASK	(1 << 2)
    268 /* This reg can be accessed by GPU commands */
    269 #define F_CMD_ACCESS	(1 << 3)
    270 /* This reg has been accessed by a VM */
    271 #define F_ACCESSED	(1 << 4)
    272 /* This reg has been accessed through GPU commands */
    273 #define F_CMD_ACCESSED	(1 << 5)
    274 /* This reg could be accessed by unaligned address */
    275 #define F_UNALIGN	(1 << 6)
    276 /* This reg is saved/restored in context */
    277 #define F_IN_CTX	(1 << 7)
    278 
    279 	struct gvt_mmio_block *mmio_block;
    280 	unsigned int num_mmio_block;
    281 
    282 	DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS);
    283 	unsigned long num_tracked_mmio;
    284 };
    285 
    286 struct intel_gvt_firmware {
    287 	void *cfg_space;
    288 	void *mmio;
    289 	bool firmware_loaded;
    290 };
    291 
    292 #define NR_MAX_INTEL_VGPU_TYPES 20
    293 struct intel_vgpu_type {
    294 	char name[16];
    295 	unsigned int avail_instance;
    296 	unsigned int low_gm_size;
    297 	unsigned int high_gm_size;
    298 	unsigned int fence;
    299 	unsigned int weight;
    300 	enum intel_vgpu_edid resolution;
    301 };
    302 
    303 struct intel_gvt {
    304 	/* GVT scope lock, protect GVT itself, and all resource currently
    305 	 * not yet protected by special locks(vgpu and scheduler lock).
    306 	 */
    307 	struct mutex lock;
    308 	/* scheduler scope lock, protect gvt and vgpu schedule related data */
    309 	struct mutex sched_lock;
    310 
    311 	struct drm_i915_private *dev_priv;
    312 	struct idr vgpu_idr;	/* vGPU IDR pool */
    313 
    314 	struct intel_gvt_device_info device_info;
    315 	struct intel_gvt_gm gm;
    316 	struct intel_gvt_fence fence;
    317 	struct intel_gvt_mmio mmio;
    318 	struct intel_gvt_firmware firmware;
    319 	struct intel_gvt_irq irq;
    320 	struct intel_gvt_gtt gtt;
    321 	struct intel_gvt_workload_scheduler scheduler;
    322 	struct notifier_block shadow_ctx_notifier_block[I915_NUM_ENGINES];
    323 	DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS);
    324 	struct intel_vgpu_type *types;
    325 	unsigned int num_types;
    326 	struct intel_vgpu *idle_vgpu;
    327 
    328 	struct task_struct *service_thread;
    329 	wait_queue_head_t service_thread_wq;
    330 
    331 	/* service_request is always used in bit operation, we should always
    332 	 * use it with atomic bit ops so that no need to use gvt big lock.
    333 	 */
    334 	unsigned long service_request;
    335 
    336 	struct {
    337 		struct engine_mmio *mmio;
    338 		int ctx_mmio_count[I915_NUM_ENGINES];
    339 		u32 *tlb_mmio_offset_list;
    340 		u32 tlb_mmio_offset_list_cnt;
    341 		u32 *mocs_mmio_offset_list;
    342 		u32 mocs_mmio_offset_list_cnt;
    343 	} engine_mmio_list;
    344 
    345 	struct dentry *debugfs_root;
    346 };
    347 
    348 static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915)
    349 {
    350 	return i915->gvt;
    351 }
    352 
    353 enum {
    354 	INTEL_GVT_REQUEST_EMULATE_VBLANK = 0,
    355 
    356 	/* Scheduling trigger by timer */
    357 	INTEL_GVT_REQUEST_SCHED = 1,
    358 
    359 	/* Scheduling trigger by event */
    360 	INTEL_GVT_REQUEST_EVENT_SCHED = 2,
    361 };
    362 
    363 static inline void intel_gvt_request_service(struct intel_gvt *gvt,
    364 		int service)
    365 {
    366 	set_bit(service, (void *)&gvt->service_request);
    367 	wake_up(&gvt->service_thread_wq);
    368 }
    369 
    370 void intel_gvt_free_firmware(struct intel_gvt *gvt);
    371 int intel_gvt_load_firmware(struct intel_gvt *gvt);
    372 
    373 /* Aperture/GM space definitions for GVT device */
    374 #define MB_TO_BYTES(mb) ((mb) << 20ULL)
    375 #define BYTES_TO_MB(b) ((b) >> 20ULL)
    376 
    377 #define HOST_LOW_GM_SIZE MB_TO_BYTES(128)
    378 #define HOST_HIGH_GM_SIZE MB_TO_BYTES(384)
    379 #define HOST_FENCE 4
    380 
    381 /* Aperture/GM space definitions for GVT device */
    382 #define gvt_aperture_sz(gvt)	  (gvt->dev_priv->ggtt.mappable_end)
    383 #define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.gmadr.start)
    384 
    385 #define gvt_ggtt_gm_sz(gvt)	  (gvt->dev_priv->ggtt.vm.total)
    386 #define gvt_ggtt_sz(gvt) \
    387 	((gvt->dev_priv->ggtt.vm.total >> PAGE_SHIFT) << 3)
    388 #define gvt_hidden_sz(gvt)	  (gvt_ggtt_gm_sz(gvt) - gvt_aperture_sz(gvt))
    389 
    390 #define gvt_aperture_gmadr_base(gvt) (0)
    391 #define gvt_aperture_gmadr_end(gvt) (gvt_aperture_gmadr_base(gvt) \
    392 				     + gvt_aperture_sz(gvt) - 1)
    393 
    394 #define gvt_hidden_gmadr_base(gvt) (gvt_aperture_gmadr_base(gvt) \
    395 				    + gvt_aperture_sz(gvt))
    396 #define gvt_hidden_gmadr_end(gvt) (gvt_hidden_gmadr_base(gvt) \
    397 				   + gvt_hidden_sz(gvt) - 1)
    398 
    399 #define gvt_fence_sz(gvt) ((gvt)->dev_priv->ggtt.num_fences)
    400 
    401 /* Aperture/GM space definitions for vGPU */
    402 #define vgpu_aperture_offset(vgpu)	((vgpu)->gm.low_gm_node.start)
    403 #define vgpu_hidden_offset(vgpu)	((vgpu)->gm.high_gm_node.start)
    404 #define vgpu_aperture_sz(vgpu)		((vgpu)->gm.aperture_sz)
    405 #define vgpu_hidden_sz(vgpu)		((vgpu)->gm.hidden_sz)
    406 
    407 #define vgpu_aperture_pa_base(vgpu) \
    408 	(gvt_aperture_pa_base(vgpu->gvt) + vgpu_aperture_offset(vgpu))
    409 
    410 #define vgpu_ggtt_gm_sz(vgpu) ((vgpu)->gm.aperture_sz + (vgpu)->gm.hidden_sz)
    411 
    412 #define vgpu_aperture_pa_end(vgpu) \
    413 	(vgpu_aperture_pa_base(vgpu) + vgpu_aperture_sz(vgpu) - 1)
    414 
    415 #define vgpu_aperture_gmadr_base(vgpu) (vgpu_aperture_offset(vgpu))
    416 #define vgpu_aperture_gmadr_end(vgpu) \
    417 	(vgpu_aperture_gmadr_base(vgpu) + vgpu_aperture_sz(vgpu) - 1)
    418 
    419 #define vgpu_hidden_gmadr_base(vgpu) (vgpu_hidden_offset(vgpu))
    420 #define vgpu_hidden_gmadr_end(vgpu) \
    421 	(vgpu_hidden_gmadr_base(vgpu) + vgpu_hidden_sz(vgpu) - 1)
    422 
    423 #define vgpu_fence_base(vgpu) (vgpu->fence.base)
    424 #define vgpu_fence_sz(vgpu) (vgpu->fence.size)
    425 
    426 struct intel_vgpu_creation_params {
    427 	__u64 handle;
    428 	__u64 low_gm_sz;  /* in MB */
    429 	__u64 high_gm_sz; /* in MB */
    430 	__u64 fence_sz;
    431 	__u64 resolution;
    432 	__s32 primary;
    433 	__u64 vgpu_id;
    434 
    435 	__u32 weight;
    436 };
    437 
    438 int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu,
    439 			      struct intel_vgpu_creation_params *param);
    440 void intel_vgpu_reset_resource(struct intel_vgpu *vgpu);
    441 void intel_vgpu_free_resource(struct intel_vgpu *vgpu);
    442 void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
    443 	u32 fence, u64 value);
    444 
    445 /* Macros for easily accessing vGPU virtual/shadow register.
    446    Explicitly seperate use for typed MMIO reg or real offset.*/
    447 #define vgpu_vreg_t(vgpu, reg) \
    448 	(*(u32 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg)))
    449 #define vgpu_vreg(vgpu, offset) \
    450 	(*(u32 *)(vgpu->mmio.vreg + (offset)))
    451 #define vgpu_vreg64_t(vgpu, reg) \
    452 	(*(u64 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg)))
    453 #define vgpu_vreg64(vgpu, offset) \
    454 	(*(u64 *)(vgpu->mmio.vreg + (offset)))
    455 
    456 #define for_each_active_vgpu(gvt, vgpu, id) \
    457 	idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \
    458 		for_each_if(vgpu->active)
    459 
    460 static inline void intel_vgpu_write_pci_bar(struct intel_vgpu *vgpu,
    461 					    u32 offset, u32 val, bool low)
    462 {
    463 	u32 *pval;
    464 
    465 	/* BAR offset should be 32 bits algiend */
    466 	offset = rounddown(offset, 4);
    467 	pval = (u32 *)(vgpu_cfg_space(vgpu) + offset);
    468 
    469 	if (low) {
    470 		/*
    471 		 * only update bit 31 - bit 4,
    472 		 * leave the bit 3 - bit 0 unchanged.
    473 		 */
    474 		*pval = (val & GENMASK(31, 4)) | (*pval & GENMASK(3, 0));
    475 	} else {
    476 		*pval = val;
    477 	}
    478 }
    479 
    480 int intel_gvt_init_vgpu_types(struct intel_gvt *gvt);
    481 void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt);
    482 
    483 struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt);
    484 void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu);
    485 struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
    486 					 struct intel_vgpu_type *type);
    487 void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
    488 void intel_gvt_release_vgpu(struct intel_vgpu *vgpu);
    489 void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
    490 				 intel_engine_mask_t engine_mask);
    491 void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu);
    492 void intel_gvt_activate_vgpu(struct intel_vgpu *vgpu);
    493 void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu);
    494 
    495 /* validating GM functions */
    496 #define vgpu_gmadr_is_aperture(vgpu, gmadr) \
    497 	((gmadr >= vgpu_aperture_gmadr_base(vgpu)) && \
    498 	 (gmadr <= vgpu_aperture_gmadr_end(vgpu)))
    499 
    500 #define vgpu_gmadr_is_hidden(vgpu, gmadr) \
    501 	((gmadr >= vgpu_hidden_gmadr_base(vgpu)) && \
    502 	 (gmadr <= vgpu_hidden_gmadr_end(vgpu)))
    503 
    504 #define vgpu_gmadr_is_valid(vgpu, gmadr) \
    505 	 ((vgpu_gmadr_is_aperture(vgpu, gmadr) || \
    506 	  (vgpu_gmadr_is_hidden(vgpu, gmadr))))
    507 
    508 #define gvt_gmadr_is_aperture(gvt, gmadr) \
    509 	 ((gmadr >= gvt_aperture_gmadr_base(gvt)) && \
    510 	  (gmadr <= gvt_aperture_gmadr_end(gvt)))
    511 
    512 #define gvt_gmadr_is_hidden(gvt, gmadr) \
    513 	  ((gmadr >= gvt_hidden_gmadr_base(gvt)) && \
    514 	   (gmadr <= gvt_hidden_gmadr_end(gvt)))
    515 
    516 #define gvt_gmadr_is_valid(gvt, gmadr) \
    517 	  (gvt_gmadr_is_aperture(gvt, gmadr) || \
    518 	    gvt_gmadr_is_hidden(gvt, gmadr))
    519 
    520 bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size);
    521 int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr);
    522 int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr);
    523 int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
    524 			     unsigned long *h_index);
    525 int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
    526 			     unsigned long *g_index);
    527 
    528 void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu,
    529 		bool primary);
    530 void intel_vgpu_reset_cfg_space(struct intel_vgpu *vgpu);
    531 
    532 int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset,
    533 		void *p_data, unsigned int bytes);
    534 
    535 int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
    536 		void *p_data, unsigned int bytes);
    537 
    538 void intel_vgpu_emulate_hotplug(struct intel_vgpu *vgpu, bool connected);
    539 
    540 static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar)
    541 {
    542 	/* We are 64bit bar. */
    543 	return (*(u64 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
    544 			PCI_BASE_ADDRESS_MEM_MASK;
    545 }
    546 
    547 void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu);
    548 int intel_vgpu_init_opregion(struct intel_vgpu *vgpu);
    549 int intel_vgpu_opregion_base_write_handler(struct intel_vgpu *vgpu, u32 gpa);
    550 
    551 int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci);
    552 void populate_pvinfo_page(struct intel_vgpu *vgpu);
    553 
    554 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload);
    555 void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason);
    556 
    557 struct intel_gvt_ops {
    558 	int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *,
    559 				unsigned int);
    560 	int (*emulate_cfg_write)(struct intel_vgpu *, unsigned int, void *,
    561 				unsigned int);
    562 	int (*emulate_mmio_read)(struct intel_vgpu *, u64, void *,
    563 				unsigned int);
    564 	int (*emulate_mmio_write)(struct intel_vgpu *, u64, void *,
    565 				unsigned int);
    566 	struct intel_vgpu *(*vgpu_create)(struct intel_gvt *,
    567 				struct intel_vgpu_type *);
    568 	void (*vgpu_destroy)(struct intel_vgpu *vgpu);
    569 	void (*vgpu_release)(struct intel_vgpu *vgpu);
    570 	void (*vgpu_reset)(struct intel_vgpu *);
    571 	void (*vgpu_activate)(struct intel_vgpu *);
    572 	void (*vgpu_deactivate)(struct intel_vgpu *);
    573 	struct intel_vgpu_type *(*gvt_find_vgpu_type)(struct intel_gvt *gvt,
    574 			const char *name);
    575 	bool (*get_gvt_attrs)(struct attribute ***type_attrs,
    576 			struct attribute_group ***intel_vgpu_type_groups);
    577 	int (*vgpu_query_plane)(struct intel_vgpu *vgpu, void *);
    578 	int (*vgpu_get_dmabuf)(struct intel_vgpu *vgpu, unsigned int);
    579 	int (*write_protect_handler)(struct intel_vgpu *, u64, void *,
    580 				     unsigned int);
    581 	void (*emulate_hotplug)(struct intel_vgpu *vgpu, bool connected);
    582 };
    583 
    584 
    585 enum {
    586 	GVT_FAILSAFE_UNSUPPORTED_GUEST,
    587 	GVT_FAILSAFE_INSUFFICIENT_RESOURCE,
    588 	GVT_FAILSAFE_GUEST_ERR,
    589 };
    590 
    591 static inline void mmio_hw_access_pre(struct drm_i915_private *dev_priv)
    592 {
    593 	intel_runtime_pm_get(&dev_priv->runtime_pm);
    594 }
    595 
    596 static inline void mmio_hw_access_post(struct drm_i915_private *dev_priv)
    597 {
    598 	intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
    599 }
    600 
    601 /**
    602  * intel_gvt_mmio_set_accessed - mark a MMIO has been accessed
    603  * @gvt: a GVT device
    604  * @offset: register offset
    605  *
    606  */
    607 static inline void intel_gvt_mmio_set_accessed(
    608 			struct intel_gvt *gvt, unsigned int offset)
    609 {
    610 	gvt->mmio.mmio_attribute[offset >> 2] |= F_ACCESSED;
    611 }
    612 
    613 /**
    614  * intel_gvt_mmio_is_cmd_accessed - mark a MMIO could be accessed by command
    615  * @gvt: a GVT device
    616  * @offset: register offset
    617  *
    618  */
    619 static inline bool intel_gvt_mmio_is_cmd_access(
    620 			struct intel_gvt *gvt, unsigned int offset)
    621 {
    622 	return gvt->mmio.mmio_attribute[offset >> 2] & F_CMD_ACCESS;
    623 }
    624 
    625 /**
    626  * intel_gvt_mmio_is_unalign - mark a MMIO could be accessed unaligned
    627  * @gvt: a GVT device
    628  * @offset: register offset
    629  *
    630  */
    631 static inline bool intel_gvt_mmio_is_unalign(
    632 			struct intel_gvt *gvt, unsigned int offset)
    633 {
    634 	return gvt->mmio.mmio_attribute[offset >> 2] & F_UNALIGN;
    635 }
    636 
    637 /**
    638  * intel_gvt_mmio_set_cmd_accessed - mark a MMIO has been accessed by command
    639  * @gvt: a GVT device
    640  * @offset: register offset
    641  *
    642  */
    643 static inline void intel_gvt_mmio_set_cmd_accessed(
    644 			struct intel_gvt *gvt, unsigned int offset)
    645 {
    646 	gvt->mmio.mmio_attribute[offset >> 2] |= F_CMD_ACCESSED;
    647 }
    648 
    649 /**
    650  * intel_gvt_mmio_has_mode_mask - if a MMIO has a mode mask
    651  * @gvt: a GVT device
    652  * @offset: register offset
    653  *
    654  * Returns:
    655  * True if a MMIO has a mode mask in its higher 16 bits, false if it isn't.
    656  *
    657  */
    658 static inline bool intel_gvt_mmio_has_mode_mask(
    659 			struct intel_gvt *gvt, unsigned int offset)
    660 {
    661 	return gvt->mmio.mmio_attribute[offset >> 2] & F_MODE_MASK;
    662 }
    663 
    664 /**
    665  * intel_gvt_mmio_is_in_ctx - check if a MMIO has in-ctx mask
    666  * @gvt: a GVT device
    667  * @offset: register offset
    668  *
    669  * Returns:
    670  * True if a MMIO has a in-context mask, false if it isn't.
    671  *
    672  */
    673 static inline bool intel_gvt_mmio_is_in_ctx(
    674 			struct intel_gvt *gvt, unsigned int offset)
    675 {
    676 	return gvt->mmio.mmio_attribute[offset >> 2] & F_IN_CTX;
    677 }
    678 
    679 /**
    680  * intel_gvt_mmio_set_in_ctx - mask a MMIO in logical context
    681  * @gvt: a GVT device
    682  * @offset: register offset
    683  *
    684  */
    685 static inline void intel_gvt_mmio_set_in_ctx(
    686 			struct intel_gvt *gvt, unsigned int offset)
    687 {
    688 	gvt->mmio.mmio_attribute[offset >> 2] |= F_IN_CTX;
    689 }
    690 
    691 void intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu);
    692 void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu);
    693 void intel_gvt_debugfs_init(struct intel_gvt *gvt);
    694 void intel_gvt_debugfs_clean(struct intel_gvt *gvt);
    695 
    696 
    697 #include "trace.h"
    698 #include "mpt.h"
    699 
    700 #endif
    701