Home | History | Annotate | Line # | Download | only in i915
i915_globals.c revision 1.1.1.1 
      1 /*	$NetBSD: i915_globals.c,v 1.1.1.1 2021/12/18 20:15:25 riastradh Exp $	*/
      2 
      3 /*
      4  * SPDX-License-Identifier: MIT
      5  *
      6  * Copyright  2019 Intel Corporation
      7  */
      8 
      9 #include <sys/cdefs.h>
     10 __KERNEL_RCSID(0, "$NetBSD: i915_globals.c,v 1.1.1.1 2021/12/18 20:15:25 riastradh Exp $");
     11 
     12 #include <linux/slab.h>
     13 #include <linux/workqueue.h>
     14 
     15 #include "i915_active.h"
     16 #include "gem/i915_gem_context.h"
     17 #include "gem/i915_gem_object.h"
     18 #include "i915_globals.h"
     19 #include "i915_request.h"
     20 #include "i915_scheduler.h"
     21 #include "i915_vma.h"
     22 
     23 static LIST_HEAD(globals);
     24 
     25 static atomic_t active;
     26 static atomic_t epoch;
     27 static struct park_work {
     28 	struct delayed_work work;
     29 	struct rcu_head rcu;
     30 	unsigned long flags;
     31 #define PENDING 0
     32 	int epoch;
     33 } park;
     34 
     35 static void i915_globals_shrink(void)
     36 {
     37 	struct i915_global *global;
     38 
     39 	/*
     40 	 * kmem_cache_shrink() discards empty slabs and reorders partially
     41 	 * filled slabs to prioritise allocating from the mostly full slabs,
     42 	 * with the aim of reducing fragmentation.
     43 	 */
     44 	list_for_each_entry(global, &globals, link)
     45 		global->shrink();
     46 }
     47 
     48 static void __i915_globals_grace(struct rcu_head *rcu)
     49 {
     50 	/* Ratelimit parking as shrinking is quite slow */
     51 	schedule_delayed_work(&park.work, round_jiffies_up_relative(2 * HZ));
     52 }
     53 
     54 static void __i915_globals_queue_rcu(void)
     55 {
     56 	park.epoch = atomic_inc_return(&epoch);
     57 	if (!atomic_read(&active)) {
     58 		init_rcu_head(&park.rcu);
     59 		call_rcu(&park.rcu, __i915_globals_grace);
     60 	}
     61 }
     62 
     63 static void __i915_globals_park(struct work_struct *work)
     64 {
     65 	destroy_rcu_head(&park.rcu);
     66 
     67 	/* Confirm nothing woke up in the last grace period */
     68 	if (park.epoch != atomic_read(&epoch)) {
     69 		__i915_globals_queue_rcu();
     70 		return;
     71 	}
     72 
     73 	clear_bit(PENDING, &park.flags);
     74 	i915_globals_shrink();
     75 }
     76 
     77 void __init i915_global_register(struct i915_global *global)
     78 {
     79 	GEM_BUG_ON(!global->shrink);
     80 	GEM_BUG_ON(!global->exit);
     81 
     82 	list_add_tail(&global->link, &globals);
     83 }
     84 
     85 static void __i915_globals_cleanup(void)
     86 {
     87 	struct i915_global *global, *next;
     88 
     89 	list_for_each_entry_safe_reverse(global, next, &globals, link)
     90 		global->exit();
     91 }
     92 
     93 static __initconst int (* const initfn[])(void) = {
     94 	i915_global_active_init,
     95 	i915_global_buddy_init,
     96 	i915_global_context_init,
     97 	i915_global_gem_context_init,
     98 	i915_global_objects_init,
     99 	i915_global_request_init,
    100 	i915_global_scheduler_init,
    101 	i915_global_vma_init,
    102 };
    103 
    104 int __init i915_globals_init(void)
    105 {
    106 	int i;
    107 
    108 	for (i = 0; i < ARRAY_SIZE(initfn); i++) {
    109 		int err;
    110 
    111 		err = initfn[i]();
    112 		if (err) {
    113 			__i915_globals_cleanup();
    114 			return err;
    115 		}
    116 	}
    117 
    118 	INIT_DELAYED_WORK(&park.work, __i915_globals_park);
    119 	return 0;
    120 }
    121 
    122 void i915_globals_park(void)
    123 {
    124 	/*
    125 	 * Defer shrinking the global slab caches (and other work) until
    126 	 * after a RCU grace period has completed with no activity. This
    127 	 * is to try and reduce the latency impact on the consumers caused
    128 	 * by us shrinking the caches the same time as they are trying to
    129 	 * allocate, with the assumption being that if we idle long enough
    130 	 * for an RCU grace period to elapse since the last use, it is likely
    131 	 * to be longer until we need the caches again.
    132 	 */
    133 	if (!atomic_dec_and_test(&active))
    134 		return;
    135 
    136 	/* Queue cleanup after the next RCU grace period has freed slabs */
    137 	if (!test_and_set_bit(PENDING, &park.flags))
    138 		__i915_globals_queue_rcu();
    139 }
    140 
    141 void i915_globals_unpark(void)
    142 {
    143 	atomic_inc(&epoch);
    144 	atomic_inc(&active);
    145 }
    146 
    147 static void __exit __i915_globals_flush(void)
    148 {
    149 	atomic_inc(&active); /* skip shrinking */
    150 
    151 	rcu_barrier(); /* wait for the work to be queued */
    152 	flush_delayed_work(&park.work);
    153 
    154 	atomic_dec(&active);
    155 }
    156 
    157 void __exit i915_globals_exit(void)
    158 {
    159 	GEM_BUG_ON(atomic_read(&active));
    160 
    161 	__i915_globals_flush();
    162 	__i915_globals_cleanup();
    163 
    164 	/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
    165 	rcu_barrier();
    166 }
    167