Home | History | Annotate | Line # | Download | only in pci
msipic.c revision 1.19
      1  1.19    hikaru /*	$NetBSD: msipic.c,v 1.19 2019/11/13 02:54:59 hikaru Exp $	*/
      2   1.1  knakahar 
      3   1.1  knakahar /*
      4   1.1  knakahar  * Copyright (c) 2015 Internet Initiative Japan Inc.
      5   1.1  knakahar  * All rights reserved.
      6   1.1  knakahar  *
      7   1.1  knakahar  * Redistribution and use in source and binary forms, with or without
      8   1.1  knakahar  * modification, are permitted provided that the following conditions
      9   1.1  knakahar  * are met:
     10   1.1  knakahar  * 1. Redistributions of source code must retain the above copyright
     11   1.1  knakahar  *    notice, this list of conditions and the following disclaimer.
     12   1.1  knakahar  * 2. Redistributions in binary form must reproduce the above copyright
     13   1.1  knakahar  *    notice, this list of conditions and the following disclaimer in the
     14   1.1  knakahar  *    documentation and/or other materials provided with the distribution.
     15   1.1  knakahar  *
     16   1.1  knakahar  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     17   1.1  knakahar  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     18   1.1  knakahar  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     19   1.1  knakahar  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     20   1.1  knakahar  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     21   1.1  knakahar  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     22   1.1  knakahar  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     23   1.1  knakahar  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     24   1.1  knakahar  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     25   1.1  knakahar  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     26   1.1  knakahar  * POSSIBILITY OF SUCH DAMAGE.
     27   1.1  knakahar  */
     28   1.1  knakahar 
     29   1.1  knakahar #include <sys/cdefs.h>
     30  1.19    hikaru __KERNEL_RCSID(0, "$NetBSD: msipic.c,v 1.19 2019/11/13 02:54:59 hikaru Exp $");
     31   1.5   msaitoh 
     32   1.5   msaitoh #include "opt_intrdebug.h"
     33   1.1  knakahar 
     34   1.1  knakahar #include <sys/types.h>
     35   1.1  knakahar #include <sys/param.h>
     36   1.1  knakahar #include <sys/systm.h>
     37   1.1  knakahar #include <sys/errno.h>
     38   1.1  knakahar #include <sys/kmem.h>
     39   1.1  knakahar #include <sys/mutex.h>
     40  1.17  knakahar #include <sys/bitops.h>
     41   1.1  knakahar 
     42   1.1  knakahar #include <dev/pci/pcivar.h>
     43   1.1  knakahar 
     44   1.1  knakahar #include <machine/i82489reg.h>
     45   1.9    nonaka #include <machine/i82489var.h>
     46   1.1  knakahar #include <machine/i82093reg.h>
     47   1.1  knakahar #include <machine/i82093var.h>
     48   1.1  knakahar #include <machine/pic.h>
     49   1.1  knakahar #include <machine/lock.h>
     50   1.1  knakahar 
     51   1.1  knakahar #include <x86/pci/msipic.h>
     52   1.1  knakahar 
     53   1.1  knakahar #ifdef INTRDEBUG
     54   1.1  knakahar #define MSIPICDEBUG
     55   1.1  knakahar #endif
     56   1.1  knakahar 
     57   1.1  knakahar #ifdef MSIPICDEBUG
     58   1.1  knakahar #define DPRINTF(msg) printf msg
     59   1.1  knakahar #else
     60   1.1  knakahar #define DPRINTF(msg)
     61   1.1  knakahar #endif
     62   1.1  knakahar 
     63   1.1  knakahar #define BUS_SPACE_WRITE_FLUSH(pc, tag) (void)bus_space_read_4(pc, tag, 0)
     64   1.1  knakahar 
     65   1.1  knakahar #define MSIPICNAMEBUF 16
     66   1.1  knakahar 
     67   1.1  knakahar /*
     68   1.1  knakahar  * A Pseudo pic for single MSI/MSI-X device.
     69   1.1  knakahar  * The pic and MSI/MSI-X device are distinbuished by "devid". The "devid"
     70   1.1  knakahar  * is managed by below "dev_seqs".
     71   1.1  knakahar  */
     72   1.1  knakahar struct msipic {
     73   1.1  knakahar 	int mp_bus;
     74   1.1  knakahar 	int mp_dev;
     75   1.1  knakahar 	int mp_fun;
     76   1.1  knakahar 
     77   1.1  knakahar 	int mp_devid; /* The device id for the MSI/MSI-X device. */
     78   1.1  knakahar 	int mp_veccnt; /* The number of MSI/MSI-X vectors. */
     79   1.1  knakahar 
     80   1.1  knakahar 	char mp_pic_name[MSIPICNAMEBUF]; /* The MSI/MSI-X device's name. */
     81   1.1  knakahar 
     82   1.1  knakahar 	struct pci_attach_args mp_pa;
     83   1.1  knakahar 	bus_space_tag_t mp_bstag;
     84   1.1  knakahar 	bus_space_handle_t mp_bshandle;
     85   1.1  knakahar 	bus_size_t mp_bssize;
     86   1.1  knakahar 	struct pic *mp_pic;
     87   1.1  knakahar 
     88   1.1  knakahar 	LIST_ENTRY(msipic) mp_list;
     89   1.1  knakahar };
     90   1.1  knakahar 
     91   1.1  knakahar static kmutex_t msipic_list_lock;
     92   1.1  knakahar 
     93   1.1  knakahar static LIST_HEAD(, msipic) msipic_list =
     94   1.1  knakahar 	LIST_HEAD_INITIALIZER(msipic_list);
     95   1.1  knakahar 
     96   1.1  knakahar /*
     97   1.1  knakahar  * This struct managements "devid" to use the same "devid" for the device
     98  1.12   msaitoh  * re-attached. If the device's bus number and device number and function
     99   1.1  knakahar  * number are equal, it is assumed re-attached.
    100   1.1  knakahar  */
    101   1.1  knakahar struct dev_last_used_seq {
    102   1.1  knakahar 	bool ds_using;
    103   1.1  knakahar 	int ds_bus;
    104   1.1  knakahar 	int ds_dev;
    105   1.1  knakahar 	int ds_fun;
    106   1.1  knakahar };
    107   1.1  knakahar /* The number of MSI/MSI-X devices supported by system. */
    108   1.1  knakahar #define NUM_MSI_DEVS 256
    109   1.1  knakahar /* Record devids to use the same devid when the device is re-attached. */
    110   1.1  knakahar static struct dev_last_used_seq dev_seqs[NUM_MSI_DEVS];
    111   1.1  knakahar 
    112   1.4  knakahar static int msipic_allocate_common_msi_devid(const struct pci_attach_args *);
    113   1.1  knakahar static void msipic_release_common_msi_devid(int);
    114   1.1  knakahar 
    115   1.1  knakahar static struct pic *msipic_find_msi_pic_locked(int);
    116   1.4  knakahar static struct pic *msipic_construct_common_msi_pic(const struct pci_attach_args *,
    117   1.1  knakahar 						   struct pic *);
    118   1.1  knakahar static void msipic_destruct_common_msi_pic(struct pic *);
    119   1.1  knakahar 
    120   1.1  knakahar static void msi_set_msictl_enablebit(struct pic *, int, int);
    121   1.1  knakahar static void msi_hwmask(struct pic *, int);
    122   1.1  knakahar static void msi_hwunmask(struct pic *, int);
    123   1.1  knakahar static void msi_addroute(struct pic *, struct cpu_info *, int, int, int);
    124   1.1  knakahar static void msi_delroute(struct pic *, struct cpu_info *, int, int, int);
    125   1.1  knakahar 
    126   1.1  knakahar static void msix_set_vecctl_mask(struct pic *, int, int);
    127   1.1  knakahar static void msix_hwmask(struct pic *, int);
    128   1.1  knakahar static void msix_hwunmask(struct pic *, int);
    129   1.1  knakahar static void msix_addroute(struct pic *, struct cpu_info *, int, int, int);
    130   1.1  knakahar static void msix_delroute(struct pic *, struct cpu_info *, int, int, int);
    131   1.1  knakahar 
    132   1.1  knakahar /*
    133   1.1  knakahar  * Return new "devid" for the device attached first.
    134   1.1  knakahar  * Return the same "devid" for the device re-attached after dettached once.
    135   1.1  knakahar  * Return -1 if the number of attached MSI/MSI-X devices is over NUM_MSI_DEVS.
    136   1.1  knakahar  */
    137   1.1  knakahar static int
    138   1.4  knakahar msipic_allocate_common_msi_devid(const struct pci_attach_args *pa)
    139   1.1  knakahar {
    140   1.1  knakahar 	pci_chipset_tag_t pc;
    141   1.1  knakahar 	pcitag_t tag;
    142   1.1  knakahar 	int bus, dev, fun, i;
    143   1.1  knakahar 
    144   1.1  knakahar 	KASSERT(mutex_owned(&msipic_list_lock));
    145   1.1  knakahar 
    146   1.1  knakahar 	pc = pa->pa_pc;
    147   1.1  knakahar 	tag = pa->pa_tag;
    148   1.1  knakahar 	pci_decompose_tag(pc, tag, &bus, &dev, &fun);
    149   1.1  knakahar 
    150   1.1  knakahar 	/* if the device was once attached, use same devid */
    151   1.1  knakahar 	for (i = 0; i < NUM_MSI_DEVS; i++) {
    152   1.1  knakahar 		/* skip host bridge */
    153   1.1  knakahar 		if (dev_seqs[i].ds_bus == 0
    154   1.1  knakahar 		    && dev_seqs[i].ds_dev == 0
    155   1.1  knakahar 		    && dev_seqs[i].ds_fun == 0)
    156   1.1  knakahar 			break;
    157   1.1  knakahar 
    158   1.1  knakahar 		if (dev_seqs[i].ds_bus == bus
    159   1.1  knakahar 		    && dev_seqs[i].ds_dev == dev
    160   1.1  knakahar 		    && dev_seqs[i].ds_fun == fun) {
    161   1.1  knakahar 			dev_seqs[i].ds_using = true;
    162   1.1  knakahar 			return i;
    163   1.1  knakahar 		}
    164   1.1  knakahar 	}
    165   1.1  knakahar 
    166   1.1  knakahar 	for (i = 0; i < NUM_MSI_DEVS; i++) {
    167   1.1  knakahar 		if (dev_seqs[i].ds_using == 0) {
    168   1.1  knakahar 			dev_seqs[i].ds_using = true;
    169   1.1  knakahar 			dev_seqs[i].ds_bus = bus;
    170   1.1  knakahar 			dev_seqs[i].ds_dev = dev;
    171   1.1  knakahar 			dev_seqs[i].ds_fun = fun;
    172   1.1  knakahar 			return i;
    173   1.1  knakahar 		}
    174   1.1  knakahar 	}
    175   1.1  knakahar 
    176   1.1  knakahar 	DPRINTF(("too many MSI devices.\n"));
    177   1.1  knakahar 	return -1;
    178   1.1  knakahar }
    179   1.1  knakahar 
    180   1.1  knakahar /*
    181   1.1  knakahar  * Set the "devid" unused, but keep reserving the "devid" to reuse when
    182   1.1  knakahar  * the device is re-attached.
    183   1.1  knakahar  */
    184   1.1  knakahar static void
    185   1.1  knakahar msipic_release_common_msi_devid(int devid)
    186   1.1  knakahar {
    187   1.1  knakahar 
    188   1.1  knakahar 	KASSERT(mutex_owned(&msipic_list_lock));
    189   1.1  knakahar 
    190   1.1  knakahar 	if (devid < 0 || NUM_MSI_DEVS <= devid) {
    191   1.1  knakahar 		DPRINTF(("%s: invalid devid.\n", __func__));
    192   1.1  knakahar 		return;
    193   1.1  knakahar 	}
    194   1.1  knakahar 
    195   1.1  knakahar 	dev_seqs[devid].ds_using = false;
    196   1.1  knakahar 	/* Keep ds_* to reuse the same devid for the same device. */
    197   1.1  knakahar }
    198   1.1  knakahar 
    199   1.1  knakahar static struct pic *
    200   1.1  knakahar msipic_find_msi_pic_locked(int devid)
    201   1.1  knakahar {
    202   1.1  knakahar 	struct msipic *mpp;
    203   1.1  knakahar 
    204   1.1  knakahar 	KASSERT(mutex_owned(&msipic_list_lock));
    205   1.1  knakahar 
    206   1.1  knakahar 	LIST_FOREACH(mpp, &msipic_list, mp_list) {
    207  1.15   msaitoh 		if (mpp->mp_devid == devid)
    208   1.1  knakahar 			return mpp->mp_pic;
    209   1.1  knakahar 	}
    210   1.1  knakahar 	return NULL;
    211   1.1  knakahar }
    212   1.1  knakahar 
    213   1.1  knakahar /*
    214   1.1  knakahar  * Return the msi_pic whose device is already registered.
    215   1.1  knakahar  * If the device is not registered yet, return NULL.
    216   1.1  knakahar  */
    217   1.1  knakahar struct pic *
    218   1.1  knakahar msipic_find_msi_pic(int devid)
    219   1.1  knakahar {
    220   1.1  knakahar 	struct pic *msipic;
    221   1.1  knakahar 
    222   1.1  knakahar 	mutex_enter(&msipic_list_lock);
    223   1.1  knakahar 	msipic = msipic_find_msi_pic_locked(devid);
    224   1.1  knakahar 	mutex_exit(&msipic_list_lock);
    225   1.1  knakahar 
    226   1.1  knakahar 	return msipic;
    227   1.1  knakahar }
    228   1.1  knakahar 
    229   1.1  knakahar /*
    230   1.1  knakahar  * A common construct process of MSI and MSI-X.
    231   1.1  knakahar  */
    232   1.1  knakahar static struct pic *
    233   1.4  knakahar msipic_construct_common_msi_pic(const struct pci_attach_args *pa,
    234   1.1  knakahar     struct pic *pic_tmpl)
    235   1.1  knakahar {
    236   1.1  knakahar 	struct pic *pic;
    237   1.1  knakahar 	struct msipic *msipic;
    238   1.1  knakahar 	int devid;
    239   1.1  knakahar 
    240   1.1  knakahar 	pic = kmem_alloc(sizeof(*pic), KM_SLEEP);
    241   1.1  knakahar 	msipic = kmem_zalloc(sizeof(*msipic), KM_SLEEP);
    242   1.1  knakahar 
    243   1.1  knakahar 	mutex_enter(&msipic_list_lock);
    244   1.1  knakahar 
    245   1.1  knakahar 	devid = msipic_allocate_common_msi_devid(pa);
    246   1.1  knakahar 	if (devid == -1) {
    247   1.1  knakahar 		mutex_exit(&msipic_list_lock);
    248   1.1  knakahar 		kmem_free(pic, sizeof(*pic));
    249   1.1  knakahar 		kmem_free(msipic, sizeof(*msipic));
    250   1.1  knakahar 		return NULL;
    251   1.1  knakahar 	}
    252   1.1  knakahar 
    253   1.1  knakahar 	memcpy(pic, pic_tmpl, sizeof(*pic));
    254  1.14   msaitoh 	pic->pic_edge_stubs
    255  1.14   msaitoh 	    = x2apic_mode ? x2apic_edge_stubs : ioapic_edge_stubs;
    256   1.1  knakahar 	pic->pic_msipic = msipic;
    257   1.1  knakahar 	msipic->mp_pic = pic;
    258   1.1  knakahar 	pci_decompose_tag(pa->pa_pc, pa->pa_tag,
    259   1.1  knakahar 	    &msipic->mp_bus, &msipic->mp_dev, &msipic->mp_fun);
    260   1.1  knakahar 	memcpy(&msipic->mp_pa, pa, sizeof(msipic->mp_pa));
    261   1.1  knakahar 	msipic->mp_devid = devid;
    262   1.1  knakahar 	/*
    263   1.1  knakahar 	 * pci_msi{,x}_alloc() must be called only once in the device driver.
    264   1.1  knakahar 	 */
    265   1.1  knakahar 	KASSERT(msipic_find_msi_pic_locked(msipic->mp_devid) == NULL);
    266   1.1  knakahar 
    267   1.1  knakahar 	LIST_INSERT_HEAD(&msipic_list, msipic, mp_list);
    268   1.1  knakahar 
    269   1.1  knakahar 	mutex_exit(&msipic_list_lock);
    270   1.1  knakahar 
    271   1.1  knakahar 	return pic;
    272   1.1  knakahar }
    273   1.1  knakahar 
    274   1.1  knakahar static void
    275   1.1  knakahar msipic_destruct_common_msi_pic(struct pic *msi_pic)
    276   1.1  knakahar {
    277   1.1  knakahar 	struct msipic *msipic;
    278   1.1  knakahar 
    279   1.1  knakahar 	if (msi_pic == NULL)
    280   1.1  knakahar 		return;
    281   1.1  knakahar 
    282   1.1  knakahar 	msipic = msi_pic->pic_msipic;
    283   1.1  knakahar 	mutex_enter(&msipic_list_lock);
    284   1.1  knakahar 	LIST_REMOVE(msipic, mp_list);
    285   1.1  knakahar 	msipic_release_common_msi_devid(msipic->mp_devid);
    286   1.1  knakahar 	mutex_exit(&msipic_list_lock);
    287   1.1  knakahar 
    288   1.1  knakahar 	kmem_free(msipic, sizeof(*msipic));
    289   1.1  knakahar 	kmem_free(msi_pic, sizeof(*msi_pic));
    290   1.1  knakahar }
    291   1.1  knakahar 
    292   1.1  knakahar /*
    293   1.1  knakahar  * The pic is MSI/MSI-X pic or not.
    294   1.1  knakahar  */
    295   1.1  knakahar bool
    296   1.1  knakahar msipic_is_msi_pic(struct pic *pic)
    297   1.1  knakahar {
    298   1.1  knakahar 
    299   1.1  knakahar 	return (pic->pic_msipic != NULL);
    300   1.1  knakahar }
    301   1.1  knakahar 
    302   1.1  knakahar /*
    303   1.1  knakahar  * Return the MSI/MSI-X devid which is unique for each devices.
    304   1.1  knakahar  */
    305   1.1  knakahar int
    306   1.1  knakahar msipic_get_devid(struct pic *pic)
    307   1.1  knakahar {
    308   1.1  knakahar 
    309   1.1  knakahar 	KASSERT(msipic_is_msi_pic(pic));
    310   1.1  knakahar 
    311   1.1  knakahar 	return pic->pic_msipic->mp_devid;
    312   1.1  knakahar }
    313   1.1  knakahar 
    314   1.1  knakahar #define MSI_MSICTL_ENABLE 1
    315   1.1  knakahar #define MSI_MSICTL_DISABLE 0
    316   1.1  knakahar static void
    317   1.1  knakahar msi_set_msictl_enablebit(struct pic *pic, int msi_vec, int flag)
    318   1.1  knakahar {
    319   1.1  knakahar 	pci_chipset_tag_t pc;
    320   1.1  knakahar 	struct pci_attach_args *pa;
    321   1.1  knakahar 	pcitag_t tag;
    322   1.1  knakahar 	pcireg_t ctl;
    323   1.3    martin 	int off, err __diagused;
    324   1.1  knakahar 
    325   1.1  knakahar 	pc = NULL;
    326   1.1  knakahar 	pa = &pic->pic_msipic->mp_pa;
    327   1.1  knakahar 	tag = pa->pa_tag;
    328   1.3    martin 	err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
    329   1.3    martin 	KASSERT(err != 0);
    330   1.1  knakahar 
    331   1.1  knakahar 	/*
    332   1.1  knakahar 	 * MSI can establish only one vector at once.
    333   1.1  knakahar 	 * So, use whole device mask bit instead of a vector mask bit.
    334   1.1  knakahar 	 */
    335   1.1  knakahar 	ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
    336   1.1  knakahar 	if (flag == MSI_MSICTL_ENABLE)
    337   1.1  knakahar 		ctl |= PCI_MSI_CTL_MSI_ENABLE;
    338   1.1  knakahar 	else
    339   1.1  knakahar 		ctl &= ~PCI_MSI_CTL_MSI_ENABLE;
    340   1.1  knakahar 
    341   1.1  knakahar 	pci_conf_write(pc, tag, off, ctl);
    342   1.1  knakahar }
    343   1.1  knakahar 
    344   1.1  knakahar static void
    345   1.1  knakahar msi_hwmask(struct pic *pic, int msi_vec)
    346   1.1  knakahar {
    347   1.1  knakahar 
    348   1.1  knakahar 	msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_DISABLE);
    349   1.1  knakahar }
    350   1.1  knakahar 
    351   1.1  knakahar /*
    352   1.1  knakahar  * Do not use pic->hwunmask() immediately after pic->delroute().
    353   1.1  knakahar  * It is required to use pic->addroute() before pic->hwunmask().
    354   1.1  knakahar  */
    355   1.1  knakahar static void
    356   1.1  knakahar msi_hwunmask(struct pic *pic, int msi_vec)
    357   1.1  knakahar {
    358   1.1  knakahar 
    359   1.1  knakahar 	msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_ENABLE);
    360   1.1  knakahar }
    361   1.1  knakahar 
    362   1.1  knakahar static void
    363   1.1  knakahar msi_addroute(struct pic *pic, struct cpu_info *ci,
    364   1.1  knakahar 	     int unused, int idt_vec, int type)
    365   1.1  knakahar {
    366   1.1  knakahar 	pci_chipset_tag_t pc;
    367   1.1  knakahar 	struct pci_attach_args *pa;
    368   1.1  knakahar 	pcitag_t tag;
    369   1.1  knakahar 	pcireg_t addr, data, ctl;
    370   1.3    martin 	int off, err __diagused;
    371   1.1  knakahar 
    372   1.1  knakahar 	pc = NULL;
    373   1.1  knakahar 	pa = &pic->pic_msipic->mp_pa;
    374   1.1  knakahar 	tag = pa->pa_tag;
    375   1.3    martin 	err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
    376   1.3    martin 	KASSERT(err != 0);
    377   1.1  knakahar 
    378   1.1  knakahar 	/*
    379   1.1  knakahar 	 * See Intel 64 and IA-32 Architectures Software Developer's Manual
    380   1.1  knakahar 	 * Volume 3 10.11 Message Signalled Interrupts.
    381   1.1  knakahar 	 */
    382   1.1  knakahar 	/*
    383   1.1  knakahar 	 * "cpuid" for MSI address is local APIC ID. In NetBSD, the ID is
    384   1.1  knakahar 	 * the same as ci->ci_cpuid.
    385   1.1  knakahar 	 */
    386   1.1  knakahar 	addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid,
    387   1.1  knakahar 	    LAPIC_MSIADDR_DSTID_MASK);
    388   1.1  knakahar 	/* If trigger mode is edge, it don't care level for trigger mode. */
    389  1.13   msaitoh 	data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
    390  1.13   msaitoh 		| LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;
    391   1.1  knakahar 
    392  1.16   msaitoh 	/*
    393  1.16   msaitoh 	 * The size of the message data register is 16bit if the extended
    394  1.16   msaitoh 	 * message data is not implemented. If it's 16bit and the per-vector
    395  1.16   msaitoh 	 * masking is not capable, the location of the upper 16bit is out of
    396  1.16   msaitoh 	 * the MSI capability structure's range. The PCI spec says the upper
    397  1.16   msaitoh 	 * 16bit is driven to 0 if the message data register is 16bit. It's the
    398  1.16   msaitoh 	 * spec, so it's OK just to write it regardless of the value of the
    399  1.16   msaitoh 	 * upper 16bit.
    400  1.16   msaitoh 	 */
    401   1.1  knakahar 	ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
    402   1.1  knakahar 	if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
    403   1.1  knakahar 		pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, addr);
    404   1.1  knakahar 		pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 0);
    405   1.1  knakahar 		pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data);
    406   1.1  knakahar 	} else {
    407   1.1  knakahar 		pci_conf_write(pc, tag, off + PCI_MSI_MADDR, addr);
    408   1.1  knakahar 		pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data);
    409   1.1  knakahar 	}
    410   1.1  knakahar 	ctl |= PCI_MSI_CTL_MSI_ENABLE;
    411   1.1  knakahar 	pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
    412   1.1  knakahar }
    413   1.1  knakahar 
    414   1.1  knakahar /*
    415   1.1  knakahar  * Do not use pic->hwunmask() immediately after pic->delroute().
    416   1.1  knakahar  * It is required to use pic->addroute() before pic->hwunmask().
    417   1.1  knakahar  */
    418   1.1  knakahar static void
    419   1.1  knakahar msi_delroute(struct pic *pic, struct cpu_info *ci,
    420   1.1  knakahar 	     int msi_vec, int idt_vec, int type)
    421   1.1  knakahar {
    422   1.1  knakahar 
    423   1.1  knakahar 	msi_hwmask(pic, msi_vec);
    424   1.1  knakahar }
    425   1.1  knakahar 
    426   1.1  knakahar /*
    427   1.1  knakahar  * Template for MSI pic.
    428   1.1  knakahar  * .pic_msipic is set later in construct_msi_pic().
    429   1.1  knakahar  */
    430   1.1  knakahar static struct pic msi_pic_tmpl = {
    431   1.1  knakahar 	.pic_type = PIC_MSI,
    432   1.1  knakahar 	.pic_vecbase = 0,
    433   1.1  knakahar 	.pic_apicid = 0,
    434   1.1  knakahar 	.pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msi_pic */
    435   1.1  knakahar 	.pic_hwmask = msi_hwmask,
    436   1.1  knakahar 	.pic_hwunmask = msi_hwunmask,
    437   1.1  knakahar 	.pic_addroute = msi_addroute,
    438   1.1  knakahar 	.pic_delroute = msi_delroute,
    439   1.1  knakahar };
    440   1.1  knakahar 
    441   1.1  knakahar /*
    442   1.1  knakahar  * Create pseudo pic for a MSI device.
    443   1.1  knakahar  */
    444   1.1  knakahar struct pic *
    445   1.4  knakahar msipic_construct_msi_pic(const struct pci_attach_args *pa)
    446   1.1  knakahar {
    447   1.1  knakahar 	struct pic *msi_pic;
    448   1.1  knakahar 	char pic_name_buf[MSIPICNAMEBUF];
    449   1.1  knakahar 
    450   1.1  knakahar 	msi_pic = msipic_construct_common_msi_pic(pa, &msi_pic_tmpl);
    451   1.1  knakahar 	if (msi_pic == NULL) {
    452   1.1  knakahar 		DPRINTF(("cannot allocate MSI pic.\n"));
    453   1.1  knakahar 		return NULL;
    454   1.1  knakahar 	}
    455   1.1  knakahar 
    456   1.1  knakahar 	memset(pic_name_buf, 0, MSIPICNAMEBUF);
    457   1.1  knakahar 	snprintf(pic_name_buf, MSIPICNAMEBUF, "msi%d",
    458   1.1  knakahar 	    msi_pic->pic_msipic->mp_devid);
    459   1.1  knakahar 	strncpy(msi_pic->pic_msipic->mp_pic_name, pic_name_buf,
    460   1.1  knakahar 	    MSIPICNAMEBUF - 1);
    461   1.1  knakahar 	msi_pic->pic_name = msi_pic->pic_msipic->mp_pic_name;
    462   1.1  knakahar 
    463   1.1  knakahar 	return msi_pic;
    464   1.1  knakahar }
    465   1.1  knakahar 
    466   1.1  knakahar /*
    467   1.1  knakahar  * Delete pseudo pic for a MSI device.
    468   1.1  knakahar  */
    469   1.1  knakahar void
    470   1.1  knakahar msipic_destruct_msi_pic(struct pic *msi_pic)
    471   1.1  knakahar {
    472   1.1  knakahar 
    473   1.1  knakahar 	msipic_destruct_common_msi_pic(msi_pic);
    474   1.1  knakahar }
    475   1.1  knakahar 
    476   1.1  knakahar #define MSIX_VECCTL_HWMASK 1
    477   1.1  knakahar #define MSIX_VECCTL_HWUNMASK 0
    478   1.1  knakahar static void
    479   1.1  knakahar msix_set_vecctl_mask(struct pic *pic, int msix_vec, int flag)
    480   1.1  knakahar {
    481   1.1  knakahar 	bus_space_tag_t bstag;
    482   1.1  knakahar 	bus_space_handle_t bshandle;
    483   1.1  knakahar 	uint64_t entry_base;
    484   1.1  knakahar 	uint32_t vecctl;
    485   1.1  knakahar 
    486   1.1  knakahar 	if (msix_vec < 0) {
    487   1.1  knakahar 		DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d",
    488   1.2  knakahar 			__func__, msipic_get_devid(pic), msix_vec));
    489   1.1  knakahar 		return;
    490   1.1  knakahar 	}
    491   1.1  knakahar 
    492   1.1  knakahar 	entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
    493   1.1  knakahar 
    494   1.1  knakahar 	bstag = pic->pic_msipic->mp_bstag;
    495   1.1  knakahar 	bshandle = pic->pic_msipic->mp_bshandle;
    496   1.1  knakahar 	vecctl = bus_space_read_4(bstag, bshandle,
    497   1.1  knakahar 	    entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL);
    498   1.1  knakahar 	if (flag == MSIX_VECCTL_HWMASK)
    499   1.8   msaitoh 		vecctl |= PCI_MSIX_VECTCTL_MASK;
    500   1.1  knakahar 	else
    501   1.8   msaitoh 		vecctl &= ~PCI_MSIX_VECTCTL_MASK;
    502   1.1  knakahar 
    503   1.1  knakahar 	bus_space_write_4(bstag, bshandle,
    504   1.1  knakahar 	    entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, vecctl);
    505   1.1  knakahar 	BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
    506   1.1  knakahar }
    507   1.1  knakahar 
    508   1.1  knakahar static void
    509   1.1  knakahar msix_hwmask(struct pic *pic, int msix_vec)
    510   1.1  knakahar {
    511   1.1  knakahar 
    512   1.1  knakahar 	msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWMASK);
    513   1.1  knakahar }
    514   1.1  knakahar 
    515   1.1  knakahar /*
    516   1.1  knakahar  * Do not use pic->hwunmask() immediately after pic->delroute().
    517   1.1  knakahar  * It is required to use pic->addroute() before pic->hwunmask().
    518   1.1  knakahar  */
    519   1.1  knakahar static void
    520   1.1  knakahar msix_hwunmask(struct pic *pic, int msix_vec)
    521   1.1  knakahar {
    522   1.1  knakahar 
    523   1.1  knakahar 	msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWUNMASK);
    524   1.1  knakahar }
    525   1.1  knakahar 
    526   1.1  knakahar static void
    527   1.1  knakahar msix_addroute(struct pic *pic, struct cpu_info *ci,
    528   1.1  knakahar 	     int msix_vec, int idt_vec, int type)
    529   1.1  knakahar {
    530   1.1  knakahar 	pci_chipset_tag_t pc;
    531   1.1  knakahar 	struct pci_attach_args *pa;
    532   1.1  knakahar 	pcitag_t tag;
    533   1.1  knakahar 	bus_space_tag_t bstag;
    534   1.1  knakahar 	bus_space_handle_t bshandle;
    535   1.1  knakahar 	uint64_t entry_base;
    536   1.1  knakahar 	pcireg_t addr, data, ctl;
    537   1.3    martin 	int off, err __diagused;
    538   1.1  knakahar 
    539   1.1  knakahar 	if (msix_vec < 0) {
    540   1.1  knakahar 		DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d",
    541   1.2  knakahar 			__func__, msipic_get_devid(pic), msix_vec));
    542   1.1  knakahar 		return;
    543   1.1  knakahar 	}
    544   1.1  knakahar 
    545   1.1  knakahar 	pa = &pic->pic_msipic->mp_pa;
    546   1.1  knakahar 	pc = pa->pa_pc;
    547   1.1  knakahar 	tag = pa->pa_tag;
    548   1.3    martin 	err = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL);
    549   1.3    martin 	KASSERT(err != 0);
    550   1.1  knakahar 
    551  1.19    hikaru 	/* Disable MSI-X before writing MSI-X table */
    552  1.19    hikaru 	ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
    553  1.19    hikaru 	ctl &= ~PCI_MSIX_CTL_ENABLE;
    554  1.19    hikaru 	pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
    555  1.19    hikaru 
    556   1.1  knakahar 	entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
    557   1.1  knakahar 
    558   1.1  knakahar 	/*
    559   1.1  knakahar 	 * See Intel 64 and IA-32 Architectures Software Developer's Manual
    560   1.1  knakahar 	 * Volume 3 10.11 Message Signalled Interrupts.
    561   1.1  knakahar 	 */
    562   1.1  knakahar 	/*
    563   1.1  knakahar 	 * "cpuid" for MSI-X address is local APIC ID. In NetBSD, the ID is
    564   1.1  knakahar 	 * the same as ci->ci_cpuid.
    565   1.1  knakahar 	 */
    566   1.1  knakahar 	addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid,
    567   1.1  knakahar 	    LAPIC_MSIADDR_DSTID_MASK);
    568   1.1  knakahar 	/* If trigger mode is edge, it don't care level for trigger mode. */
    569  1.13   msaitoh 	data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
    570  1.13   msaitoh 		| LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;
    571   1.1  knakahar 
    572   1.1  knakahar 	bstag = pic->pic_msipic->mp_bstag;
    573   1.1  knakahar 	bshandle = pic->pic_msipic->mp_bshandle;
    574   1.1  knakahar 	bus_space_write_4(bstag, bshandle,
    575   1.1  knakahar 	    entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, addr);
    576   1.1  knakahar 	bus_space_write_4(bstag, bshandle,
    577   1.1  knakahar 	    entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 0);
    578   1.1  knakahar 	bus_space_write_4(bstag, bshandle,
    579   1.1  knakahar 	    entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data);
    580   1.1  knakahar 	bus_space_write_4(bstag, bshandle,
    581   1.1  knakahar 	    entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0);
    582   1.1  knakahar 	BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
    583   1.1  knakahar 
    584   1.1  knakahar 	ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
    585   1.1  knakahar 	ctl |= PCI_MSIX_CTL_ENABLE;
    586   1.1  knakahar 	pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
    587   1.1  knakahar }
    588   1.1  knakahar 
    589   1.1  knakahar /*
    590   1.1  knakahar  * Do not use pic->hwunmask() immediately after pic->delroute().
    591   1.1  knakahar  * It is required to use pic->addroute() before pic->hwunmask().
    592   1.1  knakahar  */
    593   1.1  knakahar static void
    594   1.1  knakahar msix_delroute(struct pic *pic, struct cpu_info *ci,
    595   1.1  knakahar 	     int msix_vec, int vec, int type)
    596   1.1  knakahar {
    597   1.1  knakahar 
    598   1.1  knakahar 	msix_hwmask(pic, msix_vec);
    599   1.1  knakahar }
    600   1.1  knakahar 
    601   1.1  knakahar /*
    602   1.1  knakahar  * Template for MSI-X pic.
    603   1.1  knakahar  * .pic_msipic is set later in construct_msix_pic().
    604   1.1  knakahar  */
    605   1.1  knakahar static struct pic msix_pic_tmpl = {
    606   1.1  knakahar 	.pic_type = PIC_MSIX,
    607   1.1  knakahar 	.pic_vecbase = 0,
    608   1.1  knakahar 	.pic_apicid = 0,
    609   1.1  knakahar 	.pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msix_pic */
    610   1.1  knakahar 	.pic_hwmask = msix_hwmask,
    611   1.1  knakahar 	.pic_hwunmask = msix_hwunmask,
    612   1.1  knakahar 	.pic_addroute = msix_addroute,
    613   1.1  knakahar 	.pic_delroute = msix_delroute,
    614   1.1  knakahar };
    615   1.1  knakahar 
    616   1.1  knakahar struct pic *
    617   1.4  knakahar msipic_construct_msix_pic(const struct pci_attach_args *pa)
    618   1.1  knakahar {
    619   1.1  knakahar 	struct pic *msix_pic;
    620   1.1  knakahar 	pci_chipset_tag_t pc;
    621   1.1  knakahar 	pcitag_t tag;
    622   1.1  knakahar 	pcireg_t tbl;
    623   1.1  knakahar 	bus_space_tag_t bstag;
    624   1.1  knakahar 	bus_space_handle_t bshandle;
    625   1.1  knakahar 	bus_size_t bssize;
    626   1.1  knakahar 	size_t table_size;
    627   1.1  knakahar 	uint32_t table_offset;
    628   1.1  knakahar 	u_int memtype;
    629   1.7   msaitoh 	bus_addr_t memaddr;
    630   1.7   msaitoh 	int flags;
    631   1.1  knakahar 	int bir, bar, err, off, table_nentry;
    632   1.1  knakahar 	char pic_name_buf[MSIPICNAMEBUF];
    633   1.1  knakahar 
    634   1.6   msaitoh 	table_nentry = pci_msix_count(pa->pa_pc, pa->pa_tag);
    635   1.1  knakahar 	if (table_nentry == 0) {
    636   1.1  knakahar 		DPRINTF(("MSI-X table entry is 0.\n"));
    637   1.1  knakahar 		return NULL;
    638   1.1  knakahar 	}
    639   1.1  knakahar 
    640   1.1  knakahar 	pc = pa->pa_pc;
    641   1.1  knakahar 	tag = pa->pa_tag;
    642   1.1  knakahar 	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL) == 0) {
    643   1.1  knakahar 		DPRINTF(("%s: no msix capability", __func__));
    644   1.1  knakahar 		return NULL;
    645   1.1  knakahar 	}
    646   1.1  knakahar 
    647   1.1  knakahar 	msix_pic = msipic_construct_common_msi_pic(pa, &msix_pic_tmpl);
    648   1.1  knakahar 	if (msix_pic == NULL) {
    649   1.1  knakahar 		DPRINTF(("cannot allocate MSI-X pic.\n"));
    650   1.1  knakahar 		return NULL;
    651   1.1  knakahar 	}
    652   1.1  knakahar 
    653   1.1  knakahar 	memset(pic_name_buf, 0, MSIPICNAMEBUF);
    654   1.1  knakahar 	snprintf(pic_name_buf, MSIPICNAMEBUF, "msix%d",
    655   1.1  knakahar 	    msix_pic->pic_msipic->mp_devid);
    656   1.1  knakahar 	strncpy(msix_pic->pic_msipic->mp_pic_name, pic_name_buf,
    657   1.1  knakahar 	    MSIPICNAMEBUF - 1);
    658   1.1  knakahar 	msix_pic->pic_name = msix_pic->pic_msipic->mp_pic_name;
    659   1.1  knakahar 
    660   1.1  knakahar 	tbl = pci_conf_read(pc, tag, off + PCI_MSIX_TBLOFFSET);
    661   1.1  knakahar 	table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK;
    662   1.1  knakahar 	bir = tbl & PCI_MSIX_PBABIR_MASK;
    663  1.15   msaitoh 	switch (bir) {
    664   1.1  knakahar 	case 0:
    665   1.1  knakahar 		bar = PCI_BAR0;
    666   1.1  knakahar 		break;
    667   1.1  knakahar 	case 1:
    668   1.1  knakahar 		bar = PCI_BAR1;
    669   1.1  knakahar 		break;
    670   1.1  knakahar 	case 2:
    671   1.1  knakahar 		bar = PCI_BAR2;
    672   1.1  knakahar 		break;
    673   1.1  knakahar 	case 3:
    674   1.1  knakahar 		bar = PCI_BAR3;
    675   1.1  knakahar 		break;
    676   1.1  knakahar 	case 4:
    677   1.1  knakahar 		bar = PCI_BAR4;
    678   1.1  knakahar 		break;
    679   1.1  knakahar 	case 5:
    680   1.1  knakahar 		bar = PCI_BAR5;
    681   1.1  knakahar 		break;
    682   1.1  knakahar 	default:
    683  1.15   msaitoh 		aprint_error("detect an illegal device! "
    684  1.15   msaitoh 		    "The device use reserved BIR values.\n");
    685   1.1  knakahar 		msipic_destruct_common_msi_pic(msix_pic);
    686   1.1  knakahar 		return NULL;
    687   1.1  knakahar 	}
    688   1.1  knakahar 	memtype = pci_mapreg_type(pc, tag, bar);
    689  1.15   msaitoh 	/*
    690  1.15   msaitoh 	 * PCI_MSIX_TABLE_ENTRY_SIZE consists below
    691  1.15   msaitoh 	 *     - Vector Control (32bit)
    692  1.15   msaitoh 	 *     - Message Data (32bit)
    693  1.15   msaitoh 	 *     - Message Upper Address (32bit)
    694  1.15   msaitoh 	 *     - Message Lower Address (32bit)
    695  1.15   msaitoh 	 */
    696   1.1  knakahar 	table_size = table_nentry * PCI_MSIX_TABLE_ENTRY_SIZE;
    697   1.7   msaitoh #if 0
    698   1.1  knakahar 	err = pci_mapreg_submap(pa, bar, memtype, BUS_SPACE_MAP_LINEAR,
    699   1.1  knakahar 	    roundup(table_size, PAGE_SIZE), table_offset,
    700   1.1  knakahar 	    &bstag, &bshandle, NULL, &bssize);
    701   1.7   msaitoh #else
    702   1.7   msaitoh 	/*
    703   1.7   msaitoh 	 * Workaround for PCI prefetchable bit. Some chips (e.g. Intel 82599)
    704   1.7   msaitoh 	 * report SERR and MSI-X doesn't work. This problem might not be the
    705   1.7   msaitoh 	 * driver's bug but our PCI common part or VMs' bug. Until we find a
    706   1.7   msaitoh 	 * real reason, we ignore the prefetchable bit.
    707   1.7   msaitoh 	 */
    708   1.7   msaitoh 	if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, bar, memtype,
    709   1.7   msaitoh 		&memaddr, NULL, &flags) != 0) {
    710   1.7   msaitoh 		DPRINTF(("cannot get a map info.\n"));
    711   1.7   msaitoh 		msipic_destruct_common_msi_pic(msix_pic);
    712   1.7   msaitoh 		return NULL;
    713   1.7   msaitoh 	}
    714   1.7   msaitoh 	if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) {
    715   1.7   msaitoh 		DPRINTF(( "clear prefetchable bit\n"));
    716   1.7   msaitoh 		flags &= ~BUS_SPACE_MAP_PREFETCHABLE;
    717   1.7   msaitoh 	}
    718   1.7   msaitoh 	bssize = roundup(table_size, PAGE_SIZE);
    719  1.18       tnn 	err = _x86_memio_map(pa->pa_memt, memaddr + table_offset, bssize, flags,
    720   1.7   msaitoh 	    &bshandle);
    721   1.7   msaitoh 	bstag = pa->pa_memt;
    722   1.7   msaitoh #endif
    723   1.1  knakahar 	if (err) {
    724   1.1  knakahar 		DPRINTF(("cannot map msix table.\n"));
    725   1.1  knakahar 		msipic_destruct_common_msi_pic(msix_pic);
    726   1.1  knakahar 		return NULL;
    727   1.1  knakahar 	}
    728   1.1  knakahar 	msix_pic->pic_msipic->mp_bstag = bstag;
    729   1.1  knakahar 	msix_pic->pic_msipic->mp_bshandle = bshandle;
    730   1.1  knakahar 	msix_pic->pic_msipic->mp_bssize = bssize;
    731   1.1  knakahar 
    732   1.1  knakahar 	return msix_pic;
    733   1.1  knakahar }
    734   1.1  knakahar 
    735   1.1  knakahar /*
    736   1.1  knakahar  * Delete pseudo pic for a MSI-X device.
    737   1.1  knakahar  */
    738   1.1  knakahar void
    739   1.1  knakahar msipic_destruct_msix_pic(struct pic *msix_pic)
    740   1.1  knakahar {
    741   1.1  knakahar 	struct msipic *msipic;
    742   1.1  knakahar 
    743   1.1  knakahar 	KASSERT(msipic_is_msi_pic(msix_pic));
    744   1.1  knakahar 	KASSERT(msix_pic->pic_type == PIC_MSIX);
    745   1.1  knakahar 
    746   1.1  knakahar 	msipic = msix_pic->pic_msipic;
    747  1.18       tnn 	_x86_memio_unmap(msipic->mp_bstag, msipic->mp_bshandle,
    748  1.18       tnn 	    msipic->mp_bssize, NULL);
    749   1.1  knakahar 
    750   1.1  knakahar 	msipic_destruct_common_msi_pic(msix_pic);
    751   1.1  knakahar }
    752   1.1  knakahar 
    753   1.1  knakahar /*
    754   1.1  knakahar  * Set the number of MSI vectors for pseudo MSI pic.
    755   1.1  knakahar  */
    756   1.1  knakahar int
    757   1.1  knakahar msipic_set_msi_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs,
    758   1.1  knakahar     int count)
    759   1.1  knakahar {
    760   1.1  knakahar 
    761   1.1  knakahar 	KASSERT(msipic_is_msi_pic(msi_pic));
    762   1.1  knakahar 
    763  1.17  knakahar 	if (msi_pic->pic_type == PIC_MSI) {
    764  1.17  knakahar 		pci_chipset_tag_t pc;
    765  1.17  knakahar 		struct pci_attach_args *pa;
    766  1.17  knakahar 		pcitag_t tag;
    767  1.17  knakahar 		int off, err __diagused;
    768  1.17  knakahar 		pcireg_t ctl;
    769  1.17  knakahar 
    770  1.17  knakahar 		pc = NULL;
    771  1.17  knakahar 		pa = &msi_pic->pic_msipic->mp_pa;
    772  1.17  knakahar 		tag = pa->pa_tag;
    773  1.17  knakahar 		err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
    774  1.17  knakahar 		KASSERT(err != 0);
    775  1.17  knakahar 
    776  1.17  knakahar 		ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
    777  1.17  knakahar 		ctl &= ~PCI_MSI_CTL_MME_MASK;
    778  1.17  knakahar 		ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK);
    779  1.17  knakahar 		pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
    780  1.17  knakahar 	}
    781  1.17  knakahar 
    782   1.1  knakahar 	msi_pic->pic_msipic->mp_veccnt = count;
    783   1.1  knakahar 	return 0;
    784   1.1  knakahar }
    785   1.1  knakahar 
    786   1.1  knakahar /*
    787   1.1  knakahar  * Initialize the system to use MSI/MSI-X.
    788   1.1  knakahar  */
    789   1.1  knakahar void
    790   1.1  knakahar msipic_init(void)
    791   1.1  knakahar {
    792   1.1  knakahar 
    793   1.1  knakahar 	mutex_init(&msipic_list_lock, MUTEX_DEFAULT, IPL_NONE);
    794   1.1  knakahar }
    795