Home | History | Annotate | Line # | Download | only in netinet6
ip6_flow.c revision 1.28.2.2
      1 /*	$NetBSD: ip6_flow.c,v 1.28.2.2 2016/11/04 14:49:21 pgoyette Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by the 3am Software Foundry ("3am").  It was developed by Liam J. Foy
      9  * <liamjfoy (at) netbsd.org> and Matt Thomas <matt (at) netbsd.org>.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted provided that the following conditions
     13  * are met:
     14  * 1. Redistributions of source code must retain the above copyright
     15  *    notice, this list of conditions and the following disclaimer.
     16  * 2. Redistributions in binary form must reproduce the above copyright
     17  *    notice, this list of conditions and the following disclaimer in the
     18  *    documentation and/or other materials provided with the distribution.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30  * POSSIBILITY OF SUCH DAMAGE.
     31  *
     32  * IPv6 version was developed by Liam J. Foy. Original source existed in IPv4
     33  * format developed by Matt Thomas. Thanks to Joerg Sonnenberger, Matt
     34  * Thomas and Christos Zoulas.
     35  *
     36  * Thanks to Liverpool John Moores University, especially Dr. David Llewellyn-Jones
     37  * for providing resources (to test) and Professor Madjid Merabti.
     38  */
     39 
     40 #include <sys/cdefs.h>
     41 __KERNEL_RCSID(0, "$NetBSD: ip6_flow.c,v 1.28.2.2 2016/11/04 14:49:21 pgoyette Exp $");
     42 
     43 #ifdef _KERNEL_OPT
     44 #include "opt_net_mpsafe.h"
     45 #endif
     46 
     47 #include <sys/param.h>
     48 #include <sys/systm.h>
     49 #include <sys/malloc.h>
     50 #include <sys/mbuf.h>
     51 #include <sys/domain.h>
     52 #include <sys/protosw.h>
     53 #include <sys/socket.h>
     54 #include <sys/socketvar.h>
     55 #include <sys/time.h>
     56 #include <sys/kernel.h>
     57 #include <sys/pool.h>
     58 #include <sys/sysctl.h>
     59 #include <sys/workqueue.h>
     60 #include <sys/atomic.h>
     61 
     62 #include <net/if.h>
     63 #include <net/if_dl.h>
     64 #include <net/route.h>
     65 #include <net/pfil.h>
     66 
     67 #include <netinet/in.h>
     68 #include <netinet6/in6_var.h>
     69 #include <netinet/in_systm.h>
     70 #include <netinet/ip6.h>
     71 #include <netinet6/ip6_var.h>
     72 #include <netinet6/ip6_private.h>
     73 
     74 /*
     75  * IPv6 Fast Forward caches/hashes flows from one source to destination.
     76  *
     77  * Upon a successful forward IPv6FF caches and hashes details such as the
     78  * route, source and destination. Once another packet is received matching
     79  * the source and destination the packet is forwarded straight onto if_output
     80  * using the cached details.
     81  *
     82  * Example:
     83  * ether/fddi_input -> ip6flow_fastforward -> if_output
     84  */
     85 
     86 static struct pool ip6flow_pool;
     87 
     88 LIST_HEAD(ip6flowhead, ip6flow);
     89 
     90 /*
     91  * We could use IPv4 defines (IPFLOW_HASHBITS) but we'll
     92  * use our own (possibly for future expansion).
     93  */
     94 #define	IP6FLOW_TIMER		(5 * PR_SLOWHZ)
     95 #define	IP6FLOW_DEFAULT_HASHSIZE	(1 << IP6FLOW_HASHBITS)
     96 
     97 /*
     98  * ip6_flow.c internal lock.
     99  * If we use softnet_lock, it would cause recursive lock.
    100  *
    101  * This is a tentative workaround.
    102  * We should make it scalable somehow in the future.
    103  */
    104 static kmutex_t ip6flow_lock;
    105 static struct ip6flowhead *ip6flowtable = NULL;
    106 static struct ip6flowhead ip6flowlist;
    107 static int ip6flow_inuse;
    108 
    109 static void ip6flow_slowtimo_work(struct work *, void *);
    110 static struct workqueue	*ip6flow_slowtimo_wq;
    111 static struct work	ip6flow_slowtimo_wk;
    112 
    113 static int sysctl_net_inet6_ip6_hashsize(SYSCTLFN_PROTO);
    114 static int sysctl_net_inet6_ip6_maxflows(SYSCTLFN_PROTO);
    115 static void ip6flow_sysctl_init(struct sysctllog **);
    116 
    117 /*
    118  * Insert an ip6flow into the list.
    119  */
    120 #define	IP6FLOW_INSERT(bucket, ip6f) \
    121 do { \
    122 	LIST_INSERT_HEAD((bucket), (ip6f), ip6f_hash); \
    123 	LIST_INSERT_HEAD(&ip6flowlist, (ip6f), ip6f_list); \
    124 } while (/*CONSTCOND*/ 0)
    125 
    126 /*
    127  * Remove an ip6flow from the list.
    128  */
    129 #define	IP6FLOW_REMOVE(ip6f) \
    130 do { \
    131 	LIST_REMOVE((ip6f), ip6f_hash); \
    132 	LIST_REMOVE((ip6f), ip6f_list); \
    133 } while (/*CONSTCOND*/ 0)
    134 
    135 #ifndef IP6FLOW_DEFAULT
    136 #define	IP6FLOW_DEFAULT		256
    137 #endif
    138 
    139 int ip6_maxflows = IP6FLOW_DEFAULT;
    140 int ip6_hashsize = IP6FLOW_DEFAULT_HASHSIZE;
    141 
    142 /*
    143  * Calculate hash table position.
    144  */
    145 static size_t
    146 ip6flow_hash(const struct ip6_hdr *ip6)
    147 {
    148 	size_t hash;
    149 	uint32_t dst_sum, src_sum;
    150 	size_t idx;
    151 
    152 	src_sum = ip6->ip6_src.s6_addr32[0] + ip6->ip6_src.s6_addr32[1]
    153 	    + ip6->ip6_src.s6_addr32[2] + ip6->ip6_src.s6_addr32[3];
    154 	dst_sum = ip6->ip6_dst.s6_addr32[0] + ip6->ip6_dst.s6_addr32[1]
    155 	    + ip6->ip6_dst.s6_addr32[2] + ip6->ip6_dst.s6_addr32[3];
    156 
    157 	hash = ip6->ip6_flow;
    158 
    159 	for (idx = 0; idx < 32; idx += IP6FLOW_HASHBITS)
    160 		hash += (dst_sum >> (32 - idx)) + (src_sum >> idx);
    161 
    162 	return hash & (ip6_hashsize-1);
    163 }
    164 
    165 /*
    166  * Check to see if a flow already exists - if so return it.
    167  */
    168 static struct ip6flow *
    169 ip6flow_lookup(const struct ip6_hdr *ip6)
    170 {
    171 	size_t hash;
    172 	struct ip6flow *ip6f;
    173 
    174 	KASSERT(mutex_owned(&ip6flow_lock));
    175 
    176 	hash = ip6flow_hash(ip6);
    177 
    178 	LIST_FOREACH(ip6f, &ip6flowtable[hash], ip6f_hash) {
    179 		if (IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst, &ip6f->ip6f_dst)
    180 		    && IN6_ARE_ADDR_EQUAL(&ip6->ip6_src, &ip6f->ip6f_src)
    181 		    && ip6f->ip6f_flow == ip6->ip6_flow) {
    182 		    	/* A cached flow has been found. */
    183 			return ip6f;
    184 		}
    185 	}
    186 
    187 	return NULL;
    188 }
    189 
    190 void
    191 ip6flow_poolinit(void)
    192 {
    193 
    194 	pool_init(&ip6flow_pool, sizeof(struct ip6flow), 0, 0, 0, "ip6flowpl",
    195 			NULL, IPL_NET);
    196 }
    197 
    198 /*
    199  * Allocate memory and initialise lists. This function is called
    200  * from ip6_init and called there after to resize the hash table.
    201  * If a newly sized table cannot be malloc'ed we just continue
    202  * to use the old one.
    203  */
    204 static int
    205 ip6flow_init_locked(int table_size)
    206 {
    207 	struct ip6flowhead *new_table;
    208 	size_t i;
    209 
    210 	KASSERT(mutex_owned(&ip6flow_lock));
    211 
    212 	new_table = (struct ip6flowhead *)malloc(sizeof(struct ip6flowhead) *
    213 	    table_size, M_RTABLE, M_NOWAIT);
    214 
    215 	if (new_table == NULL)
    216 		return 1;
    217 
    218 	if (ip6flowtable != NULL)
    219 		free(ip6flowtable, M_RTABLE);
    220 
    221 	ip6flowtable = new_table;
    222 	ip6_hashsize = table_size;
    223 
    224 	LIST_INIT(&ip6flowlist);
    225 	for (i = 0; i < ip6_hashsize; i++)
    226 		LIST_INIT(&ip6flowtable[i]);
    227 
    228 	return 0;
    229 }
    230 
    231 int
    232 ip6flow_init(int table_size)
    233 {
    234 	int ret, error;
    235 
    236 	error = workqueue_create(&ip6flow_slowtimo_wq, "ip6flow_slowtimo",
    237 	    ip6flow_slowtimo_work, NULL, PRI_SOFTNET, IPL_SOFTNET, WQ_MPSAFE);
    238 	if (error != 0)
    239 		panic("%s: workqueue_create failed (%d)\n", __func__, error);
    240 
    241 	mutex_init(&ip6flow_lock, MUTEX_DEFAULT, IPL_NONE);
    242 
    243 	mutex_enter(&ip6flow_lock);
    244 	ret = ip6flow_init_locked(table_size);
    245 	mutex_exit(&ip6flow_lock);
    246 	ip6flow_sysctl_init(NULL);
    247 
    248 	return ret;
    249 }
    250 
    251 /*
    252  * IPv6 Fast Forward routine. Attempt to forward the packet -
    253  * if any problems are found return to the main IPv6 input
    254  * routine to deal with.
    255  */
    256 int
    257 ip6flow_fastforward(struct mbuf **mp)
    258 {
    259 	struct ip6flow *ip6f;
    260 	struct ip6_hdr *ip6;
    261 	struct rtentry *rt;
    262 	struct mbuf *m;
    263 	const struct sockaddr *dst;
    264 	int error;
    265 	int ret = 0;
    266 
    267 	mutex_enter(&ip6flow_lock);
    268 
    269 	/*
    270 	 * Are we forwarding packets and have flows?
    271 	 */
    272 	if (!ip6_forwarding || ip6flow_inuse == 0)
    273 		goto out;
    274 
    275 	m = *mp;
    276 	/*
    277 	 * At least size of IPv6 Header?
    278 	 */
    279 	if (m->m_len < sizeof(struct ip6_hdr))
    280 		goto out;
    281 	/*
    282 	 * Was packet received as a link-level multicast or broadcast?
    283 	 * If so, don't try to fast forward.
    284 	 */
    285 	if ((m->m_flags & (M_BCAST|M_MCAST)) != 0)
    286 		goto out;
    287 
    288 	if (IP6_HDR_ALIGNED_P(mtod(m, const void *)) == 0) {
    289 		if ((m = m_copyup(m, sizeof(struct ip6_hdr),
    290 				(max_linkhdr + 3) & ~3)) == NULL) {
    291 			goto out;
    292 		}
    293 		*mp = m;
    294 	} else if (__predict_false(m->m_len < sizeof(struct ip6_hdr))) {
    295 		if ((m = m_pullup(m, sizeof(struct ip6_hdr))) == NULL) {
    296 			goto out;
    297 		}
    298 		*mp = m;
    299 	}
    300 
    301 	ip6 = mtod(m, struct ip6_hdr *);
    302 
    303 	if ((ip6->ip6_vfc & IPV6_VERSION_MASK) != IPV6_VERSION) {
    304 		/* Bad version. */
    305 		goto out;
    306 	}
    307 
    308 	/*
    309 	 * If we have a hop-by-hop extension we must process it.
    310 	 * We just leave this up to ip6_input to deal with.
    311 	 */
    312 	if (ip6->ip6_nxt == IPPROTO_HOPOPTS)
    313 		goto out;
    314 
    315 	/*
    316 	 * Attempt to find a flow.
    317 	 */
    318 	if ((ip6f = ip6flow_lookup(ip6)) == NULL) {
    319 		/* No flow found. */
    320 		goto out;
    321 	}
    322 
    323 	/*
    324 	 * Route and interface still up?
    325 	 */
    326 	if ((rt = rtcache_validate(&ip6f->ip6f_ro)) == NULL ||
    327 	    (rt->rt_ifp->if_flags & IFF_UP) == 0 ||
    328 	    (rt->rt_flags & RTF_BLACKHOLE) != 0)
    329 		goto out;
    330 
    331 	/*
    332 	 * Packet size greater than MTU?
    333 	 */
    334 	if (m->m_pkthdr.len > rt->rt_ifp->if_mtu) {
    335 		/* Return to main IPv6 input function. */
    336 		goto out;
    337 	}
    338 
    339 	/*
    340 	 * Clear any in-bound checksum flags for this packet.
    341 	 */
    342 	m->m_pkthdr.csum_flags = 0;
    343 
    344 	if (ip6->ip6_hlim <= IPV6_HLIMDEC)
    345 		goto out;
    346 
    347 	/* Decrement hop limit (same as TTL) */
    348 	ip6->ip6_hlim -= IPV6_HLIMDEC;
    349 
    350 	if (rt->rt_flags & RTF_GATEWAY)
    351 		dst = rt->rt_gateway;
    352 	else
    353 		dst = rtcache_getdst(&ip6f->ip6f_ro);
    354 
    355 	PRT_SLOW_ARM(ip6f->ip6f_timer, IP6FLOW_TIMER);
    356 
    357 	ip6f->ip6f_uses++;
    358 
    359 	/* Send on its way - straight to the interface output routine. */
    360 	if ((error = if_output_lock(rt->rt_ifp, rt->rt_ifp, m, dst, rt)) != 0) {
    361 		ip6f->ip6f_dropped++;
    362 	} else {
    363 		ip6f->ip6f_forwarded++;
    364 	}
    365 	ret = 1;
    366  out:
    367 	mutex_exit(&ip6flow_lock);
    368 	return ret;
    369 }
    370 
    371 /*
    372  * Add the IPv6 flow statistics to the main IPv6 statistics.
    373  */
    374 static void
    375 ip6flow_addstats(const struct ip6flow *ip6f)
    376 {
    377 	struct rtentry *rt;
    378 	uint64_t *ip6s;
    379 
    380 	if ((rt = rtcache_validate(&ip6f->ip6f_ro)) != NULL)
    381 		rt->rt_use += ip6f->ip6f_uses;
    382 	ip6s = IP6_STAT_GETREF();
    383 	ip6s[IP6_STAT_FASTFORWARDFLOWS] = ip6flow_inuse;
    384 	ip6s[IP6_STAT_CANTFORWARD] += ip6f->ip6f_dropped;
    385 	ip6s[IP6_STAT_ODROPPED] += ip6f->ip6f_dropped;
    386 	ip6s[IP6_STAT_TOTAL] += ip6f->ip6f_uses;
    387 	ip6s[IP6_STAT_FORWARD] += ip6f->ip6f_forwarded;
    388 	ip6s[IP6_STAT_FASTFORWARD] += ip6f->ip6f_forwarded;
    389 	IP6_STAT_PUTREF();
    390 }
    391 
    392 /*
    393  * Add statistics and free the flow.
    394  */
    395 static void
    396 ip6flow_free(struct ip6flow *ip6f)
    397 {
    398 
    399 	KASSERT(mutex_owned(&ip6flow_lock));
    400 
    401 	/*
    402 	 * Remove the flow from the hash table (at elevated IPL).
    403 	 * Once it's off the list, we can deal with it at normal
    404 	 * network IPL.
    405 	 */
    406 	IP6FLOW_REMOVE(ip6f);
    407 
    408 	ip6flow_inuse--;
    409 	ip6flow_addstats(ip6f);
    410 	rtcache_free(&ip6f->ip6f_ro);
    411 	pool_put(&ip6flow_pool, ip6f);
    412 }
    413 
    414 static struct ip6flow *
    415 ip6flow_reap_locked(int just_one)
    416 {
    417 
    418 	KASSERT(mutex_owned(&ip6flow_lock));
    419 
    420 	while (just_one || ip6flow_inuse > ip6_maxflows) {
    421 		struct ip6flow *ip6f, *maybe_ip6f = NULL;
    422 
    423 		ip6f = LIST_FIRST(&ip6flowlist);
    424 		while (ip6f != NULL) {
    425 			/*
    426 			 * If this no longer points to a valid route -
    427 			 * reclaim it.
    428 			 */
    429 			if (rtcache_validate(&ip6f->ip6f_ro) == NULL)
    430 				goto done;
    431 			/*
    432 			 * choose the one that's been least recently
    433 			 * used or has had the least uses in the
    434 			 * last 1.5 intervals.
    435 			 */
    436 			if (maybe_ip6f == NULL ||
    437 			    ip6f->ip6f_timer < maybe_ip6f->ip6f_timer ||
    438 			    (ip6f->ip6f_timer == maybe_ip6f->ip6f_timer &&
    439 			     ip6f->ip6f_last_uses + ip6f->ip6f_uses <
    440 			         maybe_ip6f->ip6f_last_uses +
    441 			         maybe_ip6f->ip6f_uses))
    442 				maybe_ip6f = ip6f;
    443 			ip6f = LIST_NEXT(ip6f, ip6f_list);
    444 		}
    445 		ip6f = maybe_ip6f;
    446 	    done:
    447 		/*
    448 		 * Remove the entry from the flow table
    449 		 */
    450 		IP6FLOW_REMOVE(ip6f);
    451 
    452 		rtcache_free(&ip6f->ip6f_ro);
    453 		if (just_one) {
    454 			ip6flow_addstats(ip6f);
    455 			return ip6f;
    456 		}
    457 		ip6flow_inuse--;
    458 		ip6flow_addstats(ip6f);
    459 		pool_put(&ip6flow_pool, ip6f);
    460 	}
    461 	return NULL;
    462 }
    463 
    464 /*
    465  * Reap one or more flows - ip6flow_reap may remove
    466  * multiple flows if net.inet6.ip6.maxflows is reduced.
    467  */
    468 struct ip6flow *
    469 ip6flow_reap(int just_one)
    470 {
    471 	struct ip6flow *ip6f;
    472 
    473 	mutex_enter(&ip6flow_lock);
    474 	ip6f = ip6flow_reap_locked(just_one);
    475 	mutex_exit(&ip6flow_lock);
    476 	return ip6f;
    477 }
    478 
    479 static unsigned int ip6flow_work_enqueued = 0;
    480 
    481 void
    482 ip6flow_slowtimo_work(struct work *wk, void *arg)
    483 {
    484 	struct ip6flow *ip6f, *next_ip6f;
    485 
    486 	/* We can allow enqueuing another work at this point */
    487 	atomic_swap_uint(&ip6flow_work_enqueued, 0);
    488 
    489 #ifndef NET_MPSAFE
    490 	mutex_enter(softnet_lock);
    491 	KERNEL_LOCK(1, NULL);
    492 #endif
    493 	mutex_enter(&ip6flow_lock);
    494 
    495 	for (ip6f = LIST_FIRST(&ip6flowlist); ip6f != NULL; ip6f = next_ip6f) {
    496 		next_ip6f = LIST_NEXT(ip6f, ip6f_list);
    497 		if (PRT_SLOW_ISEXPIRED(ip6f->ip6f_timer) ||
    498 		    rtcache_validate(&ip6f->ip6f_ro) == NULL) {
    499 			ip6flow_free(ip6f);
    500 		} else {
    501 			ip6f->ip6f_last_uses = ip6f->ip6f_uses;
    502 			ip6flow_addstats(ip6f);
    503 			ip6f->ip6f_uses = 0;
    504 			ip6f->ip6f_dropped = 0;
    505 			ip6f->ip6f_forwarded = 0;
    506 		}
    507 	}
    508 
    509 	mutex_exit(&ip6flow_lock);
    510 #ifndef NET_MPSAFE
    511 	KERNEL_UNLOCK_ONE(NULL);
    512 	mutex_exit(softnet_lock);
    513 #endif
    514 }
    515 
    516 void
    517 ip6flow_slowtimo(void)
    518 {
    519 
    520 	/* Avoid enqueuing another work when one is already enqueued */
    521 	if (atomic_swap_uint(&ip6flow_work_enqueued, 1) == 1)
    522 		return;
    523 
    524 	workqueue_enqueue(ip6flow_slowtimo_wq, &ip6flow_slowtimo_wk, NULL);
    525 }
    526 
    527 /*
    528  * We have successfully forwarded a packet using the normal
    529  * IPv6 stack. Now create/update a flow.
    530  */
    531 void
    532 ip6flow_create(const struct route *ro, struct mbuf *m)
    533 {
    534 	const struct ip6_hdr *ip6;
    535 	struct ip6flow *ip6f;
    536 	size_t hash;
    537 
    538 	ip6 = mtod(m, const struct ip6_hdr *);
    539 
    540 #ifndef NET_MPSAFE
    541 	KERNEL_LOCK(1, NULL);
    542 #endif
    543 	mutex_enter(&ip6flow_lock);
    544 
    545 	/*
    546 	 * If IPv6 Fast Forward is disabled, don't create a flow.
    547 	 * It can be disabled by setting net.inet6.ip6.maxflows to 0.
    548 	 *
    549 	 * Don't create a flow for ICMPv6 messages.
    550 	 */
    551 	if (ip6_maxflows == 0 || ip6->ip6_nxt == IPPROTO_IPV6_ICMP)
    552 		goto out;
    553 
    554 	/*
    555 	 * See if an existing flow exists.  If so:
    556 	 *	- Remove the flow
    557 	 *	- Add flow statistics
    558 	 *	- Free the route
    559 	 *	- Reset statistics
    560 	 *
    561 	 * If a flow doesn't exist allocate a new one if
    562 	 * ip6_maxflows hasn't reached its limit. If it has
    563 	 * been reached, reap some flows.
    564 	 */
    565 	ip6f = ip6flow_lookup(ip6);
    566 	if (ip6f == NULL) {
    567 		if (ip6flow_inuse >= ip6_maxflows) {
    568 			ip6f = ip6flow_reap_locked(1);
    569 		} else {
    570 			ip6f = pool_get(&ip6flow_pool, PR_NOWAIT);
    571 			if (ip6f == NULL)
    572 				goto out;
    573 			ip6flow_inuse++;
    574 		}
    575 		memset(ip6f, 0, sizeof(*ip6f));
    576 	} else {
    577 		IP6FLOW_REMOVE(ip6f);
    578 
    579 		ip6flow_addstats(ip6f);
    580 		rtcache_free(&ip6f->ip6f_ro);
    581 		ip6f->ip6f_uses = 0;
    582 		ip6f->ip6f_last_uses = 0;
    583 		ip6f->ip6f_dropped = 0;
    584 		ip6f->ip6f_forwarded = 0;
    585 	}
    586 
    587 	/*
    588 	 * Fill in the updated/new details.
    589 	 */
    590 	rtcache_copy(&ip6f->ip6f_ro, ro);
    591 	ip6f->ip6f_dst = ip6->ip6_dst;
    592 	ip6f->ip6f_src = ip6->ip6_src;
    593 	ip6f->ip6f_flow = ip6->ip6_flow;
    594 	PRT_SLOW_ARM(ip6f->ip6f_timer, IP6FLOW_TIMER);
    595 
    596 	/*
    597 	 * Insert into the approriate bucket of the flow table.
    598 	 */
    599 	hash = ip6flow_hash(ip6);
    600 	IP6FLOW_INSERT(&ip6flowtable[hash], ip6f);
    601 
    602  out:
    603 	mutex_exit(&ip6flow_lock);
    604 #ifndef NET_MPSAFE
    605 	KERNEL_UNLOCK_ONE(NULL);
    606 #endif
    607 }
    608 
    609 /*
    610  * Invalidate/remove all flows - if new_size is positive we
    611  * resize the hash table.
    612  */
    613 int
    614 ip6flow_invalidate_all(int new_size)
    615 {
    616 	struct ip6flow *ip6f, *next_ip6f;
    617 	int error;
    618 
    619 	error = 0;
    620 
    621 	mutex_enter(&ip6flow_lock);
    622 
    623 	for (ip6f = LIST_FIRST(&ip6flowlist); ip6f != NULL; ip6f = next_ip6f) {
    624 		next_ip6f = LIST_NEXT(ip6f, ip6f_list);
    625 		ip6flow_free(ip6f);
    626 	}
    627 
    628 	if (new_size)
    629 		error = ip6flow_init_locked(new_size);
    630 
    631 	mutex_exit(&ip6flow_lock);
    632 
    633 	return error;
    634 }
    635 
    636 /*
    637  * sysctl helper routine for net.inet.ip6.maxflows. Since
    638  * we could reduce this value, call ip6flow_reap();
    639  */
    640 static int
    641 sysctl_net_inet6_ip6_maxflows(SYSCTLFN_ARGS)
    642 {
    643 	int error;
    644 
    645 	error = sysctl_lookup(SYSCTLFN_CALL(rnode));
    646 	if (error || newp == NULL)
    647 		return (error);
    648 
    649 #ifndef NET_MPSAFE
    650 	mutex_enter(softnet_lock);
    651 	KERNEL_LOCK(1, NULL);
    652 #endif
    653 
    654 	ip6flow_reap(0);
    655 
    656 #ifndef NET_MPSAFE
    657 	KERNEL_UNLOCK_ONE(NULL);
    658 	mutex_exit(softnet_lock);
    659 #endif
    660 
    661 	return (0);
    662 }
    663 
    664 static int
    665 sysctl_net_inet6_ip6_hashsize(SYSCTLFN_ARGS)
    666 {
    667 	int error, tmp;
    668 	struct sysctlnode node;
    669 
    670 	node = *rnode;
    671 	tmp = ip6_hashsize;
    672 	node.sysctl_data = &tmp;
    673 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    674 	if (error || newp == NULL)
    675 		return (error);
    676 
    677 	if ((tmp & (tmp - 1)) == 0 && tmp != 0) {
    678 		/*
    679 		 * Can only fail due to malloc()
    680 		 */
    681 #ifndef NET_MPSAFE
    682 		mutex_enter(softnet_lock);
    683 		KERNEL_LOCK(1, NULL);
    684 #endif
    685 		error = ip6flow_invalidate_all(tmp);
    686 #ifndef NET_MPSAFE
    687 		KERNEL_UNLOCK_ONE(NULL);
    688 		mutex_exit(softnet_lock);
    689 #endif
    690 	} else {
    691 		/*
    692 		 * EINVAL if not a power of 2
    693 		 */
    694 		error = EINVAL;
    695 	}
    696 
    697 	return error;
    698 }
    699 
    700 static void
    701 ip6flow_sysctl_init(struct sysctllog **clog)
    702 {
    703 
    704 	sysctl_createv(clog, 0, NULL, NULL,
    705 		       CTLFLAG_PERMANENT,
    706 		       CTLTYPE_NODE, "inet6",
    707 		       SYSCTL_DESCR("PF_INET6 related settings"),
    708 		       NULL, 0, NULL, 0,
    709 		       CTL_NET, PF_INET6, CTL_EOL);
    710 	sysctl_createv(clog, 0, NULL, NULL,
    711 		       CTLFLAG_PERMANENT,
    712 		       CTLTYPE_NODE, "ip6",
    713 		       SYSCTL_DESCR("IPv6 related settings"),
    714 		       NULL, 0, NULL, 0,
    715 		       CTL_NET, PF_INET6, IPPROTO_IPV6, CTL_EOL);
    716 
    717 	sysctl_createv(clog, 0, NULL, NULL,
    718 			CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
    719 			CTLTYPE_INT, "maxflows",
    720 			SYSCTL_DESCR("Number of flows for fast forwarding (IPv6)"),
    721 			sysctl_net_inet6_ip6_maxflows, 0, &ip6_maxflows, 0,
    722 			CTL_NET, PF_INET6, IPPROTO_IPV6,
    723 			CTL_CREATE, CTL_EOL);
    724 	sysctl_createv(clog, 0, NULL, NULL,
    725 			CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
    726 			CTLTYPE_INT, "hashsize",
    727 			SYSCTL_DESCR("Size of hash table for fast forwarding (IPv6)"),
    728 			sysctl_net_inet6_ip6_hashsize, 0, &ip6_hashsize, 0,
    729 			CTL_NET, PF_INET6, IPPROTO_IPV6,
    730 			CTL_CREATE, CTL_EOL);
    731 }
    732