Home | History | Annotate | Line # | Download | only in netinet6
ip6_flow.c revision 1.28.2.2
      1  1.28.2.2  pgoyette /*	$NetBSD: ip6_flow.c,v 1.28.2.2 2016/11/04 14:49:21 pgoyette Exp $	*/
      2       1.1  liamjfoy 
      3       1.1  liamjfoy /*-
      4       1.1  liamjfoy  * Copyright (c) 2007 The NetBSD Foundation, Inc.
      5       1.1  liamjfoy  * All rights reserved.
      6       1.1  liamjfoy  *
      7       1.1  liamjfoy  * This code is derived from software contributed to The NetBSD Foundation
      8       1.1  liamjfoy  * by the 3am Software Foundry ("3am").  It was developed by Liam J. Foy
      9       1.1  liamjfoy  * <liamjfoy (at) netbsd.org> and Matt Thomas <matt (at) netbsd.org>.
     10       1.1  liamjfoy  *
     11       1.1  liamjfoy  * Redistribution and use in source and binary forms, with or without
     12       1.1  liamjfoy  * modification, are permitted provided that the following conditions
     13       1.1  liamjfoy  * are met:
     14       1.1  liamjfoy  * 1. Redistributions of source code must retain the above copyright
     15       1.1  liamjfoy  *    notice, this list of conditions and the following disclaimer.
     16       1.1  liamjfoy  * 2. Redistributions in binary form must reproduce the above copyright
     17       1.1  liamjfoy  *    notice, this list of conditions and the following disclaimer in the
     18       1.1  liamjfoy  *    documentation and/or other materials provided with the distribution.
     19       1.1  liamjfoy  *
     20       1.1  liamjfoy  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21       1.1  liamjfoy  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22       1.1  liamjfoy  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23       1.1  liamjfoy  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24       1.1  liamjfoy  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25       1.1  liamjfoy  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26       1.1  liamjfoy  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27       1.1  liamjfoy  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28       1.1  liamjfoy  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29       1.1  liamjfoy  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30       1.1  liamjfoy  * POSSIBILITY OF SUCH DAMAGE.
     31       1.1  liamjfoy  *
     32       1.1  liamjfoy  * IPv6 version was developed by Liam J. Foy. Original source existed in IPv4
     33       1.1  liamjfoy  * format developed by Matt Thomas. Thanks to Joerg Sonnenberger, Matt
     34       1.1  liamjfoy  * Thomas and Christos Zoulas.
     35       1.1  liamjfoy  *
     36       1.1  liamjfoy  * Thanks to Liverpool John Moores University, especially Dr. David Llewellyn-Jones
     37       1.1  liamjfoy  * for providing resources (to test) and Professor Madjid Merabti.
     38       1.1  liamjfoy  */
     39       1.1  liamjfoy 
     40       1.1  liamjfoy #include <sys/cdefs.h>
     41  1.28.2.2  pgoyette __KERNEL_RCSID(0, "$NetBSD: ip6_flow.c,v 1.28.2.2 2016/11/04 14:49:21 pgoyette Exp $");
     42  1.28.2.2  pgoyette 
     43  1.28.2.2  pgoyette #ifdef _KERNEL_OPT
     44  1.28.2.2  pgoyette #include "opt_net_mpsafe.h"
     45  1.28.2.2  pgoyette #endif
     46       1.1  liamjfoy 
     47       1.1  liamjfoy #include <sys/param.h>
     48       1.1  liamjfoy #include <sys/systm.h>
     49       1.1  liamjfoy #include <sys/malloc.h>
     50       1.1  liamjfoy #include <sys/mbuf.h>
     51       1.1  liamjfoy #include <sys/domain.h>
     52       1.1  liamjfoy #include <sys/protosw.h>
     53       1.1  liamjfoy #include <sys/socket.h>
     54       1.1  liamjfoy #include <sys/socketvar.h>
     55       1.1  liamjfoy #include <sys/time.h>
     56       1.1  liamjfoy #include <sys/kernel.h>
     57       1.1  liamjfoy #include <sys/pool.h>
     58       1.1  liamjfoy #include <sys/sysctl.h>
     59      1.28     ozaki #include <sys/workqueue.h>
     60  1.28.2.1  pgoyette #include <sys/atomic.h>
     61       1.1  liamjfoy 
     62       1.1  liamjfoy #include <net/if.h>
     63       1.1  liamjfoy #include <net/if_dl.h>
     64       1.1  liamjfoy #include <net/route.h>
     65       1.1  liamjfoy #include <net/pfil.h>
     66       1.1  liamjfoy 
     67       1.1  liamjfoy #include <netinet/in.h>
     68       1.1  liamjfoy #include <netinet6/in6_var.h>
     69       1.1  liamjfoy #include <netinet/in_systm.h>
     70       1.1  liamjfoy #include <netinet/ip6.h>
     71       1.1  liamjfoy #include <netinet6/ip6_var.h>
     72      1.15   thorpej #include <netinet6/ip6_private.h>
     73       1.1  liamjfoy 
     74       1.1  liamjfoy /*
     75       1.1  liamjfoy  * IPv6 Fast Forward caches/hashes flows from one source to destination.
     76       1.1  liamjfoy  *
     77       1.1  liamjfoy  * Upon a successful forward IPv6FF caches and hashes details such as the
     78       1.1  liamjfoy  * route, source and destination. Once another packet is received matching
     79       1.1  liamjfoy  * the source and destination the packet is forwarded straight onto if_output
     80       1.1  liamjfoy  * using the cached details.
     81       1.1  liamjfoy  *
     82       1.1  liamjfoy  * Example:
     83      1.20  christos  * ether/fddi_input -> ip6flow_fastforward -> if_output
     84       1.1  liamjfoy  */
     85       1.1  liamjfoy 
     86      1.18  liamjfoy static struct pool ip6flow_pool;
     87       1.1  liamjfoy 
     88       1.1  liamjfoy LIST_HEAD(ip6flowhead, ip6flow);
     89       1.1  liamjfoy 
     90       1.1  liamjfoy /*
     91       1.1  liamjfoy  * We could use IPv4 defines (IPFLOW_HASHBITS) but we'll
     92       1.1  liamjfoy  * use our own (possibly for future expansion).
     93       1.1  liamjfoy  */
     94       1.1  liamjfoy #define	IP6FLOW_TIMER		(5 * PR_SLOWHZ)
     95       1.4  liamjfoy #define	IP6FLOW_DEFAULT_HASHSIZE	(1 << IP6FLOW_HASHBITS)
     96       1.1  liamjfoy 
     97      1.25  knakahar /*
     98      1.25  knakahar  * ip6_flow.c internal lock.
     99      1.25  knakahar  * If we use softnet_lock, it would cause recursive lock.
    100      1.25  knakahar  *
    101      1.25  knakahar  * This is a tentative workaround.
    102      1.25  knakahar  * We should make it scalable somehow in the future.
    103      1.25  knakahar  */
    104      1.25  knakahar static kmutex_t ip6flow_lock;
    105       1.4  liamjfoy static struct ip6flowhead *ip6flowtable = NULL;
    106       1.1  liamjfoy static struct ip6flowhead ip6flowlist;
    107       1.1  liamjfoy static int ip6flow_inuse;
    108       1.1  liamjfoy 
    109      1.28     ozaki static void ip6flow_slowtimo_work(struct work *, void *);
    110      1.28     ozaki static struct workqueue	*ip6flow_slowtimo_wq;
    111      1.28     ozaki static struct work	ip6flow_slowtimo_wk;
    112      1.28     ozaki 
    113  1.28.2.1  pgoyette static int sysctl_net_inet6_ip6_hashsize(SYSCTLFN_PROTO);
    114  1.28.2.1  pgoyette static int sysctl_net_inet6_ip6_maxflows(SYSCTLFN_PROTO);
    115  1.28.2.1  pgoyette static void ip6flow_sysctl_init(struct sysctllog **);
    116  1.28.2.1  pgoyette 
    117       1.1  liamjfoy /*
    118       1.1  liamjfoy  * Insert an ip6flow into the list.
    119       1.1  liamjfoy  */
    120       1.1  liamjfoy #define	IP6FLOW_INSERT(bucket, ip6f) \
    121       1.1  liamjfoy do { \
    122       1.1  liamjfoy 	LIST_INSERT_HEAD((bucket), (ip6f), ip6f_hash); \
    123       1.1  liamjfoy 	LIST_INSERT_HEAD(&ip6flowlist, (ip6f), ip6f_list); \
    124       1.1  liamjfoy } while (/*CONSTCOND*/ 0)
    125       1.1  liamjfoy 
    126       1.1  liamjfoy /*
    127       1.1  liamjfoy  * Remove an ip6flow from the list.
    128       1.1  liamjfoy  */
    129       1.1  liamjfoy #define	IP6FLOW_REMOVE(ip6f) \
    130       1.1  liamjfoy do { \
    131       1.1  liamjfoy 	LIST_REMOVE((ip6f), ip6f_hash); \
    132       1.1  liamjfoy 	LIST_REMOVE((ip6f), ip6f_list); \
    133       1.1  liamjfoy } while (/*CONSTCOND*/ 0)
    134       1.1  liamjfoy 
    135       1.1  liamjfoy #ifndef IP6FLOW_DEFAULT
    136       1.1  liamjfoy #define	IP6FLOW_DEFAULT		256
    137       1.1  liamjfoy #endif
    138       1.1  liamjfoy 
    139       1.1  liamjfoy int ip6_maxflows = IP6FLOW_DEFAULT;
    140       1.4  liamjfoy int ip6_hashsize = IP6FLOW_DEFAULT_HASHSIZE;
    141       1.1  liamjfoy 
    142       1.1  liamjfoy /*
    143       1.1  liamjfoy  * Calculate hash table position.
    144       1.1  liamjfoy  */
    145       1.1  liamjfoy static size_t
    146      1.13    dyoung ip6flow_hash(const struct ip6_hdr *ip6)
    147       1.1  liamjfoy {
    148       1.1  liamjfoy 	size_t hash;
    149       1.1  liamjfoy 	uint32_t dst_sum, src_sum;
    150       1.6  liamjfoy 	size_t idx;
    151       1.1  liamjfoy 
    152       1.1  liamjfoy 	src_sum = ip6->ip6_src.s6_addr32[0] + ip6->ip6_src.s6_addr32[1]
    153       1.1  liamjfoy 	    + ip6->ip6_src.s6_addr32[2] + ip6->ip6_src.s6_addr32[3];
    154       1.1  liamjfoy 	dst_sum = ip6->ip6_dst.s6_addr32[0] + ip6->ip6_dst.s6_addr32[1]
    155       1.1  liamjfoy 	    + ip6->ip6_dst.s6_addr32[2] + ip6->ip6_dst.s6_addr32[3];
    156       1.1  liamjfoy 
    157       1.1  liamjfoy 	hash = ip6->ip6_flow;
    158       1.1  liamjfoy 
    159       1.1  liamjfoy 	for (idx = 0; idx < 32; idx += IP6FLOW_HASHBITS)
    160       1.1  liamjfoy 		hash += (dst_sum >> (32 - idx)) + (src_sum >> idx);
    161       1.1  liamjfoy 
    162       1.4  liamjfoy 	return hash & (ip6_hashsize-1);
    163       1.1  liamjfoy }
    164       1.1  liamjfoy 
    165       1.1  liamjfoy /*
    166       1.1  liamjfoy  * Check to see if a flow already exists - if so return it.
    167       1.1  liamjfoy  */
    168       1.1  liamjfoy static struct ip6flow *
    169      1.13    dyoung ip6flow_lookup(const struct ip6_hdr *ip6)
    170       1.1  liamjfoy {
    171       1.1  liamjfoy 	size_t hash;
    172       1.1  liamjfoy 	struct ip6flow *ip6f;
    173       1.1  liamjfoy 
    174      1.25  knakahar 	KASSERT(mutex_owned(&ip6flow_lock));
    175      1.25  knakahar 
    176       1.1  liamjfoy 	hash = ip6flow_hash(ip6);
    177       1.1  liamjfoy 
    178       1.2  liamjfoy 	LIST_FOREACH(ip6f, &ip6flowtable[hash], ip6f_hash) {
    179       1.1  liamjfoy 		if (IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst, &ip6f->ip6f_dst)
    180       1.1  liamjfoy 		    && IN6_ARE_ADDR_EQUAL(&ip6->ip6_src, &ip6f->ip6f_src)
    181       1.1  liamjfoy 		    && ip6f->ip6f_flow == ip6->ip6_flow) {
    182       1.1  liamjfoy 		    	/* A cached flow has been found. */
    183       1.1  liamjfoy 			return ip6f;
    184       1.1  liamjfoy 		}
    185       1.1  liamjfoy 	}
    186       1.1  liamjfoy 
    187       1.1  liamjfoy 	return NULL;
    188       1.1  liamjfoy }
    189       1.1  liamjfoy 
    190      1.18  liamjfoy void
    191      1.18  liamjfoy ip6flow_poolinit(void)
    192      1.18  liamjfoy {
    193      1.18  liamjfoy 
    194      1.18  liamjfoy 	pool_init(&ip6flow_pool, sizeof(struct ip6flow), 0, 0, 0, "ip6flowpl",
    195      1.18  liamjfoy 			NULL, IPL_NET);
    196      1.18  liamjfoy }
    197      1.18  liamjfoy 
    198       1.1  liamjfoy /*
    199       1.4  liamjfoy  * Allocate memory and initialise lists. This function is called
    200       1.4  liamjfoy  * from ip6_init and called there after to resize the hash table.
    201       1.4  liamjfoy  * If a newly sized table cannot be malloc'ed we just continue
    202       1.4  liamjfoy  * to use the old one.
    203       1.1  liamjfoy  */
    204      1.25  knakahar static int
    205      1.25  knakahar ip6flow_init_locked(int table_size)
    206       1.1  liamjfoy {
    207       1.4  liamjfoy 	struct ip6flowhead *new_table;
    208       1.1  liamjfoy 	size_t i;
    209       1.1  liamjfoy 
    210      1.25  knakahar 	KASSERT(mutex_owned(&ip6flow_lock));
    211      1.25  knakahar 
    212       1.4  liamjfoy 	new_table = (struct ip6flowhead *)malloc(sizeof(struct ip6flowhead) *
    213       1.4  liamjfoy 	    table_size, M_RTABLE, M_NOWAIT);
    214       1.4  liamjfoy 
    215       1.4  liamjfoy 	if (new_table == NULL)
    216       1.4  liamjfoy 		return 1;
    217       1.4  liamjfoy 
    218       1.4  liamjfoy 	if (ip6flowtable != NULL)
    219       1.4  liamjfoy 		free(ip6flowtable, M_RTABLE);
    220       1.4  liamjfoy 
    221       1.4  liamjfoy 	ip6flowtable = new_table;
    222       1.4  liamjfoy 	ip6_hashsize = table_size;
    223       1.4  liamjfoy 
    224       1.1  liamjfoy 	LIST_INIT(&ip6flowlist);
    225       1.4  liamjfoy 	for (i = 0; i < ip6_hashsize; i++)
    226       1.1  liamjfoy 		LIST_INIT(&ip6flowtable[i]);
    227       1.4  liamjfoy 
    228       1.4  liamjfoy 	return 0;
    229       1.1  liamjfoy }
    230       1.1  liamjfoy 
    231      1.25  knakahar int
    232      1.25  knakahar ip6flow_init(int table_size)
    233      1.25  knakahar {
    234      1.28     ozaki 	int ret, error;
    235      1.28     ozaki 
    236      1.28     ozaki 	error = workqueue_create(&ip6flow_slowtimo_wq, "ip6flow_slowtimo",
    237      1.28     ozaki 	    ip6flow_slowtimo_work, NULL, PRI_SOFTNET, IPL_SOFTNET, WQ_MPSAFE);
    238      1.28     ozaki 	if (error != 0)
    239      1.28     ozaki 		panic("%s: workqueue_create failed (%d)\n", __func__, error);
    240      1.25  knakahar 
    241      1.25  knakahar 	mutex_init(&ip6flow_lock, MUTEX_DEFAULT, IPL_NONE);
    242      1.25  knakahar 
    243      1.25  knakahar 	mutex_enter(&ip6flow_lock);
    244      1.25  knakahar 	ret = ip6flow_init_locked(table_size);
    245      1.25  knakahar 	mutex_exit(&ip6flow_lock);
    246  1.28.2.1  pgoyette 	ip6flow_sysctl_init(NULL);
    247      1.25  knakahar 
    248      1.25  knakahar 	return ret;
    249      1.25  knakahar }
    250      1.25  knakahar 
    251       1.1  liamjfoy /*
    252       1.1  liamjfoy  * IPv6 Fast Forward routine. Attempt to forward the packet -
    253       1.1  liamjfoy  * if any problems are found return to the main IPv6 input
    254       1.1  liamjfoy  * routine to deal with.
    255       1.1  liamjfoy  */
    256       1.1  liamjfoy int
    257      1.20  christos ip6flow_fastforward(struct mbuf **mp)
    258       1.1  liamjfoy {
    259       1.1  liamjfoy 	struct ip6flow *ip6f;
    260       1.1  liamjfoy 	struct ip6_hdr *ip6;
    261       1.1  liamjfoy 	struct rtentry *rt;
    262      1.20  christos 	struct mbuf *m;
    263       1.7    dyoung 	const struct sockaddr *dst;
    264       1.1  liamjfoy 	int error;
    265      1.25  knakahar 	int ret = 0;
    266      1.25  knakahar 
    267      1.25  knakahar 	mutex_enter(&ip6flow_lock);
    268       1.1  liamjfoy 
    269       1.1  liamjfoy 	/*
    270       1.1  liamjfoy 	 * Are we forwarding packets and have flows?
    271       1.1  liamjfoy 	 */
    272       1.1  liamjfoy 	if (!ip6_forwarding || ip6flow_inuse == 0)
    273      1.25  knakahar 		goto out;
    274       1.1  liamjfoy 
    275      1.20  christos 	m = *mp;
    276       1.1  liamjfoy 	/*
    277       1.1  liamjfoy 	 * At least size of IPv6 Header?
    278       1.1  liamjfoy 	 */
    279       1.1  liamjfoy 	if (m->m_len < sizeof(struct ip6_hdr))
    280      1.25  knakahar 		goto out;
    281       1.1  liamjfoy 	/*
    282       1.1  liamjfoy 	 * Was packet received as a link-level multicast or broadcast?
    283       1.1  liamjfoy 	 * If so, don't try to fast forward.
    284       1.1  liamjfoy 	 */
    285       1.1  liamjfoy 	if ((m->m_flags & (M_BCAST|M_MCAST)) != 0)
    286      1.25  knakahar 		goto out;
    287       1.1  liamjfoy 
    288      1.13    dyoung 	if (IP6_HDR_ALIGNED_P(mtod(m, const void *)) == 0) {
    289       1.1  liamjfoy 		if ((m = m_copyup(m, sizeof(struct ip6_hdr),
    290       1.1  liamjfoy 				(max_linkhdr + 3) & ~3)) == NULL) {
    291      1.25  knakahar 			goto out;
    292       1.1  liamjfoy 		}
    293      1.20  christos 		*mp = m;
    294       1.1  liamjfoy 	} else if (__predict_false(m->m_len < sizeof(struct ip6_hdr))) {
    295       1.1  liamjfoy 		if ((m = m_pullup(m, sizeof(struct ip6_hdr))) == NULL) {
    296      1.25  knakahar 			goto out;
    297       1.1  liamjfoy 		}
    298      1.20  christos 		*mp = m;
    299       1.1  liamjfoy 	}
    300       1.1  liamjfoy 
    301       1.1  liamjfoy 	ip6 = mtod(m, struct ip6_hdr *);
    302       1.1  liamjfoy 
    303       1.1  liamjfoy 	if ((ip6->ip6_vfc & IPV6_VERSION_MASK) != IPV6_VERSION) {
    304       1.1  liamjfoy 		/* Bad version. */
    305      1.25  knakahar 		goto out;
    306       1.1  liamjfoy 	}
    307       1.1  liamjfoy 
    308       1.1  liamjfoy 	/*
    309       1.1  liamjfoy 	 * If we have a hop-by-hop extension we must process it.
    310       1.1  liamjfoy 	 * We just leave this up to ip6_input to deal with.
    311       1.1  liamjfoy 	 */
    312       1.1  liamjfoy 	if (ip6->ip6_nxt == IPPROTO_HOPOPTS)
    313      1.25  knakahar 		goto out;
    314       1.1  liamjfoy 
    315       1.1  liamjfoy 	/*
    316       1.1  liamjfoy 	 * Attempt to find a flow.
    317       1.1  liamjfoy 	 */
    318       1.1  liamjfoy 	if ((ip6f = ip6flow_lookup(ip6)) == NULL) {
    319       1.1  liamjfoy 		/* No flow found. */
    320      1.25  knakahar 		goto out;
    321       1.1  liamjfoy 	}
    322       1.1  liamjfoy 
    323       1.1  liamjfoy 	/*
    324       1.1  liamjfoy 	 * Route and interface still up?
    325       1.1  liamjfoy 	 */
    326      1.12    dyoung 	if ((rt = rtcache_validate(&ip6f->ip6f_ro)) == NULL ||
    327      1.24       roy 	    (rt->rt_ifp->if_flags & IFF_UP) == 0 ||
    328      1.24       roy 	    (rt->rt_flags & RTF_BLACKHOLE) != 0)
    329      1.25  knakahar 		goto out;
    330       1.1  liamjfoy 
    331       1.1  liamjfoy 	/*
    332       1.1  liamjfoy 	 * Packet size greater than MTU?
    333       1.1  liamjfoy 	 */
    334       1.1  liamjfoy 	if (m->m_pkthdr.len > rt->rt_ifp->if_mtu) {
    335       1.1  liamjfoy 		/* Return to main IPv6 input function. */
    336      1.25  knakahar 		goto out;
    337       1.1  liamjfoy 	}
    338       1.1  liamjfoy 
    339      1.21   msaitoh 	/*
    340      1.21   msaitoh 	 * Clear any in-bound checksum flags for this packet.
    341      1.21   msaitoh 	 */
    342      1.21   msaitoh 	m->m_pkthdr.csum_flags = 0;
    343      1.21   msaitoh 
    344       1.1  liamjfoy 	if (ip6->ip6_hlim <= IPV6_HLIMDEC)
    345      1.25  knakahar 		goto out;
    346       1.1  liamjfoy 
    347       1.1  liamjfoy 	/* Decrement hop limit (same as TTL) */
    348       1.1  liamjfoy 	ip6->ip6_hlim -= IPV6_HLIMDEC;
    349       1.1  liamjfoy 
    350       1.1  liamjfoy 	if (rt->rt_flags & RTF_GATEWAY)
    351       1.7    dyoung 		dst = rt->rt_gateway;
    352       1.1  liamjfoy 	else
    353       1.7    dyoung 		dst = rtcache_getdst(&ip6f->ip6f_ro);
    354       1.1  liamjfoy 
    355       1.1  liamjfoy 	PRT_SLOW_ARM(ip6f->ip6f_timer, IP6FLOW_TIMER);
    356       1.1  liamjfoy 
    357       1.1  liamjfoy 	ip6f->ip6f_uses++;
    358       1.1  liamjfoy 
    359       1.1  liamjfoy 	/* Send on its way - straight to the interface output routine. */
    360      1.27  knakahar 	if ((error = if_output_lock(rt->rt_ifp, rt->rt_ifp, m, dst, rt)) != 0) {
    361       1.1  liamjfoy 		ip6f->ip6f_dropped++;
    362       1.1  liamjfoy 	} else {
    363       1.1  liamjfoy 		ip6f->ip6f_forwarded++;
    364       1.1  liamjfoy 	}
    365      1.25  knakahar 	ret = 1;
    366      1.25  knakahar  out:
    367      1.25  knakahar 	mutex_exit(&ip6flow_lock);
    368      1.25  knakahar 	return ret;
    369       1.1  liamjfoy }
    370       1.1  liamjfoy 
    371       1.1  liamjfoy /*
    372       1.1  liamjfoy  * Add the IPv6 flow statistics to the main IPv6 statistics.
    373       1.1  liamjfoy  */
    374       1.1  liamjfoy static void
    375      1.13    dyoung ip6flow_addstats(const struct ip6flow *ip6f)
    376       1.1  liamjfoy {
    377      1.11    dyoung 	struct rtentry *rt;
    378      1.15   thorpej 	uint64_t *ip6s;
    379      1.11    dyoung 
    380      1.12    dyoung 	if ((rt = rtcache_validate(&ip6f->ip6f_ro)) != NULL)
    381      1.11    dyoung 		rt->rt_use += ip6f->ip6f_uses;
    382      1.15   thorpej 	ip6s = IP6_STAT_GETREF();
    383      1.15   thorpej 	ip6s[IP6_STAT_FASTFORWARDFLOWS] = ip6flow_inuse;
    384      1.15   thorpej 	ip6s[IP6_STAT_CANTFORWARD] += ip6f->ip6f_dropped;
    385      1.15   thorpej 	ip6s[IP6_STAT_ODROPPED] += ip6f->ip6f_dropped;
    386      1.15   thorpej 	ip6s[IP6_STAT_TOTAL] += ip6f->ip6f_uses;
    387      1.15   thorpej 	ip6s[IP6_STAT_FORWARD] += ip6f->ip6f_forwarded;
    388      1.15   thorpej 	ip6s[IP6_STAT_FASTFORWARD] += ip6f->ip6f_forwarded;
    389      1.15   thorpej 	IP6_STAT_PUTREF();
    390       1.1  liamjfoy }
    391       1.1  liamjfoy 
    392       1.1  liamjfoy /*
    393       1.1  liamjfoy  * Add statistics and free the flow.
    394       1.1  liamjfoy  */
    395       1.1  liamjfoy static void
    396       1.1  liamjfoy ip6flow_free(struct ip6flow *ip6f)
    397       1.1  liamjfoy {
    398       1.1  liamjfoy 
    399      1.25  knakahar 	KASSERT(mutex_owned(&ip6flow_lock));
    400      1.25  knakahar 
    401       1.1  liamjfoy 	/*
    402       1.1  liamjfoy 	 * Remove the flow from the hash table (at elevated IPL).
    403       1.1  liamjfoy 	 * Once it's off the list, we can deal with it at normal
    404       1.1  liamjfoy 	 * network IPL.
    405       1.1  liamjfoy 	 */
    406       1.1  liamjfoy 	IP6FLOW_REMOVE(ip6f);
    407      1.26  knakahar 
    408       1.1  liamjfoy 	ip6flow_inuse--;
    409       1.1  liamjfoy 	ip6flow_addstats(ip6f);
    410       1.7    dyoung 	rtcache_free(&ip6f->ip6f_ro);
    411       1.1  liamjfoy 	pool_put(&ip6flow_pool, ip6f);
    412       1.1  liamjfoy }
    413       1.1  liamjfoy 
    414      1.25  knakahar static struct ip6flow *
    415      1.25  knakahar ip6flow_reap_locked(int just_one)
    416       1.1  liamjfoy {
    417      1.25  knakahar 
    418      1.25  knakahar 	KASSERT(mutex_owned(&ip6flow_lock));
    419      1.25  knakahar 
    420       1.1  liamjfoy 	while (just_one || ip6flow_inuse > ip6_maxflows) {
    421       1.1  liamjfoy 		struct ip6flow *ip6f, *maybe_ip6f = NULL;
    422       1.1  liamjfoy 
    423       1.1  liamjfoy 		ip6f = LIST_FIRST(&ip6flowlist);
    424       1.1  liamjfoy 		while (ip6f != NULL) {
    425       1.1  liamjfoy 			/*
    426       1.1  liamjfoy 			 * If this no longer points to a valid route -
    427       1.1  liamjfoy 			 * reclaim it.
    428       1.1  liamjfoy 			 */
    429      1.12    dyoung 			if (rtcache_validate(&ip6f->ip6f_ro) == NULL)
    430       1.1  liamjfoy 				goto done;
    431       1.1  liamjfoy 			/*
    432       1.1  liamjfoy 			 * choose the one that's been least recently
    433       1.1  liamjfoy 			 * used or has had the least uses in the
    434       1.1  liamjfoy 			 * last 1.5 intervals.
    435       1.1  liamjfoy 			 */
    436       1.1  liamjfoy 			if (maybe_ip6f == NULL ||
    437       1.1  liamjfoy 			    ip6f->ip6f_timer < maybe_ip6f->ip6f_timer ||
    438       1.1  liamjfoy 			    (ip6f->ip6f_timer == maybe_ip6f->ip6f_timer &&
    439       1.1  liamjfoy 			     ip6f->ip6f_last_uses + ip6f->ip6f_uses <
    440       1.1  liamjfoy 			         maybe_ip6f->ip6f_last_uses +
    441       1.1  liamjfoy 			         maybe_ip6f->ip6f_uses))
    442       1.1  liamjfoy 				maybe_ip6f = ip6f;
    443       1.1  liamjfoy 			ip6f = LIST_NEXT(ip6f, ip6f_list);
    444       1.1  liamjfoy 		}
    445       1.1  liamjfoy 		ip6f = maybe_ip6f;
    446       1.1  liamjfoy 	    done:
    447       1.1  liamjfoy 		/*
    448       1.1  liamjfoy 		 * Remove the entry from the flow table
    449       1.1  liamjfoy 		 */
    450       1.1  liamjfoy 		IP6FLOW_REMOVE(ip6f);
    451      1.26  knakahar 
    452       1.7    dyoung 		rtcache_free(&ip6f->ip6f_ro);
    453       1.1  liamjfoy 		if (just_one) {
    454       1.1  liamjfoy 			ip6flow_addstats(ip6f);
    455       1.1  liamjfoy 			return ip6f;
    456       1.1  liamjfoy 		}
    457       1.1  liamjfoy 		ip6flow_inuse--;
    458       1.1  liamjfoy 		ip6flow_addstats(ip6f);
    459       1.1  liamjfoy 		pool_put(&ip6flow_pool, ip6f);
    460       1.1  liamjfoy 	}
    461       1.1  liamjfoy 	return NULL;
    462       1.1  liamjfoy }
    463       1.1  liamjfoy 
    464      1.25  knakahar /*
    465      1.25  knakahar  * Reap one or more flows - ip6flow_reap may remove
    466      1.25  knakahar  * multiple flows if net.inet6.ip6.maxflows is reduced.
    467      1.25  knakahar  */
    468      1.25  knakahar struct ip6flow *
    469      1.25  knakahar ip6flow_reap(int just_one)
    470      1.25  knakahar {
    471      1.25  knakahar 	struct ip6flow *ip6f;
    472      1.25  knakahar 
    473      1.25  knakahar 	mutex_enter(&ip6flow_lock);
    474      1.25  knakahar 	ip6f = ip6flow_reap_locked(just_one);
    475      1.25  knakahar 	mutex_exit(&ip6flow_lock);
    476      1.25  knakahar 	return ip6f;
    477      1.25  knakahar }
    478      1.25  knakahar 
    479  1.28.2.1  pgoyette static unsigned int ip6flow_work_enqueued = 0;
    480      1.28     ozaki 
    481       1.1  liamjfoy void
    482      1.28     ozaki ip6flow_slowtimo_work(struct work *wk, void *arg)
    483       1.1  liamjfoy {
    484       1.1  liamjfoy 	struct ip6flow *ip6f, *next_ip6f;
    485       1.1  liamjfoy 
    486  1.28.2.1  pgoyette 	/* We can allow enqueuing another work at this point */
    487  1.28.2.1  pgoyette 	atomic_swap_uint(&ip6flow_work_enqueued, 0);
    488  1.28.2.1  pgoyette 
    489  1.28.2.2  pgoyette #ifndef NET_MPSAFE
    490      1.16        ad 	mutex_enter(softnet_lock);
    491      1.16        ad 	KERNEL_LOCK(1, NULL);
    492  1.28.2.2  pgoyette #endif
    493  1.28.2.2  pgoyette 	mutex_enter(&ip6flow_lock);
    494      1.16        ad 
    495       1.1  liamjfoy 	for (ip6f = LIST_FIRST(&ip6flowlist); ip6f != NULL; ip6f = next_ip6f) {
    496       1.1  liamjfoy 		next_ip6f = LIST_NEXT(ip6f, ip6f_list);
    497       1.1  liamjfoy 		if (PRT_SLOW_ISEXPIRED(ip6f->ip6f_timer) ||
    498      1.12    dyoung 		    rtcache_validate(&ip6f->ip6f_ro) == NULL) {
    499       1.1  liamjfoy 			ip6flow_free(ip6f);
    500       1.1  liamjfoy 		} else {
    501       1.1  liamjfoy 			ip6f->ip6f_last_uses = ip6f->ip6f_uses;
    502       1.1  liamjfoy 			ip6flow_addstats(ip6f);
    503       1.1  liamjfoy 			ip6f->ip6f_uses = 0;
    504       1.1  liamjfoy 			ip6f->ip6f_dropped = 0;
    505       1.1  liamjfoy 			ip6f->ip6f_forwarded = 0;
    506       1.1  liamjfoy 		}
    507       1.1  liamjfoy 	}
    508      1.16        ad 
    509      1.25  knakahar 	mutex_exit(&ip6flow_lock);
    510  1.28.2.2  pgoyette #ifndef NET_MPSAFE
    511  1.28.2.2  pgoyette 	KERNEL_UNLOCK_ONE(NULL);
    512      1.16        ad 	mutex_exit(softnet_lock);
    513  1.28.2.2  pgoyette #endif
    514       1.1  liamjfoy }
    515       1.1  liamjfoy 
    516      1.28     ozaki void
    517      1.28     ozaki ip6flow_slowtimo(void)
    518      1.28     ozaki {
    519      1.28     ozaki 
    520      1.28     ozaki 	/* Avoid enqueuing another work when one is already enqueued */
    521  1.28.2.1  pgoyette 	if (atomic_swap_uint(&ip6flow_work_enqueued, 1) == 1)
    522      1.28     ozaki 		return;
    523      1.28     ozaki 
    524      1.28     ozaki 	workqueue_enqueue(ip6flow_slowtimo_wq, &ip6flow_slowtimo_wk, NULL);
    525      1.28     ozaki }
    526      1.28     ozaki 
    527       1.1  liamjfoy /*
    528       1.1  liamjfoy  * We have successfully forwarded a packet using the normal
    529       1.1  liamjfoy  * IPv6 stack. Now create/update a flow.
    530       1.1  liamjfoy  */
    531       1.1  liamjfoy void
    532       1.7    dyoung ip6flow_create(const struct route *ro, struct mbuf *m)
    533       1.1  liamjfoy {
    534      1.13    dyoung 	const struct ip6_hdr *ip6;
    535       1.1  liamjfoy 	struct ip6flow *ip6f;
    536       1.1  liamjfoy 	size_t hash;
    537       1.1  liamjfoy 
    538      1.13    dyoung 	ip6 = mtod(m, const struct ip6_hdr *);
    539       1.1  liamjfoy 
    540  1.28.2.2  pgoyette #ifndef NET_MPSAFE
    541  1.28.2.2  pgoyette 	KERNEL_LOCK(1, NULL);
    542  1.28.2.2  pgoyette #endif
    543  1.28.2.2  pgoyette 	mutex_enter(&ip6flow_lock);
    544  1.28.2.2  pgoyette 
    545       1.1  liamjfoy 	/*
    546       1.1  liamjfoy 	 * If IPv6 Fast Forward is disabled, don't create a flow.
    547       1.1  liamjfoy 	 * It can be disabled by setting net.inet6.ip6.maxflows to 0.
    548       1.1  liamjfoy 	 *
    549       1.1  liamjfoy 	 * Don't create a flow for ICMPv6 messages.
    550       1.1  liamjfoy 	 */
    551  1.28.2.2  pgoyette 	if (ip6_maxflows == 0 || ip6->ip6_nxt == IPPROTO_IPV6_ICMP)
    552  1.28.2.2  pgoyette 		goto out;
    553      1.22     pooka 
    554       1.1  liamjfoy 	/*
    555       1.1  liamjfoy 	 * See if an existing flow exists.  If so:
    556       1.1  liamjfoy 	 *	- Remove the flow
    557       1.1  liamjfoy 	 *	- Add flow statistics
    558       1.1  liamjfoy 	 *	- Free the route
    559       1.1  liamjfoy 	 *	- Reset statistics
    560       1.1  liamjfoy 	 *
    561       1.1  liamjfoy 	 * If a flow doesn't exist allocate a new one if
    562       1.1  liamjfoy 	 * ip6_maxflows hasn't reached its limit. If it has
    563       1.1  liamjfoy 	 * been reached, reap some flows.
    564       1.1  liamjfoy 	 */
    565       1.1  liamjfoy 	ip6f = ip6flow_lookup(ip6);
    566       1.1  liamjfoy 	if (ip6f == NULL) {
    567       1.1  liamjfoy 		if (ip6flow_inuse >= ip6_maxflows) {
    568      1.25  knakahar 			ip6f = ip6flow_reap_locked(1);
    569       1.1  liamjfoy 		} else {
    570       1.1  liamjfoy 			ip6f = pool_get(&ip6flow_pool, PR_NOWAIT);
    571       1.1  liamjfoy 			if (ip6f == NULL)
    572      1.22     pooka 				goto out;
    573       1.1  liamjfoy 			ip6flow_inuse++;
    574       1.1  liamjfoy 		}
    575       1.1  liamjfoy 		memset(ip6f, 0, sizeof(*ip6f));
    576       1.1  liamjfoy 	} else {
    577       1.1  liamjfoy 		IP6FLOW_REMOVE(ip6f);
    578      1.26  knakahar 
    579       1.1  liamjfoy 		ip6flow_addstats(ip6f);
    580       1.7    dyoung 		rtcache_free(&ip6f->ip6f_ro);
    581       1.1  liamjfoy 		ip6f->ip6f_uses = 0;
    582       1.1  liamjfoy 		ip6f->ip6f_last_uses = 0;
    583       1.1  liamjfoy 		ip6f->ip6f_dropped = 0;
    584       1.1  liamjfoy 		ip6f->ip6f_forwarded = 0;
    585       1.1  liamjfoy 	}
    586       1.1  liamjfoy 
    587       1.1  liamjfoy 	/*
    588       1.1  liamjfoy 	 * Fill in the updated/new details.
    589       1.1  liamjfoy 	 */
    590       1.7    dyoung 	rtcache_copy(&ip6f->ip6f_ro, ro);
    591       1.1  liamjfoy 	ip6f->ip6f_dst = ip6->ip6_dst;
    592       1.1  liamjfoy 	ip6f->ip6f_src = ip6->ip6_src;
    593       1.1  liamjfoy 	ip6f->ip6f_flow = ip6->ip6_flow;
    594       1.1  liamjfoy 	PRT_SLOW_ARM(ip6f->ip6f_timer, IP6FLOW_TIMER);
    595       1.1  liamjfoy 
    596       1.1  liamjfoy 	/*
    597       1.1  liamjfoy 	 * Insert into the approriate bucket of the flow table.
    598       1.1  liamjfoy 	 */
    599       1.1  liamjfoy 	hash = ip6flow_hash(ip6);
    600       1.1  liamjfoy 	IP6FLOW_INSERT(&ip6flowtable[hash], ip6f);
    601      1.22     pooka 
    602      1.22     pooka  out:
    603      1.25  knakahar 	mutex_exit(&ip6flow_lock);
    604  1.28.2.2  pgoyette #ifndef NET_MPSAFE
    605  1.28.2.2  pgoyette 	KERNEL_UNLOCK_ONE(NULL);
    606  1.28.2.2  pgoyette #endif
    607       1.1  liamjfoy }
    608       1.1  liamjfoy 
    609       1.1  liamjfoy /*
    610       1.4  liamjfoy  * Invalidate/remove all flows - if new_size is positive we
    611       1.4  liamjfoy  * resize the hash table.
    612       1.1  liamjfoy  */
    613       1.4  liamjfoy int
    614       1.4  liamjfoy ip6flow_invalidate_all(int new_size)
    615       1.1  liamjfoy {
    616       1.1  liamjfoy 	struct ip6flow *ip6f, *next_ip6f;
    617      1.26  knakahar 	int error;
    618       1.1  liamjfoy 
    619       1.4  liamjfoy 	error = 0;
    620      1.25  knakahar 
    621      1.25  knakahar 	mutex_enter(&ip6flow_lock);
    622      1.25  knakahar 
    623       1.1  liamjfoy 	for (ip6f = LIST_FIRST(&ip6flowlist); ip6f != NULL; ip6f = next_ip6f) {
    624       1.1  liamjfoy 		next_ip6f = LIST_NEXT(ip6f, ip6f_list);
    625       1.1  liamjfoy 		ip6flow_free(ip6f);
    626       1.1  liamjfoy 	}
    627       1.4  liamjfoy 
    628       1.4  liamjfoy 	if (new_size)
    629      1.25  knakahar 		error = ip6flow_init_locked(new_size);
    630       1.4  liamjfoy 
    631      1.25  knakahar 	mutex_exit(&ip6flow_lock);
    632      1.25  knakahar 
    633       1.4  liamjfoy 	return error;
    634       1.1  liamjfoy }
    635  1.28.2.1  pgoyette 
    636  1.28.2.1  pgoyette /*
    637  1.28.2.1  pgoyette  * sysctl helper routine for net.inet.ip6.maxflows. Since
    638  1.28.2.1  pgoyette  * we could reduce this value, call ip6flow_reap();
    639  1.28.2.1  pgoyette  */
    640  1.28.2.1  pgoyette static int
    641  1.28.2.1  pgoyette sysctl_net_inet6_ip6_maxflows(SYSCTLFN_ARGS)
    642  1.28.2.1  pgoyette {
    643  1.28.2.1  pgoyette 	int error;
    644  1.28.2.1  pgoyette 
    645  1.28.2.1  pgoyette 	error = sysctl_lookup(SYSCTLFN_CALL(rnode));
    646  1.28.2.1  pgoyette 	if (error || newp == NULL)
    647  1.28.2.1  pgoyette 		return (error);
    648  1.28.2.1  pgoyette 
    649  1.28.2.2  pgoyette #ifndef NET_MPSAFE
    650  1.28.2.1  pgoyette 	mutex_enter(softnet_lock);
    651  1.28.2.1  pgoyette 	KERNEL_LOCK(1, NULL);
    652  1.28.2.2  pgoyette #endif
    653  1.28.2.1  pgoyette 
    654  1.28.2.1  pgoyette 	ip6flow_reap(0);
    655  1.28.2.1  pgoyette 
    656  1.28.2.2  pgoyette #ifndef NET_MPSAFE
    657  1.28.2.1  pgoyette 	KERNEL_UNLOCK_ONE(NULL);
    658  1.28.2.1  pgoyette 	mutex_exit(softnet_lock);
    659  1.28.2.2  pgoyette #endif
    660  1.28.2.1  pgoyette 
    661  1.28.2.1  pgoyette 	return (0);
    662  1.28.2.1  pgoyette }
    663  1.28.2.1  pgoyette 
    664  1.28.2.1  pgoyette static int
    665  1.28.2.1  pgoyette sysctl_net_inet6_ip6_hashsize(SYSCTLFN_ARGS)
    666  1.28.2.1  pgoyette {
    667  1.28.2.1  pgoyette 	int error, tmp;
    668  1.28.2.1  pgoyette 	struct sysctlnode node;
    669  1.28.2.1  pgoyette 
    670  1.28.2.1  pgoyette 	node = *rnode;
    671  1.28.2.1  pgoyette 	tmp = ip6_hashsize;
    672  1.28.2.1  pgoyette 	node.sysctl_data = &tmp;
    673  1.28.2.1  pgoyette 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    674  1.28.2.1  pgoyette 	if (error || newp == NULL)
    675  1.28.2.1  pgoyette 		return (error);
    676  1.28.2.1  pgoyette 
    677  1.28.2.1  pgoyette 	if ((tmp & (tmp - 1)) == 0 && tmp != 0) {
    678  1.28.2.1  pgoyette 		/*
    679  1.28.2.1  pgoyette 		 * Can only fail due to malloc()
    680  1.28.2.1  pgoyette 		 */
    681  1.28.2.2  pgoyette #ifndef NET_MPSAFE
    682  1.28.2.1  pgoyette 		mutex_enter(softnet_lock);
    683  1.28.2.1  pgoyette 		KERNEL_LOCK(1, NULL);
    684  1.28.2.2  pgoyette #endif
    685  1.28.2.1  pgoyette 		error = ip6flow_invalidate_all(tmp);
    686  1.28.2.2  pgoyette #ifndef NET_MPSAFE
    687  1.28.2.1  pgoyette 		KERNEL_UNLOCK_ONE(NULL);
    688  1.28.2.1  pgoyette 		mutex_exit(softnet_lock);
    689  1.28.2.2  pgoyette #endif
    690  1.28.2.1  pgoyette 	} else {
    691  1.28.2.1  pgoyette 		/*
    692  1.28.2.1  pgoyette 		 * EINVAL if not a power of 2
    693  1.28.2.1  pgoyette 		 */
    694  1.28.2.1  pgoyette 		error = EINVAL;
    695  1.28.2.1  pgoyette 	}
    696  1.28.2.1  pgoyette 
    697  1.28.2.1  pgoyette 	return error;
    698  1.28.2.1  pgoyette }
    699  1.28.2.1  pgoyette 
    700  1.28.2.1  pgoyette static void
    701  1.28.2.1  pgoyette ip6flow_sysctl_init(struct sysctllog **clog)
    702  1.28.2.1  pgoyette {
    703  1.28.2.1  pgoyette 
    704  1.28.2.1  pgoyette 	sysctl_createv(clog, 0, NULL, NULL,
    705  1.28.2.1  pgoyette 		       CTLFLAG_PERMANENT,
    706  1.28.2.1  pgoyette 		       CTLTYPE_NODE, "inet6",
    707  1.28.2.1  pgoyette 		       SYSCTL_DESCR("PF_INET6 related settings"),
    708  1.28.2.1  pgoyette 		       NULL, 0, NULL, 0,
    709  1.28.2.1  pgoyette 		       CTL_NET, PF_INET6, CTL_EOL);
    710  1.28.2.1  pgoyette 	sysctl_createv(clog, 0, NULL, NULL,
    711  1.28.2.1  pgoyette 		       CTLFLAG_PERMANENT,
    712  1.28.2.1  pgoyette 		       CTLTYPE_NODE, "ip6",
    713  1.28.2.1  pgoyette 		       SYSCTL_DESCR("IPv6 related settings"),
    714  1.28.2.1  pgoyette 		       NULL, 0, NULL, 0,
    715  1.28.2.1  pgoyette 		       CTL_NET, PF_INET6, IPPROTO_IPV6, CTL_EOL);
    716  1.28.2.1  pgoyette 
    717  1.28.2.1  pgoyette 	sysctl_createv(clog, 0, NULL, NULL,
    718  1.28.2.1  pgoyette 			CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
    719  1.28.2.1  pgoyette 			CTLTYPE_INT, "maxflows",
    720  1.28.2.1  pgoyette 			SYSCTL_DESCR("Number of flows for fast forwarding (IPv6)"),
    721  1.28.2.1  pgoyette 			sysctl_net_inet6_ip6_maxflows, 0, &ip6_maxflows, 0,
    722  1.28.2.1  pgoyette 			CTL_NET, PF_INET6, IPPROTO_IPV6,
    723  1.28.2.1  pgoyette 			CTL_CREATE, CTL_EOL);
    724  1.28.2.1  pgoyette 	sysctl_createv(clog, 0, NULL, NULL,
    725  1.28.2.1  pgoyette 			CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
    726  1.28.2.1  pgoyette 			CTLTYPE_INT, "hashsize",
    727  1.28.2.1  pgoyette 			SYSCTL_DESCR("Size of hash table for fast forwarding (IPv6)"),
    728  1.28.2.1  pgoyette 			sysctl_net_inet6_ip6_hashsize, 0, &ip6_hashsize, 0,
    729  1.28.2.1  pgoyette 			CTL_NET, PF_INET6, IPPROTO_IPV6,
    730  1.28.2.1  pgoyette 			CTL_CREATE, CTL_EOL);
    731  1.28.2.1  pgoyette }
    732