Home | History | Annotate | Line # | Download | only in genfs
layer_subr.c revision 1.28.4.5
      1  1.28.4.5     rmind /*	$NetBSD: layer_subr.c,v 1.28.4.5 2011/05/30 14:57:48 rmind Exp $	*/
      2       1.1  wrstuden 
      3       1.1  wrstuden /*
      4       1.1  wrstuden  * Copyright (c) 1999 National Aeronautics & Space Administration
      5       1.1  wrstuden  * All rights reserved.
      6       1.1  wrstuden  *
      7       1.1  wrstuden  * This software was written by William Studenmund of the
      8       1.8       wiz  * Numerical Aerospace Simulation Facility, NASA Ames Research Center.
      9       1.1  wrstuden  *
     10       1.1  wrstuden  * Redistribution and use in source and binary forms, with or without
     11       1.1  wrstuden  * modification, are permitted provided that the following conditions
     12       1.1  wrstuden  * are met:
     13       1.1  wrstuden  * 1. Redistributions of source code must retain the above copyright
     14       1.1  wrstuden  *    notice, this list of conditions and the following disclaimer.
     15       1.1  wrstuden  * 2. Redistributions in binary form must reproduce the above copyright
     16       1.1  wrstuden  *    notice, this list of conditions and the following disclaimer in the
     17       1.1  wrstuden  *    documentation and/or other materials provided with the distribution.
     18       1.5     soren  * 3. Neither the name of the National Aeronautics & Space Administration
     19       1.1  wrstuden  *    nor the names of its contributors may be used to endorse or promote
     20       1.1  wrstuden  *    products derived from this software without specific prior written
     21       1.1  wrstuden  *    permission.
     22       1.1  wrstuden  *
     23       1.1  wrstuden  * THIS SOFTWARE IS PROVIDED BY THE NATIONAL AERONAUTICS & SPACE ADMINISTRATION
     24       1.1  wrstuden  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     25       1.1  wrstuden  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     26       1.1  wrstuden  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE ADMINISTRATION OR CONTRIB-
     27       1.1  wrstuden  * UTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
     28       1.1  wrstuden  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     29       1.1  wrstuden  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     30       1.1  wrstuden  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     31       1.1  wrstuden  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     32       1.1  wrstuden  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     33       1.1  wrstuden  * POSSIBILITY OF SUCH DAMAGE.
     34       1.1  wrstuden  */
     35  1.28.4.2     rmind 
     36       1.1  wrstuden /*
     37       1.1  wrstuden  * Copyright (c) 1992, 1993
     38       1.1  wrstuden  *	The Regents of the University of California.  All rights reserved.
     39       1.1  wrstuden  *
     40       1.1  wrstuden  * This code is derived from software donated to Berkeley by
     41       1.1  wrstuden  * Jan-Simon Pendry.
     42       1.1  wrstuden  *
     43       1.1  wrstuden  * Redistribution and use in source and binary forms, with or without
     44       1.1  wrstuden  * modification, are permitted provided that the following conditions
     45       1.1  wrstuden  * are met:
     46       1.1  wrstuden  * 1. Redistributions of source code must retain the above copyright
     47       1.1  wrstuden  *    notice, this list of conditions and the following disclaimer.
     48       1.1  wrstuden  * 2. Redistributions in binary form must reproduce the above copyright
     49       1.1  wrstuden  *    notice, this list of conditions and the following disclaimer in the
     50       1.1  wrstuden  *    documentation and/or other materials provided with the distribution.
     51      1.15       agc  * 3. Neither the name of the University nor the names of its contributors
     52       1.1  wrstuden  *    may be used to endorse or promote products derived from this software
     53       1.1  wrstuden  *    without specific prior written permission.
     54       1.1  wrstuden  *
     55       1.1  wrstuden  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     56       1.1  wrstuden  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     57       1.1  wrstuden  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     58       1.1  wrstuden  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     59       1.1  wrstuden  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     60       1.1  wrstuden  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     61       1.1  wrstuden  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     62       1.1  wrstuden  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     63       1.1  wrstuden  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     64       1.1  wrstuden  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     65       1.1  wrstuden  * SUCH DAMAGE.
     66       1.1  wrstuden  *
     67       1.1  wrstuden  *	from: Id: lofs_subr.c,v 1.11 1992/05/30 10:05:43 jsp Exp
     68       1.1  wrstuden  *	@(#)null_subr.c	8.7 (Berkeley) 5/14/95
     69       1.1  wrstuden  */
     70       1.9     lukem 
     71       1.9     lukem #include <sys/cdefs.h>
     72  1.28.4.5     rmind __KERNEL_RCSID(0, "$NetBSD: layer_subr.c,v 1.28.4.5 2011/05/30 14:57:48 rmind Exp $");
     73       1.1  wrstuden 
     74       1.1  wrstuden #include <sys/param.h>
     75       1.1  wrstuden #include <sys/systm.h>
     76       1.1  wrstuden #include <sys/proc.h>
     77       1.1  wrstuden #include <sys/time.h>
     78       1.1  wrstuden #include <sys/vnode.h>
     79       1.1  wrstuden #include <sys/mount.h>
     80       1.1  wrstuden #include <sys/namei.h>
     81      1.23        ad #include <sys/kmem.h>
     82       1.1  wrstuden #include <sys/malloc.h>
     83      1.23        ad 
     84       1.1  wrstuden #include <miscfs/specfs/specdev.h>
     85       1.1  wrstuden #include <miscfs/genfs/layer.h>
     86       1.1  wrstuden #include <miscfs/genfs/layer_extern.h>
     87       1.1  wrstuden 
     88      1.16       erh #ifdef LAYERFS_DIAGNOSTIC
     89      1.16       erh int layerfs_debug = 1;
     90      1.16       erh #endif
     91      1.16       erh 
     92       1.1  wrstuden /*
     93       1.1  wrstuden  * layer cache:
     94       1.1  wrstuden  * Each cache entry holds a reference to the lower vnode
     95       1.1  wrstuden  * along with a pointer to the alias vnode.  When an
     96       1.1  wrstuden  * entry is added the lower vnode is VREF'd.  When the
     97       1.1  wrstuden  * alias is removed the lower vnode is vrele'd.
     98       1.1  wrstuden  */
     99       1.1  wrstuden 
    100       1.1  wrstuden void
    101      1.27    cegger layerfs_init(void)
    102       1.1  wrstuden {
    103  1.28.4.2     rmind 	/* Nothing. */
    104       1.6  jdolecek }
    105       1.1  wrstuden 
    106       1.6  jdolecek void
    107      1.27    cegger layerfs_done(void)
    108       1.6  jdolecek {
    109  1.28.4.2     rmind 	/* Nothing. */
    110       1.1  wrstuden }
    111       1.1  wrstuden 
    112       1.1  wrstuden /*
    113  1.28.4.2     rmind  * layer_node_find: find and return alias for lower vnode or NULL.
    114  1.28.4.2     rmind  *
    115  1.28.4.2     rmind  * => Return alias vnode locked and referenced. if already exists.
    116  1.28.4.2     rmind  * => The layermp's hashlock must be held on entry, we will unlock on success.
    117       1.1  wrstuden  */
    118       1.1  wrstuden struct vnode *
    119      1.26       dsl layer_node_find(struct mount *mp, struct vnode *lowervp)
    120       1.1  wrstuden {
    121       1.1  wrstuden 	struct layer_mount *lmp = MOUNTTOLAYERMOUNT(mp);
    122       1.1  wrstuden 	struct layer_node_hashhead *hd;
    123       1.1  wrstuden 	struct layer_node *a;
    124       1.1  wrstuden 	struct vnode *vp;
    125      1.21       chs 	int error;
    126       1.1  wrstuden 
    127       1.1  wrstuden 	/*
    128  1.28.4.2     rmind 	 * Find hash bucket and search the (two-way) linked list looking
    129  1.28.4.2     rmind 	 * for a layerfs node structure which is referencing the lower vnode.
    130  1.28.4.2     rmind 	 * If found, the increment the layer_node reference count, but NOT
    131  1.28.4.2     rmind 	 * the lower vnode's reference counter.  Return vnode locked.
    132       1.1  wrstuden 	 */
    133  1.28.4.2     rmind 	KASSERT(mutex_owned(&lmp->layerm_hashlock));
    134       1.1  wrstuden 	hd = LAYER_NHASH(lmp, lowervp);
    135       1.1  wrstuden loop:
    136      1.21       chs 	LIST_FOREACH(a, hd, layer_hash) {
    137  1.28.4.2     rmind 		if (a->layer_lowervp != lowervp) {
    138  1.28.4.2     rmind 			continue;
    139  1.28.4.2     rmind 		}
    140  1.28.4.2     rmind 		vp = LAYERTOV(a);
    141  1.28.4.2     rmind 		if (vp->v_mount != mp) {
    142  1.28.4.2     rmind 			continue;
    143       1.1  wrstuden 		}
    144  1.28.4.2     rmind 		mutex_enter(vp->v_interlock);
    145  1.28.4.2     rmind 		/*
    146  1.28.4.2     rmind 		 * If we find a node being cleaned out, then ignore it and
    147  1.28.4.2     rmind 		 * continue.  A thread trying to clean out the extant layer
    148  1.28.4.2     rmind 		 * vnode needs to acquire the shared lock (i.e. the lower
    149  1.28.4.2     rmind 		 * vnode's lock), which our caller already holds.  To allow
    150  1.28.4.2     rmind 		 * the cleaning to succeed the current thread must make
    151  1.28.4.2     rmind 		 * progress.  So, for a brief time more than one vnode in a
    152  1.28.4.2     rmind 		 * layered file system may refer to a single vnode in the
    153  1.28.4.2     rmind 		 * lower file system.
    154  1.28.4.2     rmind 		 */
    155  1.28.4.2     rmind 		if ((vp->v_iflag & VI_XLOCK) != 0) {
    156  1.28.4.2     rmind 			mutex_exit(vp->v_interlock);
    157  1.28.4.2     rmind 			continue;
    158  1.28.4.2     rmind 		}
    159  1.28.4.2     rmind 		mutex_exit(&lmp->layerm_hashlock);
    160  1.28.4.2     rmind 		/*
    161  1.28.4.2     rmind 		 * We must not let vget() try to lock the layer vp, since
    162  1.28.4.2     rmind 		 * the lower vp is already locked and locking the layer vp
    163  1.28.4.2     rmind 		 * will involve locking the lower vp.
    164  1.28.4.2     rmind 		 */
    165  1.28.4.3     rmind 		error = vget(vp, LK_NOWAIT);
    166  1.28.4.2     rmind 		if (error) {
    167  1.28.4.2     rmind 			kpause("layerfs", false, 1, NULL);
    168  1.28.4.2     rmind 			mutex_enter(&lmp->layerm_hashlock);
    169  1.28.4.2     rmind 			goto loop;
    170  1.28.4.2     rmind 		}
    171  1.28.4.2     rmind 		return vp;
    172       1.1  wrstuden 	}
    173       1.1  wrstuden 	return NULL;
    174       1.1  wrstuden }
    175       1.1  wrstuden 
    176       1.1  wrstuden /*
    177  1.28.4.2     rmind  * layer_node_alloc: make a new layerfs vnode.
    178  1.28.4.2     rmind  *
    179  1.28.4.2     rmind  * => vp is the alias vnode, lowervp is the lower vnode.
    180  1.28.4.2     rmind  * => We will hold a reference to lowervp.
    181       1.1  wrstuden  */
    182       1.1  wrstuden int
    183      1.26       dsl layer_node_alloc(struct mount *mp, struct vnode *lowervp, struct vnode **vpp)
    184       1.1  wrstuden {
    185       1.1  wrstuden 	struct layer_mount *lmp = MOUNTTOLAYERMOUNT(mp);
    186       1.1  wrstuden 	struct layer_node_hashhead *hd;
    187       1.1  wrstuden 	struct layer_node *xp;
    188       1.1  wrstuden 	struct vnode *vp, *nvp;
    189       1.1  wrstuden 	int error;
    190      1.17   xtraeme 	extern int (**dead_vnodeop_p)(void *);
    191       1.1  wrstuden 
    192  1.28.4.4     rmind 	/* Get a new vnode and share its interlock with underlying vnode. */
    193  1.28.4.4     rmind 	error = getnewvnode(lmp->layerm_tag, mp, lmp->layerm_vnodeop_p,
    194  1.28.4.5     rmind 	    lowervp->v_interlock, &vp);
    195  1.28.4.2     rmind 	if (error) {
    196  1.28.4.2     rmind 		return error;
    197  1.28.4.2     rmind 	}
    198       1.1  wrstuden 	vp->v_type = lowervp->v_type;
    199  1.28.4.1     rmind 	mutex_enter(vp->v_interlock);
    200      1.22        ad 	vp->v_iflag |= VI_LAYER;
    201  1.28.4.1     rmind 	mutex_exit(vp->v_interlock);
    202       1.1  wrstuden 
    203      1.23        ad 	xp = kmem_alloc(lmp->layerm_size, KM_SLEEP);
    204      1.23        ad 	if (xp == NULL) {
    205      1.23        ad 		ungetnewvnode(vp);
    206      1.23        ad 		return ENOMEM;
    207      1.23        ad 	}
    208       1.1  wrstuden 	if (vp->v_type == VBLK || vp->v_type == VCHR) {
    209      1.25        ad 		spec_node_init(vp, lowervp->v_rdev);
    210       1.1  wrstuden 	}
    211       1.1  wrstuden 
    212       1.1  wrstuden 	vp->v_data = xp;
    213      1.23        ad 	vp->v_vflag = (vp->v_vflag & ~VV_MPSAFE) |
    214      1.23        ad 	    (lowervp->v_vflag & VV_MPSAFE);
    215       1.1  wrstuden 	xp->layer_vnode = vp;
    216       1.1  wrstuden 	xp->layer_lowervp = lowervp;
    217       1.1  wrstuden 	xp->layer_flags = 0;
    218      1.21       chs 
    219       1.1  wrstuden 	/*
    220  1.28.4.2     rmind 	 * Before inserting the node into the hash, check if other thread
    221  1.28.4.2     rmind 	 * did not race with us.  If so - return that node, destroy ours.
    222       1.1  wrstuden 	 */
    223      1.22        ad 	mutex_enter(&lmp->layerm_hashlock);
    224       1.1  wrstuden 	if ((nvp = layer_node_find(mp, lowervp)) != NULL) {
    225  1.28.4.2     rmind 		/* Free the structures we have created. */
    226       1.1  wrstuden 		if (vp->v_type == VBLK || vp->v_type == VCHR)
    227      1.25        ad 			spec_node_destroy(vp);
    228       1.1  wrstuden 
    229       1.1  wrstuden 		vp->v_type = VBAD;		/* node is discarded */
    230       1.1  wrstuden 		vp->v_op = dead_vnodeop_p;	/* so ops will still work */
    231       1.1  wrstuden 		vrele(vp);			/* get rid of it. */
    232  1.28.4.2     rmind 		kmem_free(xp, lmp->layerm_size);
    233  1.28.4.2     rmind 		*vpp = nvp;
    234  1.28.4.2     rmind 		return 0;
    235       1.1  wrstuden 	}
    236       1.1  wrstuden 
    237       1.1  wrstuden 	/*
    238      1.21       chs 	 * Insert the new node into the hash.
    239      1.21       chs 	 * Add a reference to the lower node.
    240       1.1  wrstuden 	 */
    241      1.28     pooka 	vref(lowervp);
    242       1.1  wrstuden 	hd = LAYER_NHASH(lmp, lowervp);
    243       1.1  wrstuden 	LIST_INSERT_HEAD(hd, xp, layer_hash);
    244       1.7       chs 	uvm_vnp_setsize(vp, 0);
    245      1.22        ad 	mutex_exit(&lmp->layerm_hashlock);
    246       1.1  wrstuden 
    247  1.28.4.2     rmind 	*vpp = vp;
    248  1.28.4.2     rmind 	return 0;
    249  1.28.4.2     rmind }
    250       1.1  wrstuden 
    251       1.1  wrstuden /*
    252  1.28.4.2     rmind  * layer_node_create: try to find an existing layerfs vnode refering to it,
    253  1.28.4.2     rmind  * otherwise make a new vnode which contains a reference to the lower vnode.
    254       1.1  wrstuden  *
    255  1.28.4.2     rmind  * => Caller should lock the lower node.
    256       1.1  wrstuden  */
    257       1.1  wrstuden int
    258  1.28.4.2     rmind layer_node_create(struct mount *mp, struct vnode *lowervp, struct vnode **nvpp)
    259       1.1  wrstuden {
    260       1.1  wrstuden 	struct vnode *aliasvp;
    261       1.1  wrstuden 	struct layer_mount *lmp = MOUNTTOLAYERMOUNT(mp);
    262       1.1  wrstuden 
    263  1.28.4.2     rmind 	KASSERT(VOP_ISLOCKED(lowervp));
    264  1.28.4.2     rmind 
    265      1.22        ad 	mutex_enter(&lmp->layerm_hashlock);
    266      1.21       chs 	aliasvp = layer_node_find(mp, lowervp);
    267      1.21       chs 	if (aliasvp != NULL) {
    268       1.1  wrstuden 		/*
    269  1.28.4.2     rmind 		 * Note: layer_node_find() has taken another reference to
    270  1.28.4.2     rmind 		 * the alias vnode and moved the lock holding to aliasvp.
    271       1.1  wrstuden 		 */
    272       1.1  wrstuden #ifdef LAYERFS_DIAGNOSTIC
    273      1.16       erh 		if (layerfs_debug)
    274      1.16       erh 			vprint("layer_node_create: exists", aliasvp);
    275       1.1  wrstuden #endif
    276       1.1  wrstuden 	} else {
    277       1.1  wrstuden 		int error;
    278       1.1  wrstuden 
    279      1.22        ad 		mutex_exit(&lmp->layerm_hashlock);
    280       1.1  wrstuden 		/*
    281  1.28.4.2     rmind 		 * Get a new vnode.  Make it to reference the layer_node.
    282  1.28.4.2     rmind 		 * Note: aliasvp will be return with the reference held.
    283       1.1  wrstuden 		 */
    284  1.28.4.2     rmind 		error = (lmp->layerm_alloc)(mp, lowervp, &aliasvp);
    285  1.28.4.2     rmind 		if (error)
    286  1.28.4.2     rmind 			return error;
    287       1.1  wrstuden #ifdef LAYERFS_DIAGNOSTIC
    288      1.16       erh 		if (layerfs_debug)
    289      1.16       erh 			printf("layer_node_create: create new alias vnode\n");
    290       1.1  wrstuden #endif
    291       1.1  wrstuden 	}
    292       1.1  wrstuden 
    293       1.1  wrstuden 	/*
    294  1.28.4.2     rmind 	 * Now that we acquired a reference on the upper vnode, release one
    295  1.28.4.2     rmind 	 * on the lower node.  The existence of the layer_node retains one
    296       1.1  wrstuden 	 * reference to the lower node.
    297       1.1  wrstuden 	 */
    298       1.1  wrstuden 	vrele(lowervp);
    299  1.28.4.2     rmind 	KASSERT(lowervp->v_usecount > 0);
    300       1.1  wrstuden 
    301       1.1  wrstuden #ifdef LAYERFS_DIAGNOSTIC
    302      1.16       erh 	if (layerfs_debug)
    303      1.16       erh 		vprint("layer_node_create: alias", aliasvp);
    304       1.1  wrstuden #endif
    305  1.28.4.2     rmind 	*nvpp = aliasvp;
    306  1.28.4.2     rmind 	return 0;
    307       1.1  wrstuden }
    308       1.1  wrstuden 
    309      1.16       erh #ifdef LAYERFS_DIAGNOSTIC
    310       1.1  wrstuden struct vnode *
    311      1.26       dsl layer_checkvp(struct vnode *vp, const char *fil, int lno)
    312       1.1  wrstuden {
    313       1.1  wrstuden 	struct layer_node *a = VTOLAYER(vp);
    314       1.1  wrstuden #ifdef notyet
    315       1.1  wrstuden 	/*
    316       1.1  wrstuden 	 * Can't do this check because vop_reclaim runs
    317       1.1  wrstuden 	 * with a funny vop vector.
    318       1.1  wrstuden 	 *
    319       1.1  wrstuden 	 * WRS - no it doesnt...
    320       1.1  wrstuden 	 */
    321       1.1  wrstuden 	if (vp->v_op != layer_vnodeop_p) {
    322       1.1  wrstuden 		printf ("layer_checkvp: on non-layer-node\n");
    323       1.1  wrstuden #ifdef notyet
    324       1.1  wrstuden 		while (layer_checkvp_barrier) /*WAIT*/ ;
    325       1.1  wrstuden #endif
    326       1.1  wrstuden 		panic("layer_checkvp");
    327       1.1  wrstuden 	};
    328       1.1  wrstuden #endif
    329       1.1  wrstuden 	if (a->layer_lowervp == NULL) {
    330       1.1  wrstuden 		/* Should never happen */
    331       1.1  wrstuden 		int i; u_long *p;
    332       1.1  wrstuden 		printf("vp = %p, ZERO ptr\n", vp);
    333       1.1  wrstuden 		for (p = (u_long *) a, i = 0; i < 8; i++)
    334       1.1  wrstuden 			printf(" %lx", p[i]);
    335       1.1  wrstuden 		printf("\n");
    336       1.1  wrstuden 		/* wait for debugger */
    337       1.1  wrstuden 		panic("layer_checkvp");
    338       1.1  wrstuden 	}
    339       1.1  wrstuden 	if (a->layer_lowervp->v_usecount < 1) {
    340       1.1  wrstuden 		int i; u_long *p;
    341       1.1  wrstuden 		printf("vp = %p, unref'ed lowervp\n", vp);
    342       1.1  wrstuden 		for (p = (u_long *) a, i = 0; i < 8; i++)
    343       1.1  wrstuden 			printf(" %lx", p[i]);
    344       1.1  wrstuden 		printf("\n");
    345       1.1  wrstuden 		/* wait for debugger */
    346       1.1  wrstuden 		panic ("layer with unref'ed lowervp");
    347       1.1  wrstuden 	};
    348       1.1  wrstuden #ifdef notnow
    349       1.1  wrstuden 	printf("layer %p/%d -> %p/%d [%s, %d]\n",
    350       1.1  wrstuden 	        LAYERTOV(a), LAYERTOV(a)->v_usecount,
    351       1.1  wrstuden 		a->layer_lowervp, a->layer_lowervp->v_usecount,
    352       1.1  wrstuden 		fil, lno);
    353       1.1  wrstuden #endif
    354       1.1  wrstuden 	return a->layer_lowervp;
    355       1.1  wrstuden }
    356      1.16       erh #endif
    357