OpenGrok

/*-
 * Copyright (c) 2010-2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This material is based upon work partially supported by The
 * NetBSD Foundation under a contract with Mindaugas Rasiukevicius.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * BPF byte-code generation for NPF rules.
 *
 * Overview
 *
 *	Each NPF rule is compiled into a BPF micro-program.  There is a
 *	BPF byte-code fragment for each higher-level filtering logic,
 *	e.g. to match L4 protocol, IP/mask, etc.  The generation process
 *	combines multiple BPF-byte code fragments into one program.
 *
 * Basic case
 *
 *	Consider a basic case where all filters should match.  They
 *	are expressed as logical conjunction, e.g.:
 *
 *		A and B and C and D
 *
 *	Each test (filter) criterion can be evaluated to true (match) or
 *	false (no match) and the logic is as follows:
 *
 *	- If the value is true, then jump to the "next" test (offset 0).
 *
 *	- If the value is false, then jump to the JUMP_MAGIC value (0xff).
 *	This "magic" value is used to indicate that it will have to be
 *	patched at a later stage.
 *
 *	Once all byte-code fragments are combined into one, then there
 *	are two additional steps:
 *
 *	- Two instructions are appended at the end of the program: "return
 *	success" followed by "return failure".
 *
 *	- All jumps with the JUMP_MAGIC value are patched to point to the
 *	"return failure" instruction.
 *
 *	Therefore, if all filter criteria will match, then the first
 *	instruction will be reached, indicating a successful match of the
 *	rule.  Otherwise, if any of the criteria will not match, it will
 *	take the failure path and the rule will not be matching.
 *
 * Grouping
 *
 *	Filters can have groups, which have an effect of logical
 *	disjunction, e.g.:
 *
 *		A and B and (C or D)
 *
 *	In such case, the logic inside the group has to be inverted i.e.
 *	the jump values swapped.  If the test value is true, then jump
 *	out of the group; if false, then jump "next".  At the end of the
 *	group, an addition failure path is appended and the JUMP_MAGIC
 *	uses within the group are patched to jump past the said path.
 *
 *	For multi-word comparisons (IPv6 addresses), there is another
 *	layer of grouping:
 *
 *		A and B and ((C and D) or (E and F))
 *
 *	This strains the simple-minded JUMP_MAGIC logic, so for now,
 *	when generating the jump-if-false targets for (C and D), we
 *	simply count the number of instructions left to skip over.
 *
 *	A better architecture might be to create asm-type labels for
 *	the jt and jf continuations in the first pass, and then, once
 *	their offsets are determined, go back and fill them in in the
 *	second pass.  This would simplify the logic (no need to compute
 *	exactly how many instructions we're about to generate in a
 *	chain of conditionals) and eliminate redundant RET #0
 *	instructions which are currently generated after some groups.
 */

#include <sys/cdefs.h>
__RCSID("$NetBSD: npf_bpf_comp.c,v 1.19 2025/07/10 11:44:12 joe Exp $");

#include <stdlib.h>
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <inttypes.h>
#include <err.h>
#include <assert.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#define	__FAVOR_BSD
#include <netinet/ip.h>
#include <netinet/ip6.h>
#include <netinet/udp.h>
#include <netinet/tcp.h>
#include <netinet/ip_icmp.h>
#include <netinet/icmp6.h>

#include <net/bpf.h>

#include "npfctl.h"

/*
 * Note: clear X_EQ_L4OFF when register X is invalidated i.e. it stores
 * something other than L4 header offset.  Generally, when BPF_LDX is used.
 */
#define	FETCHED_L3		0x01
#define	CHECKED_L4_PROTO	0x02
#define	X_EQ_L4OFF		0x04
#define	FETCHED_L2		0x08

struct npf_bpf {
	/*
	 * BPF program code, the allocated length (in bytes), the number
	 * of logical blocks and the flags.
	 */
	struct bpf_program	prog;
	size_t			alen;
	unsigned		nblocks;
	sa_family_t		af;
	uint32_t		flags;
	uint8_t			eth_type;

	/*
	 * Indicators whether we are inside the group and whether this
	 * group is implementing inverted logic.
	 *
	 * The current group offset (counted in BPF instructions)
	 * and block number at the start of the group.
	 */
	unsigned		ingroup;
	bool			invert;
	bool			multiword;
	unsigned		goff;
	unsigned		gblock;

	/* Track inversion (excl. mark). */
	uint32_t		invflags;

	/* BPF marks, allocated length and the real length. */
	uint32_t *		marks;
	size_t			malen;
	size_t			mlen;
};

/*
 * NPF success and failure values to be returned from BPF.
 */
#define	NPF_BPF_SUCCESS		((u_int)-1)
#define	NPF_BPF_FAILURE		0

/*
 * Magic value to indicate the failure path, which is fixed up on completion.
 * Note: this is the longest jump offset in BPF, since the offset is one byte.
 */
#define	JUMP_MAGIC		0xff

/* Reduce re-allocations by expanding in 64 byte blocks. */
#define	ALLOC_MASK		(64 - 1)
#define	ALLOC_ROUND(x)		(((x) + ALLOC_MASK) & ~ALLOC_MASK)

#ifndef IPV6_VERSION
#define	IPV6_VERSION		0x60
#endif

npf_bpf_t *
npfctl_bpf_create(void)
{
	return ecalloc(1, sizeof(npf_bpf_t));
}

static void
fixup_jumps(npf_bpf_t *ctx, u_int start, u_int end, bool swap)
{
	struct bpf_program *bp = &ctx->prog;

	for (u_int i = start; i < end; i++) {
		struct bpf_insn *insn = &bp->bf_insns[i];
		const u_int fail_off = end - i;
		bool seen_magic = false;

		if (fail_off >= JUMP_MAGIC) {
			errx(EXIT_FAILURE, "BPF generation error: "
			    "the number of instructions is over the limit");
		}
		if (BPF_CLASS(insn->code) != BPF_JMP) {
			continue;
		}
		if (BPF_OP(insn->code) == BPF_JA) {
			/*
			 * BPF_JA can be used to jump to the failure path.
			 * If we are swapping i.e. inside the group, then
			 * jump "next"; groups have a failure path appended
			 * at their end.
			 */
			if (insn->k == JUMP_MAGIC) {
				insn->k = swap ? 0 : fail_off;
			}
			continue;
		}

		/*
		 * Fixup the "magic" value.  Swap only the "magic" jumps.
		 */

		if (insn->jt == JUMP_MAGIC) {
			insn->jt = fail_off;
			seen_magic = true;
		}
		if (insn->jf == JUMP_MAGIC) {
			insn->jf = fail_off;
			seen_magic = true;
		}

		if (seen_magic && swap) {
			uint8_t jt = insn->jt;
			insn->jt = insn->jf;
			insn->jf = jt;
		}
	}
}

static void
add_insns(npf_bpf_t *ctx, struct bpf_insn *insns, size_t count)
{
	struct bpf_program *bp = &ctx->prog;
	size_t offset, len, reqlen;

	/* Note: bf_len is the count of instructions. */
	offset = bp->bf_len * sizeof(struct bpf_insn);
	len = count * sizeof(struct bpf_insn);

	/* Ensure the memory buffer for the program. */
	reqlen = ALLOC_ROUND(offset + len);
	if (reqlen > ctx->alen) {
		bp->bf_insns = erealloc(bp->bf_insns, reqlen);
		ctx->alen = reqlen;
	}

	/* Add the code block. */
	memcpy((uint8_t *)bp->bf_insns + offset, insns, len);
	bp->bf_len += count;
}

static void
add_bmarks(npf_bpf_t *ctx, const uint32_t *m, size_t len)
{
	size_t reqlen, nargs = m[1];

	if ((len / sizeof(uint32_t) - 2) != nargs) {
		errx(EXIT_FAILURE, "invalid BPF block description");
	}
	reqlen = ALLOC_ROUND(ctx->mlen + len);
	if (reqlen > ctx->malen) {
		ctx->marks = erealloc(ctx->marks, reqlen);
		ctx->malen = reqlen;
	}
	memcpy((uint8_t *)ctx->marks + ctx->mlen, m, len);
	ctx->mlen += len;
}

static void
done_block(npf_bpf_t *ctx, const uint32_t *m, size_t len)
{
	add_bmarks(ctx, m, len);
	ctx->nblocks++;
}

struct bpf_program *
npfctl_bpf_complete(npf_bpf_t *ctx)
{
	struct bpf_program *bp = &ctx->prog;
	const u_int retoff = bp->bf_len;

	/* No instructions (optimised out). */
	if (!bp->bf_len)
		return NULL;

	/* Add the return fragment (success and failure paths). */
	struct bpf_insn insns_ret[] = {
		BPF_STMT(BPF_RET+BPF_K, NPF_BPF_SUCCESS),
		BPF_STMT(BPF_RET+BPF_K, NPF_BPF_FAILURE),
	};
	add_insns(ctx, insns_ret, __arraycount(insns_ret));

	/* Fixup all jumps to the main failure path. */
	fixup_jumps(ctx, 0, retoff, false);

	return &ctx->prog;
}

const void *
npfctl_bpf_bmarks(npf_bpf_t *ctx, size_t *len)
{
	*len = ctx->mlen;
	return ctx->marks;
}

void
npfctl_bpf_destroy(npf_bpf_t *ctx)
{
	free(ctx->prog.bf_insns);
	free(ctx->marks);
	free(ctx);
}

/*
 * npfctl_bpf_group_enter: begin a logical group.  It merely uses logical
 * disjunction (OR) for comparisons within the group.
 */
void
npfctl_bpf_group_enter(npf_bpf_t *ctx, bool invert)
{
	struct bpf_program *bp = &ctx->prog;

	assert(ctx->goff == 0);
	assert(ctx->gblock == 0);

	ctx->goff = bp->bf_len;
	ctx->gblock = ctx->nblocks;
	ctx->invert = invert;
	ctx->multiword = false;
	ctx->ingroup++;
}

void
npfctl_bpf_group_exit(npf_bpf_t *ctx)
{
	struct bpf_program *bp = &ctx->prog;
	const size_t curoff = bp->bf_len;

	assert(ctx->ingroup);
	ctx->ingroup--;

	/*
	 * If we're not inverting, there were only zero or one options,
	 * and the last comparison was not a multi-word comparison
	 * requiring a fallthrough failure -- nothing to do.
	 */
	if (!ctx->invert &&
	    (ctx->nblocks - ctx->gblock) <= 1 &&
	    !ctx->multiword) {
		ctx->goff = ctx->gblock = 0;
		return;
	}

	/*
	 * If inverting, then prepend a jump over the statement below.
	 * On match, it will skip-through and the fail path will be taken.
	 */
	if (ctx->invert) {
		struct bpf_insn insns_ret[] = {
			BPF_STMT(BPF_JMP+BPF_JA, 1),
		};
		add_insns(ctx, insns_ret, __arraycount(insns_ret));
	}

	/*
	 * Append a failure return as a fall-through i.e. if there is
	 * no match within the group.
	 */
	struct bpf_insn insns_ret[] = {
		BPF_STMT(BPF_RET+BPF_K, NPF_BPF_FAILURE),
	};
	add_insns(ctx, insns_ret, __arraycount(insns_ret));

	/*
	 * Adjust jump offsets: on match - jump outside the group i.e.
	 * to the current offset.  Otherwise, jump to the next instruction
	 * which would lead to the fall-through code above if none matches.
	 */
	fixup_jumps(ctx, ctx->goff, curoff, true);
	ctx->goff = ctx->gblock = 0;
}

static void
fetch_l3(npf_bpf_t *ctx, sa_family_t af, unsigned flags)
{
	unsigned ver;

	switch (af) {
	case AF_INET:
		ver = IPVERSION;
		break;
	case AF_INET6:
		ver = IPV6_VERSION >> 4;
		break;
	case AF_UNSPEC:
		ver = 0;
		break;
	default:
		abort();
	}

	/*
	 * The memory store is populated with:
	 * - BPF_MW_IPVER: IP version (4 or 6).
	 * - BPF_MW_L4OFF: L4 header offset.
	 * - BPF_MW_L4PROTO: L4 protocol.
	 */
	if ((ctx->flags & FETCHED_L3) == 0 || (af && ctx->af == 0)) {
		const uint8_t jt = ver ? 0 : JUMP_MAGIC;
		const uint8_t jf = ver ? JUMP_MAGIC : 0;
		const bool ingroup = ctx->ingroup != 0;
		const bool invert = ctx->invert;

		/*
		 * L3 block cannot be inserted in the middle of a group.
		 * In fact, it never is.  Check and start the group after.
		 */
		if (ingroup) {
			assert(ctx->nblocks == ctx->gblock);
			npfctl_bpf_group_exit(ctx);
		}

		/*
		 * A <- IP version; A == expected-version?
		 * If no particular version specified, check for non-zero.
		 */
		struct bpf_insn insns_af[] = {
			BPF_STMT(BPF_LD+BPF_W+BPF_MEM, BPF_MW_IPVER),
			BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, ver, jt, jf),
		};
		add_insns(ctx, insns_af, __arraycount(insns_af));
		ctx->flags |= FETCHED_L3;
		ctx->af = af;

		if (af) {
			uint32_t mwords[] = { BM_IPVER, 1, af };
			add_bmarks(ctx, mwords, sizeof(mwords));
		}
		if (ingroup) {
			npfctl_bpf_group_enter(ctx, invert);
		}

	} else if (af && af != ctx->af) {
		errx(EXIT_FAILURE, "address family mismatch");
	}

	if ((flags & X_EQ_L4OFF) != 0 && (ctx->flags & X_EQ_L4OFF) == 0) {
		/* X <- IP header length */
		struct bpf_insn insns_hlen[] = {
			BPF_STMT(BPF_LDX+BPF_MEM, BPF_MW_L4OFF),
		};
		add_insns(ctx, insns_hlen, __arraycount(insns_hlen));
		ctx->flags |= X_EQ_L4OFF;
	}
}

void
fetch_ether_type(npf_bpf_t *ctx, uint16_t type)
{
	if ((ctx->flags & FETCHED_L2) != 0 || (type && ctx->eth_type != 0))
		return;

	const uint8_t jt = type ? 0 : JUMP_MAGIC;
	const uint8_t jf = type ? JUMP_MAGIC : 0;
	const bool ingroup = ctx->ingroup != 0;
	const bool invert = ctx->invert;
	unsigned off = offsetof(struct ether_header, ether_type);

	/*
	 * L2 block cannot be inserted in the middle of a group.
	 * Check and start the group after.
	 */
	if (ingroup) {
		assert(ctx->nblocks == ctx->gblock);
		npfctl_bpf_group_exit(ctx);
	}

	type = ntohs(type);

	struct bpf_insn insns_et[] = {
		BPF_STMT(BPF_LD+BPF_H+BPF_ABS, off),
		BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, type, jt, jf),
	};
	add_insns(ctx, insns_et, __arraycount(insns_et));
	ctx->flags |= FETCHED_L2;
	ctx->eth_type = type;

 	if (type) { /* bookmark ether type */
		uint32_t mwords[] = { BM_ETHER_TYPE, 1, htons(type) };
		add_bmarks(ctx, mwords, sizeof(mwords));
	}
	if (ingroup) {
		npfctl_bpf_group_enter(ctx, invert);
	}
}

static void
bm_invert_checkpoint(npf_bpf_t *ctx, const unsigned opts, uint32_t layer)
{
	uint32_t bm = 0;

	if (ctx->ingroup && ctx->invert) {
		const unsigned seen = ctx->invflags;

		if ((opts & MATCH_SRC) != 0 && (seen & MATCH_SRC) == 0) {
			bm = (layer & NPF_RULE_LAYER_3) ? BM_SRC_NEG : BM_SRC_ENEG;
		}
		if ((opts & MATCH_DST) != 0 && (seen & MATCH_DST) == 0) {
			bm = (layer & NPF_RULE_LAYER_3) ? BM_DST_NEG : BM_DST_ENEG;
		}
		ctx->invflags |= opts & (MATCH_SRC | MATCH_DST);
	}
	if (bm) {
		uint32_t mwords[] = { bm, 0 };
		add_bmarks(ctx, mwords, sizeof(mwords));
	}
}

/*
 * npfctl_bpf_ipver: match the IP version.
 */
void
npfctl_bpf_ipver(npf_bpf_t *ctx, sa_family_t af)
{
	fetch_l3(ctx, af, 0);
}

/*
 * npfctl_bpf_proto: code block to match IP version and L4 protocol.
 */
void
npfctl_bpf_proto(npf_bpf_t *ctx, unsigned proto)
{
	struct bpf_insn insns_proto[] = {
		/* A <- L4 protocol; A == expected-protocol? */
		BPF_STMT(BPF_LD+BPF_W+BPF_MEM, BPF_MW_L4PROTO),
		BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, proto, 0, JUMP_MAGIC),
	};
	add_insns(ctx, insns_proto, __arraycount(insns_proto));

	uint32_t mwords[] = { BM_PROTO, 1, proto };
	done_block(ctx, mwords, sizeof(mwords));
	ctx->flags |= CHECKED_L4_PROTO;
}

/*
 * npfctl_bpf_cidr: code block to match IPv4 or IPv6 CIDR.
 *
 * => IP address shall be in the network byte order.
 */
void
npfctl_bpf_cidr(npf_bpf_t *ctx, unsigned opts, sa_family_t af,
    const npf_addr_t *addr, const npf_netmask_t mask)
{
	const uint32_t *awords = (const uint32_t *)addr;
	unsigned nwords, origlength, length, maxmask, off;

	assert(((opts & MATCH_SRC) != 0) ^ ((opts & MATCH_DST) != 0));
	assert((mask && mask <= NPF_MAX_NETMASK) || mask == NPF_NO_NETMASK);

	switch (af) {
	case AF_INET:
		maxmask = 32;
		off = (opts & MATCH_SRC) ?
		    offsetof(struct ip, ip_src) :
		    offsetof(struct ip, ip_dst);
		nwords = sizeof(struct in_addr) / sizeof(uint32_t);
		break;
	case AF_INET6:
		maxmask = 128;
		off = (opts & MATCH_SRC) ?
		    offsetof(struct ip6_hdr, ip6_src) :
		    offsetof(struct ip6_hdr, ip6_dst);
		nwords = sizeof(struct in6_addr) / sizeof(uint32_t);
		break;
	default:
		abort();
	}

	/* Ensure address family. */
	fetch_l3(ctx, af, 0);

	length = origlength = (mask == NPF_NO_NETMASK) ? maxmask : mask;

	/* CAUTION: BPF operates in host byte-order. */
	for (unsigned i = 0; i < nwords; i++) {
		const unsigned woff = i * sizeof(uint32_t);
		uint32_t word = ntohl(awords[i]);
		uint32_t wordmask;

		if (length >= 32) {
			/* The mask is a full word - do not apply it. */
			wordmask = 0;
			length -= 32;
		} else if (length) {
			wordmask = 0xffffffff << (32 - length);
			length = 0;
		} else {
			/* The mask became zero - skip the rest. */
			break;
		}

		/* A <- IP address (or one word of it) */
		struct bpf_insn insns_ip[] = {
			BPF_STMT(BPF_LD+BPF_W+BPF_ABS, off + woff),
		};
		add_insns(ctx, insns_ip, __arraycount(insns_ip));

		/* A <- (A & MASK) */
		if (wordmask) {
			struct bpf_insn insns_mask[] = {
				BPF_STMT(BPF_ALU+BPF_AND+BPF_K, wordmask),
			};
			add_insns(ctx, insns_mask, __arraycount(insns_mask));
		}

		/*
		 * Determine how many instructions we have to jump
		 * ahead if the match fails.
		 *
		 * - If this is the last word, we jump to the final
                 *   failure, JUMP_MAGIC.
		 *
		 * - If this is not the last word, we jump past the
		 *   remaining instructions to match this sequence.
		 *   Each 32-bit word in the sequence takes two
		 *   instructions (BPF_LD and BPF_JMP).  If there is a
		 *   partial-word mask ahead, there will be one
		 *   additional instruction (BPF_ALU).
		 */
		uint8_t jf;
		if (i + 1 == (origlength + 31)/32) {
			jf = JUMP_MAGIC;
		} else {
			jf = 2*((origlength + 31)/32 - i - 1);
			if (origlength % 32 != 0 && wordmask == 0)
				jf += 1;
		}

		/* A == expected-IP-word ? */
		struct bpf_insn insns_cmp[] = {
			BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, word, 0, jf),
		};
		add_insns(ctx, insns_cmp, __arraycount(insns_cmp));
	}

	/*
	 * If we checked a chain of words in sequence, mark this as a
	 * multi-word comparison so if this is in a group there will be
	 * a fallthrough case.
	 *
	 * XXX This is a little silly; the compiler should really just
	 * record holes where conditional jumps need success/failure
	 * continuations, and go back to fill in the holes when the
	 * locations of the continuations are determined later.  But
	 * that requires restructuring this code a little more.
	 */
	ctx->multiword = (origlength + 31)/32 > 1;

	uint32_t mwords[] = {
		(opts & MATCH_SRC) ? BM_SRC_CIDR: BM_DST_CIDR, 6,
		af, mask, awords[0], awords[1], awords[2], awords[3],
	};
	bm_invert_checkpoint(ctx, opts, NPF_RULE_LAYER_3);
	done_block(ctx, mwords, sizeof(mwords));
}

/*
 * for ether address, 6 octets(a word and halfword)
 * just fetch directly using a word and halfword fetch
 */
void
npfctl_bpf_ether(npf_bpf_t *ctx, unsigned opts, struct ether_addr *ether_addr)
{
	uint32_t mac_word;
	uint16_t mac_hword;
	unsigned off;
	assert(((opts & MATCH_SRC) != 0) ^ ((opts & MATCH_DST) != 0));

	off = (opts & MATCH_SRC) ? offsetof(struct ether_header, ether_shost) :
	    offsetof(struct ether_header, ether_dhost);

	memcpy(&mac_word, ether_addr, sizeof(mac_word));
	mac_word = ntohl(mac_word);

	/* copy the last two bytes of the 6 byte ether address */
	memcpy(&mac_hword, (uint8_t *)ether_addr + sizeof(mac_word), sizeof(mac_hword));
	mac_hword = ntohs(mac_hword);

	/* load and compare first word then do same to last halfword */
	struct bpf_insn insns_ether_w[] = {
		BPF_STMT(BPF_LD+BPF_W+BPF_ABS, off),
		BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, mac_word, 0, 2),
	};
	add_insns(ctx, insns_ether_w, __arraycount(insns_ether_w));

	struct bpf_insn insns_ether_h[] = {
		BPF_STMT(BPF_LD+BPF_H+BPF_ABS, off + sizeof(mac_word)),
		BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, mac_hword, 0, JUMP_MAGIC),
	};
	add_insns(ctx, insns_ether_h, __arraycount(insns_ether_h));

	ctx->multiword = true;

	uint32_t mwords[] = {
		(opts & MATCH_SRC) ? BM_SRC_ETHER: BM_DST_ETHER, 2,
		htonl(mac_word), htons(mac_hword)
	};

	bm_invert_checkpoint(ctx, opts, NPF_RULE_LAYER_2);
	done_block(ctx, mwords, sizeof(mwords));
}

/*
 * npfctl_bpf_ports: code block to match TCP/UDP port range.
 *
 * => Port numbers shall be in the network byte order.
 */
void
npfctl_bpf_ports(npf_bpf_t *ctx, unsigned opts, in_port_t from, in_port_t to)
{
	const unsigned sport_off = offsetof(struct udphdr, uh_sport);
	const unsigned dport_off = offsetof(struct udphdr, uh_dport);
	unsigned off;

	/* TCP and UDP port offsets are the same. */
	assert(sport_off == offsetof(struct tcphdr, th_sport));
	assert(dport_off == offsetof(struct tcphdr, th_dport));
	assert(ctx->flags & CHECKED_L4_PROTO);

	assert(((opts & MATCH_SRC) != 0) ^ ((opts & MATCH_DST) != 0));
	off = (opts & MATCH_SRC) ? sport_off : dport_off;

	/* X <- IP header length */
	fetch_l3(ctx, AF_UNSPEC, X_EQ_L4OFF);

	struct bpf_insn insns_fetch[] = {
		/* A <- port */
		BPF_STMT(BPF_LD+BPF_H+BPF_IND, off),
	};
	add_insns(ctx, insns_fetch, __arraycount(insns_fetch));

	/* CAUTION: BPF operates in host byte-order. */
	from = ntohs(from);
	to = ntohs(to);

	if (from == to) {
		/* Single port case. */
		struct bpf_insn insns_port[] = {
			BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, from, 0, JUMP_MAGIC),
		};
		add_insns(ctx, insns_port, __arraycount(insns_port));
	} else {
		/* Port range case. */
		struct bpf_insn insns_range[] = {
			BPF_JUMP(BPF_JMP+BPF_JGE+BPF_K, from, 0, 1),
			BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, to, 0, 1),
			BPF_STMT(BPF_JMP+BPF_JA, JUMP_MAGIC),
		};
		add_insns(ctx, insns_range, __arraycount(insns_range));
	}

	uint32_t mwords[] = {
		(opts & MATCH_SRC) ? BM_SRC_PORTS : BM_DST_PORTS, 2, from, to
	};
	done_block(ctx, mwords, sizeof(mwords));
}

/*
 * npfctl_bpf_tcpfl: code block to match TCP flags.
 */
void
npfctl_bpf_tcpfl(npf_bpf_t *ctx, uint8_t tf, uint8_t tf_mask)
{
	const unsigned tcpfl_off = offsetof(struct tcphdr, th_flags);
	const bool usingmask = tf_mask != tf;

	/* X <- IP header length */
	fetch_l3(ctx, AF_UNSPEC, X_EQ_L4OFF);

	if ((ctx->flags & CHECKED_L4_PROTO) == 0) {
		const unsigned jf = usingmask ? 3 : 2;
		assert(ctx->ingroup == 0);

		/*
		 * A <- L4 protocol; A == TCP?  If not, jump out.
		 *
		 * Note: the TCP flag matching might be without 'proto tcp'
		 * when using a plain 'stateful' rule.  In such case it also
		 * handles other protocols, thus no strict TCP check.
		 */
		struct bpf_insn insns_tcp[] = {
			BPF_STMT(BPF_LD+BPF_W+BPF_MEM, BPF_MW_L4PROTO),
			BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, IPPROTO_TCP, 0, jf),
		};
		add_insns(ctx, insns_tcp, __arraycount(insns_tcp));
	}

	struct bpf_insn insns_tf[] = {
		/* A <- TCP flags */
		BPF_STMT(BPF_LD+BPF_B+BPF_IND, tcpfl_off),
	};
	add_insns(ctx, insns_tf, __arraycount(insns_tf));

	if (usingmask) {
		/* A <- (A & mask) */
		struct bpf_insn insns_mask[] = {
			BPF_STMT(BPF_ALU+BPF_AND+BPF_K, tf_mask),
		};
		add_insns(ctx, insns_mask, __arraycount(insns_mask));
	}

	struct bpf_insn insns_cmp[] = {
		/* A == expected-TCP-flags? */
		BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, tf, 0, JUMP_MAGIC),
	};
	add_insns(ctx, insns_cmp, __arraycount(insns_cmp));

	uint32_t mwords[] = { BM_TCPFL, 2, tf, tf_mask };
	done_block(ctx, mwords, sizeof(mwords));
}

/*
 * npfctl_bpf_icmp: code block to match ICMP type and/or code.
 * Note: suitable for both the ICMPv4 and ICMPv6.
 */
void
npfctl_bpf_icmp(npf_bpf_t *ctx, int type, int code)
{
	const u_int type_off = offsetof(struct icmp, icmp_type);
	const u_int code_off = offsetof(struct icmp, icmp_code);

	assert(ctx->flags & CHECKED_L4_PROTO);
	assert(offsetof(struct icmp6_hdr, icmp6_type) == type_off);
	assert(offsetof(struct icmp6_hdr, icmp6_code) == code_off);
	assert(type != -1 || code != -1);

	/* X <- IP header length */
	fetch_l3(ctx, AF_UNSPEC, X_EQ_L4OFF);

	if (type != -1) {
		struct bpf_insn insns_type[] = {
			BPF_STMT(BPF_LD+BPF_B+BPF_IND, type_off),
			BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, type, 0, JUMP_MAGIC),
		};
		add_insns(ctx, insns_type, __arraycount(insns_type));

		uint32_t mwords[] = { BM_ICMP_TYPE, 1, type };
		done_block(ctx, mwords, sizeof(mwords));
	}

	if (code != -1) {
		struct bpf_insn insns_code[] = {
			BPF_STMT(BPF_LD+BPF_B+BPF_IND, code_off),
			BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, code, 0, JUMP_MAGIC),
		};
		add_insns(ctx, insns_code, __arraycount(insns_code));

		uint32_t mwords[] = { BM_ICMP_CODE, 1, code };
		done_block(ctx, mwords, sizeof(mwords));
	}
}

#define	SRC_FLAG_BIT	(1U << 31)

/*
 * npfctl_bpf_table: code block to match source/destination IP address
 * against NPF table specified by ID.
 */
void
npfctl_bpf_table(npf_bpf_t *ctx, unsigned opts, unsigned tid)
{
	const bool src = (opts & MATCH_SRC) != 0;

	struct bpf_insn insns_table[] = {
		BPF_STMT(BPF_LD+BPF_IMM, (src ? SRC_FLAG_BIT : 0) | tid),
		BPF_STMT(BPF_MISC+BPF_COP, NPF_COP_TABLE),
		BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, 0, JUMP_MAGIC, 0),
	};
	add_insns(ctx, insns_table, __arraycount(insns_table));

	uint32_t mwords[] = { src ? BM_SRC_TABLE: BM_DST_TABLE, 1, tid };
	bm_invert_checkpoint(ctx, opts, NPF_RULE_LAYER_3);
	done_block(ctx, mwords, sizeof(mwords));
}